diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index 73677f33e..012b8741a 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -224,8 +224,8 @@ impl MonoApiService { self.update_parent_tree(path, tree_vec, commit) .await .unwrap(); - // remove refs start with path - storage.remove_refs(&mr.path).await.unwrap(); + // remove refs start with path exceprt mr type + storage.remove_none_mr_refs(&mr.path).await.unwrap(); // TODO: self.clean_dangling_commits().await; } // update mr @@ -315,79 +315,84 @@ impl MonoApiService { env::set_current_dir(&base_path).unwrap(); let clone_path = base_path.join(mr_link); if !fs::exists(&clone_path).unwrap() { - // fs::remove_dir_all(&clone_path).unwrap(); - Command::new("mkdir") - .arg(mr_link) - .output() - .await - .expect("Failed to mkdir"); - // cd mr - env::set_current_dir(&clone_path).unwrap(); - // libra init - Command::new("libra") - .arg("init") - .output() - .await - .expect("Failed to execute libra init"); - // libra remote add origin http://localhost:8000/project - let git_remote = if mr.path.starts_with("/") { - format!("{}{}", listen_addr, mr.path) - } else { - format!("{}/{}", listen_addr, mr.path) - }; - Command::new("libra") - .arg("remote") - .arg("add") - .arg("origin") - .arg(git_remote) - .output() - .await - .expect("Failed to execute libra remote add"); - // libra fetch origin refs/mr/PC0EPG1L - Command::new("libra") - .arg("fetch") - .arg("origin") - .arg(format!("refs/mr/{}", mr_link)) - .output() - .await - .expect("Failed to execute libra fetch"); - // libra branch PC0EPG1L origin/mr/PC0EPG1L - Command::new("libra") - .arg("branch") - .arg(mr_link) - .arg(format!("origin/mr/{}", mr_link)) - .output() - .await - .expect("Failed to execute libra branch"); - // libra switch PC0EPG1L - Command::new("libra") - .arg("switch") - .arg(mr_link) - .output() - .await - .expect("Failed to execute libra switch"); + let result = self.run_libra_diff(&mr, listen_addr, &clone_path).await; + if result.is_err() && fs::exists(&clone_path).unwrap() { + fs::remove_dir_all(&clone_path).unwrap(); + } } else { env::set_current_dir(&clone_path).unwrap(); } - // libra diff --old hash + tracing::debug!("Run libra Command: libra diff --old {}", mr.from_hash); let output = Command::new("libra") .arg("diff") .arg("--old") .arg(mr.from_hash) .output() - .await - .expect("Failed to execute libra diff"); - if output.status.success() { - return Ok(String::from_utf8_lossy(&output.stdout).to_string()); - } else { + .await?; + + let stderr_str = String::from_utf8_lossy(&output.stderr); + + if !stderr_str.trim().is_empty() || !output.status.success() { tracing::error!( "Command failed: {}", String::from_utf8_lossy(&output.stderr) ); + fs::remove_dir_all(&clone_path).unwrap(); + } else { + return Ok(String::from_utf8_lossy(&output.stdout).to_string()); } } Ok(String::new()) } + + async fn run_libra_diff( + &self, + mr: &callisto::mega_mr::Model, + listen_addr: &str, + clone_path: &PathBuf, + ) -> Result<(), anyhow::Error> { + Command::new("mkdir").arg(&mr.link).output().await?; + env::set_current_dir(clone_path).unwrap(); + Command::new("libra").arg("init").output().await?; + let git_remote = if mr.path.starts_with("/") { + format!("{}{}", listen_addr, mr.path) + } else { + format!("{}/{}", listen_addr, mr.path) + }; + tracing::debug!("Run libra Command: libra remote add origin {}", &git_remote); + Command::new("libra") + .arg("remote") + .arg("add") + .arg("origin") + .arg(git_remote) + .output() + .await?; + tracing::debug!("Run libra Command: libra fetch origin refs/mr/{}", &mr.link); + Command::new("libra") + .arg("fetch") + .arg("origin") + .arg(format!("refs/mr/{}", &mr.link)) + .output() + .await?; + tracing::debug!( + "Run libra Command: libra branch {} origin/mr/{}", + &mr.link, + &mr.link + ); + Command::new("libra") + .arg("branch") + .arg(&mr.link) + .arg(format!("origin/mr/{}", &mr.link)) + .output() + .await?; + tracing::debug!("Run libra Command: libra switch {}", &mr.link); + Command::new("libra") + .arg("switch") + .arg(&mr.link) + .output() + .await?; + Ok(()) + } } #[cfg(test)] diff --git a/ceres/src/pack/import_repo.rs b/ceres/src/pack/import_repo.rs index e8d02c765..4513d1c81 100644 --- a/ceres/src/pack/import_repo.rs +++ b/ceres/src/pack/import_repo.rs @@ -2,12 +2,18 @@ use std::{ collections::{HashMap, HashSet}, path::PathBuf, str::FromStr, - sync::atomic::{AtomicUsize, Ordering}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, }; use async_trait::async_trait; use futures::{future::join_all, StreamExt}; -use tokio::sync::mpsc::{self, UnboundedReceiver}; +use tokio::sync::{ + mpsc::{self, UnboundedReceiver}, + Semaphore, +}; use tokio_stream::wrappers::ReceiverStream; use callisto::{mega_tree, raw_blob, sea_orm_active_enums::RefTypeEnum}; @@ -58,20 +64,37 @@ impl PackHandler for ImportRepo { ) -> Result, GitError> { let storage = self.context.services.git_db_storage.clone(); let mut entry_list = vec![]; + let semaphore = Arc::new(Semaphore::new(8)); let mut join_tasks = vec![]; let repo_id = self.repo.repo_id; + while let Some(entry) = receiver.recv().await { entry_list.push(entry); - if entry_list.len() >= 10000 { + if entry_list.len() >= 1000 { let stg_clone = storage.clone(); + let acquired = semaphore.clone().acquire_owned().await.unwrap(); + let handle = tokio::spawn(async move { - stg_clone.save_entry(repo_id, entry_list).await.unwrap(); + let _acquired = acquired; + stg_clone.save_entry(repo_id, entry_list).await }); join_tasks.push(handle); entry_list = vec![]; } } - join_all(join_tasks).await; + let results = join_all(join_tasks).await; + for (i, res) in results.into_iter().enumerate() { + match res { + Ok(Ok(())) => {} + Ok(Err(e)) => { + tracing::error!("Task {} save_entry Err: {:?}", i, e); + } + Err(join_err) => { + tracing::error!("Task {} panic or cancle: {:?}", i, join_err); + } + } + } + storage.save_entry(repo_id, entry_list).await.unwrap(); self.attach_to_monorepo_parent().await.unwrap(); Ok(None) @@ -126,29 +149,28 @@ impl PackHandler for ImportRepo { } } - let mut blob_handler = vec![]; - for chunk in bids.chunks(10000) { + // let mut blob_handler = vec![]; + for chunk in bids.chunks(1000) { let raw_storage = raw_storage.clone(); let sender_clone = entry_tx.clone(); let chunk_clone = chunk.to_vec(); - let handler = tokio::spawn(async move { - let mut blob_stream = - raw_storage.get_raw_blobs_stream(chunk_clone).await.unwrap(); - while let Some(model) = blob_stream.next().await { - match model { - Ok(m) => { - // todo handle storage type - let b: Blob = m.into(); - let entry: Entry = b.into(); - sender_clone.send(entry).await.unwrap(); - } - Err(err) => eprintln!("Error: {:?}", err), + // let handler = tokio::spawn(async move { + let mut blob_stream = raw_storage.get_raw_blobs_stream(chunk_clone).await.unwrap(); + while let Some(model) = blob_stream.next().await { + match model { + Ok(m) => { + // TODO handle storage type + let b: Blob = m.into(); + let entry: Entry = b.into(); + sender_clone.send(entry).await.unwrap(); } + Err(err) => eprintln!("Error: {:?}", err), } - }); - blob_handler.push(handler); + } + // }); + // blob_handler.push(handler); } - join_all(blob_handler).await; + // join_all(blob_handler).await; tracing::info!("send blobs end"); let tags = storage.get_tags_by_repo_id(repo_id).await.unwrap(); diff --git a/ceres/src/pack/monorepo.rs b/ceres/src/pack/monorepo.rs index 1c73dc0b1..e67f37a44 100644 --- a/ceres/src/pack/monorepo.rs +++ b/ceres/src/pack/monorepo.rs @@ -48,7 +48,12 @@ impl PackHandler for MonoRepo { let storage = self.context.services.mono_storage.clone(); let result = storage.get_refs(self.path.to_str().unwrap()).await.unwrap(); - let refs = if !result.is_empty() { + + let heads_exist = result + .iter() + .any(|x| x.ref_name == common::utils::MEGA_BRANCH_NAME); + + let refs = if heads_exist { let refs: Vec = result.into_iter().map(|x| x.into()).collect(); refs } else { @@ -164,75 +169,7 @@ impl PackHandler for MonoRepo { // monorepo full pack should follow the shallow clone command 'git clone --depth=1' async fn full_pack(&self, want: Vec) -> Result>, GitError> { - let pack_config = &self.context.config.pack; - let storage = self.context.services.mono_storage.clone(); - let obj_num = AtomicUsize::new(0); - let mut trees = Vec::new(); - - let refs = storage - .get_ref(self.path.to_str().unwrap()) - .await - .unwrap() - .unwrap(); - let commit: Commit = storage - .get_commit_by_hash(&refs.ref_commit_hash) - .await - .unwrap() - .unwrap() - .into(); - let tree: Tree = storage - .get_tree_by_hash(&refs.ref_tree_hash) - .await - .unwrap() - .unwrap() - .into(); - trees.push(tree.clone()); - let mut exist_objs = HashSet::new(); - let mut counted_obj = HashSet::new(); - self.traverse_for_count(tree.clone(), &exist_objs, &mut counted_obj, &obj_num) - .await; - obj_num.fetch_add(1, Ordering::SeqCst); - - exist_objs.extend(counted_obj.clone()); - - let (entry_tx, entry_rx) = mpsc::channel(pack_config.channel_message_size); - let (stream_tx, stream_rx) = mpsc::channel(pack_config.channel_message_size); - - let want_c = want.first().unwrap(); - if refs.ref_commit_hash != *want_c { - let refs = storage - .get_ref_by_commit(self.path.to_str().unwrap(), want_c) - .await - .unwrap() - .unwrap(); - let commit: Commit = storage - .get_commit_by_hash(&refs.ref_commit_hash) - .await - .unwrap() - .unwrap() - .into(); - let tree: Tree = storage - .get_tree_by_hash(&refs.ref_tree_hash) - .await - .unwrap() - .unwrap() - .into(); - trees.push(tree.clone()); - self.traverse_for_count(tree, &exist_objs, &mut counted_obj, &obj_num) - .await; - obj_num.fetch_add(1, Ordering::SeqCst); - entry_tx.send(commit.into()).await.unwrap(); - } - - let encoder = PackEncoder::new(obj_num.into_inner(), 0, stream_tx); - encoder.encode_async(entry_rx).await.unwrap(); - let mut send_exist = HashSet::new(); - for tree in trees { - self.traverse(tree, &mut send_exist, Some(&entry_tx)).await; - } - entry_tx.send(commit.into()).await.unwrap(); - drop(entry_tx); - Ok(ReceiverStream::new(stream_rx)) + self.incremental_pack(want, Vec::new()).await } async fn incremental_pack( diff --git a/jupiter/migration/src/lib.rs b/jupiter/migration/src/lib.rs index d2cd9a190..7571a0ba9 100644 --- a/jupiter/migration/src/lib.rs +++ b/jupiter/migration/src/lib.rs @@ -3,6 +3,7 @@ use sea_orm_migration::schema::big_integer; mod m20250314_025943_init; mod m20250427_031332_add_mr_refs_tag; +mod m20250605_013340_alter_mega_mr_index; pub struct Migrator; @@ -12,6 +13,7 @@ impl MigratorTrait for Migrator { vec![ Box::new(m20250314_025943_init::Migration), Box::new(m20250427_031332_add_mr_refs_tag::Migration), + Box::new(m20250605_013340_alter_mega_mr_index::Migration), ] } } diff --git a/jupiter/migration/src/m20250605_013340_alter_mega_mr_index.rs b/jupiter/migration/src/m20250605_013340_alter_mega_mr_index.rs new file mode 100644 index 000000000..dde50cd27 --- /dev/null +++ b/jupiter/migration/src/m20250605_013340_alter_mega_mr_index.rs @@ -0,0 +1,39 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_index(Index::drop().name("idx_mr_path").to_owned()) + .await?; + + manager + .create_index( + Index::create() + .if_not_exists() + .name("idx_mr_path_link") + .unique() + .table(MegaMr::Table) + .col(MegaMr::Path) + .col(MegaMr::Link) + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, _: &SchemaManager) -> Result<(), DbErr> { + Ok(()) + } +} + +#[derive(DeriveIden)] +enum MegaMr { + Table, + Link, + Path, +} diff --git a/jupiter/src/storage/git_db_storage.rs b/jupiter/src/storage/git_db_storage.rs index cf2eae787..f9bfaae2b 100644 --- a/jupiter/src/storage/git_db_storage.rs +++ b/jupiter/src/storage/git_db_storage.rs @@ -162,21 +162,11 @@ impl GitDbStorage { let git_objects = Arc::try_unwrap(git_objects) .expect("Failed to unwrap Arc") .into_inner(); - batch_save_model(self.get_connection(), git_objects.commits) - .await - .unwrap(); - batch_save_model(self.get_connection(), git_objects.trees) - .await - .unwrap(); - batch_save_model(self.get_connection(), git_objects.blobs) - .await - .unwrap(); - batch_save_model(self.get_connection(), git_objects.raw_blobs) - .await - .unwrap(); - batch_save_model(self.get_connection(), git_objects.tags) - .await - .unwrap(); + batch_save_model(self.get_connection(), git_objects.commits).await?; + batch_save_model(self.get_connection(), git_objects.trees).await?; + batch_save_model(self.get_connection(), git_objects.blobs).await?; + batch_save_model(self.get_connection(), git_objects.raw_blobs).await?; + batch_save_model(self.get_connection(), git_objects.tags).await?; Ok(()) } diff --git a/jupiter/src/storage/mod.rs b/jupiter/src/storage/mod.rs index 8433f3a5a..dc5d4a84d 100644 --- a/jupiter/src/storage/mod.rs +++ b/jupiter/src/storage/mod.rs @@ -50,7 +50,7 @@ where .await } -pub async fn batch_save_model_with_conflict( +async fn batch_save_model_with_conflict( connection: &impl ConnectionTrait, save_models: Vec, onconflict: OnConflict, @@ -59,14 +59,12 @@ where E: EntityTrait, A: ActiveModelTrait + From<::Model> + Send, { - let mut results = Vec::new(); - for chunk in save_models.chunks(1000) { - // notice that sqlx not support packets larger than 16MB now - let res = E::insert_many(chunk.iter().cloned()) + // notice that sqlx not support packets larger than 16MB now + let futures = save_models.chunks(1000).map(|chunk| { + E::insert_many(chunk.iter().cloned()) .on_conflict(onconflict.clone()) - .exec(connection); - results.push(res); - } - futures::future::join_all(results).await; + .exec(connection) + }); + futures::future::try_join_all(futures).await?; Ok(()) } diff --git a/jupiter/src/storage/mono_storage.rs b/jupiter/src/storage/mono_storage.rs index 9bc2aaf81..63f186756 100644 --- a/jupiter/src/storage/mono_storage.rs +++ b/jupiter/src/storage/mono_storage.rs @@ -71,9 +71,10 @@ impl MonoStorage { Ok(()) } - pub async fn remove_refs(&self, path: &str) -> Result<(), MegaError> { + pub async fn remove_none_mr_refs(&self, path: &str) -> Result<(), MegaError> { mega_refs::Entity::delete_many() .filter(mega_refs::Column::Path.starts_with(path)) + .filter(mega_refs::Column::IsMr.eq(false)) .exec(self.get_connection()) .await?; Ok(())