From 893470a54ecdf8bfb0c17f656f1054a79c687106 Mon Sep 17 00:00:00 2001 From: yyjeqhc <1772413353@qq.com> Date: Tue, 15 Jul 2025 19:37:14 +0800 Subject: [PATCH] complete MR layer creation --- jupiter/src/service/mod.rs | 1 - libra/src/command/commit.rs | 7 +-- libra/tests/command/log_test.rs | 20 +++--- mercury/src/internal/model/commit.rs | 11 +--- mercury/src/internal/object/blob.rs | 1 - mercury/src/internal/object/commit.rs | 4 +- mercury/src/internal/pack/encode.rs | 2 +- scorpio/src/daemon/mod.rs | 18 ++++-- scorpio/src/fuse/mod.rs | 9 +-- scorpio/src/manager/fetch.rs | 23 +++++++ scorpio/src/manager/mod.rs | 2 +- scorpio/src/manager/mr.rs | 90 ++++++++++++++++++++------- 12 files changed, 128 insertions(+), 60 deletions(-) diff --git a/jupiter/src/service/mod.rs b/jupiter/src/service/mod.rs index d877ba631..e24d71f36 100644 --- a/jupiter/src/service/mod.rs +++ b/jupiter/src/service/mod.rs @@ -2,4 +2,3 @@ pub mod issue_service; pub mod mr_service; - diff --git a/libra/src/command/commit.rs b/libra/src/command/commit.rs index 35e5b4981..ac078d1b4 100644 --- a/libra/src/command/commit.rs +++ b/libra/src/command/commit.rs @@ -58,14 +58,14 @@ pub async fn execute(args: CommitArgs) { let user_email = UserConfig::get("user", None, "email") .await .unwrap_or_else(|| "unknown".to_string()); - + // get sign line let signoff_line = format!("Signed-off-by: {user_name} <{user_email}>"); format!("{}\n\n{signoff_line}", args.message) } else { args.message.clone() }; - + // check format(if needed) if args.conventional && !check_conventional_commits_message(&commit_message) { panic!("fatal: commit message does not follow conventional commits"); @@ -270,11 +270,10 @@ mod test { let args = args.unwrap(); assert!(args.amend); assert!(args.signoff); - } #[tokio::test] - #[serial] + #[serial] /// Tests the recursive tree creation from index entries. /// Verifies that tree objects are correctly created, saved to storage, and properly organized in a hierarchical structure. async fn test_create_tree() { diff --git a/libra/tests/command/log_test.rs b/libra/tests/command/log_test.rs index 68ac14309..bf189fb8a 100644 --- a/libra/tests/command/log_test.rs +++ b/libra/tests/command/log_test.rs @@ -132,32 +132,32 @@ async fn test_log_oneline() { let temp_path = tempdir().unwrap(); test::setup_with_new_libra_in(temp_path.path()).await; let _guard = ChangeDirGuard::new(temp_path.path()); - + // Create test commits let commit_id = create_test_commit_tree().await; let reachable_commits = get_reachable_commits(commit_id).await; - + // Test oneline format - let args = LogArgs { - number: Some(3), - oneline: true + let args = LogArgs { + number: Some(3), + oneline: true, }; - + // Since execute function writes to stdout, we'll test the logic directly let mut sorted_commits = reachable_commits.clone(); sorted_commits.sort_by(|a, b| b.committer.timestamp.cmp(&a.committer.timestamp)); - + let max_commits = std::cmp::min(args.number.unwrap_or(usize::MAX), sorted_commits.len()); - + for (i, commit) in sorted_commits.iter().take(max_commits).enumerate() { // Test short hash format (should be 7 characters) let short_hash = &commit.id.to_string()[..7]; assert_eq!(short_hash.len(), 7); - + // Test that commit message parsing works let (msg, _) = common::utils::parse_commit_msg(&commit.message); assert!(!msg.is_empty()); - + // For our test commits, verify the expected format let expected_number = 6 - i; // commits are numbered 6, 5, 4, 3, 2, 1 assert_eq!(msg.trim(), format!("Commit_{expected_number}")); diff --git a/mercury/src/internal/model/commit.rs b/mercury/src/internal/model/commit.rs index 5f9c24152..45524bf2d 100644 --- a/mercury/src/internal/model/commit.rs +++ b/mercury/src/internal/model/commit.rs @@ -1,16 +1,11 @@ - - use callisto::{git_commit, mega_commit}; use common::utils::generate_id; -use crate::{ - internal::{ - object::{commit::Commit, ObjectTrait}, - pack::entry::Entry, - }, +use crate::internal::{ + object::{commit::Commit, ObjectTrait}, + pack::entry::Entry, }; - impl From for mega_commit::Model { fn from(value: Commit) -> Self { mega_commit::Model { diff --git a/mercury/src/internal/object/blob.rs b/mercury/src/internal/object/blob.rs index 65531f75e..eb2c7af21 100644 --- a/mercury/src/internal/object/blob.rs +++ b/mercury/src/internal/object/blob.rs @@ -82,7 +82,6 @@ impl ObjectTrait for Blob { } impl Blob { - /// Create a new Blob object from the given content string. /// - This is a convenience method for creating a Blob from a string. /// - It converts the string to bytes and then calls `from_content_bytes`. diff --git a/mercury/src/internal/object/commit.rs b/mercury/src/internal/object/commit.rs index 107c97926..692827353 100644 --- a/mercury/src/internal/object/commit.rs +++ b/mercury/src/internal/object/commit.rs @@ -51,8 +51,6 @@ impl PartialEq for Commit { } } - - impl Display for Commit { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { writeln!(f, "tree: {}", self.tree_id)?; @@ -279,4 +277,4 @@ impl From for Commit { value.content, ) } -} \ No newline at end of file +} diff --git a/mercury/src/internal/pack/encode.rs b/mercury/src/internal/pack/encode.rs index 00d701d83..fc0cbf530 100644 --- a/mercury/src/internal/pack/encode.rs +++ b/mercury/src/internal/pack/encode.rs @@ -573,7 +573,7 @@ mod tests { println!("{data:?}"); let mut reader = Cursor::new(data); let (result, _) = read_offset_encoding(&mut reader).unwrap(); - println!("result: {result}" ); + println!("result: {result}"); assert_eq!(result, value as u64); } } diff --git a/scorpio/src/daemon/mod.rs b/scorpio/src/daemon/mod.rs index a3144dc31..740bd15ee 100644 --- a/scorpio/src/daemon/mod.rs +++ b/scorpio/src/daemon/mod.rs @@ -154,7 +154,7 @@ async fn mount_handler( }; let store_path = PathBuf::from(store_path).join(&temp_hash); - let _ = state.fuse.overlay_mount(inode, store_path,false).await; + let _ = state.fuse.overlay_mount(inode, store_path, false).await; let mount_info = MountInfo { hash: temp_hash.clone(), path: mono_path.clone(), @@ -174,10 +174,15 @@ async fn mount_handler( } if let Some(m) = &req.mr { - + let mr_store_path = PathBuf::from(store_path).join("mr"); // if mr is provided, we need to fetch the mr info from mono. - mr::build_mr_layer(m, store_path.into()).await.unwrap(); - + if let Err(e) = mr::build_mr_layer(m, mr_store_path).await { + return axum::Json(MountResponse { + status: FAIL.into(), + mount: MountInfo::default(), + message: format!("Failed to build mr layer: {e}"), + }); + } } // fetch the dionary node info from mono. @@ -186,7 +191,10 @@ async fn mount_handler( let store_path = PathBuf::from(store_path).join(&work_dir.hash); // checkout / mount this dictionary. - let _ = state.fuse.overlay_mount(inode, store_path,false).await; + let _ = state + .fuse + .overlay_mount(inode, store_path, req.mr.is_some()) + .await; let mount_info = MountInfo { hash: work_dir.hash, diff --git a/scorpio/src/fuse/mod.rs b/scorpio/src/fuse/mod.rs index 9c3112b80..cd8bb9183 100644 --- a/scorpio/src/fuse/mod.rs +++ b/scorpio/src/fuse/mod.rs @@ -70,7 +70,10 @@ impl MegaFuse { // mount user works. for dir in &manager.works { let _lower = PathBuf::from(store_path).join(&dir.hash); - megafuse.overlay_mount(dir.node, &_lower,false).await.unwrap(); + megafuse + .overlay_mount(dir.node, &_lower, false) + .await + .unwrap(); } megafuse @@ -92,7 +95,7 @@ impl MegaFuse { &self, inode: u64, store_path: P, - need_mr:bool, // if need mr, then create mr layer. + need_mr: bool, // if need mr, then create mr layer. ) -> std::io::Result<()> { let lower = store_path.as_ref().join("lower"); let upper = store_path.as_ref().join("upper"); @@ -151,8 +154,6 @@ impl MegaFuse { Ok(()) } - - /// Unmounts the overlay filesystem associated with a given inode asynchronously. /// /// This function removes the overlay filesystem mapped to the specified inode from diff --git a/scorpio/src/manager/fetch.rs b/scorpio/src/manager/fetch.rs index d088cd947..0d2ec44cc 100644 --- a/scorpio/src/manager/fetch.rs +++ b/scorpio/src/manager/fetch.rs @@ -328,6 +328,29 @@ pub fn enqueue_file_download(file_id: SHA1, save_path: PathBuf) { } } +/// Download multiple files for MR scenarios. +pub async fn download_mr_files( + files: Vec<(SHA1, PathBuf)>, +) -> Result<(), Box> { + let download_manager = DownloadManager::get_global(); + + // Since this is for MR files only (no directory traversal), + // we mark directory processing as complete immediately + download_manager.notify_directory_processing_complete(); + + // Enqueue all file download tasks + for (file_id, save_path) in files { + let task = DownloadTask::new(file_id, save_path); + if let Err(e) = download_manager.enqueue_download(task) { + return Err(format!("Failed to enqueue download task for file {file_id}: {e}").into()); + } + } + + // Wait for all downloads to complete + download_manager.wait_for_completion().await; + + Ok(()) +} #[allow(unused)] #[async_trait] pub trait CheckHash { diff --git a/scorpio/src/manager/mod.rs b/scorpio/src/manager/mod.rs index ca11a40e5..f1e92df2c 100644 --- a/scorpio/src/manager/mod.rs +++ b/scorpio/src/manager/mod.rs @@ -19,11 +19,11 @@ pub mod add; pub mod commit; pub mod diff; pub mod fetch; +pub mod mr; pub mod push; pub mod reset; pub mod status; pub mod store; -pub mod mr; #[derive(Serialize, Deserialize)] pub struct ScorpioManager { diff --git a/scorpio/src/manager/mr.rs b/scorpio/src/manager/mr.rs index 18278ecaa..ceed7660c 100644 --- a/scorpio/src/manager/mr.rs +++ b/scorpio/src/manager/mr.rs @@ -3,46 +3,98 @@ use std::path::PathBuf; use reqwest::Client; use serde::Deserialize; +use crate::manager::fetch::download_mr_files; use crate::util::config; /// Single file record #[derive(Debug, Deserialize)] struct FileInfo { action: String, - path: String, - _sha: String, + path: String, + sha: String, } /// Response body for /files-list endpoint #[derive(Debug, Deserialize)] struct FilesListResp { - data: Vec, + data: Vec, err_message: String, - req_result: bool, + req_result: bool, } pub async fn build_mr_layer( link: &str, - _mr_path:PathBuf -) -> Result<(), reqwest::Error> { + mr_path: PathBuf, +) -> Result<(), Box> { let files_list = fetch_files_list(link).await?; if !files_list.req_result { println!("{}", files_list.err_message); return Ok(()); } + if !mr_path.exists() { + std::fs::create_dir_all(&mr_path)?; + } + + // Collect all files that need to be downloaded + let mut download_files = Vec::new(); for file in files_list.data { - if file.action == "remove" { - // ! ! ! Attention: - // As for deleted files, the path should create a `whiteout file`, refer to create_whiteout in other files . - println!("{}", file.path); + let relative_path = file.path.strip_prefix('/').unwrap_or(&file.path); + let file_path = mr_path.join(relative_path); + + match file.action.as_str() { + "new" | "modified" => { + if let Some(parent) = file_path.parent() { + std::fs::create_dir_all(parent)?; + } + // Parse SHA1 from string and add to download queue + let file_id = file.sha.parse().expect("Invalid SHA1 format"); + download_files.push((file_id, file_path)); + } + "deleted" => { + // Create whiteout file for deleted files in overlay filesystem + // Whiteout files are character devices with 0/0 device number + if let Some(parent) = file_path.parent() { + std::fs::create_dir_all(parent)?; + let whiteout_path = parent.join(file_path.file_name().unwrap()); + + // Create whiteout as character device with 0/0 device number + // This is the standard way overlay filesystems recognize deleted files + let mode = libc::S_IFCHR | 0o777; + let dev = libc::makedev(0, 0); + + let whiteout_path_cstr = + std::ffi::CString::new(whiteout_path.to_string_lossy().as_bytes())?; + + let result = unsafe { libc::mknod(whiteout_path_cstr.as_ptr(), mode, dev) }; + + if result != 0 { + let err = std::io::Error::last_os_error(); + eprintln!("Failed to create whiteout file {}: {}", file.path, err); + } + } + + // Reserved for future use when overlay mount supports recognizing `.wh.filename` files + // if let Some(parent) = file_path.parent() { + // std::fs::create_dir_all(parent)?; + // let whiteout_name = + // format!(".wh.{}", file_path.file_name().unwrap().to_string_lossy()); + // let whiteout_path = parent.join(whiteout_name); + // std::fs::File::create(whiteout_path)?; + // } + } + _ => { + println!("Unknown action: {}", file.action); + } } } - //TODO: Implement the logic to build the MR layer - // This could involve checking out the MR branch, applying changes, etc. - // such as fetch::fetch_by_hashs(&files_list.data, &mr_path) - + + // Download all files concurrently + if !download_files.is_empty() { + download_mr_files(download_files).await?; + } + println!("MR layer built for link: {link}"); Ok(()) } @@ -52,14 +104,8 @@ pub async fn build_mr_layer( /// - `link`: unique identifier for the MR (used in the path) /// /// Returns `FilesListResp` on success, or `reqwest::Error` on failure. -async fn fetch_files_list( - link: &str, -) -> Result { - let url = format!( - "{}/api/v1/mr/{}/files-list", - config::base_url(), - link - ); +async fn fetch_files_list(link: &str) -> Result { + let url = format!("{}/api/v1/mr/{}/files-list", config::base_url(), link); Client::new() .get(&url)