From 1c9a1ea9e690c46be1b2430e859b1d2f3bb28bad Mon Sep 17 00:00:00 2001 From: Han Xiaoyang Date: Sun, 13 Jul 2025 11:00:39 +0800 Subject: [PATCH 1/3] [scorpio]:mr layer fetch. Signed-off-by: Han Xiaoyang --- scorpio/src/daemon/mod.rs | 16 +++++++-- scorpio/src/fuse/mod.rs | 12 +++++-- scorpio/src/manager/mod.rs | 2 ++ scorpio/src/manager/mr.rs | 71 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 scorpio/src/manager/mr.rs diff --git a/scorpio/src/daemon/mod.rs b/scorpio/src/daemon/mod.rs index 5daa603e1..a3144dc31 100644 --- a/scorpio/src/daemon/mod.rs +++ b/scorpio/src/daemon/mod.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use crate::fuse::MegaFuse; use crate::manager::fetch::fetch; -use crate::manager::{ScorpioManager, WorkDir}; +use crate::manager::{mr, ScorpioManager, WorkDir}; use crate::util::{config, GPath}; use axum::extract::State; use axum::routing::{get, post}; @@ -18,6 +18,7 @@ const FAIL: &str = "Fail"; #[derive(Debug, Deserialize, Serialize)] struct MountRequest { path: String, + mr: Option, // mr is the mount request, used for buck2 temp mount. } #[derive(Debug, Deserialize, Serialize)] @@ -153,7 +154,7 @@ async fn mount_handler( }; let store_path = PathBuf::from(store_path).join(&temp_hash); - let _ = state.fuse.overlay_mount(inode, store_path).await; + let _ = state.fuse.overlay_mount(inode, store_path,false).await; let mount_info = MountInfo { hash: temp_hash.clone(), path: mono_path.clone(), @@ -171,12 +172,21 @@ async fn mount_handler( message: "Directory mounted successfully".to_string(), }); } + + if let Some(m) = &req.mr { + + // if mr is provided, we need to fetch the mr info from mono. + mr::build_mr_layer(m, store_path.into()).await.unwrap(); + + } + // fetch the dionary node info from mono. let work_dir = fetch(&mut ml, inode, mono_path).await.unwrap(); + let store_path = PathBuf::from(store_path).join(&work_dir.hash); // checkout / mount this dictionary. - let _ = state.fuse.overlay_mount(inode, store_path).await; + let _ = state.fuse.overlay_mount(inode, store_path,false).await; let mount_info = MountInfo { hash: work_dir.hash, diff --git a/scorpio/src/fuse/mod.rs b/scorpio/src/fuse/mod.rs index 8b24905d2..9c3112b80 100644 --- a/scorpio/src/fuse/mod.rs +++ b/scorpio/src/fuse/mod.rs @@ -70,7 +70,7 @@ impl MegaFuse { // mount user works. for dir in &manager.works { let _lower = PathBuf::from(store_path).join(&dir.hash); - megafuse.overlay_mount(dir.node, &_lower).await.unwrap(); + megafuse.overlay_mount(dir.node, &_lower,false).await.unwrap(); } megafuse @@ -92,10 +92,16 @@ impl MegaFuse { &self, inode: u64, store_path: P, + need_mr:bool, // if need mr, then create mr layer. ) -> std::io::Result<()> { let lower = store_path.as_ref().join("lower"); let upper = store_path.as_ref().join("upper"); - let lowerdir = vec![lower]; + let mut lowerdir = vec![lower]; + if need_mr { + let mr_path = store_path.as_ref().join("mr"); + let _ = std::fs::create_dir_all(&mr_path); + lowerdir.push(mr_path); + } let upperdir = upper; let config = config::Config { @@ -145,6 +151,8 @@ impl MegaFuse { Ok(()) } + + /// Unmounts the overlay filesystem associated with a given inode asynchronously. /// /// This function removes the overlay filesystem mapped to the specified inode from diff --git a/scorpio/src/manager/mod.rs b/scorpio/src/manager/mod.rs index 47a4bdf31..ca11a40e5 100644 --- a/scorpio/src/manager/mod.rs +++ b/scorpio/src/manager/mod.rs @@ -23,6 +23,8 @@ pub mod push; pub mod reset; pub mod status; pub mod store; +pub mod mr; + #[derive(Serialize, Deserialize)] pub struct ScorpioManager { // pub url:String, diff --git a/scorpio/src/manager/mr.rs b/scorpio/src/manager/mr.rs new file mode 100644 index 000000000..18278ecaa --- /dev/null +++ b/scorpio/src/manager/mr.rs @@ -0,0 +1,71 @@ +use std::path::PathBuf; + +use reqwest::Client; +use serde::Deserialize; + +use crate::util::config; + +/// Single file record +#[derive(Debug, Deserialize)] +struct FileInfo { + action: String, + path: String, + _sha: String, +} + +/// Response body for /files-list endpoint +#[derive(Debug, Deserialize)] +struct FilesListResp { + data: Vec, + err_message: String, + req_result: bool, +} + +pub async fn build_mr_layer( + link: &str, + _mr_path:PathBuf +) -> Result<(), reqwest::Error> { + let files_list = fetch_files_list(link).await?; + + if !files_list.req_result { + println!("{}", files_list.err_message); + return Ok(()); + } + + for file in files_list.data { + if file.action == "remove" { + // ! ! ! Attention: + // As for deleted files, the path should create a `whiteout file`, refer to create_whiteout in other files . + println!("{}", file.path); + } + } + //TODO: Implement the logic to build the MR layer + // This could involve checking out the MR branch, applying changes, etc. + // such as fetch::fetch_by_hashs(&files_list.data, &mr_path) + + println!("MR layer built for link: {link}"); + Ok(()) +} + +/// Fetch the list of files for a Merge Request (MR) +/// +/// - `link`: unique identifier for the MR (used in the path) +/// +/// Returns `FilesListResp` on success, or `reqwest::Error` on failure. +async fn fetch_files_list( + link: &str, +) -> Result { + let url = format!( + "{}/api/v1/mr/{}/files-list", + config::base_url(), + link + ); + + Client::new() + .get(&url) + .send() + .await? + .error_for_status()? + .json::() + .await +} From ca744b8235ba947f29ab9fe718d7a5571ca21b0a Mon Sep 17 00:00:00 2001 From: Xiaoyang Han Date: Mon, 14 Jul 2025 11:52:16 +0800 Subject: [PATCH 2/3] [mercury]:Annotations and minor refactoring of Git objects Signed-off-by: Xiaoyang Han --- mercury/src/hash.rs | 7 +++- mercury/src/internal/model/commit.rs | 42 +------------------ mercury/src/internal/object/blob.rs | 9 +++-- mercury/src/internal/object/commit.rs | 58 +++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 45 deletions(-) diff --git a/mercury/src/hash.rs b/mercury/src/hash.rs index e691290d1..82073a17e 100644 --- a/mercury/src/hash.rs +++ b/mercury/src/hash.rs @@ -97,7 +97,12 @@ impl SHA1 { let h = sha1::Sha1::digest(data); SHA1::from_bytes(h.as_slice()) } - + /// Create a Hash from the object type and data + /// This function is used to create a SHA1 hash from the object type and data. + /// It constructs a byte vector that includes the object type, the size of the data, + /// and the data itself, and then computes the SHA1 hash of this byte vector. + /// + /// Hash compute <- {Object Type}+{ }+{Object Size（before compress）}+{\x00}+{Object Content(before compress)} pub fn from_type_and_data(object_type: ObjectType, data: &[u8]) -> SHA1 { let mut d: Vec = Vec::new(); d.extend(object_type.to_data().unwrap()); diff --git a/mercury/src/internal/model/commit.rs b/mercury/src/internal/model/commit.rs index 7d1311274..e1ed7346f 100644 --- a/mercury/src/internal/model/commit.rs +++ b/mercury/src/internal/model/commit.rs @@ -1,53 +1,15 @@ -use std::str::FromStr; + use callisto::{git_commit, mega_commit}; use common::utils::generate_id; use crate::{ - hash::SHA1, internal::{ - object::{commit::Commit, signature::Signature, ObjectTrait}, + object::{commit::Commit,ObjectTrait}, pack::entry::Entry, }, }; -impl From for Commit { - fn from(value: mega_commit::Model) -> Self { - Commit { - id: SHA1::from_str(&value.commit_id).unwrap(), - tree_id: SHA1::from_str(&value.tree).unwrap(), - parent_commit_ids: value - .parents_id - .as_array() - .unwrap() - .iter() - .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap()) - .collect(), - author: Signature::from_data(value.author.unwrap().into()).unwrap(), - committer: Signature::from_data(value.committer.unwrap().into()).unwrap(), - message: value.content.unwrap(), - } - } -} - -impl From for Commit { - fn from(value: git_commit::Model) -> Self { - Commit { - id: SHA1::from_str(&value.commit_id).unwrap(), - tree_id: SHA1::from_str(&value.tree).unwrap(), - parent_commit_ids: value - .parents_id - .as_array() - .unwrap() - .iter() - .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap()) - .collect(), - author: Signature::from_data(value.author.unwrap().into()).unwrap(), - committer: Signature::from_data(value.committer.unwrap().into()).unwrap(), - message: value.content.unwrap(), - } - } -} impl From for mega_commit::Model { fn from(value: Commit) -> Self { diff --git a/mercury/src/internal/object/blob.rs b/mercury/src/internal/object/blob.rs index 7aa6dd4ae..fc61ae018 100644 --- a/mercury/src/internal/object/blob.rs +++ b/mercury/src/internal/object/blob.rs @@ -82,10 +82,11 @@ impl ObjectTrait for Blob { } impl Blob { + + /// Create a new Blob object from the given content string. + /// - This is a convenience method for creating a Blob from a string. + /// - It converts the string to bytes and then calls `from_content_bytes`. pub fn from_content(content: &str) -> Self { - // let blob_content = Cursor::new(utils::compress_zlib(content.as_bytes()).unwrap()); - // let mut buf = ReadBoxed::new(blob_content, ObjectType::Blob, content.len()); - // Blob::from_buf_read(&mut buf, content.len()) let content = content.as_bytes().to_vec(); Blob::from_content_bytes(content) } @@ -94,7 +95,7 @@ impl Blob { /// - some file content can't be represented as a string (UTF-8), so we need to use bytes. pub fn from_content_bytes(content: Vec) -> Self { Blob { - id: SHA1::from_type_and_data(ObjectType::Blob, &content), + id: SHA1::from_type_and_data(ObjectType::Blob, &content),// Calculate the SHA1 hash from the type and content data: content, } } diff --git a/mercury/src/internal/object/commit.rs b/mercury/src/internal/object/commit.rs index aed2dfcbf..30da91302 100644 --- a/mercury/src/internal/object/commit.rs +++ b/mercury/src/internal/object/commit.rs @@ -20,6 +20,8 @@ use crate::internal::object::signature::Signature; use crate::internal::object::ObjectTrait; use crate::internal::object::ObjectType; use bstr::ByteSlice; +use callisto::git_commit; +use callisto::mega_commit; use serde::Deserialize; use serde::Serialize; @@ -34,6 +36,7 @@ use serde::Serialize; /// - The author and committer fields contain the name, email address, timestamp and timezone. /// - The message field contains the commit message, which maybe include signed or DCO. #[derive(Eq, Debug, Clone, Serialize, Deserialize)] +#[non_exhaustive] pub struct Commit { pub id: SHA1, pub tree_id: SHA1, @@ -48,6 +51,8 @@ impl PartialEq for Commit { } } + + impl Display for Commit { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { writeln!(f, "tree: {}", self.tree_id)?; @@ -76,11 +81,24 @@ impl Commit { committer, message: message.to_string(), }; + // Calculate the hash of the commit object + // The hash is calculated from the type and data of the commit object let hash = SHA1::from_type_and_data(ObjectType::Commit, &commit.to_data().unwrap()); commit.id = hash; commit } + /// Creates a new commit object from a tree ID and a list of parent commit IDs. + /// This function generates the author and committer signatures using the current time + /// and a fixed email address. + /// It also sets the commit message to the provided string. + /// # Arguments + /// - `tree_id`: The SHA1 hash of the tree object that this commit points to. + /// - `parent_commit_ids`: A vector of SHA1 hashes of the parent commits. + /// - `message`: A string containing the commit message. + /// # Returns + /// A new `Commit` object with the specified tree ID, parent commit IDs, and commit message. + /// The author and committer signatures are generated using the current time and a fixed email address. pub fn from_tree_id(tree_id: SHA1, parent_commit_ids: Vec, message: &str) -> Commit { let author = Signature::from_data( format!( @@ -214,3 +232,43 @@ impl ObjectTrait for Commit { Ok(data) } } + + + +impl From for Commit { + fn from(value: mega_commit::Model) -> Self { + Commit { + id: SHA1::from_str(&value.commit_id).unwrap(), + tree_id: SHA1::from_str(&value.tree).unwrap(), + parent_commit_ids: value + .parents_id + .as_array() + .unwrap() + .iter() + .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap()) + .collect(), + author: Signature::from_data(value.author.unwrap().into()).unwrap(), + committer: Signature::from_data(value.committer.unwrap().into()).unwrap(), + message: value.content.unwrap(), + } + } +} + +impl From for Commit { + fn from(value: git_commit::Model) -> Self { + Commit { + id: SHA1::from_str(&value.commit_id).unwrap(), + tree_id: SHA1::from_str(&value.tree).unwrap(), + parent_commit_ids: value + .parents_id + .as_array() + .unwrap() + .iter() + .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap()) + .collect(), + author: Signature::from_data(value.author.unwrap().into()).unwrap(), + committer: Signature::from_data(value.committer.unwrap().into()).unwrap(), + message: value.content.unwrap(), + } + } +} \ No newline at end of file From 0ce6b7745a472cb1be2c43b1df46e7de50c78a0d Mon Sep 17 00:00:00 2001 From: Han Xiaoyang Date: Mon, 14 Jul 2025 20:32:36 +0800 Subject: [PATCH 3/3] [mercury]:Annotations and minor refactoring of Git objects Signed-off-by: Han Xiaoyang --- mercury/Cargo.toml | 1 + mercury/src/hash.rs | 6 +-- mercury/src/internal/index.rs | 4 +- mercury/src/internal/model/commit.rs | 2 +- mercury/src/internal/object/blob.rs | 3 +- mercury/src/internal/object/commit.rs | 68 +++++++++++++++------------ mercury/src/internal/pack/decode.rs | 6 +-- mercury/src/internal/pack/encode.rs | 4 +- 8 files changed, 52 insertions(+), 42 deletions(-) diff --git a/mercury/Cargo.toml b/mercury/Cargo.toml index 10056d255..460078837 100644 --- a/mercury/Cargo.toml +++ b/mercury/Cargo.toml @@ -37,6 +37,7 @@ encoding_rs = { workspace = true } rayon = { workspace = true } reqwest = { workspace = true } ring = { workspace = true } +serde_json.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["full"] } diff --git a/mercury/src/hash.rs b/mercury/src/hash.rs index 82073a17e..ae4a3ce78 100644 --- a/mercury/src/hash.rs +++ b/mercury/src/hash.rs @@ -218,7 +218,7 @@ mod tests { Ok(hash) => { assert_eq!(hash.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); } - Err(e) => println!("Error: {}", e), + Err(e) => println!("Error: {e}"), } } @@ -230,7 +230,7 @@ mod tests { Ok(hash) => { assert_eq!(hash.to_string(), "8ab686eafeb1f44702738c8b0f24f2567c36da6d"); } - Err(e) => println!("Error: {}", e), + Err(e) => println!("Error: {e}"), } } @@ -248,7 +248,7 @@ mod tests { ] ); } - Err(e) => println!("Error: {}", e), + Err(e) => println!("Error: {e}"), } } } diff --git a/mercury/src/internal/index.rs b/mercury/src/internal/index.rs index 04178b514..ecf8ced26 100644 --- a/mercury/src/internal/index.rs +++ b/mercury/src/internal/index.rs @@ -481,7 +481,7 @@ mod tests { let index = Index::from_file("../tests/data/index/index-760").unwrap(); assert_eq!(index.size(), 760); for (_, entry) in index.entries.iter() { - println!("{}", entry); + println!("{entry}"); } } @@ -499,6 +499,6 @@ mod tests { let hash = SHA1::from_bytes(&[0; 20]); let workdir = Path::new("../"); let entry = IndexEntry::new_from_file(file, hash, workdir).unwrap(); - println!("{}", entry); + println!("{entry}"); } } diff --git a/mercury/src/internal/model/commit.rs b/mercury/src/internal/model/commit.rs index e1ed7346f..5f9c24152 100644 --- a/mercury/src/internal/model/commit.rs +++ b/mercury/src/internal/model/commit.rs @@ -5,7 +5,7 @@ use common::utils::generate_id; use crate::{ internal::{ - object::{commit::Commit,ObjectTrait}, + object::{commit::Commit, ObjectTrait}, pack::entry::Entry, }, }; diff --git a/mercury/src/internal/object/blob.rs b/mercury/src/internal/object/blob.rs index fc61ae018..65531f75e 100644 --- a/mercury/src/internal/object/blob.rs +++ b/mercury/src/internal/object/blob.rs @@ -95,7 +95,8 @@ impl Blob { /// - some file content can't be represented as a string (UTF-8), so we need to use bytes. pub fn from_content_bytes(content: Vec) -> Self { Blob { - id: SHA1::from_type_and_data(ObjectType::Blob, &content),// Calculate the SHA1 hash from the type and content + // Calculate the SHA1 hash from the type and content + id: SHA1::from_type_and_data(ObjectType::Blob, &content), data: content, } } diff --git a/mercury/src/internal/object/commit.rs b/mercury/src/internal/object/commit.rs index 30da91302..107c97926 100644 --- a/mercury/src/internal/object/commit.rs +++ b/mercury/src/internal/object/commit.rs @@ -232,43 +232,51 @@ impl ObjectTrait for Commit { Ok(data) } } - - +fn commit_from_model( + commit_id: &str, + tree: &str, + parents_id: &serde_json::Value, + author: Option, + committer: Option, + message: Option, +) -> Commit { + Commit { + id: SHA1::from_str(commit_id).unwrap(), + tree_id: SHA1::from_str(tree).unwrap(), + parent_commit_ids: parents_id + .as_array() + .unwrap() + .iter() + .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap()) + .collect(), + author: Signature::from_data(author.unwrap().into()).unwrap(), + committer: Signature::from_data(committer.unwrap().into()).unwrap(), + message: message.unwrap(), + } +} impl From for Commit { fn from(value: mega_commit::Model) -> Self { - Commit { - id: SHA1::from_str(&value.commit_id).unwrap(), - tree_id: SHA1::from_str(&value.tree).unwrap(), - parent_commit_ids: value - .parents_id - .as_array() - .unwrap() - .iter() - .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap()) - .collect(), - author: Signature::from_data(value.author.unwrap().into()).unwrap(), - committer: Signature::from_data(value.committer.unwrap().into()).unwrap(), - message: value.content.unwrap(), - } + commit_from_model( + &value.commit_id, + &value.tree, + &value.parents_id, + value.author, + value.committer, + value.content, + ) } } impl From for Commit { fn from(value: git_commit::Model) -> Self { - Commit { - id: SHA1::from_str(&value.commit_id).unwrap(), - tree_id: SHA1::from_str(&value.tree).unwrap(), - parent_commit_ids: value - .parents_id - .as_array() - .unwrap() - .iter() - .map(|id| SHA1::from_str(id.as_str().unwrap()).unwrap()) - .collect(), - author: Signature::from_data(value.author.unwrap().into()).unwrap(), - committer: Signature::from_data(value.committer.unwrap().into()).unwrap(), - message: value.content.unwrap(), - } + commit_from_model( + &value.commit_id, + &value.tree, + &value.parents_id, + value.author, + value.committer, + value.content, + ) } } \ No newline at end of file diff --git a/mercury/src/internal/pack/decode.rs b/mercury/src/internal/pack/decode.rs index 0db2f2053..4b25feb5d 100644 --- a/mercury/src/internal/pack/decode.rs +++ b/mercury/src/internal/pack/decode.rs @@ -701,7 +701,7 @@ mod tests { #[tokio::test] async fn test_pack_check_header() { let res = crate::test_utils::setup_lfs_file().await; - println!("{:?}", res); + println!("{res:?}"); let source = res .get("git-2d187177923cd618a75da6c6db45bb89d92bd504.pack") .unwrap(); @@ -733,7 +733,7 @@ mod tests { assert_eq!(bytes_read, compressed_size); assert_eq!(decompressed_data, data); } - Err(e) => panic!("Decompression failed: {:?}", e), + Err(e) => panic!("Decompression failed: {e:?}"), } } @@ -803,7 +803,7 @@ mod tests { }); if let Err(e) = rt { fs::remove_dir_all(tmp).unwrap(); - panic!("Error: {:?}", e); + panic!("Error: {e:?}"); } } // it will be stuck on dropping `Pack` on Windows if `mem_size` is None, so we need `mimalloc` diff --git a/mercury/src/internal/pack/encode.rs b/mercury/src/internal/pack/encode.rs index c54564f8c..00d701d83 100644 --- a/mercury/src/internal/pack/encode.rs +++ b/mercury/src/internal/pack/encode.rs @@ -570,10 +570,10 @@ mod tests { let value = 16389; let data = encode_offset(value); - println!("{:?}", data); + println!("{data:?}"); let mut reader = Cursor::new(data); let (result, _) = read_offset_encoding(&mut reader).unwrap(); - println!("result: {}", result); + println!("result: {result}" ); assert_eq!(result, value as u64); } }