From 39137e845481b7754f44e237adbab80be9426722 Mon Sep 17 00:00:00 2001 From: kjx52 Date: Mon, 19 May 2025 17:51:59 +0800 Subject: [PATCH 1/4] [Scorpio]: Update commit Signed-off-by: kjx52 --- scorpio/src/manager/commit.rs | 468 +++++++++++++++++----------------- scorpio/src/manager/mod.rs | 16 +- 2 files changed, 239 insertions(+), 245 deletions(-) diff --git a/scorpio/src/manager/commit.rs b/scorpio/src/manager/commit.rs index 4c9051a1a..4ee9526e2 100644 --- a/scorpio/src/manager/commit.rs +++ b/scorpio/src/manager/commit.rs @@ -1,5 +1,7 @@ use mercury::hash::SHA1; use mercury::internal::object::tree::{Tree, TreeItem, TreeItemMode}; +use std::collections::hash_map::Entry; +use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -7,225 +9,228 @@ use crate::manager::store::{TempStoreArea, TreeStore}; use super::store::ModifiedStore; -fn update_treeitem(current: &mut Tree, name: &str, hash: SHA1, item_mode: TreeItemMode) { - match current.tree_items.iter_mut().find(|i| i.name == name) { - Some(item) => { - item.id = hash; - item.mode = item_mode; - } - None => current.tree_items.push(TreeItem { - mode: item_mode, - id: hash, - name: name.to_string(), - }), - } +/// Color Extraction Macro +#[cfg(debug_assertions)] +macro_rules! color_info { + ($($arg:tt)*) => { + println!("[\x1b[34mINFO\x1b[0m] {}", format!($($arg)*)); + }; } -/// This function is used to recursively use a path to update -/// its corresponding TreeItem and all its parent Tree objects. -fn update_tree_with_index_path( - commit_db: &sled::Db, - current: &mut Tree, - index_path: &Path, - blob_hash: SHA1, - child_path: &mut PathBuf, - old_root_path: &Path, -) -> sled::Result<()> { - let mut components = index_path - .strip_prefix(&child_path) - .unwrap() - .components() - .peekable(); - - // Loop to iterate over the path. - if let Some(comp) = components.next() { - let name = comp.as_os_str().to_string_lossy().to_string(); - - // Since the temporary storage area stores files, the - // last item of the iterator is the file name. - if components.peek().is_none() { - println!(" [\x1b[34mINFO\x1b[0m] Last one."); - println!( - " [\x1b[33mDEBUG\x1b[0m] comp = {}", - comp.as_os_str().to_string_lossy() - ); - - // If the TreeItem already exists, update its Hash - // and TreeItemMode, otherwise create a new one. - update_treeitem(current, &name, blob_hash, TreeItemMode::Blob); - } else { - // Extract child path. - child_path.push(comp); - let real_child_path = old_root_path.join(&child_path); - println!( - " [\x1b[33mDEBUG\x1b[0m] child_path = {}", - child_path.display() - ); - - // Extract the subtree from the database using the - // subpath, creating a new one if it does not exist. - let mut subtree = commit_db.get_bypath(&real_child_path).unwrap_or(Tree { - id: SHA1::default(), - tree_items: Vec::new(), - }); - - println!( - " [\x1b[33mDEBUG\x1b[0m] {}.id = {}", - child_path.display(), - subtree.id._to_string() - ); - println!( - " [\x1b[33mDEBUG\x1b[0m] {}.tree_items.len() = {}", - child_path.display(), - subtree.tree_items.len() - ); - - // Recursively call the update_tree_with_blob_path - // function to enter the next level of directory. - update_tree_with_index_path( - commit_db, - &mut subtree, - index_path, - blob_hash, - child_path, - old_root_path, - )?; - println!( - " [\x1b[33mDEBUG\x1b[0m] {}.tree_items.len() = {}", - index_path.strip_prefix(&child_path).unwrap().display(), - subtree.tree_items.len() - ); - subtree - .tree_items - .iter() - .for_each(|tmp| print!("\t{}", tmp.name)); - println!(); +/// Auxiliary function, extracts the `file_name` of Path and +/// converts it to String type. +/// +/// Error: When encountering the root directory or other +/// error conditions, the root directory is also returned. +fn path_name_to_string(path: &Path) -> String { + path.file_name() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| "/".to_string()) +} - // Use the rehash function to update the Hash value - // of the subtree. - subtree.rehash(); +/// Auxiliary function, Concatenate path and `old_root_path` +/// to return the real path. +/// +/// Error: When new_path is `Path::new("")` or +/// `Path::new("/")`, returns old_root_path +fn get_real_path(path: &Path, old_root_path: &Path) -> PathBuf { + match path.as_os_str().is_empty() || path.eq(Path::new("/")) { + // The root path + true => old_root_path.to_path_buf(), + // Sub path + false => old_root_path.join(path), + } +} - // If the TreeItem already exists, update its Hash, - // otherwise create a new one. - update_treeitem(current, &name, subtree.id, TreeItemMode::Tree); +/// Auxiliary function, sort the `tree_items` in the Tree by +/// the first letter. +fn sort_tree_items(tree: &mut Tree) { + tree.tree_items + .sort_by(|item1, item2| item1.name.cmp(&item2.name)); +} - // Write the new Tree to the Db. - commit_db.insert_tree(real_child_path, subtree); - } +/// Use `item_data` to update the `tree` and return the tree +/// ID. +fn update_treeitem(tree: &mut Tree, item_data: &TreeItem) -> SHA1 { + match tree + .tree_items + .iter_mut() + .find(|i| i.name == item_data.name) + { + Some(item) => *item = item_data.clone(), + None => tree.tree_items.push(item_data.clone()), } - Ok(()) + sort_tree_items(tree); + // Rehash + tree.rehash(); + tree.id } -/// This function is used to delete the whiteout file and recursively -/// update the main Tree. -fn update_tree_with_rm_path( - commit_db: &sled::Db, - parent_path: &Path, +/// Read the new Tree into a HashMap +fn build_new_tree_map( + index_db: &sled::Db, + old_tree_db: &sled::Db, old_root_path: &Path, -) -> sled::Result { - let mut name = parent_path - .file_name() - .unwrap() - .to_string_lossy() - .to_string(); - - let mut parent_path = parent_path; - - while let Some(tmp_path) = parent_path.parent().filter(|&path| !path.eq(Path::new(""))) { - parent_path = tmp_path; - let real_parent_path = &old_root_path.join(parent_path); - // Extract the parent Tree object and filter out the TreeItem - // corresponding to the current path. - let mut parent_tree = commit_db.get_bypath(real_parent_path)?; - parent_tree.tree_items.retain(|item| item.name != name); - - // If the current Tree object no longer contains any TreeItem, - // the current Tree will be deleted directly and the next loop - // will be entered. - if parent_tree.tree_items.is_empty() { - commit_db.remove(real_parent_path.to_string_lossy().as_bytes())?; - continue; +) -> sled::Result> { + let mut res = HashMap::::new(); + + for (mut new_path, hash) in index_db.db_list()? { + // Create the Blob TreeItem. + let mut sub_item_data = TreeItem { + mode: TreeItemMode::Blob, + id: SHA1::from_str(&hash).expect("Hash parsing error in temporary storage area."), + name: path_name_to_string(&new_path), + }; + while new_path.pop() { + // println!("new_path = {}", new_path.display()); + let parent_path = get_real_path(&new_path, old_root_path); + // println!("parent_path = {}", parent_path.display()); + // println!("old_root_path = {}", old_root_path.display()); + // println!("res = {:?}", old_tree_db.get_bypath(&parent_path)); + // println!("res = {:?}", old_tree_db.get_bypath(&old_root_path)); + let mut sub_item_id = SHA1::default(); + + // Check, add and update the Tree into the HashMap + res.entry(parent_path.clone()) + // The old Tree has been read into the HashMap, update it + .and_modify(|tree| sub_item_id = update_treeitem(tree, &sub_item_data)) + // The Tree is not exist, check the old_tree_db + .or_insert_with(|| { + let mut tree = match old_tree_db.get_bypath(&parent_path) { + // The old Tree exists, update it. + Ok(tree) => { + #[cfg(debug_assertions)] + color_info!("Old Tree: \x1b[1;32m{}\x1b[0m", parent_path.display()); + tree + } + // The old Tree is not exist, create it. + Err(_) => { + #[cfg(debug_assertions)] + color_info!("New Tree: \x1b[1;32m{}\x1b[0m", parent_path.display()); + Tree { + id: SHA1::default(), + tree_items: Vec::new(), + } + } + }; + // Update the new TreeItem + sub_item_id = update_treeitem(&mut tree, &sub_item_data); + + tree + }); + + // Create a TreeItem for the current Tree + // and pass it up. + sub_item_data = TreeItem { + mode: TreeItemMode::Tree, + id: sub_item_id, + name: path_name_to_string(&parent_path), + }; } + #[cfg(debug_assertions)] + show_hashmap(&res); + } - // Otherwise update the Tree object and write to Db. - parent_tree.rehash(); - commit_db.insert_tree(real_parent_path.to_path_buf(), parent_tree); + Ok(res) +} - // When the screening is complete, update name. - name = parent_path - .file_name() - .unwrap() - .to_string_lossy() - .to_string(); +/// Check `sub_item_data` and perform delete and update on +/// `tree`. If the tree is not empty after the operation, +/// return `Some(tree.id)`, otherwise return `None`. +fn del_treeitem(tree: &mut Tree, sub_item_data: (String, Option)) -> Option { + let sub_item_name = sub_item_data.0; + match sub_item_data.1 { + Some(hash) => match tree.tree_items.iter_mut().find(|i| i.name == sub_item_name) { + Some(item) => { + item.id = hash; + } + None => panic!("Unexpected index, the rm TreeItem object not found"), + }, + None => tree.tree_items.retain(|name| name.name != sub_item_name), } - Ok(name) + if tree.tree_items.is_empty() { + None + } else { + // Rehash + tree.rehash(); + Some(tree.id) + } } -/// This function is a further encapsulation of "updating -/// the Tree object using path->hash key-value pairs". -fn update_with_index( - index_db: &sled::Db, - commit_db: &sled::Db, - root_tree: &mut Tree, +/// Read the removed files into a HashMap +fn build_removed_tree_map( + rm_db: &sled::Db, + old_tree_db: &sled::Db, + res: &mut HashMap, old_root_path: &Path, ) -> sled::Result<()> { - // Traverse all key-value pairs in the library and sort them - // by length to ensure depth priority. - let mut index_staged: Vec<_> = index_db.iter().collect::, _>>()?; - index_staged.sort_by_key(|(k, _)| k.len()); - let index_staged = index_staged; - - // Call the update_tree_with_blob_path function for all - // (path, hash) tuples to update the main Tree. - for (key_bytes, hash_bytes) in index_staged.iter() { - let index_path: PathBuf = PathBuf::from(String::from_utf8_lossy(key_bytes).to_string()); - let blob_hash: SHA1 = - SHA1::from_str(String::from_utf8(hash_bytes.to_vec()).unwrap().as_str()).unwrap(); - let mut child_path = PathBuf::new(); - - println!( - "[\x1b[33mDEBUG\x1b[0m] index_path = {}", - index_path.display() - ); - println!( - "[\x1b[33mDEBUG\x1b[0m] blob_hash = {}", - blob_hash._to_string() - ); - - update_tree_with_index_path( - commit_db, - root_tree, - &index_path, - blob_hash, - &mut child_path, - old_root_path, - )?; + // Use the HashMap to store the Path Tree + // It is expected that using HashSet to replace Vec will bring the following benefits: + // 1. Improved performance + // 2. Simplified steps + // 3. Increased operability and flexibility + for mut new_path in rm_db.path_list()? { + // Get the TreeItem name. + let mut sub_item_data: (String, Option) = (path_name_to_string(&new_path), None); + + while new_path.pop() { + let parent_path = get_real_path(&new_path, old_root_path); + + // Use the `entry` API to avoid multiple lookups + let entry = res.entry(parent_path.clone()); + // Check and del the Tree into the HashMap + sub_item_data = ( + path_name_to_string(&parent_path), + match entry { + // The old Tree has been read into the HashMap, update it + Entry::Occupied(mut occupied) => { + if let Some(hash) = del_treeitem(occupied.get_mut(), sub_item_data) { + Some(hash) + } else { + occupied.remove(); + None + } + } + // The Tree is not exist, check the old_tree_db + Entry::Vacant(vacant) => { + #[cfg(debug_assertions)] + color_info!("Old Tree: \x1b[1;32m{}\x1b[0m", parent_path.display()); + let mut tree = old_tree_db + .get_bypath(&parent_path) + .expect("Unexpected index, the rm Tree object not found"); + if let Some(hash) = del_treeitem(&mut tree, sub_item_data) { + let _ = vacant.insert(tree); + Some(hash) + } else { + None + } + } + }, + ); + } } Ok(()) } -/// This function is a further encapsulation of "updating -/// the Tree object using the whiteout path". -fn update_with_rm( - rm_db: &sled::Db, - commit_db: &sled::Db, - old_root_path: &Path, -) -> sled::Result> { - // List all whiteout files. - let rm_staged = rm_db.path_list()?; - - // Execute the update_tree_with_rm_path function for all - // whiteout files. - rm_staged - .iter() - .map(|rm_path| { - println!("[\x1b[33mDEBUG\x1b[0m] rm_path = {}", rm_path.display()); - update_tree_with_rm_path(commit_db, rm_path, old_root_path) - }) - .collect::>>() +/// This function is used to format and print HashMap +#[cfg(debug_assertions)] +fn show_hashmap(hashmap: &HashMap) { + for (tmp1, tmp2) in hashmap.iter() { + print!(" {} -> {} :", tmp1.display(), tmp2.id,); + let mut tmp2 = tmp2.clone(); + tmp2.rehash(); + for tmp3 in tmp2.tree_items.iter() { + print!( + "{{\n\tmode: {},\n\tid: {},\n\tname: {},\n}}", + tmp3.mode, + tmp3.id._to_string(), + tmp3.name + ) + } + } } /// This function is the core function of the commit operation. @@ -234,9 +239,14 @@ fn update_with_rm( /// records of the whiteout files, and use them to update the old /// version tree. pub fn commit_core( - commit_db: &sled::Db, // A copy of tree.db can be modified directly. - temp_store_area: &TempStoreArea, // The temporary storage area. - old_root_path: &Path, // The path of the main Tree in tree.db. + // Includes the old tree.db containing + //the tree structure of the previous + // version and the new tree.db. + // + // tree_dbs = (old_tree_db, new_tree_db) + tree_dbs: (&sled::Db, &sled::Db), + temp_store_area: &TempStoreArea, // The temporary storage area. + old_root_path: &Path, // The path of the main Tree in tree.db. ) -> sled::Result { // To prevent the Remove operation from affecting the // Vec of the main Tree, we now change it to performing @@ -248,45 +258,31 @@ pub fn commit_core( // The Db who storing whiteout files. let rm_db = &temp_store_area.rm_db; - let mut root_rm_file = update_with_rm(rm_db, commit_db, old_root_path)?; + let old_tree_db = tree_dbs.0; + let new_tree_db = tree_dbs.1; - println!("\x1b[34m[PART2]\x1b[0m"); - // Get the root Tree. - let mut root_tree = commit_db - .get_bypath(old_root_path) - .expect("Old root tree not found"); - - println!("\x1b[34m[PART3]\x1b[0m"); - update_with_index(index_db, commit_db, &mut root_tree, old_root_path)?; + let mut hashmap = build_new_tree_map(index_db, old_tree_db, old_root_path)?; + #[cfg(debug_assertions)] + show_hashmap(&hashmap); - let mut tmp_path = String::new(); - root_rm_file.sort(); - root_rm_file - .iter() - .filter(|&rm_file| { - if rm_file.eq(&tmp_path) { - false - } else { - tmp_path = rm_file.clone(); - true - } - }) - .for_each(|rm_path| { - // Remove the root whiteout file. - println!("[\x1b[33mDEBUG\x1b[0m] root rm file = {rm_path}"); - root_tree.tree_items.retain(|item| &item.name != rm_path); - }); - - println!("\x1b[34m[PART4]\x1b[0m"); - // Update the main Tree's Hash - root_tree.rehash(); - let res = root_tree.id; - - println!("\x1b[34m[PART5]\x1b[0m"); - // Write the new Tree to the Db. - commit_db.insert_tree(old_root_path.to_path_buf(), root_tree); - commit_db.flush()?; + println!("\x1b[34m[PART2]\x1b[0m"); + build_removed_tree_map(rm_db, old_tree_db, &mut hashmap, old_root_path)?; + #[cfg(debug_assertions)] + show_hashmap(&hashmap); + + // Insert the new Tree into the Db. + // + // Since the insert of sled::Db is an + // overwrite operation, we can update it + // with confidence. + let mut batch = sled::Batch::default(); + for (path, tree) in hashmap.iter() { + batch.insert( + path.to_string_lossy().into_owned().as_str(), + bincode::serialize(tree).unwrap(), + ); + } + new_tree_db.apply_batch(batch)?; - println!("\x1b[34m[OK]\x1b[0m"); - Ok(res) + Ok(hashmap.remove(old_root_path).unwrap().id) } diff --git a/scorpio/src/manager/mod.rs b/scorpio/src/manager/mod.rs index 71925cd9c..45a2544bf 100644 --- a/scorpio/src/manager/mod.rs +++ b/scorpio/src/manager/mod.rs @@ -66,6 +66,9 @@ impl ScorpioManager { let objectspath = path.join("objects"); let commitpath = path.join("commit"); + println!("old_dbpath = {}", old_dbpath.display()); + println!("new_dbpath = {}", new_dbpath.display()); + let modified_path = path.join("modifiedstore"); let tempstorage_path = modified_path.join("objects"); @@ -76,17 +79,12 @@ impl ScorpioManager { copy(&tempstorage_path, &objectspath, &options)?; } - if !new_dbpath.exists() { - let mut options = CopyOptions::new(); - options.copy_inside = true; - copy(&old_dbpath, &new_dbpath, &options)?; - } - - let new_db = sled::open(new_dbpath)?; + let _ = fs::remove_dir_all(&objectspath); + let old_tree_db = sled::open(old_dbpath)?; + let new_tree_db = sled::open(new_dbpath)?; let temp_store_area = TempStoreArea::new(&modified_path)?; let old_root_path = PathBuf::from(mono_path); - // let git_author = config::git_author(); let git_email = config::git_email(); let sign = Signature::new( @@ -105,7 +103,7 @@ impl ScorpioManager { }; println!("\x1b[34m[START]\x1b[0m"); - let main_tree_hash = commit_core(&new_db, &temp_store_area, &old_root_path)?; + let main_tree_hash = commit_core((&old_tree_db, &new_tree_db), &temp_store_area, &old_root_path)?; println!("\x1b[34m[DONE]\x1b[0m"); println!(" [\x1b[33mDEBUG\x1b[0m] commit.author = {}", sign.name); From 8fc8993bccf524d60ce138d89fbdfbdce4a53f8a Mon Sep 17 00:00:00 2001 From: kjx52 Date: Wed, 21 May 2025 10:02:03 +0800 Subject: [PATCH 2/4] [Scorpio]: Fix commit. Signed-off-by: kjx52 --- scorpio/src/manager/commit.rs | 9 +++++---- scorpio/src/manager/mod.rs | 4 ++-- scorpio/src/util/mod.rs | 11 +++++++++++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/scorpio/src/manager/commit.rs b/scorpio/src/manager/commit.rs index 4ee9526e2..621918f7b 100644 --- a/scorpio/src/manager/commit.rs +++ b/scorpio/src/manager/commit.rs @@ -147,7 +147,7 @@ fn del_treeitem(tree: &mut Tree, sub_item_data: (String, Option)) -> Optio } None => panic!("Unexpected index, the rm TreeItem object not found"), }, - None => tree.tree_items.retain(|name| name.name != sub_item_name), + None => tree.tree_items.retain(|item| item.name != sub_item_name), } if tree.tree_items.is_empty() { @@ -235,9 +235,10 @@ fn show_hashmap(hashmap: &HashMap) { /// This function is the core function of the commit operation. /// -/// It can extract the data in the staging area and the removal -/// records of the whiteout files, and use them to update the old -/// version tree. +/// It can use the data in the temporary storage area and the +/// deletion records of the whiteout file to update the old +/// version tree, write it to the new database, and return the +/// Hash of the main Tree. pub fn commit_core( // Includes the old tree.db containing //the tree structure of the previous diff --git a/scorpio/src/manager/mod.rs b/scorpio/src/manager/mod.rs index 5ec629b8c..6572659b6 100644 --- a/scorpio/src/manager/mod.rs +++ b/scorpio/src/manager/mod.rs @@ -72,14 +72,14 @@ impl ScorpioManager { let modified_path = path.join("modifiedstore"); let tempstorage_path = modified_path.join("objects"); - let _ = fs::remove_dir_all(&objectspath); + fs::remove_dir_all(&objectspath)?; if tempstorage_path.exists() { let mut options = CopyOptions::new(); options.copy_inside = true; copy(&tempstorage_path, &objectspath, &options)?; } - let _ = fs::remove_dir_all(&objectspath); + fs::remove_dir_all(&objectspath)?; let old_tree_db = sled::open(old_dbpath)?; let new_tree_db = sled::open(new_dbpath)?; let temp_store_area = TempStoreArea::new(&modified_path)?; diff --git a/scorpio/src/util/mod.rs b/scorpio/src/util/mod.rs index b0359dbe4..4e8c9192d 100644 --- a/scorpio/src/util/mod.rs +++ b/scorpio/src/util/mod.rs @@ -53,6 +53,17 @@ impl From for PathBuf { PathBuf::from(path_str) } } +impl From for String { + fn from(val: GPath) -> Self { + val.path.iter().fold(String::new(), |acc, part| { + match (acc.is_empty(), part.is_empty()) { + (true, _) => part.clone(), + (false, true) => acc, + (false, false) => acc + "/" + part, + } + }) + } +} impl Display for GPath { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "{}", self.path.join("/")) From 27b9d9f67f5fb7df0fd5dc33e67ccab8c7125b1e Mon Sep 17 00:00:00 2001 From: kjx52 Date: Fri, 23 May 2025 03:54:06 +0800 Subject: [PATCH 3/4] [Scorpio]: Update push. Signed-off-by: kjx52 --- scorpio/src/manager/mod.rs | 37 +++++++++++++------ scorpio/src/manager/push.rs | 71 +++++++++++++++++------------------- scorpio/src/manager/store.rs | 42 ++++++++++++++++++++- 3 files changed, 101 insertions(+), 49 deletions(-) diff --git a/scorpio/src/manager/mod.rs b/scorpio/src/manager/mod.rs index 6572659b6..57509acc4 100644 --- a/scorpio/src/manager/mod.rs +++ b/scorpio/src/manager/mod.rs @@ -1,7 +1,6 @@ use crate::manager::store::TempStoreArea; use crate::util::config; use add::add_and_del; -use bytes::Bytes; use commit::commit_core; use fs_extra::dir::{copy, CopyOptions}; use mercury::{ @@ -60,18 +59,17 @@ impl ScorpioManager { ) -> Result> { let store_path = config::store_path(); let work_dir = self.select_work(&mono_path)?; - let path = PathBuf::from(store_path).join(work_dir.hash.clone()); - let old_dbpath = path.join("tree.db"); - let new_dbpath = path.join("new_tree.db"); - let objectspath = path.join("objects"); - let commitpath = path.join("commit"); + let work_path = PathBuf::from(store_path).join(work_dir.hash.clone()); + let old_dbpath = work_path.join("tree.db"); + let new_dbpath = work_path.join("new_tree.db"); + let objectspath = work_path.join("objects"); + let commitpath = work_path.join("commit"); + let modified_path = work_path.join("modifiedstore"); + let tempstorage_path = modified_path.join("objects"); println!("old_dbpath = {}", old_dbpath.display()); println!("new_dbpath = {}", new_dbpath.display()); - let modified_path = path.join("modifiedstore"); - let tempstorage_path = modified_path.join("objects"); - fs::remove_dir_all(&objectspath)?; if tempstorage_path.exists() { let mut options = CopyOptions::new(); @@ -149,6 +147,22 @@ impl ScorpioManager { Err(Box::from("WorkDir not found")) } + pub async fn push_commit( + &self, + mono_path: &str, + ) -> Result> { + let store_path = config::store_path(); + let work_dir = self.select_work(mono_path)?; + let work_path = PathBuf::from(store_path).join(work_dir.hash.clone()); + let modified_path = work_path.join("modifiedstore"); + let temp_store_area = TempStoreArea::new(&modified_path)?; + let base_url = config::base_url(); + let url = format!("{}/{}/git-receive-pack", base_url, mono_path); + + let res= push::push(&work_path, &url, &temp_store_area.index_db).await?; + Ok(res) + } + /* pub async fn push_commit( &self, mono_path: &str, @@ -182,6 +196,7 @@ impl ScorpioManager { .await .map_err(|e| Box::new(e) as Box) } + */ pub fn check_before_mount(&self, mono_path: &str) -> Result<(), String> { for work in &self.works { @@ -213,11 +228,11 @@ impl ScorpioManager { // What is needed here is a path relative to Upper, not a path // relative to the Workdir. let work_dir = self.select_work(mono_path)?; + let store_path = config::store_path(); + let work_path = PathBuf::from(store_path).join(work_dir.hash.clone()); let mono_path = PathBuf::from(mono_path) .strip_prefix(&work_dir.path)? .to_path_buf(); - let store_path = config::store_path(); - let work_path = PathBuf::from(store_path).join(work_dir.hash.clone()); // Since index.db is the private space of the sled database, // we will combine it with objects to form a new working directory. diff --git a/scorpio/src/manager/push.rs b/scorpio/src/manager/push.rs index 6bb12b2bd..4d749864f 100644 --- a/scorpio/src/manager/push.rs +++ b/scorpio/src/manager/push.rs @@ -1,14 +1,17 @@ use bytes::BytesMut; use ceres::protocol::smart::add_pkt_line_string; -use reqwest::{header::CONTENT_TYPE, Client, Url}; -use std::{path::PathBuf, str::FromStr}; +use mercury::internal::object::{ObjectTrait, types::ObjectType}; +use reqwest::{Response, header::CONTENT_TYPE, Client, Url}; +use std::io::{Error, ErrorKind}; +use std::{path::Path, str::FromStr}; use tokio::sync::mpsc; -use crate::manager::diff::change; +use crate::manager::store::{BlobFsStore, TreeStore}; + use mercury::{ hash::SHA1, internal::{ - object::{blob::Blob, commit::Commit, signature::Signature, tree::Tree}, + object::{blob::Blob, commit::Commit, tree::Tree}, pack::encode::PackEncoder, }, }; @@ -37,36 +40,33 @@ pub async fn pack(commit: Commit, trees: Vec, blob: Vec) -> Vec } pack_data } -#[allow(unused)] -pub async fn push(path: PathBuf, monopath: PathBuf) -> std::io::Result<()> { - let mut lower = path.clone(); - lower.push("lower"); - let mut upper = path.clone(); - upper.push("upper"); - let mut dbpath = path.clone(); - dbpath.push("tree.db"); - - let db = sled::open(dbpath)?; - let mut trees = Vec::new(); - let mut blobs = Vec::new(); - let root_tree = change(upper, monopath.clone(), &mut trees, &mut blobs, &db); - trees.push(root_tree.clone()); - let default_author = Signature::from_data( - "author Quanyi Ma 1678101573 +0800" - .to_string() - .into_bytes(), - ) - .unwrap(); - let remote_hash = SHA1::from_str(path.file_name().unwrap().to_str().unwrap()).unwrap(); +pub async fn push(work_path: &Path, url: &str, index_db: &sled::Db) -> std::io::Result { + let new_dbpath = work_path.join("new_tree.db"); + let commitpath = work_path.join("commit"); - let commit = Commit::new( - default_author.clone(), - default_author, - root_tree.id, - vec![remote_hash], - "test commit ", - ); + // check path is exist + if !tokio::fs::try_exists(&commitpath).await.unwrap_or(false) { + eprintln!("Path does not exist: {}", commitpath.display()); + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("Path does not exist: {}", commitpath.display()), + )); + } + // read the file as the body to send + let commit_data = tokio::fs::read(&commitpath).await?; + let commit_hash = SHA1::from_type_and_data(ObjectType::Commit, &commit_data); + + let new_tree_db = sled::open(new_dbpath)?; + let hashmap = new_tree_db.db_tree_list()?; + + let trees = hashmap.values().cloned().collect::>(); + let blobs = work_path.to_path_buf().list_blobs(index_db)?; + + let remote_hash = SHA1::from_str(work_path.file_name().unwrap().to_str().unwrap()).unwrap(); + + let commit = Commit::from_bytes(&commit_data, commit_hash) + .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; let mut data = BytesMut::new(); add_pkt_line_string( &mut data, @@ -81,10 +81,7 @@ pub async fn push(path: PathBuf, monopath: PathBuf) -> std::io::Result<()> { let request = Client::new(); - let url = Url::from_str(&format!( - "http://localhost:8000/{}", - monopath.to_str().unwrap() - )) + let url = Url::from_str(url) .unwrap(); let res = request .post(url.join("git-receive-pack").unwrap()) @@ -99,5 +96,5 @@ pub async fn push(path: PathBuf, monopath: PathBuf) -> std::io::Result<()> { println!("[scorpio]:push seccess!") } - Ok(()) + Ok(res) } diff --git a/scorpio/src/manager/store.rs b/scorpio/src/manager/store.rs index bc15ef017..0bfbe2897 100644 --- a/scorpio/src/manager/store.rs +++ b/scorpio/src/manager/store.rs @@ -1,7 +1,10 @@ -use mercury::internal::object::{commit::Commit, tree::Tree}; +use mercury::hash::SHA1; +use mercury::internal::object::ObjectTrait; +use mercury::internal::object::{blob::Blob, commit::Commit, tree::Tree}; use std::collections::HashMap; use std::io::{Error, ErrorKind, Result}; use std::path::{Path, PathBuf}; +use std::str::FromStr; use tokio::sync::mpsc::Receiver; use crate::util::GPath; @@ -9,6 +12,7 @@ use crate::util::GPath; pub trait TreeStore { fn insert_tree(&self, path: PathBuf, tree: Tree); fn get_bypath(&self, path: &Path) -> Result; + fn db_tree_list(&self) -> Result>; } impl TreeStore for sled::Db { @@ -36,7 +40,24 @@ impl TreeStore for sled::Db { } } } + + fn db_tree_list(&self) -> Result> { + self.iter() + .map(|item| match item { + // By returning a HashMap, we avoid using a double pointer loop structure in diff.rs. + Ok((path, encoded_value)) => { + // Convert the IVec to a string and then to a PathBuf + let path = std::str::from_utf8(&path).unwrap_or("Invalid UTF8 path"); + let decoded_tree: Result = bincode::deserialize(&encoded_value) + .map_err(|_| Error::other("Deserialization error")); + Ok((PathBuf::from(path), decoded_tree?)) + } + Err(e) => Err(Error::new(ErrorKind::NotFound, e)), + }) + .collect::>>() + } } + #[allow(unused)] pub trait CommitStore { fn store_commit(&self, commit: Commit) -> Result<()>; @@ -105,6 +126,7 @@ pub trait BlobFsStore { fn add_blob_to_hash(&self, hash: &str, blob: &[u8]) -> Result<()>; fn get_blob_by_hash(&self, hash: &str) -> Result>; + fn list_blobs(&self, index_db: &sled::Db) -> Result>; } impl BlobFsStore for PathBuf { /// These functions is used to implement a storage @@ -156,6 +178,24 @@ impl BlobFsStore for PathBuf { )), } } + + fn list_blobs(&self, index_db: &sled::Db) -> Result> { + let hashmap = index_db.db_list()?; + let object_path = self.join("objects"); + hashmap + .values() + .map(|hash| { + let hash_path = object_path.join(&hash[0..2]); + let sha_hash = + SHA1::from_str(hash).map_err(|e| Error::new(ErrorKind::InvalidInput, e))?; + + let data_path = hash_path.join(&hash[2..]); + let data = std::fs::read(&data_path)?; + let blob = Blob::from_bytes(&data, sha_hash); + blob.map_err(|e| Error::new(ErrorKind::InvalidData, e)) + }) + .collect::>>() + } } pub trait ModifiedStore { From b72e64bd48c47bc2c5f029861e678e857bbac2e2 Mon Sep 17 00:00:00 2001 From: kjx52 Date: Tue, 27 May 2025 06:54:17 +0800 Subject: [PATCH 4/4] [Scorpio]: Update push. Signed-off-by: kjx52 --- scorpio/src/manager/mod.rs | 7 ++- scorpio/src/manager/push.rs | 106 ++++++++++++++++++++++++++++++++---- 2 files changed, 100 insertions(+), 13 deletions(-) diff --git a/scorpio/src/manager/mod.rs b/scorpio/src/manager/mod.rs index 57509acc4..57828e34c 100644 --- a/scorpio/src/manager/mod.rs +++ b/scorpio/src/manager/mod.rs @@ -70,14 +70,14 @@ impl ScorpioManager { println!("old_dbpath = {}", old_dbpath.display()); println!("new_dbpath = {}", new_dbpath.display()); - fs::remove_dir_all(&objectspath)?; + let _ = fs::remove_dir_all(&objectspath); if tempstorage_path.exists() { + println!("tempstorage_path = {}, Copy", tempstorage_path.display()); let mut options = CopyOptions::new(); options.copy_inside = true; copy(&tempstorage_path, &objectspath, &options)?; } - fs::remove_dir_all(&objectspath)?; let old_tree_db = sled::open(old_dbpath)?; let new_tree_db = sled::open(new_dbpath)?; let temp_store_area = TempStoreArea::new(&modified_path)?; @@ -156,10 +156,13 @@ impl ScorpioManager { let work_path = PathBuf::from(store_path).join(work_dir.hash.clone()); let modified_path = work_path.join("modifiedstore"); let temp_store_area = TempStoreArea::new(&modified_path)?; + println!("OK1"); let base_url = config::base_url(); let url = format!("{}/{}/git-receive-pack", base_url, mono_path); + println!("START"); let res= push::push(&work_path, &url, &temp_store_area.index_db).await?; + println!("END"); Ok(res) } /* diff --git a/scorpio/src/manager/push.rs b/scorpio/src/manager/push.rs index 4d749864f..e594dd574 100644 --- a/scorpio/src/manager/push.rs +++ b/scorpio/src/manager/push.rs @@ -1,7 +1,7 @@ use bytes::BytesMut; use ceres::protocol::smart::add_pkt_line_string; -use mercury::internal::object::{ObjectTrait, types::ObjectType}; -use reqwest::{Response, header::CONTENT_TYPE, Client, Url}; +use regex::Regex; +use reqwest::{header::CONTENT_TYPE, Client, Response, Url}; use std::io::{Error, ErrorKind}; use std::{path::Path, str::FromStr}; use tokio::sync::mpsc; @@ -11,7 +11,12 @@ use crate::manager::store::{BlobFsStore, TreeStore}; use mercury::{ hash::SHA1, internal::{ - object::{blob::Blob, commit::Commit, tree::Tree}, + object::{ + blob::Blob, + commit::Commit, + signature::{Signature, SignatureType}, + tree::Tree, + }, pack::encode::PackEncoder, }, }; @@ -41,10 +46,83 @@ pub async fn pack(commit: Commit, trees: Vec, blob: Vec) -> Vec pack_data } +fn string_to_sha(hash: &str) -> std::io::Result { + SHA1::from_str(hash).map_err(|e| Error::new(ErrorKind::InvalidData, e)) +} + +fn extract_commit_from_bytes(commitpath: &Path) -> std::io::Result { + let commit_string = std::fs::read_to_string(&commitpath)?; + // This function uses regular expressions to extract the + // required data, which may be split later to improve fault + // tolerance. + let regex_rule = Regex::new( + r####"(?x) + # trees' hash + tree[^0-9a-z]+(?P[0-9a-z]{40})\n + parent[^0-9a-z]+(?P[0-9a-z]{40})\n + + # author + author[[:blank:]]+ + (?P[a-zA-Z0-9_-]+) + [[:blank:]]+ + <(?P[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[[:alpha:]]{2,})>\n + .*\n + \n + + # committer + committer[[:blank:]]+ + (?P[a-zA-Z0-9_-]+) + [[:blank:]]+ + <(?P[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[[:alpha:]]{2,})>\n + .*\n + \n + + # commit message + (?s)(?P.*) + "####, + ) + .unwrap(); + match regex_rule.captures(&commit_string) { + Some(commit_data) => { + let extract_data = |name: &str| -> String { + commit_data + .name(name) + .map_or("", |data| data.as_str()) + .to_string() + }; + + let current_hash = extract_data("current_hash"); + let parent_hash = extract_data("parent_hash"); + let author = extract_data("author"); + let author_email = extract_data("author_email"); + let committer = extract_data("committer"); + let committer_email = extract_data("committer_email"); + let message = extract_data("message"); + + let author_sign = Signature::new(SignatureType::Author, author, author_email); + let committer_sign = + Signature::new(SignatureType::Committer, committer, committer_email); + + Ok(Commit::new( + author_sign, + committer_sign, + string_to_sha(¤t_hash)?, + vec![string_to_sha(&parent_hash)?], + &message, + )) + } + None => Err(Error::new( + ErrorKind::InvalidData, + "Commit data is missing or format is incorrect", + )), + } +} + pub async fn push(work_path: &Path, url: &str, index_db: &sled::Db) -> std::io::Result { let new_dbpath = work_path.join("new_tree.db"); let commitpath = work_path.join("commit"); + println!("PART1"); // check path is exist if !tokio::fs::try_exists(&commitpath).await.unwrap_or(false) { eprintln!("Path does not exist: {}", commitpath.display()); @@ -54,19 +132,25 @@ pub async fn push(work_path: &Path, url: &str, index_db: &sled::Db) -> std::io:: )); } // read the file as the body to send - let commit_data = tokio::fs::read(&commitpath).await?; - let commit_hash = SHA1::from_type_and_data(ObjectType::Commit, &commit_data); + let commit = extract_commit_from_bytes(&commitpath)?; + println!("commit id = {}", commit.id._to_string()); + println!("commit tree_id = {}", commit.tree_id._to_string()); + println!("PART2"); + println!("new_dbpath = {}", new_dbpath.display()); let new_tree_db = sled::open(new_dbpath)?; + println!("Fin"); let hashmap = new_tree_db.db_tree_list()?; let trees = hashmap.values().cloned().collect::>(); - let blobs = work_path.to_path_buf().list_blobs(index_db)?; + let blobs = work_path + .to_path_buf() + .list_blobs(index_db) + .unwrap_or(Vec::new()); - let remote_hash = SHA1::from_str(work_path.file_name().unwrap().to_str().unwrap()).unwrap(); + let remote_hash = string_to_sha(work_path.file_name().unwrap().to_str().unwrap())?; - let commit = Commit::from_bytes(&commit_data, commit_hash) - .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; + println!("PART3"); let mut data = BytesMut::new(); add_pkt_line_string( &mut data, @@ -79,10 +163,10 @@ pub async fn push(work_path: &Path, url: &str, index_db: &sled::Db) -> std::io:: tracing::debug!("{:?}", data); data.extend(pack(commit, trees, blobs).await); + println!("PART4"); let request = Client::new(); - let url = Url::from_str(url) - .unwrap(); + let url = Url::from_str(url).unwrap(); let res = request .post(url.join("git-receive-pack").unwrap()) .header(CONTENT_TYPE, "application/x-git-receive-pack-request")