From 6aaa1f74d2ede8c969f01620566840f163ece447 Mon Sep 17 00:00:00 2001 From: AidCheng Date: Fri, 1 Aug 2025 20:59:12 +0100 Subject: [PATCH 01/15] [libra]REC: diff logic x -> new engine/diff-engine module Created a new engine module to handle some of the shared mechanisms between app layers Reconstructed the diff logic inside libra/commands/diff.rs, and moved the new logic to the diff_engine Signed-off-by: AidCheng --- Cargo.toml | 2 + engine/Cargo.toml | 12 ++ engine/src/diff_engine.rs | 255 +++++++++++++++++++++++++++++++++++++ engine/src/lib.rs | 3 + libra/Cargo.toml | 1 + libra/src/command/diff.rs | 258 ++++---------------------------------- 6 files changed, 294 insertions(+), 237 deletions(-) create mode 100644 engine/Cargo.toml create mode 100644 engine/src/diff_engine.rs create mode 100644 engine/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 526530e3e..b61fad38b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ members = [ "extensions/observatory", "scorpio", "context", + "engine", ] default-members = ["mega", "mono", "libra", "aries", "orion", "orion-server"] resolver = "1" @@ -36,6 +37,7 @@ mono = { path = "mono" } observatory = { path = "extensions/observatory" } orion = { path = "orion" } context = { path = "context" } +engine = { path = "engine" } anyhow = "1.0.98" serde = "1.0.219" diff --git a/engine/Cargo.toml b/engine/Cargo.toml new file mode 100644 index 000000000..53cf300b6 --- /dev/null +++ b/engine/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "engine" +version = "0.1.0" +edition = "2021" + +[dependencies] +imara-diff = { workspace = true } +infer = { workspace = true } +mercury = { workspace = true } +path-absolutize = { workspace = true } +tracing = { workspace = true } +tokio = {workspace = true} \ No newline at end of file diff --git a/engine/src/diff_engine.rs b/engine/src/diff_engine.rs new file mode 100644 index 000000000..a16f5178f --- /dev/null +++ b/engine/src/diff_engine.rs @@ -0,0 +1,255 @@ +use std::collections::HashMap; +use std:: { + io::{self}, + path::{Path, PathBuf} +}; +use std::collections::HashSet; +use imara_diff::{ + Algorithm, + Diff, + UnifiedDiffConfig, + BasicLineDiffPrinter +}; +use imara_diff::InternedInput; +use mercury::{ + hash::SHA1 +}; +use infer; +use path_absolutize::Absolutize; + +pub struct DiffEngine; + +impl DiffEngine { + pub async fn diff( + old_blobs: Vec<(PathBuf, SHA1)>, + new_blobs: Vec<(PathBuf, SHA1)>, + algorithm: String, + filter: Vec, + w: &mut dyn io::Write, + read_content: &dyn Fn(&PathBuf, &SHA1) -> Vec, + ){ + let old_blobs: HashMap = old_blobs.into_iter().collect(); + let new_blobs: HashMap = new_blobs.into_iter().collect(); + + // union set + let union_files: HashSet = old_blobs.keys().chain(new_blobs.keys()).cloned().collect(); + tracing::debug!( + "old_blobs: {:?}, new_blobs: {:?}, union_files: {:?}", + old_blobs.len(), + new_blobs.len(), + union_files.len() + ); + + + + // filter files, cross old and new files, and pathspec + for file in union_files { + if Self::should_process(&file, &filter, &old_blobs, &new_blobs) { + Self::write_diff_for_file(&file, &old_blobs, &new_blobs, algorithm.as_str(), w, &read_content); + } + } + } + + fn should_process( + file: &PathBuf, + filter: &[PathBuf], + old_blobs: &HashMap, + new_blobs: &HashMap, + ) -> bool { + // Skip if not in filter paths + if !filter.is_empty() && !filter.iter().any(|path| Self::sub_of(file, path).unwrap_or(false)) { + return false; + } + // Skip if hashes are equal or both absent + old_blobs.get(file) != new_blobs.get(file) + } + + fn sub_of(path: &PathBuf, parent: &PathBuf) -> Result { + let path_abs: PathBuf = path.absolutize()?.to_path_buf(); + let parent_abs: PathBuf = parent.absolutize()?.to_path_buf(); + Ok(path_abs.starts_with(parent_abs)) + } + + fn write_diff_for_file( + file: &PathBuf, + old_blobs: &HashMap, + new_blobs: &HashMap, + algorithm: &str, + w: &mut dyn io::Write, + read_content: &dyn Fn(&PathBuf, &SHA1) -> Vec, + ) { + let new_hash = new_blobs.get(file); + let old_hash = old_blobs.get(file); + + let old_content = old_hash.map_or_else(Vec::new, |h| read_content(file, h)); + let new_content = new_hash.map_or_else(Vec::new, |h| read_content(file, h)); + + writeln!( + w, + "diff --git a/{} b/{}", + file.display(), + file.display() + ).unwrap(); + + if old_hash.is_none() { + writeln!(w, "new file mode 100644").unwrap(); + } else if new_hash.is_none() { + writeln!(w, "deleted file mode 100644").unwrap(); + } + + let old_index = old_hash.map_or("0000000".to_string(), |h| h.to_string()[0..8].to_string()); + let new_index = new_hash.map_or("0000000".to_string(), |h| h.to_string()[0..8].to_string()); + writeln!(w, "index {old_index}..{new_index}").unwrap(); + + let old_type = infer::get(&old_content); + let new_type = infer::get(&new_content); + match (String::from_utf8(old_content), String::from_utf8(new_content)) { + (Ok(old_text), Ok(new_text)) => { + let (old_prefix, new_prefix) = if old_text.is_empty() { + ( + "/dev/null".to_string(), + format!("b/{}", Self::file_display(file, new_hash, new_type)), + ) + } else if new_text.is_empty() { + ( + format!("a/{}", Self::file_display(file, old_hash, old_type)), + "/dev/null".to_string(), + ) + } else { + ( + format!("a/{}", Self::file_display(file, old_hash, old_type)), + format!("b/{}", Self::file_display(file, new_hash, new_type)), + ) + }; + writeln!(w, "--- {old_prefix}").unwrap(); + writeln!(w, "+++ {new_prefix}").unwrap(); + Self::imara_diff_result(&old_text, &new_text, algorithm, w); + } + _ => { + // TODO: Handle non-UTF-8 data as binary for now; consider optimization in the future. + writeln!( + w, + "Binary files a/{} and b/{} differ", + Self::file_display(file, old_hash, old_type), + Self::file_display(file, new_hash, new_type) + ).unwrap(); + } + } + } + + // display file with type + fn file_display(file: &Path, hash: Option<&SHA1>, file_type: Option) -> String { + let file_name = match hash { + Some(_) => file.display().to_string(), + None => "dev/null".to_string(), + }; + + if let Some(file_type) = file_type { + // Check if the file type is displayable in browser, like image, audio, video, etc. + if matches!( + file_type.matcher_type(), + infer::MatcherType::Audio | infer::MatcherType::Video | infer::MatcherType::Image + ) { + return format!("{} ({})", file_name, file_type.mime_type()).to_string(); + } + } + file_name + } + + fn imara_diff_result(old: &str, new: &str, algorithm: &str, w: &mut dyn io::Write) { + let input = InternedInput::new(old, new); + + let algo = match algorithm { + "myers" => Algorithm::Myers, + "myersMinimal" => Algorithm::MyersMinimal, + // default is the histogram algo + _ => Algorithm::Histogram, + }; + tracing::debug!("libra [diff]: choose the algorithm: {:?}", algo); + + let mut diff = Diff::compute(algo, &input); + + // did the postprocess_lines + diff.postprocess_lines(&input); + + let result = diff + .unified_diff( + &BasicLineDiffPrinter(&input.interner), + UnifiedDiffConfig::default(), + &input, + ) + .to_string(); + + write!(w, "{result}").unwrap(); + } + +} + +mod tests { + #[test] + fn test_diff_algorithms_correctness_and_efficiency() { + let old = r#"function foo() { + if (condition) { + doSomething(); + doSomethingElse(); + andAnotherThing(); + } else { + alternative(); + } +}"#; + + let new = r#"function foo() { + if (condition) { + // Added comment + doSomething(); + // Modified this line + modifiedSomethingElse(); + andAnotherThing(); + } else { + alternative(); + } + + // Added new block + addedNewFunctionality(); +}"#; + let mut outputs = Vec::new(); + + let algos = ["histogram", "myers", "myersMinimal"]; + + // test the different algo benchmark + for algo in algos { + let mut buf = Vec::new(); + let start = tokio::time::Instant::now(); + crate::DiffEngine::imara_diff_result(old, new, algo, &mut buf); + let elapse = start.elapsed(); + let ouput = String::from_utf8(buf).expect("Invalid UTF-8 in diff ouput"); + + println!("libra diff algorithm: {algo:?} Spend Time: {elapse:?}"); + assert!( + !ouput.is_empty(), + "libra diff algorithm: {algo} produce a empty output" + ); + assert!( + ouput.contains("@@"), + "libra diff algorithm: {algo}, ouput missing diff markers" + ); + + outputs.push((algo, ouput)); + } + + // check the line counter difference + for (algo, output) in outputs { + let plus_line = output.lines().filter(|line| line.starts_with("+")).count(); + let minus_line = output.lines().filter(|line| line.starts_with("-")).count(); + assert_eq!( + plus_line, 6, + "libra diff algorithm {algo}, expect plus_line: 6, got {plus_line} " + ); + assert_eq!( + minus_line, 1, + "libra diff algorithm {algo}, expect minus_line: 1, got {minus_line} " + ); + } + } +} + diff --git a/engine/src/lib.rs b/engine/src/lib.rs new file mode 100644 index 000000000..36407148b --- /dev/null +++ b/engine/src/lib.rs @@ -0,0 +1,3 @@ +pub mod diff_engine; + +pub use diff_engine::DiffEngine; \ No newline at end of file diff --git a/libra/Cargo.toml b/libra/Cargo.toml index c373376d7..4935f4028 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -17,6 +17,7 @@ ceres = { workspace = true } clap = { workspace = true, features = ["derive"] } colored = { workspace = true } common = { workspace = true } +engine = { workspace = true } flate2 = { workspace = true } # add features = ["zlib"] if slow futures = { workspace = true } futures-util = { workspace = true } diff --git a/libra/src/command/diff.rs b/libra/src/command/diff.rs index 4da778f1e..71757de4e 100644 --- a/libra/src/command/diff.rs +++ b/libra/src/command/diff.rs @@ -1,12 +1,10 @@ use std::{ - collections::{HashMap, HashSet}, fmt, io::{self, Write}, - path::{Path, PathBuf}, + path::{PathBuf}, }; use clap::Parser; -use imara_diff::{Algorithm, BasicLineDiffPrinter, Diff, InternedInput, UnifiedDiffConfig}; use mercury::{ hash::SHA1, internal::{ @@ -15,6 +13,7 @@ use mercury::{ pack::utils::calculate_object_hash, }, }; +use engine::DiffEngine; use similar; use crate::{ @@ -28,8 +27,7 @@ use crate::{ #[cfg(unix)] use std::process::{Command, Stdio}; - -use crate::utils::path_ext::PathExt; +use crate::utils::util::{to_workdir_path}; #[derive(Parser, Debug)] pub struct DiffArgs { @@ -125,13 +123,29 @@ pub async fn execute(args: DiffArgs) { let paths: Vec = args.pathspec.iter().map(util::to_workdir_path).collect(); let mut buf: Vec = Vec::new(); + let read_content = |file: &PathBuf, hash: &SHA1| { + // read content from blob or file + match load_object::(hash) { + Ok(blob) => blob.data, + Err(_) => { + let file = to_workdir_path(file); + std::fs::read(&file) + .map_err(|e| { + eprintln!("fatal: could not read file '{}': {}", file.display(), e); + }) + .unwrap() + } + } + }; + // filter files, cross old and new files, and pathspec - diff( + DiffEngine::diff( old_blobs, new_blobs, args.algorithm.unwrap_or_default(), paths.into_iter().collect(), &mut buf, + &read_content, ) .await; @@ -161,123 +175,6 @@ pub async fn execute(args: DiffArgs) { } } -pub async fn diff( - old_blobs: Vec<(PathBuf, SHA1)>, - new_blobs: Vec<(PathBuf, SHA1)>, - algorithm: String, - filter: Vec, - w: &mut dyn io::Write, -) { - let old_blobs: HashMap = old_blobs.into_iter().collect(); - let new_blobs: HashMap = new_blobs.into_iter().collect(); - // unison set - let union_files: HashSet = old_blobs.keys().chain(new_blobs.keys()).cloned().collect(); - tracing::debug!( - "old blobs {:?}, new blobs {:?}, union files {:?}", - old_blobs.len(), - new_blobs.len(), - union_files.len() - ); - - let read_content = |file: &PathBuf, hash: &SHA1| { - // read content from blob or file - match load_object::(hash) { - Ok(blob) => blob.data, - Err(_) => { - let file = util::workdir_to_absolute(file); - std::fs::read(&file) - .map_err(|e| { - eprintln!("fatal: could not read file '{}': {}", file.display(), e); - }) - .unwrap() - } - } - }; - - // filter files, cross old and new files, and pathspec - for file in union_files { - // if new_file did't start with any path in filter, skip it - if !filter.is_empty() && !filter.iter().any(|path| file.sub_of(path)) { - continue; - } - - let new_hash = new_blobs.get(&file); - let old_hash = old_blobs.get(&file); - if new_hash == old_hash { - continue; - } - - let old_content = match old_hash.as_ref() { - Some(hash) => read_content(&file, hash), - None => Vec::new(), - }; - let new_content = match new_hash.as_ref() { - Some(hash) => read_content(&file, hash), - None => Vec::new(), - }; - - writeln!( - w, - "diff --git a/{} b/{}", - file.display(), - file.display() // files name is always the same, current did't support rename - ) - .unwrap(); - - if old_hash.is_none() { - writeln!(w, "new file mode 100644").unwrap(); - } else if new_hash.is_none() { - writeln!(w, "deleted file mode 100644").unwrap(); - } - - let old_index = old_hash.map_or("0000000".to_string(), |h| h.to_string()[0..8].to_string()); - let new_index = new_hash.map_or("0000000".to_string(), |h| h.to_string()[0..8].to_string()); - writeln!(w, "index {old_index}..{new_index}").unwrap(); - // check is the content is valid utf-8 or maybe binary - let old_type = infer::get(&old_content); - let new_type = infer::get(&new_content); - match ( - String::from_utf8(old_content), - String::from_utf8(new_content), - ) { - (Ok(old_text), Ok(new_text)) => { - let (old_prefix, new_prefix) = if old_text.is_empty() { - // New file - ( - "/dev/null".to_string(), - format!("b/{}", file_display(&file, new_hash, new_type)), - ) - } else if new_text.is_empty() { - // Remove file - ( - format!("a/{}", file_display(&file, old_hash, old_type)), - "/dev/null".to_string(), - ) - } else { - // Update file - ( - format!("a/{}", file_display(&file, old_hash, old_type)), - format!("b/{}", file_display(&file, new_hash, new_type)), - ) - }; - writeln!(w, "--- {old_prefix}").unwrap(); - writeln!(w, "+++ {new_prefix}").unwrap(); - imara_diff_result(&old_text, &new_text, algorithm.as_str(), w); - } - _ => { - // TODO: Handle non-UTF-8 data as binary for now; consider optimization in the future. - writeln!( - w, - "Binary files a/{} and b/{} differ", - file_display(&file, old_hash, old_type), - file_display(&file, new_hash, new_type) - ) - .unwrap(); - } - } - } -} - async fn get_commit_blobs(commit_hash: &SHA1) -> Vec<(PathBuf, SHA1)> { let commit = load_object::(commit_hash).unwrap(); let tree = load_object::(&commit.tree_id).unwrap(); @@ -301,25 +198,6 @@ fn get_files_blobs(files: &[PathBuf]) -> Vec<(PathBuf, SHA1)> { .collect() } -// display file with type -fn file_display(file: &Path, hash: Option<&SHA1>, file_type: Option) -> String { - let file_name = match hash { - Some(_) => file.display().to_string(), - None => "dev/null".to_string(), - }; - - if let Some(file_type) = file_type { - // Check if the file type is displayable in browser, like image, audio, video, etc. - if matches!( - file_type.matcher_type(), - infer::MatcherType::Audio | infer::MatcherType::Video | infer::MatcherType::Image - ) { - return format!("{} ({})", file_name, file_type.mime_type()).to_string(); - } - } - file_name -} - struct Line(Option); impl fmt::Display for Line { @@ -362,39 +240,11 @@ fn similar_diff_result(old: &str, new: &str, w: &mut dyn io::Write) { } } -fn imara_diff_result(old: &str, new: &str, algorithm: &str, w: &mut dyn io::Write) { - let input = InternedInput::new(old, new); - - let algo = match algorithm { - "myers" => Algorithm::Myers, - "myersMinimal" => Algorithm::MyersMinimal, - // default is the histogram algo - _ => Algorithm::Histogram, - }; - tracing::debug!("libra [diff]: choose the algorithm: {:?}", algo); - - let mut diff = Diff::compute(algo, &input); - - // did the postprocess_lines - diff.postprocess_lines(&input); - - let result = diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(); - - write!(w, "{result}").unwrap(); -} - #[cfg(test)] mod test { use crate::utils::test; use serial_test::serial; use std::fs; - use std::time::Instant; use tempfile::tempdir; use super::*; @@ -487,70 +337,4 @@ mod test { assert_eq!(blob.len(), 1); assert_eq!(blob[0].0, PathBuf::from("not_ignore")); } - - #[test] - fn test_diff_algorithms_correctness_and_efficiency() { - let old = r#"function foo() { - if (condition) { - doSomething(); - doSomethingElse(); - andAnotherThing(); - } else { - alternative(); - } -}"#; - - let new = r#"function foo() { - if (condition) { - // Added comment - doSomething(); - // Modified this line - modifiedSomethingElse(); - andAnotherThing(); - } else { - alternative(); - } - - // Added new block - addedNewFunctionality(); -}"#; - let mut outputs = Vec::new(); - - let algos = ["histogram", "myers", "myersMinimal"]; - - // test the different algo benchmark - for algo in algos { - let mut buf = Vec::new(); - let start = Instant::now(); - imara_diff_result(old, new, algo, &mut buf); - let elapse = start.elapsed(); - let ouput = String::from_utf8(buf).expect("Invalid UTF-8 in diff ouput"); - - println!("libra diff algorithm: {algo:?} Spend Time: {elapse:?}"); - assert!( - !ouput.is_empty(), - "libra diff algorithm: {algo} produce a empty output" - ); - assert!( - ouput.contains("@@"), - "libra diff algorithm: {algo}, ouput missing diff markers" - ); - - outputs.push((algo, ouput)); - } - - // check the line counter difference - for (algo, output) in outputs { - let plus_line = output.lines().filter(|line| line.starts_with("+")).count(); - let minus_line = output.lines().filter(|line| line.starts_with("-")).count(); - assert_eq!( - plus_line, 6, - "libra diff algorithm {algo}, expect plus_line: 6, got {plus_line} " - ); - assert_eq!( - minus_line, 1, - "libra diff algorithm {algo}, expect minus_line: 1, got {minus_line} " - ); - } - } -} +} \ No newline at end of file From 86200d955ff8baa14d876bc107a26028fb9ac3f1 Mon Sep 17 00:00:00 2001 From: AidCheng Date: Tue, 5 Aug 2025 16:45:27 +0100 Subject: [PATCH 02/15] [mono]FEAT: Reconstructed mono mr_files_changed api to use the new diff_engine Signed-off-by: AidCheng --- mono/Cargo.toml | 2 + mono/src/api/mr/mr_router.rs | 134 +++++++++++++++++++++++++++++++++-- 2 files changed, 131 insertions(+), 5 deletions(-) diff --git a/mono/Cargo.toml b/mono/Cargo.toml index 393cad29e..8505d81e5 100644 --- a/mono/Cargo.toml +++ b/mono/Cargo.toml @@ -23,6 +23,7 @@ ceres = { workspace = true } vault = { workspace = true } saturn = { workspace = true } context = { workspace = true } +engine = { workspace = true } anyhow = { workspace = true } axum = { workspace = true } @@ -62,6 +63,7 @@ utoipa-swagger-ui = { workspace = true, features = ["axum"] } uuid = { workspace = true, features = ["v4"] } + [target.'cfg(not(windows))'.dependencies] jemallocator = { workspace = true } diff --git a/mono/src/api/mr/mr_router.rs b/mono/src/api/mr/mr_router.rs index 0b3656315..57ccd1034 100644 --- a/mono/src/api/mr/mr_router.rs +++ b/mono/src/api/mr/mr_router.rs @@ -1,8 +1,7 @@ use std::{collections::HashMap, path::PathBuf}; use axum::{ - extract::{Path, State}, - Json, + extract::{Path, State}, Json, }; use jupiter::service::mr_service::MRService; use utoipa_axum::{router::OpenApiRouter, routes}; @@ -273,8 +272,8 @@ async fn mr_files_changed( Path(link): Path, state: State, ) -> Result>, ApiError> { - let listen_addr = &state.listen_addr; - let diff_res = state.monorepo().content_diff(&link, listen_addr).await?; + // let listen_addr = &state.listen_addr; + let diff_res = state.monorepo().content_diff(&link).await?; let diff_files = extract_files_with_status(&diff_res); let mut paths = vec![]; @@ -465,8 +464,8 @@ fn build_forest(paths: Vec) -> Vec { #[cfg(test)] mod test { use std::collections::HashMap; - use crate::api::mr::mr_router::{build_forest, extract_files_with_status}; + use crate::api::mr::FilesChangedList; #[test] fn test_parse_diff_result_to_filelist() { @@ -515,4 +514,129 @@ mod test { let forest = build_forest(paths); println!("{}", serde_json::to_string_pretty(&forest).unwrap()); } + + #[test] + fn test_mr_files_changed_logic() { + // Test the core logic of mr_files_changed function + // This tests the data transformation logic without needing the full state + + let sample_diff_output = r#"diff --git a/src/main.rs b/src/main.rs + new file mode 100644 + index 0000000..abc1234 + --- /dev/null + +++ b/src/main.rs + @@ -0,0 +1,5 @@ + +fn main() { + + println!("Hello, world!"); + +} + diff --git a/src/lib.rs b/src/lib.rs + index def5678..ghi9012 100644 + --- a/src/lib.rs + +++ b/src/lib.rs + @@ -1,3 +1,4 @@ + +// Added a comment + pub fn add(left: usize, right: usize) -> usize { + left + right + } + diff --git a/README.md b/README.md + deleted file mode 100644 + index 1234567..0000000"#; + + // Test extract_files_with_status + let diff_files = extract_files_with_status(sample_diff_output); + + assert_eq!(diff_files.len(), 3); + assert_eq!(diff_files.get("src/main.rs"), Some(&"new".to_string())); + assert_eq!(diff_files.get("src/lib.rs"), Some(&"modified".to_string())); + assert_eq!(diff_files.get("README.md"), Some(&"deleted".to_string())); + + // Test path extraction and tree building + let mut paths = vec![]; + for (path, _) in diff_files { + paths.push(path); + } + + let mui_trees = build_forest(paths); + + // Verify the tree structure + assert!(!mui_trees.is_empty()); + + // Check that we have the expected root nodes + let root_labels: Vec<&str> = mui_trees.iter().map(|tree| tree.label.as_str()).collect(); + assert!(root_labels.contains(&"src")); + assert!(root_labels.contains(&"README.md")); + + // Test the complete response structure + let files_changed_list = FilesChangedList { + mui_trees, + content: sample_diff_output.to_string(), + }; + + assert!(!files_changed_list.mui_trees.is_empty()); + assert_eq!(files_changed_list.content, sample_diff_output); + } + + #[test] + fn test_extract_files_with_status_edge_cases() { + // Test with empty diff output + let empty_diff = ""; + let result = extract_files_with_status(empty_diff); + assert!(result.is_empty()); + + // Test with malformed diff output + let malformed_diff = "not a valid diff output"; + let result = extract_files_with_status(malformed_diff); + assert!(result.is_empty()); + + // Test with diff containing only additions + let additions_only = r#"diff --git a/new_file.txt b/new_file.txt +new file mode 100644 +index 0000000..1234567 +--- /dev/null ++++ b/new_file.txt"#; + + let result = extract_files_with_status(additions_only); + assert_eq!(result.len(), 1); + assert_eq!(result.get("new_file.txt"), Some(&"new".to_string())); + + // Test with diff containing only deletions + let deletions_only = r#"diff --git a/old_file.txt b/old_file.txt +deleted file mode 100644 +index 1234567..0000000"#; + + let result = extract_files_with_status(deletions_only); + assert_eq!(result.len(), 1); + assert_eq!(result.get("old_file.txt"), Some(&"deleted".to_string())); + } + + #[test] + fn test_build_forest_edge_cases() { + // Test with empty paths + let empty_paths = vec![]; + let result = build_forest(empty_paths); + assert!(result.is_empty()); + + // Test with single file + let single_file = vec!["single_file.txt".to_string()]; + let result = build_forest(single_file); + assert_eq!(result.len(), 1); + assert_eq!(result[0].label, "single_file.txt"); + + // Test with deeply nested paths + let nested_paths = vec![ + "a/b/c/d/e/file.txt".to_string(), + "a/b/different.txt".to_string(), + "a/another.txt".to_string(), + ]; + let result = build_forest(nested_paths); + assert_eq!(result.len(), 1); // Should have one root "a" + assert_eq!(result[0].label, "a"); + + // The tree should have nested structure + let root = &result[0]; + assert!(root.children.is_some()); + let children = root.children.as_ref().unwrap(); + assert!(children.iter().any(|child| child.label == "b")); + assert!(children.iter().any(|child| child.label == "another.txt")); + } } From c345a73c8d332a8bfff1c01128952ee61dfbc8c6 Mon Sep 17 00:00:00 2001 From: AidCheng Date: Tue, 5 Aug 2025 16:47:05 +0100 Subject: [PATCH 03/15] [mono]FEAT: complement missing files from the last commit Signed-off-by: AidCheng --- Cargo.toml | 2 +- ceres/Cargo.toml | 1 + ceres/src/api_service/mono_api_service.rs | 234 ++++++++++++++-------- engine/src/diff_engine.rs | 161 ++++++++++++++- 4 files changed, 312 insertions(+), 86 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b61fad38b..d9438ab86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ futures = "0.3.31" futures-util = "0.3.31" go-defer = "0.1.0" russh = "0.52.1" -axum = "0.8.4" +axum = {version="0.8.4", features=["macros", "json"]} axum-extra = "0.10.1" tower-http = "0.6.4" tower = "0.5.2" diff --git a/ceres/Cargo.toml b/ceres/Cargo.toml index 9cef0dc54..b238611ab 100644 --- a/ceres/Cargo.toml +++ b/ceres/Cargo.toml @@ -15,6 +15,7 @@ common = { workspace = true } jupiter = { workspace = true } callisto = { workspace = true } mercury = { workspace = true } +engine = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true, features = ["net", "process"] } diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index 915da74a1..f16013245 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -38,14 +38,13 @@ use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::str::FromStr; -use std::{env, fs}; use async_trait::async_trait; -use tokio::process::Command; use callisto::sea_orm_active_enums::ConvTypeEnum; use callisto::{mega_blob, mega_mr, mega_tree, raw_blob}; use common::errors::MegaError; +use engine::diff_engine::DiffEngine; use jupiter::storage::base_storage::StorageConnector; use jupiter::storage::Storage; use jupiter::utils::converter::generate_git_keep_with_timestamp; @@ -284,7 +283,7 @@ impl MonoApiService { if mr.path != "/" { let path = PathBuf::from(mr.path.clone()); - // beacuse only parent tree is needed so we skip current directory + // because only parent tree is needed so we skip current directory let (tree_vec, _) = self .search_tree_for_update(path.parent().unwrap()) .await @@ -292,7 +291,7 @@ impl MonoApiService { self.update_parent_tree(path, tree_vec, commit) .await .unwrap(); - // remove refs start with path exceprt mr type + // remove refs start with path except mr type storage.remove_none_mr_refs(&mr.path).await.unwrap(); // TODO: self.clean_dangling_commits().await; } @@ -372,92 +371,163 @@ impl MonoApiService { Ok(p_commit_id) } - pub async fn content_diff(&self, mr_link: &str, listen_addr: &str) -> Result { + /// Generate diff content directly from two blob sets using the diff engine + /// + /// # Arguments + /// * `mr_link` - Merge request link identifier + /// + /// # Returns + /// String containing the unified diff output + pub async fn content_diff( + &self, + mr_link: &str, + ) -> Result { let stg = self.storage.mr_storage(); - if let Some(mr) = stg.get_mr(mr_link).await.unwrap() { - let base_path = self.storage.config().base_dir.clone(); - env::set_current_dir(&base_path).unwrap(); - let clone_path = base_path.join(mr_link); - if !fs::exists(&clone_path).unwrap() { - let result = self.run_libra_diff(&mr, listen_addr, &clone_path).await; - if result.is_err() && fs::exists(&clone_path).unwrap() { - fs::remove_dir_all(&clone_path).unwrap(); + let mr = stg.get_mr(mr_link).await.unwrap().ok_or_else(|| { + GitError::CustomError(format!("Merge request not found: {}", mr_link)) + })?; + + let old_blobs = self.get_commit_blobs(&mr.from_hash).await.map_err(|e| { + GitError::CustomError(format!("Failed to get old commit blobs: {}", e)) + })?; + let new_blobs = self.get_commit_blobs(&mr.to_hash).await.map_err(|e| { + GitError::CustomError(format!("Failed to get new commit blobs: {}", e)) + })?; + + + let diff_output = Vec::new(); + let algo = "histogram".to_string(); // Default diff algorithm + let filter_paths: Vec = Vec::new(); // No filtering by default + + // Pre-fetch all blob contents to avoid async issues in the closure + let mut blob_cache: HashMap> = HashMap::new(); + + // Collect all unique hashes + let mut all_hashes = HashSet::new(); + for (_, hash) in &old_blobs { + all_hashes.insert(*hash); + } + for (_, hash) in &new_blobs { + all_hashes.insert(*hash); + } + + // Fetch all blobs concurrently + for hash in all_hashes { + match self.get_raw_blob_by_hash(&hash.to_string()).await { + Ok(Some(blob)) => { + blob_cache.insert(hash, blob.data.unwrap_or_default()); + } + _ => { + blob_cache.insert(hash, Vec::new()); } - } else { - env::set_current_dir(&clone_path).unwrap(); - } - tracing::debug!("Run libra Command: libra diff --old {}", mr.from_hash); - let output = Command::new("libra") - .arg("diff") - .arg("--old") - .arg(mr.from_hash) - .output() - .await?; - - let stderr_str = String::from_utf8_lossy(&output.stderr); - - if !stderr_str.trim().is_empty() || !output.status.success() { - tracing::error!( - "Command failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - fs::remove_dir_all(&clone_path).unwrap(); - } else { - return Ok(String::from_utf8_lossy(&output.stdout).to_string()); } } - Ok(String::new()) - } - async fn run_libra_diff( - &self, - mr: &callisto::mega_mr::Model, - listen_addr: &str, - clone_path: &PathBuf, - ) -> Result<(), anyhow::Error> { - Command::new("mkdir").arg(&mr.link).output().await?; - env::set_current_dir(clone_path).unwrap(); - Command::new("libra").arg("init").output().await?; - let git_remote = if mr.path.starts_with("/") { - format!("{}{}", listen_addr, mr.path) - } else { - format!("{}/{}", listen_addr, mr.path) + // Simple synchronous closure that uses the pre-fetched cache + let read_content = |_file: &PathBuf, hash: &SHA1| -> Vec { + blob_cache.get(hash).cloned().unwrap_or_default() }; - tracing::debug!("Run libra Command: libra remote add origin {}", &git_remote); - Command::new("libra") - .arg("remote") - .arg("add") - .arg("origin") - .arg(git_remote) - .output() - .await?; - tracing::debug!("Run libra Command: libra fetch origin refs/mr/{}", &mr.link); - Command::new("libra") - .arg("fetch") - .arg("origin") - .arg(format!("refs/mr/{}", &mr.link)) - .output() - .await?; - tracing::debug!( - "Run libra Command: libra branch {} origin/mr/{}", - &mr.link, - &mr.link - ); - Command::new("libra") - .arg("branch") - .arg(&mr.link) - .arg(format!("origin/mr/{}", &mr.link)) - .output() - .await?; - tracing::debug!("Run libra Command: libra switch {}", &mr.link); - Command::new("libra") - .arg("switch") - .arg(&mr.link) - .output() - .await?; - Ok(()) + + DiffEngine::mono_diff( + old_blobs, + new_blobs, + algo, + filter_paths, + read_content, + ).await; + + //TODO: handle errors to avoid axum warning + String::from_utf8(diff_output).map_err(|e| { + GitError::CustomError(format!("Failed to convert diff output to string: {}", e)) + }) } + // pub async fn content_diff(&self, mr_link: &str, listen_addr: &str) -> Result { + // let stg = self.storage.mr_storage(); + // if let Some(mr) = stg.get_mr(mr_link).await.unwrap() { + // let base_path = self.storage.config().base_dir.clone(); + // env::set_current_dir(&base_path).unwrap(); + // let clone_path = base_path.join(mr_link); + // if !fs::exists(&clone_path).unwrap() { + // let result = self.run_libra_diff(&mr, listen_addr, &clone_path).await; + // if result.is_err() && fs::exists(&clone_path).unwrap() { + // fs::remove_dir_all(&clone_path).unwrap(); + // } + // } else { + // env::set_current_dir(&clone_path).unwrap(); + // } + // tracing::debug!("Run libra Command: libra diff --old {}", mr.from_hash); + // let output: std::process::Output = Command::new("libra") + // .arg("diff") + // .arg("--old") + // .arg(mr.from_hash) + // .output() + // .await?; + // + // let stderr_str = String::from_utf8_lossy(&output.stderr); + // + // if !stderr_str.trim().is_empty() || !output.status.success() { + // tracing::error!( + // "Command failed: {}", + // String::from_utf8_lossy(&output.stderr) + // ); + // fs::remove_dir_all(&clone_path).unwrap(); + // } else { + // return Ok(String::from_utf8_lossy(&output.stdout).to_string()); + // } + // } + // Ok(String::new()) + // } + + // async fn run_libra_diff( + // &self, + // mr: &callisto::mega_mr::Model, + // listen_addr: &str, + // clone_path: &PathBuf, + // ) -> Result<(), anyhow::Error> { + // Command::new("mkdir").arg(&mr.link).output().await?; + // env::set_current_dir(clone_path).unwrap(); + // Command::new("libra").arg("init").output().await?; + // let git_remote = if mr.path.starts_with("/") { + // format!("{}{}", listen_addr, mr.path) + // } else { + // format!("{}/{}", listen_addr, mr.path) + // }; + // tracing::debug!("Run libra Command: libra remote add origin {}", &git_remote); + // Command::new("libra") + // .arg("remote") + // .arg("add") + // .arg("origin") + // .arg(git_remote) + // .output() + // .await?; + // tracing::debug!("Run libra Command: libra fetch origin refs/mr/{}", &mr.link); + // Command::new("libra") + // .arg("fetch") + // .arg("origin") + // .arg(format!("refs/mr/{}", &mr.link)) + // .output() + // .await?; + // tracing::debug!( + // "Run libra Command: libra branch {} origin/mr/{}", + // &mr.link, + // &mr.link + // ); + // Command::new("libra") + // .arg("branch") + // .arg(&mr.link) + // .arg(format!("origin/mr/{}", &mr.link)) + // .output() + // .await?; + // tracing::debug!("Run libra Command: libra switch {}", &mr.link); + // Command::new("libra") + // .arg("switch") + // .arg(&mr.link) + // .output() + // .await?; + // Ok(()) + // } + pub async fn mr_files_list( &self, old_files: Vec<(PathBuf, SHA1)>, diff --git a/engine/src/diff_engine.rs b/engine/src/diff_engine.rs index a16f5178f..6204590f4 100644 --- a/engine/src/diff_engine.rs +++ b/engine/src/diff_engine.rs @@ -1,7 +1,8 @@ use std::collections::HashMap; use std:: { io::{self}, - path::{Path, PathBuf} + path::{Path, PathBuf}, + fmt::Write }; use std::collections::HashSet; use imara_diff::{ @@ -17,9 +18,35 @@ use mercury::{ use infer; use path_absolutize::Absolutize; + +/// The main diff engine responsible for computing and formatting file differences. +/// +/// `DiffEngine` provides static methods to compare files between two states (old and new) +/// and generate unified diff output. It supports various diff algorithms and handles +/// both text and binary files appropriately. pub struct DiffEngine; impl DiffEngine { + /// Computes and writes unified diffs for changed files between two blob sets. + /// + /// This is the main entry point for the diff engine. It compares files between + /// old and new blob collections, applies filtering, and writes the results in + /// unified diff format. + /// + /// # Arguments + /// + /// * `old_blobs` - Vector of (path, hash) tuples representing the old file state + /// * `new_blobs` - Vector of (path, hash) tuples representing the new file state + /// * `algorithm` - Diff algorithm to use ("myers", "myersMinimal", or "histogram") + /// * `filter` - List of paths to filter; empty means process all files + /// * `w` - Writer to output the diff results + /// * `read_content` - Function to read file content given a path and hash + /// + /// # Algorithm Options + /// + /// - `"myers"` - Standard Myers algorithm + /// - `"myersMinimal"` - Myers algorithm optimized for minimal diffs + /// - `"histogram"` - Histogram algorithm (default, generally fastest) pub async fn diff( old_blobs: Vec<(PathBuf, SHA1)>, new_blobs: Vec<(PathBuf, SHA1)>, @@ -40,8 +67,6 @@ impl DiffEngine { union_files.len() ); - - // filter files, cross old and new files, and pathspec for file in union_files { if Self::should_process(&file, &filter, &old_blobs, &new_blobs) { @@ -50,6 +75,46 @@ impl DiffEngine { } } + pub async fn mono_diff( + old_blobs: Vec<(PathBuf, SHA1)>, + new_blobs: Vec<(PathBuf, SHA1)>, + algorithm: String, + filter: Vec, + read_content: F, + ) -> Vec + where + F: Fn(&PathBuf, &SHA1) -> Vec, + { + let old_blobs: HashMap = old_blobs.into_iter().collect(); + let new_blobs: HashMap = new_blobs.into_iter().collect(); + + // union set + let union_files: HashSet = old_blobs.keys().chain(new_blobs.keys()).cloned().collect(); + tracing::debug!( + "old_blobs: {:?}, new_blobs: {:?}, union_files: {:?}", + old_blobs.len(), + new_blobs.len(), + union_files.len() + ); + + let mut diffs = Vec::new(); + + for file in union_files { + if Self::should_process(&file, &filter, &old_blobs, &new_blobs) { + let diff = Self::diff_for_file_string( + &file, + &old_blobs, + &new_blobs, + algorithm.as_str(), + &read_content, + ); + diffs.push(diff); + } + } + + diffs + } + fn should_process( file: &PathBuf, filter: &[PathBuf], @@ -70,6 +135,96 @@ impl DiffEngine { Ok(path_abs.starts_with(parent_abs)) } + pub fn diff_for_file_string( + file: &PathBuf, + old_blobs: &HashMap, + new_blobs: &HashMap, + _algorithm: &str, + read_content: &dyn Fn(&PathBuf, &SHA1) -> Vec, + ) -> String { + let mut out = String::new(); + + // Look up hashes + let new_hash = new_blobs.get(file); + let old_hash = old_blobs.get(file); + + // Read contents or empty + let old_bytes = old_hash.map_or_else(Vec::new, |h| read_content(file, h)); + let new_bytes = new_hash.map_or_else(Vec::new, |h| read_content(file, h)); + + // diff header + writeln!(out, "diff --git a/{} b/{}", file.display(), file.display()).unwrap(); + + // file-mode lines + if old_hash.is_none() { + writeln!(out, "new file mode 100644").unwrap(); + } else if new_hash.is_none() { + writeln!(out, "deleted file mode 100644").unwrap(); + } + + // index line + let old_index = old_hash + .map(|h| h.to_string()[0..8].to_string()) + .unwrap_or_else(|| "00000000".into()); + let new_index = new_hash + .map(|h| h.to_string()[0..8].to_string()) + .unwrap_or_else(|| "00000000".into()); + writeln!(out, "index {old_index}..{new_index}").unwrap(); + + // infer MIME / text vs binary + let _old_type = infer::get(&old_bytes); + let _new_type = infer::get(&new_bytes); + + // Try UTF-8 first + match (String::from_utf8(old_bytes.clone()), String::from_utf8(new_bytes.clone())) { + (Ok(old_text), Ok(new_text)) => { + // a/ and b/ prefixes + let (old_pref, new_pref) = if old_text.is_empty() { + ("/dev/null".to_string(), + format!("b/{}", file.display())) + } else if new_text.is_empty() { + (format!("a/{}", file.display()), "/dev/null".to_string()) + } else { + (format!("a/{}", file.display()), format!("b/{}", file.display())) + }; + + writeln!(out, "--- {old_pref}").unwrap(); + writeln!(out, "+++ {new_pref}").unwrap(); + + // call your diff engine; here I'll inline a placeholder + // replace this with your actual diff routine, e.g.: + // imara_diff_result(&old_text, &new_text, algorithm, &mut out); + // + // For demonstration, we'll just show unified header: + writeln!( + out, + "@@ -1,{} +1,{} @@", + old_text.lines().count(), + new_text.lines().count(), + ).unwrap(); + for line in old_text.lines() { + writeln!(out, "-{line}").unwrap(); + } + for line in new_text.lines() { + writeln!(out, "+{line}").unwrap(); + } + } + + // Binary fallback + _ => { + writeln!( + out, + "Binary files a/{} and b/{} differ", + file.display(), + file.display() + ) + .unwrap(); + } + } + + out + } + fn write_diff_for_file( file: &PathBuf, old_blobs: &HashMap, From cd82a738ad66966aadfa5d6bd363f83746bc5792 Mon Sep 17 00:00:00 2001 From: AidCheng Date: Tue, 5 Aug 2025 21:24:09 +0100 Subject: [PATCH 04/15] [engine]FEAT: cleaned engine structure. Make diff can be called from both libra and mono which returns String Also modified libra to handle String return instead of using a buffer Signed-off-by: AidCheng --- ceres/src/api_service/mono_api_service.rs | 9 +-- engine/src/diff_engine.rs | 94 ++++++++++------------- libra/src/command/diff.rs | 16 ++-- 3 files changed, 50 insertions(+), 69 deletions(-) diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index f16013245..212f52b72 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -395,7 +395,6 @@ impl MonoApiService { })?; - let diff_output = Vec::new(); let algo = "histogram".to_string(); // Default diff algorithm let filter_paths: Vec = Vec::new(); // No filtering by default @@ -428,7 +427,8 @@ impl MonoApiService { blob_cache.get(hash).cloned().unwrap_or_default() }; - DiffEngine::mono_diff( + // Use the unified diff function that returns a single string + let diff_output = DiffEngine::diff( old_blobs, new_blobs, algo, @@ -436,10 +436,7 @@ impl MonoApiService { read_content, ).await; - //TODO: handle errors to avoid axum warning - String::from_utf8(diff_output).map_err(|e| { - GitError::CustomError(format!("Failed to convert diff output to string: {}", e)) - }) + Ok(diff_output) } // pub async fn content_diff(&self, mr_link: &str, listen_addr: &str) -> Result { diff --git a/engine/src/diff_engine.rs b/engine/src/diff_engine.rs index 6204590f4..4e645f17b 100644 --- a/engine/src/diff_engine.rs +++ b/engine/src/diff_engine.rs @@ -27,11 +27,11 @@ use path_absolutize::Absolutize; pub struct DiffEngine; impl DiffEngine { - /// Computes and writes unified diffs for changed files between two blob sets. + /// Computes and returns unified diffs for changed files between two blob sets as a single string. /// - /// This is the main entry point for the diff engine. It compares files between - /// old and new blob collections, applies filtering, and writes the results in - /// unified diff format. + /// This is the unified diff engine that handles all diff operations and returns a single + /// string containing all the diff output. Both libra and mono can use this function and + /// then handle the string output according to their needs. /// /// # Arguments /// @@ -39,7 +39,6 @@ impl DiffEngine { /// * `new_blobs` - Vector of (path, hash) tuples representing the new file state /// * `algorithm` - Diff algorithm to use ("myers", "myersMinimal", or "histogram") /// * `filter` - List of paths to filter; empty means process all files - /// * `w` - Writer to output the diff results /// * `read_content` - Function to read file content given a path and hash /// /// # Algorithm Options @@ -47,72 +46,59 @@ impl DiffEngine { /// - `"myers"` - Standard Myers algorithm /// - `"myersMinimal"` - Myers algorithm optimized for minimal diffs /// - `"histogram"` - Histogram algorithm (default, generally fastest) - pub async fn diff( + /// + /// # Returns + /// + /// A single string containing all diff output. Users can then write this to a file, + /// display it in terminal, or process it further as needed. + pub async fn diff( old_blobs: Vec<(PathBuf, SHA1)>, new_blobs: Vec<(PathBuf, SHA1)>, algorithm: String, filter: Vec, - w: &mut dyn io::Write, - read_content: &dyn Fn(&PathBuf, &SHA1) -> Vec, - ){ - let old_blobs: HashMap = old_blobs.into_iter().collect(); - let new_blobs: HashMap = new_blobs.into_iter().collect(); - - // union set - let union_files: HashSet = old_blobs.keys().chain(new_blobs.keys()).cloned().collect(); - tracing::debug!( - "old_blobs: {:?}, new_blobs: {:?}, union_files: {:?}", - old_blobs.len(), - new_blobs.len(), - union_files.len() - ); - - // filter files, cross old and new files, and pathspec - for file in union_files { - if Self::should_process(&file, &filter, &old_blobs, &new_blobs) { - Self::write_diff_for_file(&file, &old_blobs, &new_blobs, algorithm.as_str(), w, &read_content); - } + read_content: F, + ) -> String + where + F: Fn(&PathBuf, &SHA1) -> Vec, + { + let (processed_files, old_blobs_map, new_blobs_map) = + Self::prepare_diff_data(old_blobs, new_blobs, &filter); + + let mut diff_results = Vec::new(); + for file in processed_files { + let diff = Self::diff_for_file_string(&file, &old_blobs_map, &new_blobs_map, algorithm.as_str(), &read_content); + diff_results.push(diff); } + + diff_results.join("") } - pub async fn mono_diff( + /// Extracts common diff preparation logic + fn prepare_diff_data( old_blobs: Vec<(PathBuf, SHA1)>, new_blobs: Vec<(PathBuf, SHA1)>, - algorithm: String, - filter: Vec, - read_content: F, - ) -> Vec - where - F: Fn(&PathBuf, &SHA1) -> Vec, - { - let old_blobs: HashMap = old_blobs.into_iter().collect(); - let new_blobs: HashMap = new_blobs.into_iter().collect(); + filter: &[PathBuf], + ) -> (Vec, HashMap, HashMap) { + let old_blobs_map: HashMap = old_blobs.into_iter().collect(); + let new_blobs_map: HashMap = new_blobs.into_iter().collect(); // union set - let union_files: HashSet = old_blobs.keys().chain(new_blobs.keys()).cloned().collect(); + let union_files: HashSet = old_blobs_map.keys().chain(new_blobs_map.keys()).cloned().collect(); + tracing::debug!( "old_blobs: {:?}, new_blobs: {:?}, union_files: {:?}", - old_blobs.len(), - new_blobs.len(), + old_blobs_map.len(), + new_blobs_map.len(), union_files.len() ); - let mut diffs = Vec::new(); - - for file in union_files { - if Self::should_process(&file, &filter, &old_blobs, &new_blobs) { - let diff = Self::diff_for_file_string( - &file, - &old_blobs, - &new_blobs, - algorithm.as_str(), - &read_content, - ); - diffs.push(diff); - } - } + // filter files that should be processed + let processed_files: Vec = union_files + .into_iter() + .filter(|file| Self::should_process(file, filter, &old_blobs_map, &new_blobs_map)) + .collect(); - diffs + (processed_files, old_blobs_map, new_blobs_map) } fn should_process( diff --git a/libra/src/command/diff.rs b/libra/src/command/diff.rs index 71757de4e..2fd788d80 100644 --- a/libra/src/command/diff.rs +++ b/libra/src/command/diff.rs @@ -122,7 +122,6 @@ pub async fn execute(args: DiffArgs) { // use pathspec to filter files let paths: Vec = args.pathspec.iter().map(util::to_workdir_path).collect(); - let mut buf: Vec = Vec::new(); let read_content = |file: &PathBuf, hash: &SHA1| { // read content from blob or file match load_object::(hash) { @@ -138,25 +137,24 @@ pub async fn execute(args: DiffArgs) { } }; - // filter files, cross old and new files, and pathspec - DiffEngine::diff( + // Get diff output as string using the unified diff function + let diff_output = DiffEngine::diff( old_blobs, new_blobs, args.algorithm.unwrap_or_default(), paths.into_iter().collect(), - &mut buf, - &read_content, + read_content, ) .await; + // Handle output - libra processes the string according to its needs match w { Some(ref mut file) => { - file.write_all(&buf).unwrap(); + file.write_all(diff_output.as_bytes()).unwrap(); } None => { #[cfg(unix)] { - #[cfg(unix)] let mut child = Command::new("less") .arg("-R") .arg("-F") @@ -164,12 +162,12 @@ pub async fn execute(args: DiffArgs) { .spawn() .expect("failed to execute process"); let stdin = child.stdin.as_mut().unwrap(); - stdin.write_all(&buf).unwrap(); + stdin.write_all(diff_output.as_bytes()).unwrap(); child.wait().unwrap(); } #[cfg(not(unix))] { - io::stdout().write_all(&buf).unwrap(); + io::stdout().write_all(diff_output.as_bytes()).unwrap(); } } } From 174614f8858dfb7a767b1b3c99630c0e4bd914fe Mon Sep 17 00:00:00 2001 From: AidCheng Date: Tue, 5 Aug 2025 22:02:55 +0100 Subject: [PATCH 05/15] [engine]: engine module is changed to neptune_engine.rs Signed-off-by: AidCheng --- Cargo.toml | 11 +- ceres/Cargo.toml | 4 +- ceres/src/api_service/mono_api_service.rs | 6 +- engine/src/lib.rs | 3 - libra/Cargo.toml | 4 +- libra/src/command/diff.rs | 7 +- mono/Cargo.toml | 1 - {engine => neptune}/Cargo.toml | 6 +- neptune/src/lib.rs | 3 + .../src/neptune_engine.rs | 205 +----------------- 10 files changed, 22 insertions(+), 228 deletions(-) delete mode 100644 engine/src/lib.rs rename {engine => neptune}/Cargo.toml (59%) create mode 100644 neptune/src/lib.rs rename engine/src/diff_engine.rs => neptune/src/neptune_engine.rs (51%) diff --git a/Cargo.toml b/Cargo.toml index d9438ab86..dc508943c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ members = [ "extensions/observatory", "scorpio", "context", - "engine", + "neptune", ] default-members = ["mega", "mono", "libra", "aries", "orion", "orion-server"] resolver = "1" @@ -37,7 +37,7 @@ mono = { path = "mono" } observatory = { path = "extensions/observatory" } orion = { path = "orion" } context = { path = "context" } -engine = { path = "engine" } +neptune = { path = "neptune" } anyhow = "1.0.98" serde = "1.0.219" @@ -51,10 +51,8 @@ rand_chacha = "0.9.0" smallvec = "1.15.0" tokio = "1.45.1" tokio-stream = "0.1.17" -tokio-test = "0.4.4" tokio-util = "0.7.15" clap = "4.5.39" -async-std = "1.13.1" async-trait = "0.1.88" async-stream = "0.3.6" bytes = "1.10.1" @@ -84,8 +82,6 @@ uuid = "1.17.0" regex = "1.11.1" ed25519-dalek = "2.1.1" ctrlc = "3.4.7" -git2 = "0.20.2" -home = "0.5.11" ring = "0.17.14" cedar-policy = "4.4.1" secp256k1 = "0.30.0" @@ -115,9 +111,7 @@ serial_test = "3.2.0" sysinfo = "0.36.0" http = "1.3.1" byte-unit = "5.1.6" -color-backtrace = "0.7.0" ignore = "0.4.23" -imara-diff = "0.2.0" indicatif = "0.18.0" infer = "0.19.0" path-absolutize = "3.1.1" @@ -127,7 +121,6 @@ similar = "2.7.0" url = "2.5.4" wax = "0.6.0" pager = "0.16.1" -tracing-test = "0.2.5" jemallocator = "0.5.4" mimalloc = "0.1.46" assert_cmd = "2.0.17" diff --git a/ceres/Cargo.toml b/ceres/Cargo.toml index b238611ab..448aabb47 100644 --- a/ceres/Cargo.toml +++ b/ceres/Cargo.toml @@ -15,7 +15,7 @@ common = { workspace = true } jupiter = { workspace = true } callisto = { workspace = true } mercury = { workspace = true } -engine = { workspace = true } +neptune = { workspace = true } anyhow = { workspace = true } tokio = { workspace = true, features = ["net", "process"] } @@ -24,13 +24,11 @@ axum = { workspace = true } tracing = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -clap = { workspace = true, features = ["derive"] } chrono = { workspace = true } futures = { workspace = true } bytes = { workspace = true } async-trait = { workspace = true } rand = { workspace = true } -sea-orm = { workspace = true } ring = { workspace = true } hex = { workspace = true } sysinfo = { workspace = true } diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index 212f52b72..d1d11737d 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -44,7 +44,7 @@ use async_trait::async_trait; use callisto::sea_orm_active_enums::ConvTypeEnum; use callisto::{mega_blob, mega_mr, mega_tree, raw_blob}; use common::errors::MegaError; -use engine::diff_engine::DiffEngine; +use neptune::neptune_engine::Diff; use jupiter::storage::base_storage::StorageConnector; use jupiter::storage::Storage; use jupiter::utils::converter::generate_git_keep_with_timestamp; @@ -371,7 +371,7 @@ impl MonoApiService { Ok(p_commit_id) } - /// Generate diff content directly from two blob sets using the diff engine + /// Generate diff content directly from two blob sets using the diff neptune /// /// # Arguments /// * `mr_link` - Merge request link identifier @@ -428,7 +428,7 @@ impl MonoApiService { }; // Use the unified diff function that returns a single string - let diff_output = DiffEngine::diff( + let diff_output = Diff::diff( old_blobs, new_blobs, algo, diff --git a/engine/src/lib.rs b/engine/src/lib.rs deleted file mode 100644 index 36407148b..000000000 --- a/engine/src/lib.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod diff_engine; - -pub use diff_engine::DiffEngine; \ No newline at end of file diff --git a/libra/Cargo.toml b/libra/Cargo.toml index 4935f4028..0bec242f2 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -17,18 +17,16 @@ ceres = { workspace = true } clap = { workspace = true, features = ["derive"] } colored = { workspace = true } common = { workspace = true } -engine = { workspace = true } flate2 = { workspace = true } # add features = ["zlib"] if slow futures = { workspace = true } futures-util = { workspace = true } gemini = { workspace = true, optional = true } hex = { workspace = true } -imara-diff = { workspace = true } indicatif = { workspace = true } -infer = { workspace = true } lazy_static = { workspace = true } lru-mem = { workspace = true } mercury = { workspace = true } +neptune = { workspace = true } once_cell = { workspace = true } path-absolutize = { workspace = true } pathdiff = { workspace = true } diff --git a/libra/src/command/diff.rs b/libra/src/command/diff.rs index 2fd788d80..da1a1091a 100644 --- a/libra/src/command/diff.rs +++ b/libra/src/command/diff.rs @@ -13,13 +13,12 @@ use mercury::{ pack::utils::calculate_object_hash, }, }; -use engine::DiffEngine; +use neptune::Diff; use similar; use crate::{ command::{ - get_target_commit, load_object, - status::{self, changes_to_be_committed}, + get_target_commit, load_object, status::{self, changes_to_be_committed} }, internal::head::Head, utils::{object_ext::TreeExt, path, util}, @@ -138,7 +137,7 @@ pub async fn execute(args: DiffArgs) { }; // Get diff output as string using the unified diff function - let diff_output = DiffEngine::diff( + let diff_output = Diff::diff( old_blobs, new_blobs, args.algorithm.unwrap_or_default(), diff --git a/mono/Cargo.toml b/mono/Cargo.toml index 8505d81e5..0f452cb9f 100644 --- a/mono/Cargo.toml +++ b/mono/Cargo.toml @@ -23,7 +23,6 @@ ceres = { workspace = true } vault = { workspace = true } saturn = { workspace = true } context = { workspace = true } -engine = { workspace = true } anyhow = { workspace = true } axum = { workspace = true } diff --git a/engine/Cargo.toml b/neptune/Cargo.toml similarity index 59% rename from engine/Cargo.toml rename to neptune/Cargo.toml index 53cf300b6..2bbc27c7b 100644 --- a/engine/Cargo.toml +++ b/neptune/Cargo.toml @@ -1,12 +1,10 @@ [package] -name = "engine" +name = "neptune" version = "0.1.0" edition = "2021" [dependencies] -imara-diff = { workspace = true } infer = { workspace = true } mercury = { workspace = true } path-absolutize = { workspace = true } -tracing = { workspace = true } -tokio = {workspace = true} \ No newline at end of file +tracing = { workspace = true } \ No newline at end of file diff --git a/neptune/src/lib.rs b/neptune/src/lib.rs new file mode 100644 index 000000000..6e8a0eb95 --- /dev/null +++ b/neptune/src/lib.rs @@ -0,0 +1,3 @@ +pub mod neptune_engine; + +pub use neptune_engine::Diff; \ No newline at end of file diff --git a/engine/src/diff_engine.rs b/neptune/src/neptune_engine.rs similarity index 51% rename from engine/src/diff_engine.rs rename to neptune/src/neptune_engine.rs index 4e645f17b..64b7176fa 100644 --- a/engine/src/diff_engine.rs +++ b/neptune/src/neptune_engine.rs @@ -1,17 +1,9 @@ use std::collections::HashMap; use std:: { - io::{self}, - path::{Path, PathBuf}, + path::{PathBuf}, fmt::Write }; use std::collections::HashSet; -use imara_diff::{ - Algorithm, - Diff, - UnifiedDiffConfig, - BasicLineDiffPrinter -}; -use imara_diff::InternedInput; use mercury::{ hash::SHA1 }; @@ -19,17 +11,17 @@ use infer; use path_absolutize::Absolutize; -/// The main diff engine responsible for computing and formatting file differences. +/// The main diff neptune responsible for computing and formatting file differences. /// /// `DiffEngine` provides static methods to compare files between two states (old and new) /// and generate unified diff output. It supports various diff algorithms and handles /// both text and binary files appropriately. -pub struct DiffEngine; +pub struct Diff; -impl DiffEngine { +impl Diff { /// Computes and returns unified diffs for changed files between two blob sets as a single string. /// - /// This is the unified diff engine that handles all diff operations and returns a single + /// This is the unified diff neptune that handles all diff operations and returns a single /// string containing all the diff output. Both libra and mono can use this function and /// then handle the string output according to their needs. /// @@ -177,7 +169,7 @@ impl DiffEngine { writeln!(out, "--- {old_pref}").unwrap(); writeln!(out, "+++ {new_pref}").unwrap(); - // call your diff engine; here I'll inline a placeholder + // call your diff neptune; here I'll inline a placeholder // replace this with your actual diff routine, e.g.: // imara_diff_result(&old_text, &new_text, algorithm, &mut out); // @@ -210,187 +202,4 @@ impl DiffEngine { out } - - fn write_diff_for_file( - file: &PathBuf, - old_blobs: &HashMap, - new_blobs: &HashMap, - algorithm: &str, - w: &mut dyn io::Write, - read_content: &dyn Fn(&PathBuf, &SHA1) -> Vec, - ) { - let new_hash = new_blobs.get(file); - let old_hash = old_blobs.get(file); - - let old_content = old_hash.map_or_else(Vec::new, |h| read_content(file, h)); - let new_content = new_hash.map_or_else(Vec::new, |h| read_content(file, h)); - - writeln!( - w, - "diff --git a/{} b/{}", - file.display(), - file.display() - ).unwrap(); - - if old_hash.is_none() { - writeln!(w, "new file mode 100644").unwrap(); - } else if new_hash.is_none() { - writeln!(w, "deleted file mode 100644").unwrap(); - } - - let old_index = old_hash.map_or("0000000".to_string(), |h| h.to_string()[0..8].to_string()); - let new_index = new_hash.map_or("0000000".to_string(), |h| h.to_string()[0..8].to_string()); - writeln!(w, "index {old_index}..{new_index}").unwrap(); - - let old_type = infer::get(&old_content); - let new_type = infer::get(&new_content); - match (String::from_utf8(old_content), String::from_utf8(new_content)) { - (Ok(old_text), Ok(new_text)) => { - let (old_prefix, new_prefix) = if old_text.is_empty() { - ( - "/dev/null".to_string(), - format!("b/{}", Self::file_display(file, new_hash, new_type)), - ) - } else if new_text.is_empty() { - ( - format!("a/{}", Self::file_display(file, old_hash, old_type)), - "/dev/null".to_string(), - ) - } else { - ( - format!("a/{}", Self::file_display(file, old_hash, old_type)), - format!("b/{}", Self::file_display(file, new_hash, new_type)), - ) - }; - writeln!(w, "--- {old_prefix}").unwrap(); - writeln!(w, "+++ {new_prefix}").unwrap(); - Self::imara_diff_result(&old_text, &new_text, algorithm, w); - } - _ => { - // TODO: Handle non-UTF-8 data as binary for now; consider optimization in the future. - writeln!( - w, - "Binary files a/{} and b/{} differ", - Self::file_display(file, old_hash, old_type), - Self::file_display(file, new_hash, new_type) - ).unwrap(); - } - } - } - - // display file with type - fn file_display(file: &Path, hash: Option<&SHA1>, file_type: Option) -> String { - let file_name = match hash { - Some(_) => file.display().to_string(), - None => "dev/null".to_string(), - }; - - if let Some(file_type) = file_type { - // Check if the file type is displayable in browser, like image, audio, video, etc. - if matches!( - file_type.matcher_type(), - infer::MatcherType::Audio | infer::MatcherType::Video | infer::MatcherType::Image - ) { - return format!("{} ({})", file_name, file_type.mime_type()).to_string(); - } - } - file_name - } - - fn imara_diff_result(old: &str, new: &str, algorithm: &str, w: &mut dyn io::Write) { - let input = InternedInput::new(old, new); - - let algo = match algorithm { - "myers" => Algorithm::Myers, - "myersMinimal" => Algorithm::MyersMinimal, - // default is the histogram algo - _ => Algorithm::Histogram, - }; - tracing::debug!("libra [diff]: choose the algorithm: {:?}", algo); - - let mut diff = Diff::compute(algo, &input); - - // did the postprocess_lines - diff.postprocess_lines(&input); - - let result = diff - .unified_diff( - &BasicLineDiffPrinter(&input.interner), - UnifiedDiffConfig::default(), - &input, - ) - .to_string(); - - write!(w, "{result}").unwrap(); - } - -} - -mod tests { - #[test] - fn test_diff_algorithms_correctness_and_efficiency() { - let old = r#"function foo() { - if (condition) { - doSomething(); - doSomethingElse(); - andAnotherThing(); - } else { - alternative(); - } -}"#; - - let new = r#"function foo() { - if (condition) { - // Added comment - doSomething(); - // Modified this line - modifiedSomethingElse(); - andAnotherThing(); - } else { - alternative(); - } - - // Added new block - addedNewFunctionality(); -}"#; - let mut outputs = Vec::new(); - - let algos = ["histogram", "myers", "myersMinimal"]; - - // test the different algo benchmark - for algo in algos { - let mut buf = Vec::new(); - let start = tokio::time::Instant::now(); - crate::DiffEngine::imara_diff_result(old, new, algo, &mut buf); - let elapse = start.elapsed(); - let ouput = String::from_utf8(buf).expect("Invalid UTF-8 in diff ouput"); - - println!("libra diff algorithm: {algo:?} Spend Time: {elapse:?}"); - assert!( - !ouput.is_empty(), - "libra diff algorithm: {algo} produce a empty output" - ); - assert!( - ouput.contains("@@"), - "libra diff algorithm: {algo}, ouput missing diff markers" - ); - - outputs.push((algo, ouput)); - } - - // check the line counter difference - for (algo, output) in outputs { - let plus_line = output.lines().filter(|line| line.starts_with("+")).count(); - let minus_line = output.lines().filter(|line| line.starts_with("-")).count(); - assert_eq!( - plus_line, 6, - "libra diff algorithm {algo}, expect plus_line: 6, got {plus_line} " - ); - assert_eq!( - minus_line, 1, - "libra diff algorithm {algo}, expect minus_line: 1, got {minus_line} " - ); - } - } -} - +} \ No newline at end of file From 4a39987478da1aa3233acd3246d0f51bed8eab26 Mon Sep 17 00:00:00 2001 From: AidCheng Date: Tue, 5 Aug 2025 22:54:46 +0100 Subject: [PATCH 06/15] [neptune]FEAT: Added line check feature for diff File diff over 1k lines will be automatically replaced with a marker (containing the file name) Signed-off-by: AidCheng --- neptune/src/neptune_engine.rs | 58 ++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/neptune/src/neptune_engine.rs b/neptune/src/neptune_engine.rs index 64b7176fa..7a4172520 100644 --- a/neptune/src/neptune_engine.rs +++ b/neptune/src/neptune_engine.rs @@ -10,7 +10,6 @@ use mercury::{ use infer; use path_absolutize::Absolutize; - /// The main diff neptune responsible for computing and formatting file differences. /// /// `DiffEngine` provides static methods to compare files between two states (old and new) @@ -19,6 +18,10 @@ use path_absolutize::Absolutize; pub struct Diff; impl Diff { + const MAX_DIFF_LINES: usize = 1000; // Limit to avoid excessive output + const LARGE_FILE_MARKER: &'static str = "<<, read_content: F, - ) -> String + ) -> String where F: Fn(&PathBuf, &SHA1) -> Vec, { @@ -58,13 +61,59 @@ impl Diff { let mut diff_results = Vec::new(); for file in processed_files { - let diff = Self::diff_for_file_string(&file, &old_blobs_map, &new_blobs_map, algorithm.as_str(), &read_content); - diff_results.push(diff); + if let Some(large_file_marker) = Self::is_large_file( + &file, + &old_blobs_map, + &new_blobs_map, + &read_content + ) { + diff_results.push(large_file_marker); + } else { + let diff = Self::diff_for_file_string(&file, &old_blobs_map, &new_blobs_map, algorithm.as_str(), &read_content); + diff_results.push(diff); + } } diff_results.join("") } + + /// Checks if a file is large and returns a message if it is. + fn is_large_file ( + file: &PathBuf, + old_blobs: &HashMap, + new_blobs: &HashMap, + read_content: &F + ) -> Option + where + F: Fn(&PathBuf, &SHA1) -> Vec, + { + // Check if file is large based on some criteria, e.g. number of lines + let old_hash = old_blobs.get(file); + let new_hash = new_blobs.get(file); + + let old_bytes = old_hash.map_or_else(Vec::new, |h| read_content(file, h)); + let new_bytes = new_hash.map_or_else(Vec::new, |h| read_content(file, h)); + + let old_lines = String::from_utf8_lossy(&old_bytes).lines().count(); + let new_lines = String::from_utf8_lossy(&new_bytes).lines().count(); + let total_lines = old_lines + new_lines; + + if total_lines > Self::MAX_DIFF_LINES { + Some(format!( + "{}{}:{}:{}{}\n", + Self::LARGE_FILE_MARKER, + file.display(), + total_lines, + Self::MAX_DIFF_LINES, + Self::LARGE_FILE_END + )) + } else { + None + } + } + + /// Extracts common diff preparation logic fn prepare_diff_data( old_blobs: Vec<(PathBuf, SHA1)>, @@ -103,6 +152,7 @@ impl Diff { if !filter.is_empty() && !filter.iter().any(|path| Self::sub_of(file, path).unwrap_or(false)) { return false; } + // Skip if hashes are equal or both absent old_blobs.get(file) != new_blobs.get(file) } From 66ec1b9f1a87eee41c12f9c0d36672c981bd5d4e Mon Sep 17 00:00:00 2001 From: AidCheng Date: Wed, 6 Aug 2025 00:23:42 +0100 Subject: [PATCH 07/15] [mono]FIX: clippy warning Signed-off-by: AidCheng --- ceres/src/api_service/mono_api_service.rs | 92 +---------------------- 1 file changed, 3 insertions(+), 89 deletions(-) diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index d1d11737d..94bac9675 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -384,14 +384,14 @@ impl MonoApiService { ) -> Result { let stg = self.storage.mr_storage(); let mr = stg.get_mr(mr_link).await.unwrap().ok_or_else(|| { - GitError::CustomError(format!("Merge request not found: {}", mr_link)) + GitError::CustomError(format!("Merge request not found: {mr_link}")) })?; let old_blobs = self.get_commit_blobs(&mr.from_hash).await.map_err(|e| { - GitError::CustomError(format!("Failed to get old commit blobs: {}", e)) + GitError::CustomError(format!("Failed to get old commit blobs: {e}")) })?; let new_blobs = self.get_commit_blobs(&mr.to_hash).await.map_err(|e| { - GitError::CustomError(format!("Failed to get new commit blobs: {}", e)) + GitError::CustomError(format!("Failed to get new commit blobs: {e}")) })?; @@ -439,92 +439,6 @@ impl MonoApiService { Ok(diff_output) } - // pub async fn content_diff(&self, mr_link: &str, listen_addr: &str) -> Result { - // let stg = self.storage.mr_storage(); - // if let Some(mr) = stg.get_mr(mr_link).await.unwrap() { - // let base_path = self.storage.config().base_dir.clone(); - // env::set_current_dir(&base_path).unwrap(); - // let clone_path = base_path.join(mr_link); - // if !fs::exists(&clone_path).unwrap() { - // let result = self.run_libra_diff(&mr, listen_addr, &clone_path).await; - // if result.is_err() && fs::exists(&clone_path).unwrap() { - // fs::remove_dir_all(&clone_path).unwrap(); - // } - // } else { - // env::set_current_dir(&clone_path).unwrap(); - // } - // tracing::debug!("Run libra Command: libra diff --old {}", mr.from_hash); - // let output: std::process::Output = Command::new("libra") - // .arg("diff") - // .arg("--old") - // .arg(mr.from_hash) - // .output() - // .await?; - // - // let stderr_str = String::from_utf8_lossy(&output.stderr); - // - // if !stderr_str.trim().is_empty() || !output.status.success() { - // tracing::error!( - // "Command failed: {}", - // String::from_utf8_lossy(&output.stderr) - // ); - // fs::remove_dir_all(&clone_path).unwrap(); - // } else { - // return Ok(String::from_utf8_lossy(&output.stdout).to_string()); - // } - // } - // Ok(String::new()) - // } - - // async fn run_libra_diff( - // &self, - // mr: &callisto::mega_mr::Model, - // listen_addr: &str, - // clone_path: &PathBuf, - // ) -> Result<(), anyhow::Error> { - // Command::new("mkdir").arg(&mr.link).output().await?; - // env::set_current_dir(clone_path).unwrap(); - // Command::new("libra").arg("init").output().await?; - // let git_remote = if mr.path.starts_with("/") { - // format!("{}{}", listen_addr, mr.path) - // } else { - // format!("{}/{}", listen_addr, mr.path) - // }; - // tracing::debug!("Run libra Command: libra remote add origin {}", &git_remote); - // Command::new("libra") - // .arg("remote") - // .arg("add") - // .arg("origin") - // .arg(git_remote) - // .output() - // .await?; - // tracing::debug!("Run libra Command: libra fetch origin refs/mr/{}", &mr.link); - // Command::new("libra") - // .arg("fetch") - // .arg("origin") - // .arg(format!("refs/mr/{}", &mr.link)) - // .output() - // .await?; - // tracing::debug!( - // "Run libra Command: libra branch {} origin/mr/{}", - // &mr.link, - // &mr.link - // ); - // Command::new("libra") - // .arg("branch") - // .arg(&mr.link) - // .arg(format!("origin/mr/{}", &mr.link)) - // .output() - // .await?; - // tracing::debug!("Run libra Command: libra switch {}", &mr.link); - // Command::new("libra") - // .arg("switch") - // .arg(&mr.link) - // .output() - // .await?; - // Ok(()) - // } - pub async fn mr_files_list( &self, old_files: Vec<(PathBuf, SHA1)>, From 706921bace6c8343975e1853e695ce3a636dec32 Mon Sep 17 00:00:00 2001 From: Aid C <57825561+AidCheng@users.noreply.github.com> Date: Wed, 6 Aug 2025 01:52:44 +0100 Subject: [PATCH 08/15] Update neptune/src/neptune_engine.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Aid C. <57825561+AidCheng@users.noreply.github.com> --- neptune/src/neptune_engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neptune/src/neptune_engine.rs b/neptune/src/neptune_engine.rs index 7a4172520..4f5366a4e 100644 --- a/neptune/src/neptune_engine.rs +++ b/neptune/src/neptune_engine.rs @@ -10,7 +10,7 @@ use mercury::{ use infer; use path_absolutize::Absolutize; -/// The main diff neptune responsible for computing and formatting file differences. +/// The main diff engine responsible for computing and formatting file differences. /// /// `DiffEngine` provides static methods to compare files between two states (old and new) /// and generate unified diff output. It supports various diff algorithms and handles From cd5f353a598493f111e93436870a0670e948e462 Mon Sep 17 00:00:00 2001 From: Aid C <57825561+AidCheng@users.noreply.github.com> Date: Wed, 6 Aug 2025 01:55:49 +0100 Subject: [PATCH 09/15] Update neptune/src/neptune_engine.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Aid C. <57825561+AidCheng@users.noreply.github.com> --- neptune/src/neptune_engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neptune/src/neptune_engine.rs b/neptune/src/neptune_engine.rs index 4f5366a4e..38c82f0f2 100644 --- a/neptune/src/neptune_engine.rs +++ b/neptune/src/neptune_engine.rs @@ -24,7 +24,7 @@ impl Diff { /// Computes and returns unified diffs for changed files between two blob sets as a single string. /// - /// This is the unified diff neptune that handles all diff operations and returns a single + /// This is the unified diff function that handles all diff operations and returns a single /// string containing all the diff output. Both libra and mono can use this function and /// then handle the string output according to their needs. /// From a72a99cac7b52a58598cf810edc5fb69a4a6de12 Mon Sep 17 00:00:00 2001 From: Aid C <57825561+AidCheng@users.noreply.github.com> Date: Wed, 6 Aug 2025 01:55:56 +0100 Subject: [PATCH 10/15] Update neptune/src/neptune_engine.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Aid C. <57825561+AidCheng@users.noreply.github.com> --- neptune/src/neptune_engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neptune/src/neptune_engine.rs b/neptune/src/neptune_engine.rs index 38c82f0f2..05c6d458b 100644 --- a/neptune/src/neptune_engine.rs +++ b/neptune/src/neptune_engine.rs @@ -219,7 +219,7 @@ impl Diff { writeln!(out, "--- {old_pref}").unwrap(); writeln!(out, "+++ {new_pref}").unwrap(); - // call your diff neptune; here I'll inline a placeholder + // call your diff engine; here I'll inline a placeholder // replace this with your actual diff routine, e.g.: // imara_diff_result(&old_text, &new_text, algorithm, &mut out); // From be207bb62b445d309a6a9204660610ccf8652fc1 Mon Sep 17 00:00:00 2001 From: Aid C <57825561+AidCheng@users.noreply.github.com> Date: Wed, 6 Aug 2025 01:57:06 +0100 Subject: [PATCH 11/15] Update ceres/src/api_service/mono_api_service.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Aid C. <57825561+AidCheng@users.noreply.github.com> --- ceres/src/api_service/mono_api_service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ceres/src/api_service/mono_api_service.rs b/ceres/src/api_service/mono_api_service.rs index 94bac9675..6884da2b2 100644 --- a/ceres/src/api_service/mono_api_service.rs +++ b/ceres/src/api_service/mono_api_service.rs @@ -371,7 +371,7 @@ impl MonoApiService { Ok(p_commit_id) } - /// Generate diff content directly from two blob sets using the diff neptune + /// Generate diff content directly from two blob sets using the diff engine /// /// # Arguments /// * `mr_link` - Merge request link identifier From d9d3632b4e198d0d4a6e825cf7dfe71048ade1f2 Mon Sep 17 00:00:00 2001 From: Aid C <57825561+AidCheng@users.noreply.github.com> Date: Wed, 6 Aug 2025 02:04:46 +0100 Subject: [PATCH 12/15] Update neptune_engine.rs Signed-off-by: Aid C. <57825561+AidCheng@users.noreply.github.com> --- neptune/src/neptune_engine.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/neptune/src/neptune_engine.rs b/neptune/src/neptune_engine.rs index 05c6d458b..8cd456d1c 100644 --- a/neptune/src/neptune_engine.rs +++ b/neptune/src/neptune_engine.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std:: { +use std::{ path::{PathBuf}, fmt::Write }; @@ -219,11 +219,6 @@ impl Diff { writeln!(out, "--- {old_pref}").unwrap(); writeln!(out, "+++ {new_pref}").unwrap(); - // call your diff engine; here I'll inline a placeholder - // replace this with your actual diff routine, e.g.: - // imara_diff_result(&old_text, &new_text, algorithm, &mut out); - // - // For demonstration, we'll just show unified header: writeln!( out, "@@ -1,{} +1,{} @@", @@ -252,4 +247,4 @@ impl Diff { out } -} \ No newline at end of file +} From 75cb513fc5230365ebc2d07ff80d9d33f552bed8 Mon Sep 17 00:00:00 2001 From: AidCheng Date: Wed, 6 Aug 2025 01:59:23 +0100 Subject: [PATCH 13/15] FIX: copilot review Signed-off-by: AidCheng --- neptune/src/neptune_engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neptune/src/neptune_engine.rs b/neptune/src/neptune_engine.rs index 8cd456d1c..9d04b8d81 100644 --- a/neptune/src/neptune_engine.rs +++ b/neptune/src/neptune_engine.rs @@ -24,7 +24,7 @@ impl Diff { /// Computes and returns unified diffs for changed files between two blob sets as a single string. /// - /// This is the unified diff function that handles all diff operations and returns a single + /// This is the unified diff engine that handles all diff operations and returns a single /// string containing all the diff output. Both libra and mono can use this function and /// then handle the string output according to their needs. /// From b7dffee566720b50e40570a7d6e0d4f187d3de7b Mon Sep 17 00:00:00 2001 From: AidCheng Date: Wed, 6 Aug 2025 02:14:49 +0100 Subject: [PATCH 14/15] UPDATE: copilot suggestion Signed-off-by: AidCheng --- neptune/src/neptune_engine.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/neptune/src/neptune_engine.rs b/neptune/src/neptune_engine.rs index 9d04b8d81..8aaa00bf9 100644 --- a/neptune/src/neptune_engine.rs +++ b/neptune/src/neptune_engine.rs @@ -92,8 +92,8 @@ impl Diff { let old_hash = old_blobs.get(file); let new_hash = new_blobs.get(file); - let old_bytes = old_hash.map_or_else(Vec::new, |h| read_content(file, h)); - let new_bytes = new_hash.map_or_else(Vec::new, |h| read_content(file, h)); + let old_bytes = old_hash.map_or_else(|| Vec::new(), |h| read_content(file, h)); + let new_bytes = new_hash.map_or_else(|| Vec::new(), |h| read_content(file, h)); let old_lines = String::from_utf8_lossy(&old_bytes).lines().count(); let new_lines = String::from_utf8_lossy(&new_bytes).lines().count(); @@ -177,8 +177,8 @@ impl Diff { let old_hash = old_blobs.get(file); // Read contents or empty - let old_bytes = old_hash.map_or_else(Vec::new, |h| read_content(file, h)); - let new_bytes = new_hash.map_or_else(Vec::new, |h| read_content(file, h)); + let old_bytes = old_hash.map_or_else(|| Vec::new(), |h| read_content(file, h)); + let new_bytes = new_hash.map_or_else(|| Vec::new(), |h| read_content(file, h)); // diff header writeln!(out, "diff --git a/{} b/{}", file.display(), file.display()).unwrap(); @@ -193,10 +193,10 @@ impl Diff { // index line let old_index = old_hash .map(|h| h.to_string()[0..8].to_string()) - .unwrap_or_else(|| "00000000".into()); + .unwrap_or_else(|| "0000000".into()); let new_index = new_hash .map(|h| h.to_string()[0..8].to_string()) - .unwrap_or_else(|| "00000000".into()); + .unwrap_or_else(|| "0000000".into()); writeln!(out, "index {old_index}..{new_index}").unwrap(); // infer MIME / text vs binary From b8308fb913de49ad05fdeb4e75793ceb4397bba5 Mon Sep 17 00:00:00 2001 From: AidCheng Date: Wed, 6 Aug 2025 10:55:33 +0100 Subject: [PATCH 15/15] UPDATE: Clippy Error Fix Signed-off-by: AidCheng --- neptune/src/neptune_engine.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/neptune/src/neptune_engine.rs b/neptune/src/neptune_engine.rs index 8aaa00bf9..9d04b8d81 100644 --- a/neptune/src/neptune_engine.rs +++ b/neptune/src/neptune_engine.rs @@ -92,8 +92,8 @@ impl Diff { let old_hash = old_blobs.get(file); let new_hash = new_blobs.get(file); - let old_bytes = old_hash.map_or_else(|| Vec::new(), |h| read_content(file, h)); - let new_bytes = new_hash.map_or_else(|| Vec::new(), |h| read_content(file, h)); + let old_bytes = old_hash.map_or_else(Vec::new, |h| read_content(file, h)); + let new_bytes = new_hash.map_or_else(Vec::new, |h| read_content(file, h)); let old_lines = String::from_utf8_lossy(&old_bytes).lines().count(); let new_lines = String::from_utf8_lossy(&new_bytes).lines().count(); @@ -177,8 +177,8 @@ impl Diff { let old_hash = old_blobs.get(file); // Read contents or empty - let old_bytes = old_hash.map_or_else(|| Vec::new(), |h| read_content(file, h)); - let new_bytes = new_hash.map_or_else(|| Vec::new(), |h| read_content(file, h)); + let old_bytes = old_hash.map_or_else(Vec::new, |h| read_content(file, h)); + let new_bytes = new_hash.map_or_else(Vec::new, |h| read_content(file, h)); // diff header writeln!(out, "diff --git a/{} b/{}", file.display(), file.display()).unwrap(); @@ -193,10 +193,10 @@ impl Diff { // index line let old_index = old_hash .map(|h| h.to_string()[0..8].to_string()) - .unwrap_or_else(|| "0000000".into()); + .unwrap_or_else(|| "00000000".into()); let new_index = new_hash .map(|h| h.to_string()[0..8].to_string()) - .unwrap_or_else(|| "0000000".into()); + .unwrap_or_else(|| "00000000".into()); writeln!(out, "index {old_index}..{new_index}").unwrap(); // infer MIME / text vs binary