[neptune]FEAT: new module that reconstructs diff function from mono and libra

Cargo.toml

@@ -18,6 +18,7 @@ members = [
     "extensions/observatory",
     "scorpio",
     "context",
+    "neptune",
 ]
 default-members = ["mega", "mono", "libra", "aries", "orion", "orion-server"]
 resolver = "1"
@@ -36,6 +37,7 @@ mono = { path = "mono" }
 observatory = { path = "extensions/observatory" }
 orion = { path = "orion" }
 context = { path = "context" }
+neptune = { path = "neptune" }
 
 anyhow = "1.0.98"
 serde = "1.0.219"
@@ -49,10 +51,8 @@ rand_chacha = "0.9.0"
 smallvec = "1.15.0"
 tokio = "1.45.1"
 tokio-stream = "0.1.17"
-tokio-test = "0.4.4"
 tokio-util = "0.7.15"
 clap = "4.5.39"
-async-std = "1.13.1"
 async-trait = "0.1.88"
 async-stream = "0.3.6"
 bytes = "1.10.1"
@@ -63,7 +63,7 @@ futures = "0.3.31"
 futures-util = "0.3.31"
 go-defer = "0.1.0"
 russh = "0.52.1"
-axum = "0.8.4"
+axum = {version="0.8.4", features=["macros", "json"]}
 axum-extra = "0.10.1"
 tower-http = "0.6.4"
 tower = "0.5.2"
@@ -82,8 +82,6 @@ uuid = "1.17.0"
 regex = "1.11.1"
 ed25519-dalek = "2.1.1"
 ctrlc = "3.4.7"
-git2 = "0.20.2"
-home = "0.5.11"
 ring = "0.17.14"
 cedar-policy = "4.4.1"
 secp256k1 = "0.30.0"
@@ -113,9 +111,7 @@ serial_test = "3.2.0"
 sysinfo = "0.36.0"
 http = "1.3.1"
 byte-unit = "5.1.6"
-color-backtrace = "0.7.0"
 ignore = "0.4.23"
-imara-diff = "0.2.0"
 indicatif = "0.18.0"
 infer = "0.19.0"
 path-absolutize = "3.1.1"
@@ -125,7 +121,6 @@ similar = "2.7.0"
 url = "2.5.4"
 wax = "0.6.0"
 pager = "0.16.1"
-tracing-test = "0.2.5"
 jemallocator = "0.5.4"
 mimalloc = "0.1.46"
 assert_cmd = "2.0.17"

ceres/Cargo.toml

@@ -15,6 +15,7 @@ common = { workspace = true }
 jupiter = { workspace = true }
 callisto = { workspace = true }
 mercury = { workspace = true }
+neptune = { workspace = true }
 
 anyhow = { workspace = true }
 tokio = { workspace = true, features = ["net", "process"] }
@@ -23,13 +24,11 @@ axum = { workspace = true }
 tracing = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-clap = { workspace = true, features = ["derive"] }
 chrono = { workspace = true }
 futures = { workspace = true }
 bytes = { workspace = true }
 async-trait = { workspace = true }
 rand = { workspace = true }
-sea-orm = { workspace = true }
 ring = { workspace = true }
 hex = { workspace = true }
 sysinfo = { workspace = true }

ceres/src/api_service/mono_api_service.rs

@@ -38,14 +38,13 @@
 use std::collections::{HashMap, HashSet};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::{env, fs};
 
 use async_trait::async_trait;
-use tokio::process::Command;
 
 use callisto::sea_orm_active_enums::ConvTypeEnum;
 use callisto::{mega_blob, mega_mr, mega_tree, raw_blob};
 use common::errors::MegaError;
+use neptune::neptune_engine::Diff;
 use jupiter::storage::base_storage::StorageConnector;
 use jupiter::storage::Storage;
 use jupiter::utils::converter::generate_git_keep_with_timestamp;
@@ -284,15 +283,15 @@ impl MonoApiService {
 
             if mr.path != "/" {
                 let path = PathBuf::from(mr.path.clone());
-                // beacuse only parent tree is needed so we skip current directory
+                // because only parent tree is needed so we skip current directory
                 let (tree_vec, _) = self
                     .search_tree_for_update(path.parent().unwrap())
                     .await
                     .unwrap();
                 self.update_parent_tree(path, tree_vec, commit)
                     .await
                     .unwrap();
-                // remove refs start with path exceprt mr type
+                // remove refs start with path except mr type
                 storage.remove_none_mr_refs(&mr.path).await.unwrap();
                 // TODO: self.clean_dangling_commits().await;
             }
@@ -372,90 +371,72 @@ impl MonoApiService {
         Ok(p_commit_id)
     }
 
-    pub async fn content_diff(&self, mr_link: &str, listen_addr: &str) -> Result<String, GitError> {
+    /// Generate diff content directly from two blob sets using the diff engine
+    ///
+    /// # Arguments
+    /// * `mr_link` - Merge request link identifier
+    ///
+    /// # Returns
+    /// String containing the unified diff output
+    pub async fn content_diff(
+        &self,
+        mr_link: &str,
+    ) -> Result<String, GitError> {
         let stg = self.storage.mr_storage();
-        if let Some(mr) = stg.get_mr(mr_link).await.unwrap() {
-            let base_path = self.storage.config().base_dir.clone();
-            env::set_current_dir(&base_path).unwrap();
-            let clone_path = base_path.join(mr_link);
-            if !fs::exists(&clone_path).unwrap() {
-                let result = self.run_libra_diff(&mr, listen_addr, &clone_path).await;
-                if result.is_err() && fs::exists(&clone_path).unwrap() {
-                    fs::remove_dir_all(&clone_path).unwrap();
+        let mr = stg.get_mr(mr_link).await.unwrap().ok_or_else(|| {
+            GitError::CustomError(format!("Merge request not found: {mr_link}"))
+        })?;
+
+        let old_blobs = self.get_commit_blobs(&mr.from_hash).await.map_err(|e| {
+            GitError::CustomError(format!("Failed to get old commit blobs: {e}"))
+        })?;
+        let new_blobs = self.get_commit_blobs(&mr.to_hash).await.map_err(|e| {
+            GitError::CustomError(format!("Failed to get new commit blobs: {e}"))
+        })?;
+
+
+        let algo = "histogram".to_string(); // Default diff algorithm
+        let filter_paths: Vec<PathBuf> = Vec::new(); // No filtering by default
+
+        // Pre-fetch all blob contents to avoid async issues in the closure
+        let mut blob_cache: HashMap<SHA1, Vec<u8>> = HashMap::new();
+
+        // Collect all unique hashes
+        let mut all_hashes = HashSet::new();
+        for (_, hash) in &old_blobs {
+            all_hashes.insert(*hash);
+        }
+        for (_, hash) in &new_blobs {
+            all_hashes.insert(*hash);
+        }
+
+        // Fetch all blobs concurrently
+        for hash in all_hashes {
+            match self.get_raw_blob_by_hash(&hash.to_string()).await {
+                Ok(Some(blob)) => {
+                    blob_cache.insert(hash, blob.data.unwrap_or_default());
+                }
+                _ => {
+                    blob_cache.insert(hash, Vec::new());
                 }
-            } else {
-                env::set_current_dir(&clone_path).unwrap();
-            }
-            tracing::debug!("Run libra Command: libra diff --old {}", mr.from_hash);
-            let output = Command::new("libra")
-                .arg("diff")
-                .arg("--old")
-                .arg(mr.from_hash)
-                .output()
-                .await?;
-
-            let stderr_str = String::from_utf8_lossy(&output.stderr);
-
-            if !stderr_str.trim().is_empty() || !output.status.success() {
-                tracing::error!(
-                    "Command failed: {}",
-                    String::from_utf8_lossy(&output.stderr)
-                );
-                fs::remove_dir_all(&clone_path).unwrap();
-            } else {
-                return Ok(String::from_utf8_lossy(&output.stdout).to_string());
             }
         }
-        Ok(String::new())
-    }
 
-    async fn run_libra_diff(
-        &self,
-        mr: &callisto::mega_mr::Model,
-        listen_addr: &str,
-        clone_path: &PathBuf,
-    ) -> Result<(), anyhow::Error> {
-        Command::new("mkdir").arg(&mr.link).output().await?;
-        env::set_current_dir(clone_path).unwrap();
-        Command::new("libra").arg("init").output().await?;
-        let git_remote = if mr.path.starts_with("/") {
-            format!("{}{}", listen_addr, mr.path)
-        } else {
-            format!("{}/{}", listen_addr, mr.path)
+        // Simple synchronous closure that uses the pre-fetched cache
+        let read_content = |_file: &PathBuf, hash: &SHA1| -> Vec<u8> {
+            blob_cache.get(hash).cloned().unwrap_or_default()
         };
-        tracing::debug!("Run libra Command: libra remote add origin {}", &git_remote);
-        Command::new("libra")
-            .arg("remote")
-            .arg("add")
-            .arg("origin")
-            .arg(git_remote)
-            .output()
-            .await?;
-        tracing::debug!("Run libra Command: libra fetch origin refs/mr/{}", &mr.link);
-        Command::new("libra")
-            .arg("fetch")
-            .arg("origin")
-            .arg(format!("refs/mr/{}", &mr.link))
-            .output()
-            .await?;
-        tracing::debug!(
-            "Run libra Command: libra branch {} origin/mr/{}",
-            &mr.link,
-            &mr.link
-        );
-        Command::new("libra")
-            .arg("branch")
-            .arg(&mr.link)
-            .arg(format!("origin/mr/{}", &mr.link))
-            .output()
-            .await?;
-        tracing::debug!("Run libra Command: libra switch {}", &mr.link);
-        Command::new("libra")
-            .arg("switch")
-            .arg(&mr.link)
-            .output()
-            .await?;
-        Ok(())
+
+        // Use the unified diff function that returns a single string
+        let diff_output = Diff::diff(
+            old_blobs,
+            new_blobs,
+            algo,
+            filter_paths,
+            read_content,
+        ).await;
+
+        Ok(diff_output)
     }
 
     pub async fn mr_files_list(

libra/Cargo.toml

@@ -22,12 +22,11 @@ futures = { workspace = true }
 futures-util = { workspace = true }
 gemini = { workspace = true, optional = true }
 hex = { workspace = true }
-imara-diff = { workspace = true }
 indicatif = { workspace = true }
-infer = { workspace = true }
 lazy_static = { workspace = true }
 lru-mem = { workspace = true }
 mercury = { workspace = true }
+neptune = { workspace = true }
 once_cell = { workspace = true }
 path-absolutize = { workspace = true }
 pathdiff = { workspace = true }