add Tests to Actions & test lfs_split

.github/workflows/base.yml

@@ -60,6 +60,31 @@ jobs:
           command: clippy
           args: --workspace --all-targets --all-features -- -D warnings
 
+  #
+  test:
+    name: Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: Swatinem/rust-cache@v2
+      - run: |
+          sudo apt-get update
+          sudo apt-get install -y git-lfs
+          git lfs install
+          git config --global user.email "mega@github.com"
+          git config --global user.name "Mega"
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --workspace --test '*' -- --nocapture
+
   #
   doc:
     name: Doc

ceres/src/lfs/handler.rs

@@ -196,11 +196,7 @@ pub async fn lfs_process_batch(
         // Found
         let found = meta.is_ok();
         let mut meta = meta.unwrap_or_default();
-        if found
-            && config
-                .lfs_storage
-                .exist_object(&config.repo_name, &meta.oid)
-        {
+        if found && lfs_file_exist(config, &meta).await {
             // originla download method, split mode use ``
             response_objects.push(represent(object, &meta, true, false, false, &server_url).await);
             continue;
@@ -470,6 +466,28 @@ fn create_link(href: &str, header: &HashMap<String, String>) -> Link {
     }
 }
 
+/// check if meta file exist in storage.
+async fn lfs_file_exist(config: &LfsConfig, meta: &MetaObject) -> bool {
+    if meta.splited && config.enable_split {
+        let relations = config
+            .context
+            .services
+            .lfs_storage
+            .get_lfs_relations(meta.oid.clone())
+            .await
+            .unwrap();
+        relations.iter().all(|relation| {
+            config
+                .lfs_storage
+                .exist_object(&config.repo_name, &relation.sub_oid)
+        })
+    } else {
+        config
+            .lfs_storage
+            .exist_object(&config.repo_name, &meta.oid)
+    }
+}
+
 async fn lfs_get_filtered_locks(
     storage: Arc<LfsStorage>,
     refspec: &str,

mega/Cargo.toml

@@ -40,6 +40,7 @@ bytes = { workspace = true }
 go-defer = { workspace = true }
 git2 = "0.19.0"
 toml = "0.8.13"
+tempfile = "3.10.1"
 
 [build-dependencies]
 shadow-rs = { workspace = true }

mega/tests/lfs_test.rs

@@ -0,0 +1,118 @@
+/// integration tests for the mega module
+use std::process::Command;
+use std::{env, fs, io, thread};
+use std::io::Write;
+use std::path::Path;
+use rand::Rng;
+use tempfile::TempDir;
+
+const PORT: u16 = 8000; // mega server port
+/// check if git lfs is installed
+fn check_git_lfs() -> bool {
+    let status = Command::new("git")
+        .args(["lfs", "version"])
+        .status()
+        .expect("Failed to execute git lfs version");
+
+    status.success()
+}
+
+fn run_git_cmd(args: &[&str]) {
+    let status = Command::new("git")
+        .args(args)
+        .status()
+        .unwrap();
+
+    assert!(status.success(), "Git command failed: git {}", args.join(" "));
+}
+
+fn is_port_in_use(port: u16) -> bool {
+    let output = Command::new("lsof")
+        .arg(format!("-i:{}", port))
+        .output()
+        .expect("Failed to execute command");
+
+    !output.stdout.is_empty()
+}
+
+fn run_mega_server() {
+    thread::spawn(|| {
+        let args = vec!["service", "multi", "http"];
+        mega::cli::parse(Some(args)).expect("Failed to start mega service");
+    });
+    // loop check until 8000 port to be ready
+    let mut i = 0;
+    while !is_port_in_use(PORT) && (i < 10) {
+        thread::sleep(std::time::Duration::from_secs(1));
+        i += 1;
+    }
+    assert!(is_port_in_use(PORT), "mega server not started");
+    println!("mega server started in {} secs", i);
+}
+
+fn generate_large_file(path: &str, size_mb: usize) -> io::Result<()> {
+    let mut file = fs::File::create(path)?;
+    let mut rng = rand::thread_rng();
+
+    const BUFFER_SIZE: usize = 1024 * 1024; // 1MB buffer
+    let mut buffer = [0u8; BUFFER_SIZE];
+
+    for _ in 0..size_mb {
+        rng.fill(&mut buffer[..]);
+        file.write_all(&buffer)?;
+    }
+
+    Ok(())
+}
+
+fn lfs_push(url: &str) -> io::Result<()> {
+    let temp_dir = TempDir::new()?;
+    let repo_path = temp_dir.path().to_str().unwrap();
+    println!("repo_path: {}", repo_path);
+
+    // git init
+    run_git_cmd(&["init", repo_path]);
+
+    env::set_current_dir(repo_path)?;
+
+    // track Large file
+    run_git_cmd(&["lfs", "track", "*.bin"]);
+
+    // create large file
+    generate_large_file("large_file.bin", 60)?;
+
+    // add & commit
+    run_git_cmd(&["add", "."]);
+    run_git_cmd(&["commit", "-m", "add large file"]);
+
+    // push to mega server
+    run_git_cmd(&["remote", "add", "mega", url]);
+    run_git_cmd(&["push", "--all", "mega"]);
+
+    Ok(())
+}
+
+fn lfs_clone(url: &str) -> io::Result<()> {
+    let temp_dir = TempDir::new()?;
+    println!("clone temp_dir: {:?}", temp_dir.path());
+    env::set_current_dir(temp_dir.path())?;
+    // git clone url
+    run_git_cmd(&["clone", url]);
+
+    assert!(Path::new("lfs/large_file.bin").exists(), "Failed to clone large file");
+    Ok(())
+}
+
+#[test]
+fn lfs_split_with_git() {
+    assert!(check_git_lfs(), "git lfs is not installed");
+
+    let mega_dir = TempDir::new().unwrap();
+    env::set_var("MEGA_BASE_DIR", mega_dir.path());
+    // start mega server at background
+    run_mega_server();
+
+    let url = &format!("http://localhost:{}/third-part/lfs.git", PORT);
+    lfs_push(url).expect("Failed to push large file to mega server");
+    lfs_clone(url).expect("Failed to clone large file from mega server");
+}