diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 47403edaf..82fe88d08 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -60,6 +60,31 @@ jobs: command: clippy args: --workspace --all-targets --all-features -- -D warnings + # + test: + name: Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v2 + - run: | + sudo apt-get update + sudo apt-get install -y git-lfs + git lfs install + git config --global user.email "mega@github.com" + git config --global user.name "Mega" + - uses: actions-rs/cargo@v1 + with: + command: test + args: --workspace --test '*' -- --nocapture + # doc: name: Doc diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index dce0b3d54..64df64795 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -196,11 +196,7 @@ pub async fn lfs_process_batch( // Found let found = meta.is_ok(); let mut meta = meta.unwrap_or_default(); - if found - && config - .lfs_storage - .exist_object(&config.repo_name, &meta.oid) - { + if found && lfs_file_exist(config, &meta).await { // originla download method, split mode use `` response_objects.push(represent(object, &meta, true, false, false, &server_url).await); continue; @@ -470,6 +466,28 @@ fn create_link(href: &str, header: &HashMap) -> Link { } } +/// check if meta file exist in storage. +async fn lfs_file_exist(config: &LfsConfig, meta: &MetaObject) -> bool { + if meta.splited && config.enable_split { + let relations = config + .context + .services + .lfs_storage + .get_lfs_relations(meta.oid.clone()) + .await + .unwrap(); + relations.iter().all(|relation| { + config + .lfs_storage + .exist_object(&config.repo_name, &relation.sub_oid) + }) + } else { + config + .lfs_storage + .exist_object(&config.repo_name, &meta.oid) + } +} + async fn lfs_get_filtered_locks( storage: Arc, refspec: &str, diff --git a/mega/Cargo.toml b/mega/Cargo.toml index 6cb33524f..c0eeabc55 100644 --- a/mega/Cargo.toml +++ b/mega/Cargo.toml @@ -40,6 +40,7 @@ bytes = { workspace = true } go-defer = { workspace = true } git2 = "0.19.0" toml = "0.8.13" +tempfile = "3.10.1" [build-dependencies] shadow-rs = { workspace = true } diff --git a/mega/tests/lfs_test.rs b/mega/tests/lfs_test.rs new file mode 100644 index 000000000..0f601f34e --- /dev/null +++ b/mega/tests/lfs_test.rs @@ -0,0 +1,118 @@ +/// integration tests for the mega module +use std::process::Command; +use std::{env, fs, io, thread}; +use std::io::Write; +use std::path::Path; +use rand::Rng; +use tempfile::TempDir; + +const PORT: u16 = 8000; // mega server port +/// check if git lfs is installed +fn check_git_lfs() -> bool { + let status = Command::new("git") + .args(["lfs", "version"]) + .status() + .expect("Failed to execute git lfs version"); + + status.success() +} + +fn run_git_cmd(args: &[&str]) { + let status = Command::new("git") + .args(args) + .status() + .unwrap(); + + assert!(status.success(), "Git command failed: git {}", args.join(" ")); +} + +fn is_port_in_use(port: u16) -> bool { + let output = Command::new("lsof") + .arg(format!("-i:{}", port)) + .output() + .expect("Failed to execute command"); + + !output.stdout.is_empty() +} + +fn run_mega_server() { + thread::spawn(|| { + let args = vec!["service", "multi", "http"]; + mega::cli::parse(Some(args)).expect("Failed to start mega service"); + }); + // loop check until 8000 port to be ready + let mut i = 0; + while !is_port_in_use(PORT) && (i < 10) { + thread::sleep(std::time::Duration::from_secs(1)); + i += 1; + } + assert!(is_port_in_use(PORT), "mega server not started"); + println!("mega server started in {} secs", i); +} + +fn generate_large_file(path: &str, size_mb: usize) -> io::Result<()> { + let mut file = fs::File::create(path)?; + let mut rng = rand::thread_rng(); + + const BUFFER_SIZE: usize = 1024 * 1024; // 1MB buffer + let mut buffer = [0u8; BUFFER_SIZE]; + + for _ in 0..size_mb { + rng.fill(&mut buffer[..]); + file.write_all(&buffer)?; + } + + Ok(()) +} + +fn lfs_push(url: &str) -> io::Result<()> { + let temp_dir = TempDir::new()?; + let repo_path = temp_dir.path().to_str().unwrap(); + println!("repo_path: {}", repo_path); + + // git init + run_git_cmd(&["init", repo_path]); + + env::set_current_dir(repo_path)?; + + // track Large file + run_git_cmd(&["lfs", "track", "*.bin"]); + + // create large file + generate_large_file("large_file.bin", 60)?; + + // add & commit + run_git_cmd(&["add", "."]); + run_git_cmd(&["commit", "-m", "add large file"]); + + // push to mega server + run_git_cmd(&["remote", "add", "mega", url]); + run_git_cmd(&["push", "--all", "mega"]); + + Ok(()) +} + +fn lfs_clone(url: &str) -> io::Result<()> { + let temp_dir = TempDir::new()?; + println!("clone temp_dir: {:?}", temp_dir.path()); + env::set_current_dir(temp_dir.path())?; + // git clone url + run_git_cmd(&["clone", url]); + + assert!(Path::new("lfs/large_file.bin").exists(), "Failed to clone large file"); + Ok(()) +} + +#[test] +fn lfs_split_with_git() { + assert!(check_git_lfs(), "git lfs is not installed"); + + let mega_dir = TempDir::new().unwrap(); + env::set_var("MEGA_BASE_DIR", mega_dir.path()); + // start mega server at background + run_mega_server(); + + let url = &format!("http://localhost:{}/third-part/lfs.git", PORT); + lfs_push(url).expect("Failed to push large file to mega server"); + lfs_clone(url).expect("Failed to clone large file from mega server"); +} \ No newline at end of file