From a8c161436d19db7f03631c970b60ee8a63e3af9f Mon Sep 17 00:00:00 2001 From: HouXiaoxuan Date: Mon, 22 Jul 2024 14:35:36 +0800 Subject: [PATCH 1/5] fix LFS process batch: only check for existence of original file Signed-off-by: HouXiaoxuan --- ceres/src/lfs/handler.rs | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index dce0b3d54..64df64795 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -196,11 +196,7 @@ pub async fn lfs_process_batch( // Found let found = meta.is_ok(); let mut meta = meta.unwrap_or_default(); - if found - && config - .lfs_storage - .exist_object(&config.repo_name, &meta.oid) - { + if found && lfs_file_exist(config, &meta).await { // originla download method, split mode use `` response_objects.push(represent(object, &meta, true, false, false, &server_url).await); continue; @@ -470,6 +466,28 @@ fn create_link(href: &str, header: &HashMap) -> Link { } } +/// check if meta file exist in storage. +async fn lfs_file_exist(config: &LfsConfig, meta: &MetaObject) -> bool { + if meta.splited && config.enable_split { + let relations = config + .context + .services + .lfs_storage + .get_lfs_relations(meta.oid.clone()) + .await + .unwrap(); + relations.iter().all(|relation| { + config + .lfs_storage + .exist_object(&config.repo_name, &relation.sub_oid) + }) + } else { + config + .lfs_storage + .exist_object(&config.repo_name, &meta.oid) + } +} + async fn lfs_get_filtered_locks( storage: Arc, refspec: &str, From fb12ad3286b305edf2242ee271b52a7e3e90a029 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 22 Jul 2024 15:30:41 +0800 Subject: [PATCH 2/5] add integration test for `lfs_split` Signed-off-by: Qihang Cai --- mega/Cargo.toml | 1 + mega/tests/lfs_test.rs | 118 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 mega/tests/lfs_test.rs diff --git a/mega/Cargo.toml b/mega/Cargo.toml index 6cb33524f..c0eeabc55 100644 --- a/mega/Cargo.toml +++ b/mega/Cargo.toml @@ -40,6 +40,7 @@ bytes = { workspace = true } go-defer = { workspace = true } git2 = "0.19.0" toml = "0.8.13" +tempfile = "3.10.1" [build-dependencies] shadow-rs = { workspace = true } diff --git a/mega/tests/lfs_test.rs b/mega/tests/lfs_test.rs new file mode 100644 index 000000000..980c0c11f --- /dev/null +++ b/mega/tests/lfs_test.rs @@ -0,0 +1,118 @@ +/// integration tests for the mega module +use std::process::Command; +use std::{env, fs, io, thread}; +use std::io::Write; +use std::path::Path; +use rand::Rng; +use tempfile::TempDir; + +const PORT: u16 = 8000; // mega server port +/// check if git lfs is installed +fn check_git_lfs() -> bool { + let status = Command::new("git") + .args(&["lfs", "version"]) + .status() + .expect("Failed to execute git lfs version"); + + status.success() +} + +fn run_git_cmd(args: &[&str]) { + let status = Command::new("git") + .args(args) + .status() + .unwrap(); + + assert!(status.success(), "Git command failed: git {}", args.join(" ")); +} + +fn is_port_in_use(port: u16) -> bool { + let output = Command::new("lsof") + .arg(format!("-i:{}", port)) + .output() + .expect("Failed to execute command"); + + !output.stdout.is_empty() +} + +fn run_mega_server() { + thread::spawn(|| { + let args = vec!["service", "multi", "http"]; + mega::cli::parse(Some(args)).expect("Failed to start mega service"); + }); + // loop check until 8000 port to be ready + let mut i = 0; + while !is_port_in_use(PORT) && (i < 10) { + thread::sleep(std::time::Duration::from_secs(1)); + i += 1; + } + assert!(is_port_in_use(PORT), "mega server not started"); + println!("mega server started in {} secs", i); +} + +fn generate_large_file(path: &str, size_mb: usize) -> io::Result<()> { + let mut file = fs::File::create(path)?; + let mut rng = rand::thread_rng(); + + const BUFFER_SIZE: usize = 1024 * 1024; // 1MB buffer + let mut buffer = [0u8; BUFFER_SIZE]; + + for _ in 0..size_mb { + rng.fill(&mut buffer[..]); + file.write_all(&buffer)?; + } + + Ok(()) +} + +fn lfs_push(url: &str) -> io::Result<()> { + let temp_dir = TempDir::new()?; + let repo_path = temp_dir.path().to_str().unwrap(); + println!("repo_path: {}", repo_path); + + // git init + run_git_cmd(&["init", repo_path]); + + env::set_current_dir(repo_path)?; + + // track Large file + run_git_cmd(&["lfs", "track", "*.bin"]); + + // create large file + generate_large_file("large_file.bin", 60)?; + + // add & commit + run_git_cmd(&["add", "."]); + run_git_cmd(&["commit", "-m", "add large file"]); + + // push to mega server + run_git_cmd(&["remote", "add", "mega", url]); + run_git_cmd(&["push", "--all", "mega"]); + + Ok(()) +} + +fn lfs_clone(url: &str) -> io::Result<()> { + let temp_dir = TempDir::new()?; + println!("clone temp_dir: {:?}", temp_dir.path()); + env::set_current_dir(temp_dir.path())?; + // git clone url + run_git_cmd(&["clone", url]); + + assert!(Path::new("lfs/large_file.bin").exists(), "Failed to clone large file"); + Ok(()) +} + +#[test] +fn lfs_split_with_git() { + assert!(check_git_lfs(), "git lfs is not installed"); + + let mega_dir = TempDir::new().unwrap(); + env::set_var("MEGA_BASE_DIR", mega_dir.path()); + // start mega server at background + run_mega_server(); + + let url = &format!("http://localhost:{}/third-part/lfs.git", PORT); + lfs_push(url).expect("Failed to push large file to mega server"); + lfs_clone(url).expect("Failed to clone large file from mega server"); +} \ No newline at end of file From c3e89c18b32d402302474e8710fb59cdc6e2044a Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 22 Jul 2024 15:48:57 +0800 Subject: [PATCH 3/5] update GitHub Actions, add Tests Signed-off-by: Qihang Cai --- .github/workflows/base.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 47403edaf..e5cf1bbc8 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -60,6 +60,29 @@ jobs: command: clippy args: --workspace --all-targets --all-features -- -D warnings + # + test: + name: Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v2 + - run: | + sudo apt-get update + sudo apt-get install -y git-lfs + git lfs install + - uses: actions-rs/cargo@v1 + with: + command: test + args: --workspace --test '*' -- --nocapture + # doc: name: Doc From bda72658b763ec44b66b89b13cac21c7dd9d6c7f Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 22 Jul 2024 15:52:21 +0800 Subject: [PATCH 4/5] clear clippy warning Signed-off-by: Qihang Cai --- mega/tests/lfs_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mega/tests/lfs_test.rs b/mega/tests/lfs_test.rs index 980c0c11f..0f601f34e 100644 --- a/mega/tests/lfs_test.rs +++ b/mega/tests/lfs_test.rs @@ -10,7 +10,7 @@ const PORT: u16 = 8000; // mega server port /// check if git lfs is installed fn check_git_lfs() -> bool { let status = Command::new("git") - .args(&["lfs", "version"]) + .args(["lfs", "version"]) .status() .expect("Failed to execute git lfs version"); From ccd20b3f5a84dcc2ae1264fb9e9f7da14d31647d Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 22 Jul 2024 16:16:50 +0800 Subject: [PATCH 5/5] update actions, add git email & name Signed-off-by: Qihang Cai --- .github/workflows/base.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index e5cf1bbc8..82fe88d08 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -78,6 +78,8 @@ jobs: sudo apt-get update sudo apt-get install -y git-lfs git lfs install + git config --global user.email "mega@github.com" + git config --global user.name "Mega" - uses: actions-rs/cargo@v1 with: command: test