From 92f2d94d2e5197c1bf20f25d31a65f4ad150125d Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Fri, 16 Aug 2024 23:17:59 +0800 Subject: [PATCH 01/14] libra: add Command `lfs track` Signed-off-by: Qihang Cai --- libra/Cargo.toml | 2 + libra/src/command/lfs.rs | 106 +++++++++++++++++++++++++++++++++++++++ libra/src/command/mod.rs | 1 + libra/src/main.rs | 3 ++ libra/src/utils/path.rs | 5 ++ libra/src/utils/util.rs | 1 + 6 files changed, 118 insertions(+) create mode 100644 libra/src/command/lfs.rs diff --git a/libra/Cargo.toml b/libra/Cargo.toml index 3f203f4fc..2c08bb799 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -35,6 +35,8 @@ url = "2.5.0" futures-util = "0.3.30" rpassword = "7.3.1" indicatif = "0.17.8" +wax = "0.6.0" +regex = { workspace = true } [target.'cfg(unix)'.dependencies] # only on Unix pager = "0.16.0" diff --git a/libra/src/command/lfs.rs b/libra/src/command/lfs.rs new file mode 100644 index 000000000..29dcc5c68 --- /dev/null +++ b/libra/src/command/lfs.rs @@ -0,0 +1,106 @@ +use clap::Subcommand; +use regex::Regex; +use std::fs::{File, OpenOptions}; +use std::io; +use std::io::{BufRead, Read, Seek, SeekFrom, Write}; +use std::path::Path; +use crate::utils::{path, util}; +use crate::utils::path_ext::PathExt; + +#[derive(Subcommand, Debug)] +pub enum LfsCmds { + /// View or add LFS paths to Libra Attributes + Track { + pattern: Option>, + }, + /// Remove LFS paths from Libra Attributes + Untrack { + path: Vec, + }, +} + +pub async fn execute(cmd: LfsCmds) { + match cmd { + LfsCmds::Track { pattern } => { + let attr_path = path::attributes().to_string_or_panic(); + match pattern { + Some(pattern) => { + add_lfs_patterns(&attr_path, pattern).unwrap(); + } + None => { + let lfs_patterns = extract_lfs_patterns(&attr_path).unwrap(); + if !lfs_patterns.is_empty() { + println!("Listing tracked patterns"); + for p in lfs_patterns { + println!(" {} ({})", p, util::ATTRIBUTES); // '\t' seems to be 8 spaces, :( + } + } + } + } + } + LfsCmds::Untrack { path } => { + println!("{:?}", path); + } + } +} + +fn add_lfs_patterns(file_path: &str, patterns: Vec) -> io::Result<()> { + let mut file = OpenOptions::new() + .create(true) + .read(true) + .append(true) + .open(file_path)?; + + if file.metadata()?.len() > 0 { + file.seek(SeekFrom::End(-1))?; + + let mut last_byte = [0; 1]; + file.read_exact(&mut last_byte)?; + + // ensure the last byte is '\n' + if last_byte[0] != b'\n' { + file.write_all(b"\n")?; + } + } + + let lfs_patterns = extract_lfs_patterns(&file_path)?; + for pattern in patterns { + if lfs_patterns.contains(&pattern) { + continue; + } + println!("Tracking {}", pattern); + let pattern = format!("{} filter=lfs diff=lfs merge=lfs -text\n", pattern.replace(" ", r"\ ")); + file.write_all(pattern.as_bytes())?; + } + + Ok(()) +} + +fn extract_lfs_patterns(file_path: &str) -> io::Result> { + let path = Path::new(file_path); + if !path.exists() { + return Ok(Vec::new()); + } + let file = File::open(&path)?; + let reader = io::BufReader::new(file); + + // ' ' needs '\' before it to be escaped + let re = Regex::new(r"^\s*(([^\s#\\]|\\ )+)").unwrap(); + + let mut patterns = Vec::new(); + + for line in reader.lines() { + let line = line?; + if !line.contains("filter=lfs") { + continue; + } + if let Some(cap) = re.captures(&line) { + if let Some(pattern) = cap.get(1) { + let pattern = pattern.as_str().replace(r"\ ", " "); + patterns.push(pattern); + } + } + } + + Ok(patterns) +} \ No newline at end of file diff --git a/libra/src/command/mod.rs b/libra/src/command/mod.rs index 07aae11d7..dc60c848d 100644 --- a/libra/src/command/mod.rs +++ b/libra/src/command/mod.rs @@ -14,6 +14,7 @@ pub mod remove; pub mod restore; pub mod status; pub mod switch; +pub mod lfs; use crate::internal::protocol::https_client::BasicAuth; use crate::utils::util; diff --git a/libra/src/main.rs b/libra/src/main.rs index c711cb4f2..477bf1762 100644 --- a/libra/src/main.rs +++ b/libra/src/main.rs @@ -39,6 +39,8 @@ enum Commands { Restore(command::restore::RestoreArgs), #[command(about = "Show the working tree status")] Status, + #[command(subcommand, about = "Large File Storage")] + Lfs(command::lfs::LfsCmds), #[command(about = "Show commit logs")] Log(command::log::LogArgs), #[command(about = "List, create, or delete branches")] @@ -96,6 +98,7 @@ async fn main() { Commands::Rm(args) => command::remove::execute(args).unwrap(), Commands::Restore(args) => command::restore::execute(args).await, Commands::Status => command::status::execute().await, + Commands::Lfs(cmd) => command::lfs::execute(cmd).await, Commands::Log(args) => command::log::execute(args).await, Commands::Branch(args) => command::branch::execute(args).await, Commands::Commit(args) => command::commit::execute(args).await, diff --git a/libra/src/utils/path.rs b/libra/src/utils/path.rs index dd7c36be1..812f39c97 100644 --- a/libra/src/utils/path.rs +++ b/libra/src/utils/path.rs @@ -4,10 +4,15 @@ use crate::utils::util; pub fn index() -> PathBuf { util::storage_path().join("index") } + pub fn objects() -> PathBuf { util::storage_path().join("objects") } pub fn database() -> PathBuf { util::storage_path().join(util::DATABASE) +} + +pub fn attributes() -> PathBuf { + util::working_dir().join(util::ATTRIBUTES) } \ No newline at end of file diff --git a/libra/src/utils/util.rs b/libra/src/utils/util.rs index 7b64cf427..c9eba7690 100644 --- a/libra/src/utils/util.rs +++ b/libra/src/utils/util.rs @@ -13,6 +13,7 @@ use crate::utils::path_ext::PathExt; pub const ROOT_DIR: &str = ".libra"; pub const DATABASE: &str = "libra.db"; +pub const ATTRIBUTES: &str = ".libra_attributes"; /// Returns the current working directory as a `PathBuf`. /// From 439464c51ff64845c45ff8a4a4e6e4cbaff52bf4 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sat, 17 Aug 2024 00:41:35 +0800 Subject: [PATCH 02/14] libra: add Command `lfs untrack` Signed-off-by: Qihang Cai --- libra/src/command/lfs.rs | 54 ++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/libra/src/command/lfs.rs b/libra/src/command/lfs.rs index 29dcc5c68..603a36548 100644 --- a/libra/src/command/lfs.rs +++ b/libra/src/command/lfs.rs @@ -2,7 +2,7 @@ use clap::Subcommand; use regex::Regex; use std::fs::{File, OpenOptions}; use std::io; -use std::io::{BufRead, Read, Seek, SeekFrom, Write}; +use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; use std::path::Path; use crate::utils::{path, util}; use crate::utils::path_ext::PathExt; @@ -21,7 +21,7 @@ pub enum LfsCmds { pub async fn execute(cmd: LfsCmds) { match cmd { - LfsCmds::Track { pattern } => { + LfsCmds::Track { pattern } => { // TODO: deduplicate let attr_path = path::attributes().to_string_or_panic(); match pattern { Some(pattern) => { @@ -39,7 +39,8 @@ pub async fn execute(cmd: LfsCmds) { } } LfsCmds::Untrack { path } => { - println!("{:?}", path); + let attr_path = path::attributes().to_string_or_panic(); + untrack_lfs_patterns(&attr_path, path).unwrap(); } } } @@ -63,12 +64,12 @@ fn add_lfs_patterns(file_path: &str, patterns: Vec) -> io::Result<()> { } } - let lfs_patterns = extract_lfs_patterns(&file_path)?; + let lfs_patterns = extract_lfs_patterns(file_path)?; for pattern in patterns { if lfs_patterns.contains(&pattern) { continue; } - println!("Tracking {}", pattern); + println!("Tracking \"{}\"", pattern); let pattern = format!("{} filter=lfs diff=lfs merge=lfs -text\n", pattern.replace(" ", r"\ ")); file.write_all(pattern.as_bytes())?; } @@ -76,13 +77,52 @@ fn add_lfs_patterns(file_path: &str, patterns: Vec) -> io::Result<()> { Ok(()) } +fn untrack_lfs_patterns(file_path: &str, patterns: Vec) -> io::Result<()> { + if !Path::new(file_path).exists() { + return Ok(()); + } + let file = File::open(file_path)?; + let reader = BufReader::new(file); + + let mut lines: Vec = Vec::new(); + for line in reader.lines() { + let line = line?; + let mut matched_pattern = None; + // delete the specified lfs patterns + for pattern in &patterns { + let pattern = pattern.replace(" ", r"\ "); + if line.trim_start().starts_with(&pattern) && line.contains("filter=lfs") { + matched_pattern = Some(pattern); + break; + } + } + match matched_pattern { + Some(pattern) => println!("Untracking \"{}\"", pattern), + None => lines.push(line), + } + } + + // clear the file + let mut file = OpenOptions::new() + .write(true) + .truncate(true) + .open(file_path)?; + + for line in lines { + file.write_all(line.as_bytes())?; + file.write_all(b"\n")?; + } + + Ok(()) +} + fn extract_lfs_patterns(file_path: &str) -> io::Result> { let path = Path::new(file_path); if !path.exists() { return Ok(Vec::new()); } - let file = File::open(&path)?; - let reader = io::BufReader::new(file); + let file = File::open(path)?; + let reader = BufReader::new(file); // ' ' needs '\' before it to be escaped let re = Regex::new(r"^\s*(([^\s#\\]|\\ )+)").unwrap(); From 5d6a6d46635b58476ee06ec71da95f96677ae8e0 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 19 Aug 2024 14:57:44 +0800 Subject: [PATCH 03/14] lfs: `track`/`untrack` support sub-dirs, but all patterns record in root attributes file (to improve) Signed-off-by: Qihang Cai --- libra/src/command/add.rs | 3 ++- libra/src/command/lfs.rs | 51 ++++++++++++---------------------------- libra/src/utils/lfs.rs | 46 ++++++++++++++++++++++++++++++++++++ libra/src/utils/mod.rs | 3 ++- 4 files changed, 65 insertions(+), 38 deletions(-) create mode 100644 libra/src/utils/lfs.rs diff --git a/libra/src/command/add.rs b/libra/src/command/add.rs index fd032340c..cf388da6b 100644 --- a/libra/src/command/add.rs +++ b/libra/src/command/add.rs @@ -97,7 +97,8 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) { if verbose { println!("removed: {}", file_str); } - } else { + } else { // FIXME: unreachable code! This situation is not included in `status::changes_to_be_staged()` + // FIXME: should check files in original input paths // TODO do this check earlier, once fatal occurs, nothing should be done // file is not tracked && not exists, which means wrong pathspec println!("fatal: pathspec '{}' did not match any files", file.display()); diff --git a/libra/src/command/lfs.rs b/libra/src/command/lfs.rs index 603a36548..04a079072 100644 --- a/libra/src/command/lfs.rs +++ b/libra/src/command/lfs.rs @@ -1,15 +1,14 @@ use clap::Subcommand; -use regex::Regex; use std::fs::{File, OpenOptions}; use std::io; use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; use std::path::Path; -use crate::utils::{path, util}; +use crate::utils::{lfs, path, util}; use crate::utils::path_ext::PathExt; #[derive(Subcommand, Debug)] pub enum LfsCmds { - /// View or add LFS paths to Libra Attributes + /// View or add LFS paths to Libra Attributes (root) Track { pattern: Option>, }, @@ -20,15 +19,17 @@ pub enum LfsCmds { } pub async fn execute(cmd: LfsCmds) { + // TODO: attributes file should be created in current dir, NOT root dir + let attr_path = path::attributes().to_string_or_panic(); match cmd { LfsCmds::Track { pattern } => { // TODO: deduplicate - let attr_path = path::attributes().to_string_or_panic(); match pattern { Some(pattern) => { + let pattern = convert_patterns_to_workdir(pattern); // add_lfs_patterns(&attr_path, pattern).unwrap(); } None => { - let lfs_patterns = extract_lfs_patterns(&attr_path).unwrap(); + let lfs_patterns = lfs::extract_lfs_patterns(&attr_path).unwrap(); if !lfs_patterns.is_empty() { println!("Listing tracked patterns"); for p in lfs_patterns { @@ -39,12 +40,19 @@ pub async fn execute(cmd: LfsCmds) { } } LfsCmds::Untrack { path } => { - let attr_path = path::attributes().to_string_or_panic(); + let path = convert_patterns_to_workdir(path); // untrack_lfs_patterns(&attr_path, path).unwrap(); } } } +/// temp +fn convert_patterns_to_workdir(patterns: Vec) -> Vec { + patterns.into_iter().map(|p| { + util::to_workdir_path(&p).to_string_or_panic() + }).collect() +} + fn add_lfs_patterns(file_path: &str, patterns: Vec) -> io::Result<()> { let mut file = OpenOptions::new() .create(true) @@ -64,7 +72,7 @@ fn add_lfs_patterns(file_path: &str, patterns: Vec) -> io::Result<()> { } } - let lfs_patterns = extract_lfs_patterns(file_path)?; + let lfs_patterns = lfs::extract_lfs_patterns(file_path)?; for pattern in patterns { if lfs_patterns.contains(&pattern) { continue; @@ -114,33 +122,4 @@ fn untrack_lfs_patterns(file_path: &str, patterns: Vec) -> io::Result<() } Ok(()) -} - -fn extract_lfs_patterns(file_path: &str) -> io::Result> { - let path = Path::new(file_path); - if !path.exists() { - return Ok(Vec::new()); - } - let file = File::open(path)?; - let reader = BufReader::new(file); - - // ' ' needs '\' before it to be escaped - let re = Regex::new(r"^\s*(([^\s#\\]|\\ )+)").unwrap(); - - let mut patterns = Vec::new(); - - for line in reader.lines() { - let line = line?; - if !line.contains("filter=lfs") { - continue; - } - if let Some(cap) = re.captures(&line) { - if let Some(pattern) = cap.get(1) { - let pattern = pattern.as_str().replace(r"\ ", " "); - patterns.push(pattern); - } - } - } - - Ok(patterns) } \ No newline at end of file diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs new file mode 100644 index 000000000..e9e08666c --- /dev/null +++ b/libra/src/utils/lfs.rs @@ -0,0 +1,46 @@ +use std::fs::File; +use std::io; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use regex::Regex; + +/// Check if a file is LFS tracked +/// - only check root attributes file now, should check all attributes files in sub-dirs +// pub fn is_lfs_tracked

(path: P) -> bool +// where +// P: AsRef, +// { +// // check .libra_attributes +// let path = path.as_ref(); +// +// } + +/// Extract LFS patterns from `.libra_attributes` file +pub fn extract_lfs_patterns(file_path: &str) -> io::Result> { + let path = Path::new(file_path); + if !path.exists() { + return Ok(Vec::new()); + } + let file = File::open(path)?; + let reader = BufReader::new(file); + + // ' ' needs '\' before it to be escaped + let re = Regex::new(r"^\s*(([^\s#\\]|\\ )+)").unwrap(); + + let mut patterns = Vec::new(); + + for line in reader.lines() { + let line = line?; + if !line.contains("filter=lfs") { + continue; + } + if let Some(cap) = re.captures(&line) { + if let Some(pattern) = cap.get(1) { + let pattern = pattern.as_str().replace(r"\ ", " "); + patterns.push(pattern); + } + } + } + + Ok(patterns) +} \ No newline at end of file diff --git a/libra/src/utils/mod.rs b/libra/src/utils/mod.rs index 5035c1a48..5427ad3f1 100644 --- a/libra/src/utils/mod.rs +++ b/libra/src/utils/mod.rs @@ -3,4 +3,5 @@ pub(crate) mod test; pub(crate) mod path; pub(crate) mod object_ext; pub(crate) mod path_ext; -pub(crate) mod client_storage; \ No newline at end of file +pub(crate) mod client_storage; +pub(crate) mod lfs; \ No newline at end of file From 8c164873699260ae4eb9d8959755833d4452b5b5 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 19 Aug 2024 17:38:16 +0800 Subject: [PATCH 04/14] stage pointer file & backup while "add" new `lfs` file Signed-off-by: Qihang Cai --- libra/Cargo.toml | 3 ++ libra/src/command/add.rs | 11 +++- libra/src/utils/lfs.rs | 109 +++++++++++++++++++++++++++++++++++---- 3 files changed, 111 insertions(+), 12 deletions(-) diff --git a/libra/Cargo.toml b/libra/Cargo.toml index 2c08bb799..e24c6f8de 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -36,7 +36,10 @@ futures-util = "0.3.30" rpassword = "7.3.1" indicatif = "0.17.8" wax = "0.6.0" +lazy_static = { workspace = true } regex = { workspace = true } +sha2 = "0.10.8" +hex = { workspace = true } [target.'cfg(unix)'.dependencies] # only on Unix pager = "0.16.0" diff --git a/libra/src/command/add.rs b/libra/src/command/add.rs index cf388da6b..e51f58930 100644 --- a/libra/src/command/add.rs +++ b/libra/src/command/add.rs @@ -5,7 +5,7 @@ use crate::command::status; use mercury::internal::index::{Index, IndexEntry}; use crate::utils::object_ext::BlobExt; -use crate::utils::{path, util}; +use crate::utils::{lfs, path, util}; #[derive(Parser, Debug)] pub struct AddArgs { @@ -107,7 +107,14 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) { // file exists if !index.tracked(file_str, 0) { // file is not tracked - let blob = Blob::from_file(&file_abs); + let blob = if lfs::is_lfs_tracked(file) { + let (pointer, oid) = lfs::generate_pointer_file(&file_abs); + tracing::debug!("\n{}", pointer); + lfs::backup_lfs_file(&file_abs, &oid).unwrap(); + Blob::from_content(&pointer) + } else { + Blob::from_file(&file_abs) + }; blob.save(); index.add(IndexEntry::new_from_file(file, blob.id, &workdir).unwrap()); if verbose { diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index e9e08666c..2ea3f076b 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -1,19 +1,96 @@ use std::fs::File; -use std::io; -use std::io::{BufRead, BufReader}; +use std::{fs, io}; +use std::io::{BufRead, BufReader, Read}; use std::path::Path; +use lazy_static::lazy_static; +use path_abs::{PathInfo, PathOps}; use regex::Regex; +use sha2::{Digest, Sha256}; +use wax::Pattern; +use crate::utils::{path, util}; +use crate::utils::path_ext::PathExt; + +lazy_static! { + static ref LFS_PATTERNS: Vec = { // cache + let attr_path = path::attributes().to_string_or_panic(); + extract_lfs_patterns(&attr_path).unwrap() + }; +} /// Check if a file is LFS tracked +/// - support Glob pattern matching (TODO: support .gitignore patterns) /// - only check root attributes file now, should check all attributes files in sub-dirs -// pub fn is_lfs_tracked

(path: P) -> bool -// where -// P: AsRef, -// { -// // check .libra_attributes -// let path = path.as_ref(); -// -// } +/// - absolute path or relative path to workdir +pub fn is_lfs_tracked

(path: P) -> bool +where + P: AsRef, +{ + if LFS_PATTERNS.is_empty() { + return false; + } + + let path = util::to_workdir_path(path); + let glob = wax::any(LFS_PATTERNS.iter().map(|s| s.as_str()).collect::>()).unwrap(); + glob.is_match(path.to_str().unwrap()) +} + +const LFS_VERSION: &str = "https://git-lfs.github.com/spec/v1"; +const LFS_HASH_ALGO: &str = "sha256"; + +/// Generate lfs pointer file +/// - return (pointer content, file hash) +/// - absolute path +pub fn generate_pointer_file

(path: P) -> (String, String) +where + P: AsRef, +{ + let path = path.as_ref(); + // calc file hash without type + let file_hash = calc_lfs_file_hash(path).unwrap(); + + let pointer = format!("version {}\noid {}:{}\nsize {}\n", + LFS_VERSION, LFS_HASH_ALGO, file_hash, path.metadata().unwrap().len()); + (pointer, file_hash) +} + +/// Copy LFS file to `.libra/lfs/objects` +/// - absolute path +pub fn backup_lfs_file

(path: P, hash: &str) -> io::Result<()> +where + P: AsRef, +{ + let path = path.as_ref(); + let backup_path = util::storage_path() + .join("lfs/objects") + .join(hash[..2].to_string()) + .join(hash[2..4].to_string()) + .join(hash); + fs::create_dir_all(backup_path.parent().unwrap())?; + fs::copy(path, backup_path)?; + Ok(()) +} + +/// SHA256 without type +/// TODO: performance optimization, 200MB 4s now, slower than `sha256sum` +pub fn calc_lfs_file_hash

(path: P) -> io::Result +where + P: AsRef, +{ + let path = path.as_ref(); + let mut hash = Sha256::new(); + let file = File::open(path)?; + let mut reader = BufReader::new(file); + let mut buffer = [0; 65536]; + loop { + let n = reader.read(&mut buffer)?; + if n == 0 { + break; + } + hash.update(&buffer[..n]); + } + let file_hash = hex::encode(hash.finalize()); + Ok(file_hash) +} /// Extract LFS patterns from `.libra_attributes` file pub fn extract_lfs_patterns(file_path: &str) -> io::Result> { @@ -43,4 +120,16 @@ pub fn extract_lfs_patterns(file_path: &str) -> io::Result> { } Ok(patterns) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_pointer_file() { + let path = Path::new("../tests/data/packs/git-2d187177923cd618a75da6c6db45bb89d92bd504.pack"); + let (pointer, _oid) = generate_pointer_file(path); + print!("{}", pointer); + } } \ No newline at end of file From 35bcb38d1c212ef6aaff1a010fa90b3bb3e107e9 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 19 Aug 2024 18:27:09 +0800 Subject: [PATCH 05/14] stage pointer file & backup while "add" modified `lfs` file Signed-off-by: Qihang Cai --- libra/src/command/add.rs | 22 +++++++++++++--------- libra/src/utils/lfs.rs | 17 ++++++++--------- libra/src/utils/object_ext.rs | 13 ++++++++++++- 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/libra/src/command/add.rs b/libra/src/command/add.rs index e51f58930..606ab2b17 100644 --- a/libra/src/command/add.rs +++ b/libra/src/command/add.rs @@ -107,14 +107,7 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) { // file exists if !index.tracked(file_str, 0) { // file is not tracked - let blob = if lfs::is_lfs_tracked(file) { - let (pointer, oid) = lfs::generate_pointer_file(&file_abs); - tracing::debug!("\n{}", pointer); - lfs::backup_lfs_file(&file_abs, &oid).unwrap(); - Blob::from_content(&pointer) - } else { - Blob::from_file(&file_abs) - }; + let blob = gen_blob_from_file(&file_abs); blob.save(); index.add(IndexEntry::new_from_file(file, blob.id, &workdir).unwrap()); if verbose { @@ -124,7 +117,7 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) { // file is tracked, maybe modified if index.is_modified(file_str, 0, &workdir) { // file is modified(meta), but content may not change - let blob = Blob::from_file(&file_abs); + let blob = gen_blob_from_file(&file_abs); if !index.verify_hash(file_str, 0, &blob.id) { // content is changed blob.save(); @@ -137,6 +130,17 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) { } } } + +/// Generate a `Blob` from a file +/// - if the file is tracked by LFS, generate a `Blob` with pointer file +fn gen_blob_from_file(path: impl AsRef) -> Blob { + if lfs::is_lfs_tracked(&path) { + Blob::from_lfs_file(&path) + } else { + Blob::from_file(&path) + } +} + #[cfg(test)] mod test { use super::*; diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index 2ea3f076b..8f3e206c3 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -37,13 +37,10 @@ where const LFS_VERSION: &str = "https://git-lfs.github.com/spec/v1"; const LFS_HASH_ALGO: &str = "sha256"; -/// Generate lfs pointer file +/// Generate lfs pointer file string /// - return (pointer content, file hash) /// - absolute path -pub fn generate_pointer_file

(path: P) -> (String, String) -where - P: AsRef, -{ +pub fn generate_pointer_file(path: impl AsRef) -> (String, String) { let path = path.as_ref(); // calc file hash without type let file_hash = calc_lfs_file_hash(path).unwrap(); @@ -62,11 +59,13 @@ where let path = path.as_ref(); let backup_path = util::storage_path() .join("lfs/objects") - .join(hash[..2].to_string()) - .join(hash[2..4].to_string()) + .join(&hash[..2]) + .join(&hash[2..4]) .join(hash); - fs::create_dir_all(backup_path.parent().unwrap())?; - fs::copy(path, backup_path)?; + if !backup_path.exists() { + fs::create_dir_all(backup_path.parent().unwrap())?; + fs::copy(path, backup_path)?; + } Ok(()) } diff --git a/libra/src/utils/object_ext.rs b/libra/src/utils/object_ext.rs index bba717d3b..c276bfd6e 100644 --- a/libra/src/utils/object_ext.rs +++ b/libra/src/utils/object_ext.rs @@ -6,7 +6,7 @@ use mercury::internal::object::commit::Commit; use mercury::internal::object::ObjectTrait; use mercury::internal::object::tree::{Tree, TreeItemMode}; -use crate::utils::util; +use crate::utils::{lfs, util}; pub trait TreeExt { fn load(hash: &SHA1) -> Tree; @@ -20,6 +20,7 @@ pub trait CommitExt { pub trait BlobExt { fn load(hash: &SHA1) -> Blob; fn from_file(path: impl AsRef) -> Blob; + fn from_lfs_file(path: impl AsRef) -> Blob; fn save(&self) -> SHA1; } @@ -75,6 +76,16 @@ impl BlobExt for Blob { Blob::from_content(&file_content) } + /// Create a blob from an LFS file + /// - include: create a pointer file & copy the file to `.libra/lfs/objects` + /// - `path`: absolute or relative path to current dir + fn from_lfs_file(path: impl AsRef) -> Blob { + let (pointer, oid) = lfs::generate_pointer_file(&path); + tracing::debug!("\n{}", pointer); + lfs::backup_lfs_file(&path, &oid).unwrap(); + Blob::from_content(&pointer) + } + fn save(&self) -> SHA1 { let storage = util::objects_storage(); let id = self.id; From 186d00282cde94b0ecc35c96771726f3bb399d41 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Wed, 21 Aug 2024 16:34:48 +0800 Subject: [PATCH 06/14] add `LFSClient::push_objects`, get lfs objs info & send Batch request; TODO: upload LFS files Signed-off-by: Qihang Cai --- libra/Cargo.toml | 1 + libra/src/internal/protocol/lfs_client.rs | 77 ++++++++++++++++++ libra/src/internal/protocol/mod.rs | 2 + libra/src/utils/lfs.rs | 97 +++++++++++++++++++++-- 4 files changed, 169 insertions(+), 8 deletions(-) create mode 100644 libra/src/internal/protocol/lfs_client.rs diff --git a/libra/Cargo.toml b/libra/Cargo.toml index e24c6f8de..43da68709 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -40,6 +40,7 @@ lazy_static = { workspace = true } regex = { workspace = true } sha2 = "0.10.8" hex = { workspace = true } +serde_json = { workspace = true } [target.'cfg(unix)'.dependencies] # only on Unix pager = "0.16.0" diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs new file mode 100644 index 000000000..91c5ac71b --- /dev/null +++ b/libra/src/internal/protocol/lfs_client.rs @@ -0,0 +1,77 @@ +use reqwest::Client; +use url::Url; +use ceres::lfs::lfs_structs::{BatchRequest, RequestVars}; +use mercury::internal::object::types::ObjectType; +use mercury::internal::pack::entry::Entry; +use crate::internal::protocol::https_client::BasicAuth; +use crate::internal::protocol::ProtocolClient; +use crate::utils::lfs; + +pub struct LFSClient { + pub url: Url, + pub client: Client, +} + +impl ProtocolClient for LFSClient { + /// Construct LFSClient from a given Repo URL. + fn from_url(repo_url: &Url) -> Self { + let lfs_server = Url::parse(&lfs::generate_lfs_server_url(repo_url.to_string())).unwrap(); + let client = Client::builder() + .http1_only() + .default_headers(lfs::LFS_HEADERS.clone()) + .build() + .unwrap(); + Self { + url: lfs_server.join("/objects/batch").unwrap(), + client, + } + } +} + +impl LFSClient { + /// push LFS objects to remote server + pub async fn push_objects<'a, I>(&self, objs: I, auth: Option) + where + I: IntoIterator + { + // filter pointer file within blobs + let mut lfs_oids = Vec::new(); + for blob in objs.into_iter().filter(|e| e.obj_type == ObjectType::Blob) { + let oid = lfs::parse_pointer_data(&blob.data); + if let Some(oid) = oid { + lfs_oids.push(oid); + } + } + + let mut lfs_objs = Vec::new(); + for oid in &lfs_oids { + let path = lfs::lfs_object_path(oid); + if !path.exists() { + eprintln!("fatal: LFS object not found: {}", oid); + return; + } + let size = path.metadata().unwrap().len() as i64; + lfs_objs.push(RequestVars { + oid: oid.to_owned(), + size, + ..Default::default() + }) + } + + let batch_request = BatchRequest { + operation: "upload".to_string(), + transfers: vec![lfs::LFS_TRANSFER_API.to_string()], + objects: lfs_objs, + hash_algo: lfs::LFS_HASH_ALGO.to_string(), + enable_split: None, + }; + + let mut request = self.client.post(self.url.clone()).json(&batch_request); + if let Some(auth) = auth { + request = request.basic_auth(auth.username, Some(auth.password)); + } + + let response = request.send().await.unwrap(); + tracing::debug!("LFS push response: {:?}", response.json::().await.unwrap()); + } +} \ No newline at end of file diff --git a/libra/src/internal/protocol/mod.rs b/libra/src/internal/protocol/mod.rs index 206132edc..c64529366 100644 --- a/libra/src/internal/protocol/mod.rs +++ b/libra/src/internal/protocol/mod.rs @@ -1,6 +1,8 @@ use url::Url; pub mod https_client; +pub mod lfs_client; + #[allow(dead_code)] // todo: unimplemented pub trait ProtocolClient { /// create client from url diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index 8f3e206c3..55789c3f2 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -1,10 +1,11 @@ use std::fs::File; use std::{fs, io}; use std::io::{BufRead, BufReader, Read}; -use std::path::Path; +use std::path::{Path, PathBuf}; use lazy_static::lazy_static; use path_abs::{PathInfo, PathOps}; use regex::Regex; +use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_TYPE}; use sha2::{Digest, Sha256}; use wax::Pattern; use crate::utils::{path, util}; @@ -15,6 +16,13 @@ lazy_static! { let attr_path = path::attributes().to_string_or_panic(); extract_lfs_patterns(&attr_path).unwrap() }; + + pub static ref LFS_HEADERS: HeaderMap = { + let mut headers = HeaderMap::new(); + headers.insert(ACCEPT, HeaderValue::from_static("application/vnd.git-lfs+json")); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/vnd.git-lfs+json")); + headers + }; } /// Check if a file is LFS tracked @@ -35,7 +43,11 @@ where } const LFS_VERSION: &str = "https://git-lfs.github.com/spec/v1"; -const LFS_HASH_ALGO: &str = "sha256"; +/// This is the original & default transfer adapter. All Git LFS clients and servers SHOULD support it. +pub const LFS_TRANSFER_API: &str = "basic"; +pub const LFS_HASH_ALGO: &str = "sha256"; +const LFS_OID_LEN: usize = 64; +const LFS_POINTER_MAX_SIZE: usize = 300; // bytes /// Generate lfs pointer file string /// - return (pointer content, file hash) @@ -50,18 +62,49 @@ pub fn generate_pointer_file(path: impl AsRef) -> (String, String) { (pointer, file_hash) } +/// Generate LFS Server Url from repo Url. +/// By default, Git LFS will append `.git/info/lfs` to the end of a Git remote url to build the LFS server URL. +/// [doc: server-discovery](https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md) +/// - like https://git-server.com/foo/bar.git/info/lfs +/// - support ssh & https & git@ format +pub fn generate_lfs_server_url(mut url: String) -> String { + if url.ends_with('/') { + url.pop(); + } + if !url.ends_with(".git") { + url.push_str(".git"); + } + url.push_str("/info/lfs"); + + if url.starts_with("git@") { + // git@git-server.com:foo/bar.git + url = "https://".to_string() + &url[4..].replace(":", "/"); + } else if url.starts_with("ssh://") { + // ssh://git-server.com/foo/bar.git + url = "https://".to_string() + &url[6..]; + } + + url +} + +/// Generate LFS cache path, in `.libra/lfs/objects` +pub fn lfs_object_path(oid: &str) -> PathBuf { + util::storage_path() + .join("lfs/objects") + .join(&oid[..2]) + .join(&oid[2..4]) + .join(oid) + .into() +} + /// Copy LFS file to `.libra/lfs/objects` /// - absolute path -pub fn backup_lfs_file

(path: P, hash: &str) -> io::Result<()> +pub fn backup_lfs_file

(path: P, oid: &str) -> io::Result<()> where P: AsRef, { let path = path.as_ref(); - let backup_path = util::storage_path() - .join("lfs/objects") - .join(&hash[..2]) - .join(&hash[2..4]) - .join(hash); + let backup_path = lfs_object_path(oid); if !backup_path.exists() { fs::create_dir_all(backup_path.parent().unwrap())?; fs::copy(path, backup_path)?; @@ -91,6 +134,22 @@ where Ok(file_hash) } +/// Check if `data` is an LFS pointer, return `oid` +pub fn parse_pointer_data(data: &[u8]) -> Option { + if data.len() > LFS_POINTER_MAX_SIZE { + return None; + } + // Start with format `version ...` + if let Some(data) = data.strip_prefix(format!("version {}\noid {}:", LFS_VERSION, LFS_HASH_ALGO).as_bytes()) { + if data[LFS_OID_LEN] == b'\n' { + // check `oid` length + let oid = String::from_utf8(data[..LFS_OID_LEN].to_vec()).unwrap(); + return Some(oid); + } + } + None +} + /// Extract LFS patterns from `.libra_attributes` file pub fn extract_lfs_patterns(file_path: &str) -> io::Result> { let path = Path::new(file_path); @@ -131,4 +190,26 @@ mod tests { let (pointer, _oid) = generate_pointer_file(path); print!("{}", pointer); } + + #[test] + fn test_is_pointer_file() { + let data = b"version https://git-lfs.github.com/spec/v1\noid sha256:1234567890abcdef\nsize 1234\n"; + assert!(parse_pointer_data(data).is_some()); + } + + #[test] + fn test_gen_lfs_server_url() { + const LFS_SERVER_URL: &str = "https://github.com/web3infra-foundation/mega.git/info/lfs"; + let url = "https://github.com/web3infra-foundation/mega".to_owned(); + assert_eq!(generate_lfs_server_url(url), LFS_SERVER_URL); + + let url = "https://github.com/web3infra-foundation/mega.git".to_owned(); + assert_eq!(generate_lfs_server_url(url), LFS_SERVER_URL); + + let url = "git@github.com:web3infra-foundation/mega.git".to_owned(); + assert_eq!(generate_lfs_server_url(url), LFS_SERVER_URL); + + let url = "ssh://github.com/web3infra-foundation/mega.git".to_owned(); + assert_eq!(generate_lfs_server_url(url), LFS_SERVER_URL); + } } \ No newline at end of file From e2e07621379b243f6697ae18d7c68b5a5e3a8f9b Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Fri, 23 Aug 2024 14:57:19 +0800 Subject: [PATCH 07/14] ignore "actions" in LFS Batch response if "operation" == "upload" & file exists. see (https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md) Signed-off-by: Qihang Cai --- ceres/src/lfs/handler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index 37fe7f04a..08786f539 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -198,7 +198,7 @@ pub async fn lfs_process_batch( let mut meta = meta.unwrap_or_default(); if found && lfs_file_exist(config, &meta).await { // originla download method, split mode use `` - response_objects.push(represent(object, &meta, true, false, false, &server_url).await); + response_objects.push(represent(object, &meta, batch_vars.operation == "download", false, false, &server_url).await); continue; } // Not found From 0f85912147fc5e2fad8ccb3ea4d0dde494cef068 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Fri, 23 Aug 2024 17:20:27 +0800 Subject: [PATCH 08/14] complete `LFSClient::push_objects`: upload all LFS files while `push` Signed-off-by: Qihang Cai --- libra/Cargo.toml | 1 + libra/src/command/push.rs | 8 ++- libra/src/internal/protocol/lfs_client.rs | 59 ++++++++++++++++++++++- libra/src/utils/lfs.rs | 1 - 4 files changed, 65 insertions(+), 4 deletions(-) diff --git a/libra/Cargo.toml b/libra/Cargo.toml index 43da68709..681da3c8e 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -40,6 +40,7 @@ lazy_static = { workspace = true } regex = { workspace = true } sha2 = "0.10.8" hex = { workspace = true } +serde = { workspace = true } serde_json = { workspace = true } [target.'cfg(unix)'.dependencies] # only on Unix diff --git a/libra/src/command/push.rs b/libra/src/command/push.rs index 7f8aad6f4..ddefa115a 100644 --- a/libra/src/command/push.rs +++ b/libra/src/command/push.rs @@ -20,6 +20,7 @@ use crate::internal::branch::Branch; use crate::internal::config::Config; use crate::internal::head::Head; use crate::internal::protocol::https_client::{BasicAuth, HttpsClient}; +use crate::internal::protocol::lfs_client::LFSClient; use crate::internal::protocol::ProtocolClient; use crate::utils::object_ext::{BlobExt, CommitExt, TreeExt}; @@ -117,6 +118,11 @@ pub async fn execute(args: PushArgs) { SHA1::from_str(&remote_hash).unwrap() ); + { // upload lfs files + let client = LFSClient::from_url(&url); + client.push_objects(&objs, auth.clone()).await; + } + // let (tx, rx) = mpsc::channel::(); let (entry_tx, entry_rx) = mpsc::channel(1_000_000); let (stream_tx, mut stream_rx) = mpsc::channel(1_000_000); @@ -138,7 +144,7 @@ pub async fn execute(args: PushArgs) { data.extend_from_slice(&pack_data); println!("Delta compression done."); - let res = client.send_pack(data.freeze(), auth).await.unwrap(); + let res = client.send_pack(data.freeze(), auth).await.unwrap(); // TODO: send stream if res.status() != 200 { eprintln!("status code: {}", res.status()); diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index 91c5ac71b..fc8b4332a 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -1,6 +1,7 @@ use reqwest::Client; +use serde::{Deserialize, Serialize}; use url::Url; -use ceres::lfs::lfs_structs::{BatchRequest, RequestVars}; +use ceres::lfs::lfs_structs::{BatchRequest, Representation, RequestVars}; use mercury::internal::object::types::ObjectType; use mercury::internal::pack::entry::Entry; use crate::internal::protocol::https_client::BasicAuth; @@ -12,6 +13,14 @@ pub struct LFSClient { pub client: Client, } +/// see [successful-responses](https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#successful-responses) +#[derive(Serialize, Deserialize)] +struct LfsBatchResponse { + transfer: Option, + objects: Vec, + hash_algo: Option, +} + impl ProtocolClient for LFSClient { /// Construct LFSClient from a given Repo URL. fn from_url(repo_url: &Url) -> Self { @@ -72,6 +81,52 @@ impl LFSClient { } let response = request.send().await.unwrap(); - tracing::debug!("LFS push response: {:?}", response.json::().await.unwrap()); + + let resp = response.json::().await.unwrap(); + tracing::debug!("LFS push response:\n {:#?}", serde_json::to_value(&resp).unwrap()); + + // TODO: parallel upload + for obj in resp.objects { + self.upload_object(obj).await; + } + println!("LFS objects push completed."); + } + + /// upload (PUT) one LFS file to remote server + async fn upload_object(&self, object: Representation) { + if let Some(err) = object.error { + eprintln!("fatal: LFS upload failed. Code: {}, Message: {}", err.code, err.message); + return; + } + + if let Some(actions) = object.actions { + let upload_link = actions.get("upload"); + if upload_link.is_none() { + eprintln!("fatal: LFS upload failed. No upload action found"); + return; + } + + let link = upload_link.unwrap(); + let mut request = self.client.put(link.href.clone()); + for (k, v) in &link.header { + request = request.header(k, v); + } + + let file_path = lfs::lfs_object_path(&object.oid); + let file = tokio::fs::File::open(file_path).await.unwrap(); + println!("Uploading LFS file: {}", object.oid); + let resp = request + .body(reqwest::Body::wrap_stream(tokio_util::io::ReaderStream::new(file))) + .send() + .await + .unwrap(); + if !resp.status().is_success() { + eprintln!("fatal: LFS upload failed. Status: {}, Message: {}", resp.status(), resp.text().await.unwrap()); + return; + } + println!("Uploaded."); + } else { + tracing::debug!("LFS file {} already exists on remote server", object.oid); + } } } \ No newline at end of file diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index 55789c3f2..81142a4e8 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -94,7 +94,6 @@ pub fn lfs_object_path(oid: &str) -> PathBuf { .join(&oid[..2]) .join(&oid[2..4]) .join(oid) - .into() } /// Copy LFS file to `.libra/lfs/objects` From d17b4ecbf22b04e2b36346ffa09dcc990251976d Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sun, 25 Aug 2024 13:00:32 +0800 Subject: [PATCH 09/14] fix `libra`: deal with files that can't be cast to `UTF-8`, using `Vec` rather than `String` Signed-off-by: Qihang Cai --- libra/src/utils/object_ext.rs | 9 +++++++-- mercury/src/internal/object/blob.rs | 6 ++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/libra/src/utils/object_ext.rs b/libra/src/utils/object_ext.rs index c276bfd6e..6e56f01b5 100644 --- a/libra/src/utils/object_ext.rs +++ b/libra/src/utils/object_ext.rs @@ -1,3 +1,5 @@ +use std::fs; +use std::io::{BufReader, Read}; use std::path::{Path, PathBuf}; use mercury::hash::SHA1; @@ -72,8 +74,11 @@ impl BlobExt for Blob { /// Create a blob from a file /// - `path`: absolute or relative path to current dir fn from_file(path: impl AsRef) -> Blob { - let file_content = std::fs::read_to_string(path).unwrap(); - Blob::from_content(&file_content) + let mut data = Vec::new(); + let file = fs::File::open(path).unwrap(); + let mut reader = BufReader::new(file); + reader.read_to_end(&mut data).unwrap(); + Blob::from_content_bytes(data) } /// Create a blob from an LFS file diff --git a/mercury/src/internal/object/blob.rs b/mercury/src/internal/object/blob.rs index 32c00ca6d..73f37d068 100644 --- a/mercury/src/internal/object/blob.rs +++ b/mercury/src/internal/object/blob.rs @@ -89,6 +89,12 @@ impl Blob { // let mut buf = ReadBoxed::new(blob_content, ObjectType::Blob, content.len()); // Blob::from_buf_read(&mut buf, content.len()) let content = content.as_bytes().to_vec(); + Blob::from_content_bytes(content) + } + + /// Create a new Blob object from the given content bytes. + /// - some file content can't be represented as a string (UTF-8), so we need to use bytes. + pub fn from_content_bytes(content: Vec) -> Self { Blob { id: SHA1::from_type_and_data(ObjectType::Blob, &content), data: content, From 8096b73f5e00c0ce28002dcbc7693b96226ddeb6 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sun, 25 Aug 2024 15:37:43 +0800 Subject: [PATCH 10/14] improve `calc_file_blob_hash`: support `LFS` file Signed-off-by: Qihang Cai --- libra/src/command/mod.rs | 16 ++++++++++++++++ libra/src/command/restore.rs | 3 ++- libra/src/command/status.rs | 3 ++- libra/src/utils/lfs.rs | 8 ++++---- libra/src/utils/util.rs | 10 +--------- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/libra/src/command/mod.rs b/libra/src/command/mod.rs index dc60c848d..a98f6101f 100644 --- a/libra/src/command/mod.rs +++ b/libra/src/command/mod.rs @@ -22,6 +22,10 @@ use mercury::{errors::GitError, hash::SHA1, internal::object::ObjectTrait}; use rpassword::read_password; use std::io; use std::io::Write; +use std::path::Path; +use mercury::internal::object::blob::Blob; +use crate::utils; +use crate::utils::object_ext::BlobExt; // impl load for all objects fn load_object(hash: &SHA1) -> Result @@ -112,6 +116,18 @@ pub fn parse_commit_msg(msg_gpg: &str) -> (String, Option) { } } +/// Calculate the hash of a file blob +/// - for `lfs` file: calculate hash of the pointer data +pub fn calc_file_blob_hash(path: impl AsRef) -> io::Result { + let blob = if utils::lfs::is_lfs_tracked(&path) { + let (pointer, _) = utils::lfs::generate_pointer_file(&path); + Blob::from_content(&pointer) + } else { + Blob::from_file(&path) + }; + Ok(blob.id) +} + #[cfg(test)] mod test { use mercury::internal::object::commit::Commit; diff --git a/libra/src/command/restore.rs b/libra/src/command/restore.rs index 2ebc56604..0b7551640 100644 --- a/libra/src/command/restore.rs +++ b/libra/src/command/restore.rs @@ -13,6 +13,7 @@ use mercury::internal::object::blob::Blob; use mercury::internal::object::commit::Commit; use mercury::internal::object::tree::Tree; use mercury::internal::object::types::ObjectType; +use crate::command::calc_file_blob_hash; #[derive(Parser, Debug)] pub struct RestoreArgs { @@ -206,7 +207,7 @@ pub fn restore_worktree(filter: &Vec, target_blobs: &[(PathBuf, SHA1)]) } else { // file exists let path_wd_str = path_wd.to_string_or_panic(); - let hash = util::calc_file_blob_hash(&path_abs).unwrap(); + let hash = calc_file_blob_hash(&path_abs).unwrap(); if target_blobs.contains_key(path_wd) { // both in target & worktree: 1. modified 2. same if hash != target_blobs[path_wd] { diff --git a/libra/src/command/status.rs b/libra/src/command/status.rs index 0a1462850..0aa75e706 100644 --- a/libra/src/command/status.rs +++ b/libra/src/command/status.rs @@ -9,6 +9,7 @@ use mercury::internal::object::tree::Tree; use crate::internal::head::Head; use mercury::internal::index::Index; +use crate::command::calc_file_blob_hash; use crate::utils::object_ext::{CommitExt, TreeExt}; use crate::utils::{path, util}; @@ -159,7 +160,7 @@ pub fn changes_to_be_staged() -> Changes { changes.deleted.push(file.clone()); } else if index.is_modified(file_str, 0, &workdir) { // only calc the hash if the file is modified (metadata), for optimization - let file_hash = util::calc_file_blob_hash(&file_abs).unwrap(); + let file_hash = calc_file_blob_hash(&file_abs).unwrap(); if !index.verify_hash(file_str, 0, &file_hash) { changes.modified.push(file.clone()); } diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index 81142a4e8..121b31d2f 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -50,16 +50,16 @@ const LFS_OID_LEN: usize = 64; const LFS_POINTER_MAX_SIZE: usize = 300; // bytes /// Generate lfs pointer file string -/// - return (pointer content, file hash) +/// - return (pointer content, lfs oid) /// - absolute path pub fn generate_pointer_file(path: impl AsRef) -> (String, String) { let path = path.as_ref(); // calc file hash without type - let file_hash = calc_lfs_file_hash(path).unwrap(); + let oid = calc_lfs_file_hash(path).unwrap(); let pointer = format!("version {}\noid {}:{}\nsize {}\n", - LFS_VERSION, LFS_HASH_ALGO, file_hash, path.metadata().unwrap().len()); - (pointer, file_hash) + LFS_VERSION, LFS_HASH_ALGO, oid, path.metadata().unwrap().len()); + (pointer, oid) } /// Generate LFS Server Url from repo Url. diff --git a/libra/src/utils/util.rs b/libra/src/utils/util.rs index c9eba7690..f520b8928 100644 --- a/libra/src/utils/util.rs +++ b/libra/src/utils/util.rs @@ -1,6 +1,6 @@ use path_abs::{PathAbs, PathInfo}; use std::collections::HashSet; -use std::io::{BufReader, Read, Write}; +use std::io::Write; use std::path::{Path, PathBuf}; use std::{env, fs, io}; @@ -193,14 +193,6 @@ where workdir_to_relative(path, cur_dir()) } -pub fn calc_file_blob_hash(path: impl AsRef) -> io::Result { - let file = fs::File::open(path)?; - let mut reader = BufReader::new(file); - let mut data = Vec::new(); - reader.read_to_end(&mut data)?; - Ok(SHA1::from_type_and_data(ObjectType::Blob, &data)) -} - /// List all files in the given dir and its sub_dir, except `.libra` /// - input `path`: absolute path or relative path to the current dir /// - output: to workdir path From 0cf31b3dd2b8045a9ed599f89e2ff67de296a98f Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sun, 25 Aug 2024 19:55:21 +0800 Subject: [PATCH 11/14] update `libra restore`: support `LFS`, download LFS file or copy from cache to replace pointer file Signed-off-by: Qihang Cai --- libra/Cargo.toml | 2 +- libra/src/command/push.rs | 4 +- libra/src/command/restore.rs | 38 +++++++++++--- libra/src/internal/config.rs | 28 +++++++++++ libra/src/internal/protocol/lfs_client.rs | 60 ++++++++++++++++++++++- libra/src/utils/lfs.rs | 21 +++++++- 6 files changed, 139 insertions(+), 14 deletions(-) diff --git a/libra/Cargo.toml b/libra/Cargo.toml index 681da3c8e..f081544bc 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -24,7 +24,7 @@ sha1 = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } futures = { workspace = true } -reqwest = { workspace = true, features = ["stream"] } +reqwest = { workspace = true, features = ["stream", "json"] } tokio-util = { version = "0.7.11", features = ["io"] } color-backtrace = "0.6.1" colored = "2.1.0" diff --git a/libra/src/command/push.rs b/libra/src/command/push.rs index ddefa115a..4c44f6837 100644 --- a/libra/src/command/push.rs +++ b/libra/src/command/push.rs @@ -56,7 +56,7 @@ pub async fn execute(args: PushArgs) { Some(repo) => repo, None => { // e.g. [branch "master"].remote = origin - let remote = Config::get("branch", Some(&branch), "remote").await; + let remote = Config::get_remote(&branch).await; if let Some(remote) = remote { remote } else { @@ -65,7 +65,7 @@ pub async fn execute(args: PushArgs) { } } }; - let repo_url = Config::get("remote", Some(&repository), "url").await; + let repo_url = Config::get_remote_url(&repository).await; if repo_url.is_none() { eprintln!("fatal: remote '{}' not found, please use 'libra remote add'", repository); return; diff --git a/libra/src/command/restore.rs b/libra/src/command/restore.rs index 0b7551640..873847677 100644 --- a/libra/src/command/restore.rs +++ b/libra/src/command/restore.rs @@ -3,10 +3,10 @@ use crate::internal::head::Head; use mercury::internal::index::{Index, IndexEntry}; use crate::utils::object_ext::{BlobExt, CommitExt, TreeExt}; use crate::utils::path_ext::PathExt; -use crate::utils::{path, util}; +use crate::utils::{lfs, path, util}; use clap::Parser; use std::collections::{HashMap, HashSet}; -use std::fs; +use std::{fs, io}; use std::path::PathBuf; use mercury::hash::SHA1; use mercury::internal::object::blob::Blob; @@ -14,6 +14,7 @@ use mercury::internal::object::commit::Commit; use mercury::internal::object::tree::Tree; use mercury::internal::object::types::ObjectType; use crate::command::calc_file_blob_hash; +use crate::internal::protocol::lfs_client::LFSClient; #[derive(Parser, Debug)] pub struct RestoreArgs { @@ -118,7 +119,7 @@ pub async fn execute(args: RestoreArgs) { // The order is very important // `restore_worktree` will decide whether to delete the file based on whether it is tracked in the index. if worktree { - restore_worktree(&paths, &target_blobs); + restore_worktree(&paths, &target_blobs).await; } if staged { restore_index(&paths, &target_blobs); @@ -137,10 +138,31 @@ fn preprocess_blobs(blobs: &[(PathBuf, SHA1)]) -> HashMap { /// restore a blob to file /// - `path` : to workdir -fn restore_to_file(hash: &SHA1, path: &PathBuf) { +async fn restore_to_file(hash: &SHA1, path: &PathBuf) -> io::Result<()> { let blob = Blob::load(hash); let path_abs = util::workdir_to_absolute(path); - util::write_file(&blob.data, &path_abs).unwrap(); + if let Some(parent) = path_abs.parent() { + fs::create_dir_all(parent)?; + } + match lfs::parse_pointer_data(&blob.data) { + Some((oid, size)) => { + // LFS file + let lfs_obj_path = lfs::lfs_object_path(&oid); + if lfs_obj_path.exists() { + // found in local cache + fs::copy(&lfs_obj_path, &path_abs)?; + } else { + // not exist, download from server + let lfs_client = LFSClient::new().await; + lfs_client.download_object(&oid, size, &path_abs).await; + } + } + None => { + // normal file + util::write_file(&blob.data, &path_abs)?; + } + } + Ok(()) } /// Get the deleted files in the worktree(vs Index), filtered by `filters` @@ -163,7 +185,7 @@ fn get_worktree_deleted_files_in_filters( /// Restore the worktree /// - `filter`: abs or relative to current (user input) /// - `target_blobs`: to workdir path -pub fn restore_worktree(filter: &Vec, target_blobs: &[(PathBuf, SHA1)]) { +pub async fn restore_worktree(filter: &Vec, target_blobs: &[(PathBuf, SHA1)]) { let target_blobs = preprocess_blobs(target_blobs); let deleted_files = get_worktree_deleted_files_in_filters(filter, &target_blobs); @@ -199,7 +221,7 @@ pub fn restore_worktree(filter: &Vec, target_blobs: &[(PathBuf, SHA1)]) // file not exist, deleted or illegal if target_blobs.contains_key(path_wd) { // file in target_blobs (deleted), need to restore - restore_to_file(&target_blobs[path_wd], path_wd); + restore_to_file(&target_blobs[path_wd], path_wd).await.unwrap(); } else { // not in target_commit and workdir (illegal path), user input unreachable!("It should be checked before"); @@ -212,7 +234,7 @@ pub fn restore_worktree(filter: &Vec, target_blobs: &[(PathBuf, SHA1)]) // both in target & worktree: 1. modified 2. same if hash != target_blobs[path_wd] { // modified - restore_to_file(&target_blobs[path_wd], path_wd); + restore_to_file(&target_blobs[path_wd], path_wd).await.unwrap(); } // else: same, keep } else { // not in target but in worktree: New file diff --git a/libra/src/internal/config.rs b/libra/src/internal/config.rs index a88e9f770..0929b8e97 100644 --- a/libra/src/internal/config.rs +++ b/libra/src/internal/config.rs @@ -5,6 +5,7 @@ use sea_orm::ActiveValue::Set; use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter}; use crate::internal::db::get_db_conn_instance; +use crate::internal::head::Head; use crate::internal::model::config; use crate::internal::model::config::Model; @@ -54,6 +55,33 @@ impl Config { values.first().map(|c| c.value.to_owned()) } + /// Get remote repo name by branch name + pub async fn get_remote(branch: &str) -> Option { + // e.g. [branch "master"].remote = origin + Config::get("branch", Some(branch), "remote").await + } + + /// Get remote repo name of current branch + pub async fn get_current_remote() -> Option { + match Head::current().await { + Head::Branch(name) => { + Config::get_remote(&name).await + }, + Head::Detached(_) => None, + } + } + + pub async fn get_remote_url(remote: &str) -> Option { + Config::get("remote", Some(remote), "url").await + } + + pub async fn get_current_remote_url() -> Option { + match Config::get_current_remote().await { + Some(remote) => Config::get_remote_url(&remote).await, + None => None, + } + } + /// Get all configuration values /// - e.g. remote.origin.url can be multiple pub async fn get_all(configuration: &str, name: Option<&str>, key: &str) -> Vec { diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index fc8b4332a..8f1d9caf9 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -1,9 +1,13 @@ +use std::path::Path; +use futures_util::StreamExt; use reqwest::Client; use serde::{Deserialize, Serialize}; +use tokio::io::AsyncWriteExt; use url::Url; use ceres::lfs::lfs_structs::{BatchRequest, Representation, RequestVars}; use mercury::internal::object::types::ObjectType; use mercury::internal::pack::entry::Entry; +use crate::internal::config::Config; use crate::internal::protocol::https_client::BasicAuth; use crate::internal::protocol::ProtocolClient; use crate::utils::lfs; @@ -38,6 +42,15 @@ impl ProtocolClient for LFSClient { } impl LFSClient { + /// Construct LFSClient from current remote URL. + pub async fn new() -> Self { + let url = Config::get_current_remote_url().await; + match url { + Some(url) => LFSClient::from_url(&Url::parse(&url).unwrap()), + None => panic!("fatal: current remote url not found"), + } + } + /// push LFS objects to remote server pub async fn push_objects<'a, I>(&self, objs: I, auth: Option) where @@ -53,7 +66,7 @@ impl LFSClient { } let mut lfs_objs = Vec::new(); - for oid in &lfs_oids { + for (oid, _) in &lfs_oids { let path = lfs::lfs_object_path(oid); if !path.exists() { eprintln!("fatal: LFS object not found: {}", oid); @@ -129,4 +142,49 @@ impl LFSClient { tracing::debug!("LFS file {} already exists on remote server", object.oid); } } + + /// download (GET) one LFS file from remote server + pub async fn download_object(&self, oid: &str, size: u64, path: impl AsRef) { + let batch_request = BatchRequest { + operation: "download".to_string(), + transfers: vec![lfs::LFS_TRANSFER_API.to_string()], + objects: vec![RequestVars { + oid: oid.to_owned(), + size: size as i64, + ..Default::default() + }], + hash_algo: lfs::LFS_HASH_ALGO.to_string(), + enable_split: None, + }; + + let request = self.client.post(self.url.clone()).json(&batch_request); + let response = request.send().await.unwrap(); + + let resp = response.json::().await.unwrap(); + tracing::debug!("LFS download response:\n {:#?}", serde_json::to_value(&resp).unwrap()); + + let link = resp.objects[0].actions.as_ref().unwrap().get("download").unwrap(); + + let mut request = self.client.get(link.href.clone()); + for (k, v) in &link.header { + request = request.header(k, v); + } + + let response = request.send().await.unwrap(); + + if !response.status().is_success() { + eprintln!("fatal: LFS download failed. Status: {}, Message: {}", response.status(), response.text().await.unwrap()); + return; + } + + println!("Downloading LFS file: {}", oid); + let mut file = tokio::fs::File::create(path).await.unwrap(); + let mut stream = response.bytes_stream(); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.unwrap(); + file.write_all(&chunk).await.unwrap(); + } + println!("Downloaded."); // TODO: checksum + } } \ No newline at end of file diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index 121b31d2f..3f54efb1c 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -134,7 +134,7 @@ where } /// Check if `data` is an LFS pointer, return `oid` -pub fn parse_pointer_data(data: &[u8]) -> Option { +pub fn parse_pointer_data(data: &[u8]) -> Option<(String, u64)> { if data.len() > LFS_POINTER_MAX_SIZE { return None; } @@ -143,7 +143,12 @@ pub fn parse_pointer_data(data: &[u8]) -> Option { if data[LFS_OID_LEN] == b'\n' { // check `oid` length let oid = String::from_utf8(data[..LFS_OID_LEN].to_vec()).unwrap(); - return Some(oid); + if let Some(data) = data.strip_prefix(format!("{}\nsize ", oid).as_bytes()) { + let data = String::from_utf8(data[..].to_vec()).unwrap(); + if let Ok(size) = data.trim_end().parse::() { + return Some((oid, size)); + } + } } } None @@ -211,4 +216,16 @@ mod tests { let url = "ssh://github.com/web3infra-foundation/mega.git".to_owned(); assert_eq!(generate_lfs_server_url(url), LFS_SERVER_URL); } + + #[test] + fn test_parse_pointer_data() { + let data = r#"version https://git-lfs.github.com/spec/v1 +oid sha256:4859402c258b836d02e955d1090e29f586e58b2040504d68afec3d8d43757bba +size 10 +"#; + let res = parse_pointer_data(data.as_bytes()).unwrap(); + println!("{:?}", res); + assert_eq!(res.0, "4859402c258b836d02e955d1090e29f586e58b2040504d68afec3d8d43757bba"); + assert_eq!(res.1, 10); + } } \ No newline at end of file From 2e09b20dabd5b82f381af1e9853ec25a390a92cf Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 26 Aug 2024 15:24:17 +0800 Subject: [PATCH 12/14] improve: use `async_static!` to replace `lazy_static!` for async function Signed-off-by: Qihang Cai --- libra/Cargo.toml | 2 ++ libra/src/command/restore.rs | 8 ++++---- libra/src/internal/protocol/lfs_client.rs | 5 +++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/libra/Cargo.toml b/libra/Cargo.toml index f081544bc..2c4e60a07 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -42,6 +42,8 @@ sha2 = "0.10.8" hex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +async_static = "0.1.3" +once_cell = "1.19.0" [target.'cfg(unix)'.dependencies] # only on Unix pager = "0.16.0" diff --git a/libra/src/command/restore.rs b/libra/src/command/restore.rs index 873847677..8cf359a9f 100644 --- a/libra/src/command/restore.rs +++ b/libra/src/command/restore.rs @@ -14,7 +14,7 @@ use mercury::internal::object::commit::Commit; use mercury::internal::object::tree::Tree; use mercury::internal::object::types::ObjectType; use crate::command::calc_file_blob_hash; -use crate::internal::protocol::lfs_client::LFSClient; +use crate::internal::protocol::lfs_client::LFS_CLIENT; #[derive(Parser, Debug)] pub struct RestoreArgs { @@ -136,7 +136,8 @@ fn preprocess_blobs(blobs: &[(PathBuf, SHA1)]) -> HashMap { .collect() } -/// restore a blob to file +/// Restore a blob to file. +/// If blob is an LFS pointer, download the actual file from LFS server. /// - `path` : to workdir async fn restore_to_file(hash: &SHA1, path: &PathBuf) -> io::Result<()> { let blob = Blob::load(hash); @@ -153,8 +154,7 @@ async fn restore_to_file(hash: &SHA1, path: &PathBuf) -> io::Result<()> { fs::copy(&lfs_obj_path, &path_abs)?; } else { // not exist, download from server - let lfs_client = LFSClient::new().await; - lfs_client.download_object(&oid, size, &path_abs).await; + LFS_CLIENT.await.download_object(&oid, size, &path_abs).await; } } None => { diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index 8f1d9caf9..77c21a04d 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -1,4 +1,5 @@ use std::path::Path; +use async_static::async_static; use futures_util::StreamExt; use reqwest::Client; use serde::{Deserialize, Serialize}; @@ -12,6 +13,10 @@ use crate::internal::protocol::https_client::BasicAuth; use crate::internal::protocol::ProtocolClient; use crate::utils::lfs; +async_static! { + pub static ref LFS_CLIENT: LFSClient = LFSClient::new().await; +} + pub struct LFSClient { pub url: Url, pub client: Client, From 173e492afeb249757242c7bbacbf1349f61856a8 Mon Sep 17 00:00:00 2001 From: HouXiaoxuan Date: Wed, 21 Aug 2024 19:20:47 +0800 Subject: [PATCH 13/14] fix(ceres): Avoid handling empty relations in `lfs_get_relations`, fix `lfs_file_exist` unwrap error when invoking `upload` twice Signed-off-by: HouXiaoxuan --- ceres/src/lfs/handler.rs | 8 ++++++++ jupiter/src/storage/lfs_storage.rs | 5 ----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index 08786f539..4641a7438 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -355,6 +355,11 @@ pub async fn lfs_download_object( Ok(meta) => { // client didn't support split, splice the object and return it. let relations = relation_db.get_lfs_relations(meta.oid).await.unwrap(); + if relations.is_empty() { + return Err(GitLFSError::GeneralError( + "oid didn't have chunks".to_string(), + )); + } let mut bytes = vec![0u8; meta.size as usize]; for relation in relations { let sub_bytes = config @@ -476,6 +481,9 @@ async fn lfs_file_exist(config: &LfsConfig, meta: &MetaObject) -> bool { .get_lfs_relations(meta.oid.clone()) .await .unwrap(); + if relations.is_empty() { + return false; + } relations.iter().all(|relation| { config .lfs_storage diff --git a/jupiter/src/storage/lfs_storage.rs b/jupiter/src/storage/lfs_storage.rs index 0b2cf21ca..4b5ff4b15 100644 --- a/jupiter/src/storage/lfs_storage.rs +++ b/jupiter/src/storage/lfs_storage.rs @@ -71,11 +71,6 @@ impl LfsStorage { .all(self.get_connection()) .await .unwrap(); - if result.is_empty() { - return Err(MegaError::with_message( - "Object relation not found, maybe have not been uploaded yet", - )); - } Ok(result) } From d6ace077c841294cfa33907470543d2ee14291a8 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 26 Aug 2024 16:21:40 +0800 Subject: [PATCH 14/14] update libra README Signed-off-by: Qihang Cai --- libra/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libra/README.md b/libra/README.md index 8dfd4c811..dff111a53 100644 --- a/libra/README.md +++ b/libra/README.md @@ -70,6 +70,7 @@ achieving unified management. - [ ] `rebase` - [x] `index-pack` - [x] `remote` +- [x] `lfs` - [ ] `config` #### Remote - [x] `push` @@ -78,8 +79,9 @@ achieving unified management. - [x] `fetch` ### Others -- [ ] `.gitignore` and `.gitattributes` -- [ ] `lfs` +- [ ] `.gitignore` +- [x] `.gitattributes` (only for `lfs` now) +- [x] `LFS` (embedded) - [ ] `ssh` ## Development