add LFS support for Libra

ceres/src/lfs/handler.rs

@@ -198,7 +198,7 @@ pub async fn lfs_process_batch(
         let mut meta = meta.unwrap_or_default();
         if found && lfs_file_exist(config, &meta).await {
             // originla download method, split mode use ``
-            response_objects.push(represent(object, &meta, true, false, false, &server_url).await);
+            response_objects.push(represent(object, &meta, batch_vars.operation == "download", false, false, &server_url).await);
             continue;
         }
         // Not found
@@ -355,6 +355,11 @@ pub async fn lfs_download_object(
             Ok(meta) => {
                 // client didn't support split, splice the object and return it.
                 let relations = relation_db.get_lfs_relations(meta.oid).await.unwrap();
+                if relations.is_empty() {
+                    return Err(GitLFSError::GeneralError(
+                        "oid didn't have chunks".to_string(),
+                    ));
+                }
                 let mut bytes = vec![0u8; meta.size as usize];
                 for relation in relations {
                     let sub_bytes = config
@@ -476,6 +481,9 @@ async fn lfs_file_exist(config: &LfsConfig, meta: &MetaObject) -> bool {
             .get_lfs_relations(meta.oid.clone())
             .await
             .unwrap();
+        if relations.is_empty() {
+            return false;
+        }
         relations.iter().all(|relation| {
             config
                 .lfs_storage

jupiter/src/storage/lfs_storage.rs

@@ -71,11 +71,6 @@ impl LfsStorage {
             .all(self.get_connection())
             .await
             .unwrap();
-        if result.is_empty() {
-            return Err(MegaError::with_message(
-                "Object relation not found, maybe have not been uploaded yet",
-            ));
-        }
         Ok(result)
     }
 

libra/Cargo.toml

@@ -24,7 +24,7 @@ sha1 = { workspace = true }
 bytes = { workspace = true }
 chrono = { workspace = true }
 futures = { workspace = true }
-reqwest = { workspace = true, features = ["stream"] }
+reqwest = { workspace = true, features = ["stream", "json"] }
 tokio-util = { version = "0.7.11", features = ["io"] }
 color-backtrace = "0.6.1"
 colored = "2.1.0"
@@ -35,6 +35,15 @@ url = "2.5.0"
 futures-util = "0.3.30"
 rpassword = "7.3.1"
 indicatif = "0.17.8"
+wax = "0.6.0"
+lazy_static = { workspace = true }
+regex = { workspace = true }
+sha2 = "0.10.8"
+hex = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+async_static = "0.1.3"
+once_cell = "1.19.0"
 
 [target.'cfg(unix)'.dependencies] # only on Unix
 pager = "0.16.0"

libra/README.md

@@ -70,6 +70,7 @@ achieving unified management.
 - [ ] `rebase`
 - [x] `index-pack`
 - [x] `remote`
+- [x] `lfs`
 - [ ] `config`
 #### Remote
 - [x] `push`
@@ -78,8 +79,9 @@ achieving unified management.
 - [x] `fetch`
 
 ### Others
-- [ ] `.gitignore` and `.gitattributes`
-- [ ] `lfs`
+- [ ] `.gitignore`
+- [x] `.gitattributes` (only for `lfs` now)
+- [x] `LFS` (embedded)
 - [ ] `ssh`
 
 ## Development

libra/src/command/add.rs

@@ -5,7 +5,7 @@ use crate::command::status;
 use mercury::internal::index::{Index, IndexEntry};
 use crate::utils::object_ext::BlobExt;
 
-use crate::utils::{path, util};
+use crate::utils::{lfs, path, util};
 
 #[derive(Parser, Debug)]
 pub struct AddArgs {
@@ -97,7 +97,8 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) {
             if verbose {
                 println!("removed: {}", file_str);
             }
-        } else {
+        } else { // FIXME: unreachable code! This situation is not included in `status::changes_to_be_staged()`
+            // FIXME: should check files in original input paths
             // TODO do this check earlier, once fatal occurs, nothing should be done
             // file is not tracked && not exists, which means wrong pathspec
             println!("fatal: pathspec '{}' did not match any files", file.display());
@@ -106,7 +107,7 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) {
         // file exists
         if !index.tracked(file_str, 0) {
             // file is not tracked
-            let blob = Blob::from_file(&file_abs);
+            let blob = gen_blob_from_file(&file_abs);
             blob.save();
             index.add(IndexEntry::new_from_file(file, blob.id, &workdir).unwrap());
             if verbose {
@@ -116,7 +117,7 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) {
             // file is tracked, maybe modified
             if index.is_modified(file_str, 0, &workdir) {
                 // file is modified(meta), but content may not change
-                let blob = Blob::from_file(&file_abs);
+                let blob = gen_blob_from_file(&file_abs);
                 if !index.verify_hash(file_str, 0, &blob.id) {
                     // content is changed
                     blob.save();
@@ -129,6 +130,17 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) {
         }
     }
 }
+
+/// Generate a `Blob` from a file
+/// - if the file is tracked by LFS, generate a `Blob` with pointer file
+fn gen_blob_from_file(path: impl AsRef<Path>) -> Blob {
+    if lfs::is_lfs_tracked(&path) {
+        Blob::from_lfs_file(&path)
+    } else {
+        Blob::from_file(&path)
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

libra/src/command/lfs.rs

@@ -0,0 +1,125 @@
+use clap::Subcommand;
+use std::fs::{File, OpenOptions};
+use std::io;
+use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write};
+use std::path::Path;
+use crate::utils::{lfs, path, util};
+use crate::utils::path_ext::PathExt;
+
+#[derive(Subcommand, Debug)]
+pub enum LfsCmds {
+    /// View or add LFS paths to Libra Attributes (root)
+    Track {
+        pattern: Option<Vec<String>>,
+    },
+    /// Remove LFS paths from Libra Attributes
+    Untrack {
+        path: Vec<String>,
+    },
+}
+
+pub async fn execute(cmd: LfsCmds) {
+    // TODO: attributes file should be created in current dir, NOT root dir
+    let attr_path = path::attributes().to_string_or_panic();
+    match cmd {
+        LfsCmds::Track { pattern } => { // TODO: deduplicate
+            match pattern {
+                Some(pattern) => {
+                    let pattern = convert_patterns_to_workdir(pattern); //
+                    add_lfs_patterns(&attr_path, pattern).unwrap();
+                }
+                None => {
+                    let lfs_patterns = lfs::extract_lfs_patterns(&attr_path).unwrap();
+                    if !lfs_patterns.is_empty() {
+                        println!("Listing tracked patterns");
+                        for p in lfs_patterns {
+                            println!("    {} ({})", p, util::ATTRIBUTES); // '\t' seems to be 8 spaces, :(
+                        }
+                    }
+                }
+            }
+        }
+        LfsCmds::Untrack { path } => {
+            let path = convert_patterns_to_workdir(path); //
+            untrack_lfs_patterns(&attr_path, path).unwrap();
+        }
+    }
+}
+
+/// temp
+fn convert_patterns_to_workdir(patterns: Vec<String>) -> Vec<String> {
+    patterns.into_iter().map(|p| {
+        util::to_workdir_path(&p).to_string_or_panic()
+    }).collect()
+}
+
+fn add_lfs_patterns(file_path: &str, patterns: Vec<String>) -> io::Result<()> {
+    let mut file = OpenOptions::new()
+        .create(true)
+        .read(true)
+        .append(true)
+        .open(file_path)?;
+
+    if file.metadata()?.len() > 0 {
+        file.seek(SeekFrom::End(-1))?;
+
+        let mut last_byte = [0; 1];
+        file.read_exact(&mut last_byte)?;
+
+        // ensure the last byte is '\n'
+        if last_byte[0] != b'\n' {
+            file.write_all(b"\n")?;
+        }
+    }
+
+    let lfs_patterns = lfs::extract_lfs_patterns(file_path)?;
+    for pattern in patterns {
+        if lfs_patterns.contains(&pattern) {
+            continue;
+        }
+        println!("Tracking \"{}\"", pattern);
+        let pattern = format!("{} filter=lfs diff=lfs merge=lfs -text\n", pattern.replace(" ", r"\ "));
+        file.write_all(pattern.as_bytes())?;
+    }
+
+    Ok(())
+}
+
+fn untrack_lfs_patterns(file_path: &str, patterns: Vec<String>) -> io::Result<()> {
+    if !Path::new(file_path).exists() {
+        return Ok(());
+    }
+    let file = File::open(file_path)?;
+    let reader = BufReader::new(file);
+
+    let mut lines: Vec<String> = Vec::new();
+    for line in reader.lines() {
+        let line = line?;
+        let mut matched_pattern = None;
+        // delete the specified lfs patterns
+        for pattern in &patterns {
+            let pattern = pattern.replace(" ", r"\ ");
+            if line.trim_start().starts_with(&pattern) && line.contains("filter=lfs") {
+                matched_pattern = Some(pattern);
+                break;
+            }
+        }
+        match matched_pattern {
+            Some(pattern) => println!("Untracking \"{}\"", pattern),
+            None => lines.push(line),
+        }
+    }
+
+    // clear the file
+    let mut file = OpenOptions::new()
+        .write(true)
+        .truncate(true)
+        .open(file_path)?;
+
+    for line in lines {
+        file.write_all(line.as_bytes())?;
+        file.write_all(b"\n")?;
+    }
+
+    Ok(())
+}

libra/src/command/mod.rs

@@ -14,13 +14,18 @@ pub mod remove;
 pub mod restore;
 pub mod status;
 pub mod switch;
+pub mod lfs;
 
 use crate::internal::protocol::https_client::BasicAuth;
 use crate::utils::util;
 use mercury::{errors::GitError, hash::SHA1, internal::object::ObjectTrait};
 use rpassword::read_password;
 use std::io;
 use std::io::Write;
+use std::path::Path;
+use mercury::internal::object::blob::Blob;
+use crate::utils;
+use crate::utils::object_ext::BlobExt;
 
 // impl load for all objects
 fn load_object<T>(hash: &SHA1) -> Result<T, GitError>
@@ -111,6 +116,18 @@ pub fn parse_commit_msg(msg_gpg: &str) -> (String, Option<String>) {
     }
 }
 
+/// Calculate the hash of a file blob
+/// - for `lfs` file: calculate hash of the pointer data
+pub fn calc_file_blob_hash(path: impl AsRef<Path>) -> io::Result<SHA1> {
+    let blob =  if utils::lfs::is_lfs_tracked(&path) {
+        let (pointer, _) = utils::lfs::generate_pointer_file(&path);
+        Blob::from_content(&pointer)
+    } else {
+        Blob::from_file(&path)
+    };
+    Ok(blob.id)
+}
+
 #[cfg(test)]
 mod test {
     use mercury::internal::object::commit::Commit;

libra/src/command/push.rs

@@ -20,6 +20,7 @@ use crate::internal::branch::Branch;
 use crate::internal::config::Config;
 use crate::internal::head::Head;
 use crate::internal::protocol::https_client::{BasicAuth, HttpsClient};
+use crate::internal::protocol::lfs_client::LFSClient;
 use crate::internal::protocol::ProtocolClient;
 use crate::utils::object_ext::{BlobExt, CommitExt, TreeExt};
 
@@ -55,7 +56,7 @@ pub async fn execute(args: PushArgs) {
         Some(repo) => repo,
         None => {
             // e.g. [branch "master"].remote = origin
-            let remote = Config::get("branch", Some(&branch), "remote").await;
+            let remote = Config::get_remote(&branch).await;
             if let Some(remote) = remote {
                 remote
             } else {
@@ -64,7 +65,7 @@ pub async fn execute(args: PushArgs) {
             }
         }
     };
-    let repo_url = Config::get("remote", Some(&repository), "url").await;
+    let repo_url = Config::get_remote_url(&repository).await;
     if repo_url.is_none() {
         eprintln!("fatal: remote '{}' not found, please use 'libra remote add'", repository);
         return;
@@ -117,6 +118,11 @@ pub async fn execute(args: PushArgs) {
         SHA1::from_str(&remote_hash).unwrap()
     );
 
+    { // upload lfs files
+        let client = LFSClient::from_url(&url);
+        client.push_objects(&objs, auth.clone()).await;
+    }
+
     // let (tx, rx) = mpsc::channel::<Entry>();
     let (entry_tx, entry_rx) = mpsc::channel(1_000_000);
     let (stream_tx, mut stream_rx) = mpsc::channel(1_000_000);
@@ -138,7 +144,7 @@ pub async fn execute(args: PushArgs) {
     data.extend_from_slice(&pack_data);
     println!("Delta compression done.");
 
-    let res = client.send_pack(data.freeze(), auth).await.unwrap();
+    let res = client.send_pack(data.freeze(), auth).await.unwrap(); // TODO: send stream
 
     if res.status() != 200 {
         eprintln!("status code: {}", res.status());