Improve libra: better feedback & add integration test

.github/workflows/base.yml

@@ -82,6 +82,10 @@ jobs:
           git config --global user.email "mega@github.com"
           git config --global user.name "Mega"
           git config --global lfs.url http://localhost:8000
+      - uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --bin mega --bin libra
       - uses: actions-rs/cargo@v1
         with:
           command: test

libra/Cargo.toml

@@ -46,6 +46,7 @@ serde_json = { workspace = true }
 async_static = "0.1.3"
 once_cell = "1.19.0"
 byte-unit = "5.1.4"
+scopeguard = "1.2.0"
 
 [target.'cfg(unix)'.dependencies] # only on Unix
 pager = "0.16.0"

libra/src/command/clone.rs

@@ -1,13 +1,14 @@
 use std::path::PathBuf;
 use std::{env, fs};
-
+use std::cell::Cell;
 use crate::command;
 use crate::command::restore::RestoreArgs;
 use crate::internal::branch::Branch;
 use crate::internal::config::{Config, RemoteConfig};
 use crate::internal::head::Head;
 use clap::Parser;
-
+use colored::Colorize;
+use scopeguard::defer;
 use crate::utils::path_ext::PathExt;
 use crate::utils::util;
 
@@ -59,6 +60,15 @@ pub async fn execute(args: CloneArgs) {
         println!("Cloning into '{}'", repo_name);
     }
 
+    let is_success = Cell::new(false);
+    // clean up the directory if panic
+    defer! {
+        if !is_success.get() {
+            fs::remove_dir_all(&local_path).unwrap();
+            eprintln!("{}", "fatal: clone failed, delete repo directory automatically".red());
+        }
+    }
+
     // CAUTION: change [current_dir] to the repo directory
     env::set_current_dir(&local_path).unwrap();
     command::init::execute().await;
@@ -72,6 +82,8 @@ pub async fn execute(args: CloneArgs) {
 
     /* setup */
     setup(remote_repo.clone()).await;
+
+    is_success.set(true);
 }
 
 async fn setup(remote_repo: String) {

libra/src/command/fetch.rs

@@ -1,7 +1,7 @@
 use std::io;
 use std::vec;
 use std::{collections::HashSet, fs, io::Write};
-
+use std::time::Instant;
 use ceres::protocol::ServiceType::UploadPack;
 use clap::Parser;
 use indicatif::ProgressBar;
@@ -22,6 +22,7 @@ use crate::{
     },
     utils::{self, path_ext::PathExt},
 };
+use crate::utils::util;
 
 #[derive(Parser, Debug)]
 pub struct FetchArgs {
@@ -105,6 +106,7 @@ pub async fn fetch_repository(remote_config: &RemoteConfig) {
     let mut pack_data = Vec::new();
     let mut reach_pack = false;
     let bar = ProgressBar::new_spinner();
+    let time = Instant::now();
     loop {
         let (len, data) = read_pkt_line(&mut reader).await.unwrap();
         if len == 0 {
@@ -115,6 +117,10 @@ pub async fn fetch_repository(remote_config: &RemoteConfig) {
             tracing::debug!("Receiving PACK data...");
         }
         if reach_pack { // 2.PACK data
+            let bytes_per_sec = pack_data.len() as f64 / time.elapsed().as_secs_f64();
+            let total = util::auto_unit_bytes(pack_data.len() as u64);
+            let bps = util::auto_unit_bytes(bytes_per_sec as u64);
+            bar.set_message(format!("Receiving objects: {total:.2} | {bps:.2}/s"));
             bar.tick();
             // Side-Band Capability, should be enabled if Server Support
             let code = data[0];

libra/src/command/lfs.rs

@@ -3,7 +3,6 @@ use std::fs::{File, OpenOptions};
 use std::io;
 use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write};
 use std::path::Path;
-use byte_unit::UnitType;
 use reqwest::StatusCode;
 use ceres::lfs::lfs_structs::LockListQuery;
 use mercury::internal::index::Index;
@@ -185,8 +184,7 @@ pub async fn execute(cmd: LfsCmds) {
                         let _type = if is_pointer || !path_abs.exists() { "-" } else { "*" };
                         let oid = if long { oid } else { oid[..10].to_owned() };
                         let tail = if size {
-                            let byte = byte_unit::Byte::from(lfs_size);
-                            let byte = byte.get_appropriate_unit(UnitType::Decimal);
+                            let byte = util::auto_unit_bytes(lfs_size);
                             format!(" ({byte:.2})")
                         } else {
                             "".to_string()

libra/src/command/push.rs

@@ -66,11 +66,6 @@ pub async fn execute(args: PushArgs) {
         }
     };
     let repo_url = Config::get_remote_url(&repository).await;
-    if repo_url.is_none() {
-        eprintln!("fatal: remote '{}' not found, please use 'libra remote add'", repository);
-        return;
-    }
-    let repo_url = repo_url.unwrap();
 
     let branch = args.refspec.unwrap_or(branch);
     let commit_hash = Branch::find_branch(&branch, None).await.unwrap().commit.to_plain_str();
@@ -301,7 +296,7 @@ fn diff_tree_objs(old_tree: Option<&SHA1>, new_tree: &SHA1) -> HashSet<Entry> {
                 }
                 _ => { // TODO: submodule (TreeItemMode: Commit)
                     if item.mode == TreeItemMode::Commit { // (160000)| Gitlink (Submodule)
-                        eprintln!("fatal: submodule not supported");
+                        eprintln!("{}", "Warning: Submodule is not supported yet".red());
                     }
                     let blob = Blob::load(&item.id);
                     objs.insert(blob.into());

libra/src/internal/config.rs

@@ -63,22 +63,29 @@ impl Config {
     }
 
     /// Get remote repo name of current branch
-    pub async fn get_current_remote() -> Option<String> {
+    pub async fn get_current_remote() -> Result<Option<String>, ()> {
         match Head::current().await {
             Head::Branch(name) => {
-                Config::get_remote(&name).await
+                Ok(Config::get_remote(&name).await)
+            },
+            Head::Detached(_) => {
+                eprintln!("fatal: HEAD is detached, cannot get remote");
+                Err(())
             },
-            Head::Detached(_) => None,
         }
     }
 
-    pub async fn get_remote_url(remote: &str) -> Option<String> {
-        Config::get("remote", Some(remote), "url").await
+    pub async fn get_remote_url(remote: &str) -> String {
+        match Config::get("remote", Some(remote), "url").await {
+            Some(url) => url,
+            None => panic!("fatal: No URL configured for remote '{}'.", remote),
+        }
     }
 
+    /// return `None` if no remote is set
     pub async fn get_current_remote_url() -> Option<String> {
-        match Config::get_current_remote().await {
-            Some(remote) => Config::get_remote_url(&remote).await,
+        match Config::get_current_remote().await.unwrap() {
+            Some(remote) => Some(Config::get_remote_url(&remote).await),
             None => None,
         }
     }

libra/src/internal/protocol/lfs_client.rs

@@ -57,7 +57,7 @@ impl LFSClient {
         let url = Config::get_current_remote_url().await;
         match url {
             Some(url) => LFSClient::from_url(&Url::parse(&url).unwrap()),
-            None => panic!("fatal: current remote url not found"),
+            None => panic!("fatal: no remote set for current branch, use `libra branch --set-upstream-to <remote>/<branch>`"),
         }
     }
 
@@ -152,24 +152,24 @@ impl LFSClient {
 
         // TODO: parallel upload
         for obj in resp.objects {
-            self.upload_object(obj).await;
+            self.upload_object(obj).await?;
         }
         println!("LFS objects push completed.");
         Ok(())
     }
 
     /// upload (PUT) one LFS file to remote server
-    async fn upload_object(&self, object: Representation) {
+    async fn upload_object(&self, object: Representation) -> Result<(), ()> {
         if let Some(err) = object.error {
             eprintln!("fatal: LFS upload failed. Code: {}, Message: {}", err.code, err.message);
-            return;
+            return Err(());
         }
 
         if let Some(actions) = object.actions {
             let upload_link = actions.get("upload");
             if upload_link.is_none() {
                 eprintln!("fatal: LFS upload failed. No upload action found");
-                return;
+                return Err(());
             }
 
             let link = upload_link.unwrap();
@@ -188,12 +188,13 @@ impl LFSClient {
                 .unwrap();
             if !resp.status().is_success() {
                 eprintln!("fatal: LFS upload failed. Status: {}, Message: {}", resp.status(), resp.text().await.unwrap());
-                return;
+                return Err(());
             }
             println!("Uploaded.");
         } else {
             tracing::debug!("LFS file {} already exists on remote server", object.oid);
         }
+        Ok(())
     }
 
     /// download (GET) one LFS file from remote server
@@ -262,7 +263,10 @@ impl LFSClient {
         if checksum == oid {
             println!("Downloaded.");
         } else {
-            eprintln!("fatal: LFS download failed. Checksum mismatch: {} != {}", checksum, oid);
+            eprintln!("fatal: LFS download failed. Checksum mismatch: {} != {}. Fallback to pointer file.", checksum, oid);
+            let pointer = lfs::format_pointer_string(oid, size);
+            file.set_len(0).await.unwrap(); // clear
+            file.write_all(pointer.as_bytes()).await.unwrap();
         }
     }
 

libra/src/utils/lfs.rs

@@ -59,11 +59,14 @@ pub fn generate_pointer_file(path: impl AsRef<Path>) -> (String, String) {
     // calc file hash without type
     let oid = calc_lfs_file_hash(path).unwrap();
 
-    let pointer = format!("version {}\noid {}:{}\nsize {}\n",
-                          LFS_VERSION, LFS_HASH_ALGO, oid, path.metadata().unwrap().len());
+    let pointer = format_pointer_string(&oid, path.metadata().unwrap().len());
     (pointer, oid)
 }
 
+pub fn format_pointer_string(oid: &str, size: u64) -> String {
+    format!("version {}\noid {}:{}\nsize {}\n", LFS_VERSION, LFS_HASH_ALGO, oid, size)
+}
+
 /// Generate LFS Server Url from repo Url.
 /// By default, Git LFS will append `.git/info/lfs` to the end of a Git remote url to build the LFS server URL.
 /// [doc: server-discovery](https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md)

libra/src/utils/object_ext.rs

@@ -1,7 +1,7 @@
 use std::fs;
 use std::io::{BufReader, Read};
 use std::path::{Path, PathBuf};
-
+use colored::Colorize;
 use mercury::hash::SHA1;
 use mercury::internal::object::blob::Blob;
 use mercury::internal::object::commit::Commit;
@@ -38,6 +38,9 @@ impl TreeExt for Tree {
         let mut items = Vec::new();
         for item in self.tree_items.iter() {
             if item.mode != TreeItemMode::Tree { // Not Tree, maybe Blob, link, etc.
+                if item.mode == TreeItemMode::Commit { // submodule
+                    eprintln!("{}", "Warning: Submodule is not supported yet".red());
+                }
                 items.push((PathBuf::from(item.name.clone()), item.id));
             } else {
                 let sub_tree = Tree::load(&item.id);

libra/src/utils/util.rs

@@ -318,6 +318,15 @@ pub fn get_repo_name_from_url(mut url: &str) -> Option<&str> {
     Some(&url[repo_start..repo_end])
 }
 
+/// Find the appropriate unit and value for Bytes.
+/// ### Examples
+/// - 1024 bytes -> 1 KiB
+/// - 1024 * 1024 bytes -> 1 MiB
+pub fn auto_unit_bytes(bytes: u64) -> byte_unit::AdjustedByte {
+    let bytes = byte_unit::Byte::from(bytes);
+    bytes.get_appropriate_unit(byte_unit::UnitType::Binary)
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

mega/Cargo.toml

@@ -44,6 +44,8 @@ bytes = { workspace = true }
 go-defer = { workspace = true }
 git2 = { workspace = true }
 tempfile = { workspace = true }
+serial_test = "3.1.1"
+lazy_static = {workspace = true}
 
 [build-dependencies]
 shadow-rs = { workspace = true }