From 5b6d10881cf34f7239f11a35c86b76f75cc6b4dc Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Fri, 30 Aug 2024 18:27:40 +0800 Subject: [PATCH 01/16] add `libra lfs locks`: get all locks of current branch (refspec) from server Signed-off-by: Qihang Cai --- libra/src/command/lfs.rs | 35 +++++++++++++++++++++ libra/src/internal/config.rs | 1 + libra/src/internal/protocol/lfs_client.rs | 38 ++++++++++++++++++++--- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/libra/src/command/lfs.rs b/libra/src/command/lfs.rs index 04a079072..6f8f87773 100644 --- a/libra/src/command/lfs.rs +++ b/libra/src/command/lfs.rs @@ -3,6 +3,9 @@ use std::fs::{File, OpenOptions}; use std::io; use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; use std::path::Path; +use ceres::lfs::lfs_structs::LockListQuery; +use crate::internal::head::Head; +use crate::internal::protocol::lfs_client::LFS_CLIENT; use crate::utils::{lfs, path, util}; use crate::utils::path_ext::PathExt; @@ -16,6 +19,15 @@ pub enum LfsCmds { Untrack { path: Vec, }, + /// Lists currently locked files from the Git LFS server. (Current Branch) + Locks { + #[clap(long, short)] + id: Option, + #[clap(long, short)] + path: Option, + #[clap(long, short)] + limit: Option, + } } pub async fn execute(cmd: LfsCmds) { @@ -43,6 +55,29 @@ pub async fn execute(cmd: LfsCmds) { let path = convert_patterns_to_workdir(path); // untrack_lfs_patterns(&attr_path, path).unwrap(); } + LfsCmds::Locks { id, path, limit } => { + let refspec = match Head::current().await { + Head::Branch(name) => format!("refs/heads/{}", name), + Head::Detached(_) => { + println!("fatal: HEAD is detached"); + return; + } + }; + tracing::debug!("refspec: {}", refspec); + let query = LockListQuery { + id: id.unwrap_or_default(), + path: path.unwrap_or_default(), + limit: limit.map(|l| l.to_string()).unwrap_or_default(), + cursor: "".to_string(), + refspec, + }; + let locks = LFS_CLIENT.await.get_locks(query).await.locks; + if !locks.is_empty() { + for lock in locks { + println!("{} {} {} {}", lock.id, lock.path, lock.locked_at, lock.owner.unwrap().name); + } + } + } } } diff --git a/libra/src/internal/config.rs b/libra/src/internal/config.rs index 0929b8e97..287445951 100644 --- a/libra/src/internal/config.rs +++ b/libra/src/internal/config.rs @@ -56,6 +56,7 @@ impl Config { } /// Get remote repo name by branch name + /// - You may need to `[branch::set-upstream]` if return `None` pub async fn get_remote(branch: &str) -> Option { // e.g. [branch "master"].remote = origin Config::get("branch", Some(branch), "remote").await diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index 77c21a04d..507d2671a 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -5,7 +5,7 @@ use reqwest::Client; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; use url::Url; -use ceres::lfs::lfs_structs::{BatchRequest, Representation, RequestVars}; +use ceres::lfs::lfs_structs::{BatchRequest, LockList, LockListQuery, Representation, RequestVars}; use mercury::internal::object::types::ObjectType; use mercury::internal::pack::entry::Entry; use crate::internal::config::Config; @@ -18,7 +18,8 @@ async_static! { } pub struct LFSClient { - pub url: Url, + pub batch_url: Url, + pub lfs_url: Url, pub client: Client, } @@ -40,7 +41,8 @@ impl ProtocolClient for LFSClient { .build() .unwrap(); Self { - url: lfs_server.join("/objects/batch").unwrap(), + batch_url: lfs_server.join("/objects/batch").unwrap(), + lfs_url: lfs_server, client, } } @@ -93,7 +95,7 @@ impl LFSClient { enable_split: None, }; - let mut request = self.client.post(self.url.clone()).json(&batch_request); + let mut request = self.client.post(self.batch_url.clone()).json(&batch_request); if let Some(auth) = auth { request = request.basic_auth(auth.username, Some(auth.password)); } @@ -162,7 +164,7 @@ impl LFSClient { enable_split: None, }; - let request = self.client.post(self.url.clone()).json(&batch_request); + let request = self.client.post(self.batch_url.clone()).json(&batch_request); let response = request.send().await.unwrap(); let resp = response.json::().await.unwrap(); @@ -192,4 +194,30 @@ impl LFSClient { } println!("Downloaded."); // TODO: checksum } +} + +// LFS locks API +impl LFSClient { + pub async fn get_locks(&self, query: LockListQuery) -> LockList { + let url = self.lfs_url.join("/locks").unwrap(); + let mut request = self.client.get(url); + request = request.query(&[ + ("id", query.id), + ("path", query.path), + ("limit", query.limit), + ("cursor", query.cursor), + ("refspec", query.refspec) + ]); + let response = request.send().await.unwrap(); + + if !response.status().is_success() { + eprintln!("fatal: LFS get locks failed. Status: {}, Message: {}", response.status(), response.text().await.unwrap()); + return LockList { + locks: Vec::new(), + next_cursor: String::default(), + }; + } + + response.json::().await.unwrap() + } } \ No newline at end of file From 1c53383edcf733535f66931f38a837d925da328c Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Fri, 30 Aug 2024 21:41:59 +0800 Subject: [PATCH 02/16] add `libra lfs lock`: lock a file on server Signed-off-by: Qihang Cai --- ceres/src/lfs/handler.rs | 2 +- libra/src/command/lfs.rs | 55 +++++++++++++++++++---- libra/src/internal/protocol/lfs_client.rs | 18 +++++++- 3 files changed, 63 insertions(+), 12 deletions(-) diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index 6737b897f..754bacc92 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -598,7 +598,7 @@ async fn lfs_add_lock( }); let d = serde_json::to_string(&locks_from_data).unwrap(); - d.clone_into(&mut val.data); + d.clone_into(&mut val.data); // FIXME: must turn into `ActiveModel` before modify, or update failed. let res = storage.update_lock(val).await; match res.is_ok() { true => Ok(()), diff --git a/libra/src/command/lfs.rs b/libra/src/command/lfs.rs index 6f8f87773..052d80c0d 100644 --- a/libra/src/command/lfs.rs +++ b/libra/src/command/lfs.rs @@ -3,7 +3,9 @@ use std::fs::{File, OpenOptions}; use std::io; use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; use std::path::Path; +use reqwest::StatusCode; use ceres::lfs::lfs_structs::LockListQuery; +use crate::command::ask_basic_auth; use crate::internal::head::Head; use crate::internal::protocol::lfs_client::LFS_CLIENT; use crate::utils::{lfs, path, util}; @@ -19,7 +21,7 @@ pub enum LfsCmds { Untrack { path: Vec, }, - /// Lists currently locked files from the Git LFS server. (Current Branch) + /// Lists currently locked files from the Libra LFS server. (Current Branch) Locks { #[clap(long, short)] id: Option, @@ -27,6 +29,11 @@ pub enum LfsCmds { path: Option, #[clap(long, short)] limit: Option, + }, + /// Set a file as "locked" on the Libra LFS server + Lock { + /// String path name of the locked file. This should be relative to the root of the repository working directory + path: String, } } @@ -56,13 +63,7 @@ pub async fn execute(cmd: LfsCmds) { untrack_lfs_patterns(&attr_path, path).unwrap(); } LfsCmds::Locks { id, path, limit } => { - let refspec = match Head::current().await { - Head::Branch(name) => format!("refs/heads/{}", name), - Head::Detached(_) => { - println!("fatal: HEAD is detached"); - return; - } - }; + let refspec = current_refspec().await.unwrap(); tracing::debug!("refspec: {}", refspec); let query = LockListQuery { id: id.unwrap_or_default(), @@ -74,10 +75,46 @@ pub async fn execute(cmd: LfsCmds) { let locks = LFS_CLIENT.await.get_locks(query).await.locks; if !locks.is_empty() { for lock in locks { - println!("{} {} {} {}", lock.id, lock.path, lock.locked_at, lock.owner.unwrap().name); + println!("{}\tID:{}", lock.path, lock.id); } } } + LfsCmds::Lock { path } => { + if !lfs::is_lfs_tracked(&path) { + eprintln!("fatal: {} is not an LFS tracked file", path); + return; + } + + let refspec = current_refspec().await.unwrap(); + let mut auth = None; + loop { + let resp = LFS_CLIENT.await.lock(path.clone(), refspec.clone(), auth.clone()).await; + let code = resp.status(); + if code.is_success() { + println!("Locked {}", path); + return; + } else if code == StatusCode::FORBIDDEN { + eprintln!("Forbidden: You must have push access to create a lock"); + auth = Some(ask_basic_auth()); + continue; + } else if code == StatusCode::CONFLICT { + eprintln!("Conflict: already created lock"); + } else if !code.is_success() { + eprintln!("fatal: LFS lock failed. Code: {}, Message: {}", code, resp.text().await.unwrap()); + } + break; + } + } + } +} + +async fn current_refspec() -> Option { + match Head::current().await { + Head::Branch(name) => Some(format!("refs/heads/{}", name)), + Head::Detached(_) => { + println!("fatal: HEAD is detached"); + None + } } } diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index 507d2671a..8d7a0c300 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -1,11 +1,11 @@ use std::path::Path; use async_static::async_static; use futures_util::StreamExt; -use reqwest::Client; +use reqwest::{Client, Response, StatusCode}; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; use url::Url; -use ceres::lfs::lfs_structs::{BatchRequest, LockList, LockListQuery, Representation, RequestVars}; +use ceres::lfs::lfs_structs::{BatchRequest, LockList, LockListQuery, LockRequest, Ref, Representation, RequestVars}; use mercury::internal::object::types::ObjectType; use mercury::internal::pack::entry::Entry; use crate::internal::config::Config; @@ -220,4 +220,18 @@ impl LFSClient { response.json::().await.unwrap() } + + /// lock an LFS file + /// - `refspec` is must in Mega Server, but optional in Git Doc + pub async fn lock(&self, path: String, refspec: String, basic_auth: Option) -> Response { + let url = self.lfs_url.join("/locks").unwrap(); + let mut request = self.client.post(url).json(&LockRequest { + path, + refs: Ref { name: refspec }, + }); + if let Some(auth) = basic_auth { + request = request.basic_auth(auth.username, Some(auth.password)); + } + request.send().await.unwrap() + } } \ No newline at end of file From 0fb2f243b0a0c81e6a6ac3897b7a9da759add82a Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Thu, 5 Sep 2024 16:17:52 +0800 Subject: [PATCH 03/16] update LFS server url generation method for only supporting monorepo Signed-off-by: Qihang Cai --- libra/src/utils/lfs.rs | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index 3f54efb1c..bce0dc165 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -7,6 +7,7 @@ use path_abs::{PathInfo, PathOps}; use regex::Regex; use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_TYPE}; use sha2::{Digest, Sha256}; +use url::Url; use wax::Pattern; use crate::utils::{path, util}; use crate::utils::path_ext::PathExt; @@ -67,7 +68,9 @@ pub fn generate_pointer_file(path: impl AsRef) -> (String, String) { /// [doc: server-discovery](https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md) /// - like https://git-server.com/foo/bar.git/info/lfs /// - support ssh & https & git@ format -pub fn generate_lfs_server_url(mut url: String) -> String { +#[deprecated(note = "It's for git, not monorepo")] +#[allow(dead_code)] +pub fn generate_git_lfs_server_url(mut url: String) -> String { if url.ends_with('/') { url.pop(); } @@ -87,6 +90,24 @@ pub fn generate_lfs_server_url(mut url: String) -> String { url } +/// Generate Mono LFS Server Url from repo Url. +/// - Just get domain with port +/// ### Example +/// https://github.com/git-lfs/git-lfs/blob/main/docs/api/locking.md -> https://github.com +/// +/// http://localhost:8000/xxx/yyy -> http://localhost:8000 +pub fn generate_lfs_server_url(url: String) -> String { + let url = Url::parse(&url).unwrap(); + match url.port() { + None => { + format!("{}://{}", url.scheme(), url.host().unwrap()) + } + Some(port) => { + format!("{}://{}:{}", url.scheme(), url.host().unwrap(), port) + } + } +} + /// Generate LFS cache path, in `.libra/lfs/objects` pub fn lfs_object_path(oid: &str) -> PathBuf { util::storage_path() @@ -202,7 +223,7 @@ mod tests { } #[test] - fn test_gen_lfs_server_url() { + fn test_gen_git_lfs_server_url() { const LFS_SERVER_URL: &str = "https://github.com/web3infra-foundation/mega.git/info/lfs"; let url = "https://github.com/web3infra-foundation/mega".to_owned(); assert_eq!(generate_lfs_server_url(url), LFS_SERVER_URL); @@ -217,6 +238,14 @@ mod tests { assert_eq!(generate_lfs_server_url(url), LFS_SERVER_URL); } + #[test] + fn test_gen_mono_lfs_server_url() { + const LFS_SERVER_URL: &str = "https://github.com/web3infra-foundation/mega.git/info/lfs"; + assert_eq!(generate_lfs_server_url(LFS_SERVER_URL.to_owned()), "https://github.com"); + const LOCAL_LFS_SERVER_URL: &str = "http://localhost:8000/xxx/yyy"; + assert_eq!(generate_lfs_server_url(LOCAL_LFS_SERVER_URL.to_owned()), "http://localhost:8000"); + } + #[test] fn test_parse_pointer_data() { let data = r#"version https://git-lfs.github.com/spec/v1 From 8b3c2ea844679a91b711215d3d512dedaaa63c9a Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Thu, 5 Sep 2024 18:47:27 +0800 Subject: [PATCH 04/16] add `libra lfs unlock` Signed-off-by: Qihang Cai --- libra/src/command/lfs.rs | 64 ++++++++++++++++++++--- libra/src/command/status.rs | 7 +++ libra/src/internal/protocol/lfs_client.rs | 30 +++++++++-- 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/libra/src/command/lfs.rs b/libra/src/command/lfs.rs index 052d80c0d..f36a1e46b 100644 --- a/libra/src/command/lfs.rs +++ b/libra/src/command/lfs.rs @@ -5,7 +5,7 @@ use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; use std::path::Path; use reqwest::StatusCode; use ceres::lfs::lfs_structs::LockListQuery; -use crate::command::ask_basic_auth; +use crate::command::{ask_basic_auth, status}; use crate::internal::head::Head; use crate::internal::protocol::lfs_client::LFS_CLIENT; use crate::utils::{lfs, path, util}; @@ -34,6 +34,14 @@ pub enum LfsCmds { Lock { /// String path name of the locked file. This should be relative to the root of the repository working directory path: String, + }, + /// Remove "locked" setting for a file on the Libra LFS server + Unlock { + path: String, + #[clap(long, short)] + force: bool, + #[clap(long, short)] + id: Option } } @@ -80,27 +88,67 @@ pub async fn execute(cmd: LfsCmds) { } } LfsCmds::Lock { path } => { - if !lfs::is_lfs_tracked(&path) { - eprintln!("fatal: {} is not an LFS tracked file", path); + // Only check existence + if !Path::new(&path).exists() { + eprintln!("fatal: pathspec '{}' did not match any files", path); return; } let refspec = current_refspec().await.unwrap(); let mut auth = None; loop { - let resp = LFS_CLIENT.await.lock(path.clone(), refspec.clone(), auth.clone()).await; - let code = resp.status(); + let code = LFS_CLIENT.await.lock(path.clone(), refspec.clone(), auth.clone()).await; if code.is_success() { println!("Locked {}", path); - return; } else if code == StatusCode::FORBIDDEN { eprintln!("Forbidden: You must have push access to create a lock"); auth = Some(ask_basic_auth()); continue; } else if code == StatusCode::CONFLICT { eprintln!("Conflict: already created lock"); - } else if !code.is_success() { - eprintln!("fatal: LFS lock failed. Code: {}, Message: {}", code, resp.text().await.unwrap()); + } + break; + } + } + LfsCmds::Unlock { path, force, id } => { + if !force { + if !Path::new(&path).exists() { + eprintln!("fatal: pathspec '{}' did not match any files", path); + return; + } + if !status::is_clean().await { + eprintln!("fatal: working tree not clean"); + return; + } + } + let refspec = current_refspec().await.unwrap(); + let id = match id { + None => { + // get id by path + let locks = LFS_CLIENT.await.get_locks(LockListQuery { + refspec: refspec.clone(), + path: path.clone(), + id: "".to_string(), + cursor: "".to_string(), + limit: "".to_string(), + }).await.locks; + if locks.is_empty() { + eprintln!("fatal: no lock found for path '{}'", path); + return; + } + locks[0].id.clone() + } + Some(id) => id + }; + let mut auth = None; + loop { + let code = LFS_CLIENT.await.unlock(id.clone(), refspec.clone(), force, auth.clone()).await; + if code.is_success() { + println!("Unlocked {}", path); + } else if code == StatusCode::FORBIDDEN { + eprintln!("Forbidden: You must have push access to unlock"); + auth = Some(ask_basic_auth()); + continue; } break; } diff --git a/libra/src/command/status.rs b/libra/src/command/status.rs index 0aa75e706..3303e7151 100644 --- a/libra/src/command/status.rs +++ b/libra/src/command/status.rs @@ -108,6 +108,13 @@ pub async fn execute() { } } +/// Check if the working tree is clean +pub async fn is_clean() -> bool { + let staged = changes_to_be_committed().await; + let unstaged = changes_to_be_staged(); + staged.is_empty() && unstaged.is_empty() +} + /** * Compare the difference between `index` and the last `Commit Tree` */ diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index 8d7a0c300..fa732ff66 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -1,11 +1,11 @@ use std::path::Path; use async_static::async_static; use futures_util::StreamExt; -use reqwest::{Client, Response, StatusCode}; +use reqwest::{Client, StatusCode}; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; use url::Url; -use ceres::lfs::lfs_structs::{BatchRequest, LockList, LockListQuery, LockRequest, Ref, Representation, RequestVars}; +use ceres::lfs::lfs_structs::{BatchRequest, LockList, LockListQuery, LockRequest, Ref, Representation, RequestVars, UnlockRequest}; use mercury::internal::object::types::ObjectType; use mercury::internal::pack::entry::Entry; use crate::internal::config::Config; @@ -223,7 +223,7 @@ impl LFSClient { /// lock an LFS file /// - `refspec` is must in Mega Server, but optional in Git Doc - pub async fn lock(&self, path: String, refspec: String, basic_auth: Option) -> Response { + pub async fn lock(&self, path: String, refspec: String, basic_auth: Option) -> StatusCode { let url = self.lfs_url.join("/locks").unwrap(); let mut request = self.client.post(url).json(&LockRequest { path, @@ -232,6 +232,28 @@ impl LFSClient { if let Some(auth) = basic_auth { request = request.basic_auth(auth.username, Some(auth.password)); } - request.send().await.unwrap() + let resp = request.send().await.unwrap(); + let code = resp.status(); + if !resp.status().is_success() { + eprintln!("fatal: LFS lock failed. Status: {}, Message: {}", code, resp.text().await.unwrap()); + } + code + } + + pub async fn unlock(&self, id: String, refspec: String, force: bool, basic_auth: Option) -> StatusCode { + let url = self.lfs_url.join(&format!("/locks/{}/unlock", id)).unwrap(); + let mut request = self.client.post(url).json(&UnlockRequest { + force: Some(force), + refs: Ref { name: refspec }, + }); + if let Some(auth) = basic_auth.clone() { + request = request.basic_auth(auth.username, Some(auth.password)); + } + let resp = request.send().await.unwrap(); + let code = resp.status(); + if !resp.status().is_success() { + eprintln!("fatal: LFS unlock failed. Status: {}, Message: {}", code, resp.text().await.unwrap()); + } + code } } \ No newline at end of file From f6a3703e9b24a82f23606983efc3bf53deb4cc4a Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sat, 7 Sep 2024 19:57:55 +0800 Subject: [PATCH 05/16] Verify Locks before pushing LFS files Signed-off-by: Qihang Cai --- libra/src/command/lfs.rs | 2 +- libra/src/command/push.rs | 6 ++- libra/src/internal/protocol/lfs_client.rs | 63 +++++++++++++++++++++-- libra/src/utils/lfs.rs | 14 ++++- 4 files changed, 79 insertions(+), 6 deletions(-) diff --git a/libra/src/command/lfs.rs b/libra/src/command/lfs.rs index f36a1e46b..c863ca926 100644 --- a/libra/src/command/lfs.rs +++ b/libra/src/command/lfs.rs @@ -156,7 +156,7 @@ pub async fn execute(cmd: LfsCmds) { } } -async fn current_refspec() -> Option { +pub(crate) async fn current_refspec() -> Option { match Head::current().await { Head::Branch(name) => Some(format!("refs/heads/{}", name)), Head::Detached(_) => { diff --git a/libra/src/command/push.rs b/libra/src/command/push.rs index 4c44f6837..5d4e59cd5 100644 --- a/libra/src/command/push.rs +++ b/libra/src/command/push.rs @@ -120,7 +120,11 @@ pub async fn execute(args: PushArgs) { { // upload lfs files let client = LFSClient::from_url(&url); - client.push_objects(&objs, auth.clone()).await; + let res = client.push_objects(&objs, auth.clone()).await; + if res.is_err() { + eprintln!("fatal: LFS files upload failed, stop pushing"); + return; + } } // let (tx, rx) = mpsc::channel::(); diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index fa732ff66..3c7b1cc19 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::path::Path; use async_static::async_static; use futures_util::StreamExt; @@ -5,9 +6,10 @@ use reqwest::{Client, StatusCode}; use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; use url::Url; -use ceres::lfs::lfs_structs::{BatchRequest, LockList, LockListQuery, LockRequest, Ref, Representation, RequestVars, UnlockRequest}; +use ceres::lfs::lfs_structs::{BatchRequest, LockList, LockListQuery, LockRequest, Ref, Representation, RequestVars, UnlockRequest, VerifiableLockList, VerifiableLockRequest}; use mercury::internal::object::types::ObjectType; use mercury::internal::pack::entry::Entry; +use crate::command; use crate::internal::config::Config; use crate::internal::protocol::https_client::BasicAuth; use crate::internal::protocol::ProtocolClient; @@ -59,7 +61,7 @@ impl LFSClient { } /// push LFS objects to remote server - pub async fn push_objects<'a, I>(&self, objs: I, auth: Option) + pub async fn push_objects<'a, I>(&self, objs: I, auth: Option) -> Result<(), ()> where I: IntoIterator { @@ -77,7 +79,7 @@ impl LFSClient { let path = lfs::lfs_object_path(oid); if !path.exists() { eprintln!("fatal: LFS object not found: {}", oid); - return; + continue; } let size = path.metadata().unwrap().len() as i64; lfs_objs.push(RequestVars { @@ -87,6 +89,42 @@ impl LFSClient { }) } + // verify locks + match self.verify_locks(VerifiableLockRequest { + refs: Ref { name: command::lfs::current_refspec().await.unwrap() }, + ..Default::default() + }, auth.clone()).await { + Ok((_, locks)) => { + tracing::debug!("LFS verify locks response:\n {:?}", locks); + let oids: HashSet = lfs_oids.iter().map(|(oid, _)| oid.clone()).collect(); + let ours = locks.ours.iter().filter(|l| { + let oid = lfs::get_oid_by_path(&l.path); + oids.contains(&oid) + }).collect::>(); + if !ours.is_empty() { + println!("The following files are locked by you, consider unlocking them:"); + for lock in ours { + println!(" - {}", lock.path); + } + } + let theirs = locks.theirs.iter().filter(|l| { + let oid = lfs::get_oid_by_path(&l.path); + oids.contains(&oid) + }).collect::>(); + if !theirs.is_empty() { + eprintln!("Locking failed: The following files are locked by another user:"); + for lock in theirs { + eprintln!(" - {}", lock.path); + } + return Err(()); + } + } + Err(_) => { + eprintln!("fatal: Get LFS verify locks failed"); + return Err(()); + } + } + let batch_request = BatchRequest { operation: "upload".to_string(), transfers: vec![lfs::LFS_TRANSFER_API.to_string()], @@ -110,6 +148,7 @@ impl LFSClient { self.upload_object(obj).await; } println!("LFS objects push completed."); + Ok(()) } /// upload (PUT) one LFS file to remote server @@ -256,4 +295,22 @@ impl LFSClient { } code } + + /// List Locks for Verification + pub async fn verify_locks(&self, query: VerifiableLockRequest, basic_auth: Option) + -> Result<(StatusCode, VerifiableLockList), ()> + { + let url = self.lfs_url.join("/locks/verify").unwrap(); + let mut request = self.client.post(url).json(&query); + if let Some(auth) = basic_auth { + request = request.basic_auth(auth.username, Some(auth.password)); + } + let resp = request.send().await.unwrap(); + let code = resp.status(); + if !resp.status().is_success() { + eprintln!("fatal: LFS verify locks failed. Status: {}, Message: {}", code, resp.text().await.unwrap()); + return Err(()); + } + Ok((code, resp.json::().await.unwrap())) + } } \ No newline at end of file diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index bce0dc165..f114d77eb 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -9,6 +9,7 @@ use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_TYPE}; use sha2::{Digest, Sha256}; use url::Url; use wax::Pattern; +use mercury::internal::index::Index; use crate::utils::{path, util}; use crate::utils::path_ext::PathExt; @@ -117,6 +118,17 @@ pub fn lfs_object_path(oid: &str) -> PathBuf { .join(oid) } +/// Get LFS file oid by path (through `Index`), NOT re-calculate +pub fn get_oid_by_path(path: &str) -> String { + let index_file = path::index(); + let index = Index::load(&index_file).unwrap(); + let hash = index.get_hash(path, 0).unwrap(); + let storage = util::objects_storage(); + let data = storage.get(&hash).unwrap(); + let (oid, _) = parse_pointer_data(&data).unwrap(); + oid +} + /// Copy LFS file to `.libra/lfs/objects` /// - absolute path pub fn backup_lfs_file

(path: P, oid: &str) -> io::Result<()> @@ -154,7 +166,7 @@ where Ok(file_hash) } -/// Check if `data` is an LFS pointer, return `oid` +/// Check if `data` is an LFS pointer, return `oid` & `size` pub fn parse_pointer_data(data: &[u8]) -> Option<(String, u64)> { if data.len() > LFS_POINTER_MAX_SIZE { return None; From bb152e7d404df708816b2a0c08fb755003bff861 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sun, 8 Sep 2024 00:01:58 +0800 Subject: [PATCH 06/16] Remove useless `enable_split` field in `BatchRequest` Signed-off-by: Qihang Cai --- ceres/src/lfs/lfs_structs.rs | 1 - libra/src/internal/protocol/lfs_client.rs | 2 -- 2 files changed, 3 deletions(-) diff --git a/ceres/src/lfs/lfs_structs.rs b/ceres/src/lfs/lfs_structs.rs index da5ca27f2..7d874fb7f 100644 --- a/ceres/src/lfs/lfs_structs.rs +++ b/ceres/src/lfs/lfs_structs.rs @@ -88,7 +88,6 @@ pub struct BatchRequest { pub transfers: Vec, pub objects: Vec, pub hash_algo: String, - pub enable_split: Option, } #[derive(Serialize, Deserialize)] diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index 3c7b1cc19..7b5fbbcdf 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -130,7 +130,6 @@ impl LFSClient { transfers: vec![lfs::LFS_TRANSFER_API.to_string()], objects: lfs_objs, hash_algo: lfs::LFS_HASH_ALGO.to_string(), - enable_split: None, }; let mut request = self.client.post(self.batch_url.clone()).json(&batch_request); @@ -200,7 +199,6 @@ impl LFSClient { ..Default::default() }], hash_algo: lfs::LFS_HASH_ALGO.to_string(), - enable_split: None, }; let request = self.client.post(self.batch_url.clone()).json(&batch_request); From 3b7aae503d5e37b9a9024dfb8c41848812db89ba Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sun, 8 Sep 2024 15:33:39 +0800 Subject: [PATCH 07/16] fix `LFSClient`: deal with `404` & `403` correctly Signed-off-by: Qihang Cai --- libra/src/internal/protocol/lfs_client.rs | 45 +++++++++++++++-------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index 7b5fbbcdf..47961592d 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -89,12 +89,23 @@ impl LFSClient { }) } - // verify locks - match self.verify_locks(VerifiableLockRequest { - refs: Ref { name: command::lfs::current_refspec().await.unwrap() }, - ..Default::default() - }, auth.clone()).await { - Ok((_, locks)) => { + { // verify locks + let (code, locks) = self.verify_locks(VerifiableLockRequest { + refs: Ref { name: command::lfs::current_refspec().await.unwrap() }, + ..Default::default() + }, auth.clone()).await; + + if code == StatusCode::FORBIDDEN { + eprintln!("fatal: Forbidden: You must have push access to verify locks"); + return Err(()); + } else if code == StatusCode::NOT_FOUND { + // By default, an LFS server that doesn't implement any locking endpoints should return 404. + // This response will not halt any Git pushes. + } else if !code.is_success() { + eprintln!("fatal: LFS verify locks failed. Status: {}", code); + return Err(()); + } else { + // success tracing::debug!("LFS verify locks response:\n {:?}", locks); let oids: HashSet = lfs_oids.iter().map(|(oid, _)| oid.clone()).collect(); let ours = locks.ours.iter().filter(|l| { @@ -119,10 +130,6 @@ impl LFSClient { return Err(()); } } - Err(_) => { - eprintln!("fatal: Get LFS verify locks failed"); - return Err(()); - } } let batch_request = BatchRequest { @@ -271,7 +278,7 @@ impl LFSClient { } let resp = request.send().await.unwrap(); let code = resp.status(); - if !resp.status().is_success() { + if !resp.status().is_success() && code != StatusCode::FORBIDDEN { eprintln!("fatal: LFS lock failed. Status: {}, Message: {}", code, resp.text().await.unwrap()); } code @@ -288,7 +295,7 @@ impl LFSClient { } let resp = request.send().await.unwrap(); let code = resp.status(); - if !resp.status().is_success() { + if !resp.status().is_success() && code != StatusCode::FORBIDDEN { eprintln!("fatal: LFS unlock failed. Status: {}, Message: {}", code, resp.text().await.unwrap()); } code @@ -296,7 +303,7 @@ impl LFSClient { /// List Locks for Verification pub async fn verify_locks(&self, query: VerifiableLockRequest, basic_auth: Option) - -> Result<(StatusCode, VerifiableLockList), ()> + -> (StatusCode, VerifiableLockList) { let url = self.lfs_url.join("/locks/verify").unwrap(); let mut request = self.client.post(url).json(&query); @@ -305,10 +312,16 @@ impl LFSClient { } let resp = request.send().await.unwrap(); let code = resp.status(); - if !resp.status().is_success() { + // By default, an LFS server that doesn't implement any locking endpoints should return 404. + // This response will not halt any Git pushes. + if !code.is_success() && code != StatusCode::NOT_FOUND && code != StatusCode::FORBIDDEN { eprintln!("fatal: LFS verify locks failed. Status: {}, Message: {}", code, resp.text().await.unwrap()); - return Err(()); + return (code, VerifiableLockList { + ours: Vec::new(), + theirs: Vec::new(), + next_cursor: String::default(), + }); } - Ok((code, resp.json::().await.unwrap())) + (code, resp.json::().await.unwrap()) } } \ No newline at end of file From 82530fc1e8d68fdf486a215fd8533cd9b13fe8b6 Mon Sep 17 00:00:00 2001 From: HouXiaoxuan Date: Sun, 8 Sep 2024 20:20:38 +0800 Subject: [PATCH 08/16] feat: delete RequestVars args for `lfs_fetch_chunk_ids` Signed-off-by: HouXiaoxuan --- ceres/src/lfs/handler.rs | 11 ++++------- mono/src/api/lfs/lfs_router.rs | 3 +-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index 96c7c8944..34a4feca7 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -238,7 +238,7 @@ pub async fn lfs_process_batch( /// else return an error. pub async fn lfs_fetch_chunk_ids( context: &Context, - fetch_vars: &RequestVars, + oid: &String, ) -> Result, GitLFSError> { let config = context.config.lfs.clone(); @@ -249,13 +249,13 @@ pub async fn lfs_fetch_chunk_ids( } let storage = context.services.lfs_storage.clone(); - let meta = lfs_get_meta(storage.clone(), &fetch_vars.oid) + let meta = lfs_get_meta(storage.clone(), oid) .await .map_err(|_| GitLFSError::GeneralError("".to_string()))?; assert!(meta.splited, "database didn't match the split mode"); let relations = storage - .get_lfs_relations(fetch_vars.oid.clone()) + .get_lfs_relations(oid.to_owned()) .await .map_err(|_| GitLFSError::GeneralError("".to_string()))?; @@ -272,10 +272,7 @@ pub async fn lfs_fetch_chunk_ids( let tmp_request_vars = RequestVars { oid: relation.sub_oid.clone(), size: relation.size, - authorization: fetch_vars.authorization.clone(), - password: fetch_vars.password.clone(), - user: fetch_vars.user.clone(), - repo: fetch_vars.repo.clone(), + ..Default::default() }; response_objects.push(ChunkRepresentation { sub_oid: relation.sub_oid, diff --git a/mono/src/api/lfs/lfs_router.rs b/mono/src/api/lfs/lfs_router.rs index 5f89725fc..6ebb7cd05 100644 --- a/mono/src/api/lfs/lfs_router.rs +++ b/mono/src/api/lfs/lfs_router.rs @@ -207,9 +207,8 @@ pub async fn lfs_process_batch( pub async fn lfs_fetch_chunk_ids( state: State, Path(oid): Path, - Json(json): Json, ) -> Result, (StatusCode, String)> { - let result = handler::lfs_fetch_chunk_ids(&state.context, &json).await; + let result = handler::lfs_fetch_chunk_ids(&state.context, &oid).await; match result { Ok(response) => { let size = response.iter().fold(0, |acc, chunk| acc + chunk.size); From 2db450bdb3f94bf43ac668c75fa0671fd2ab3c69 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Sun, 8 Sep 2024 20:09:20 +0800 Subject: [PATCH 09/16] add `libra lfs ls-files`: list LFS files (current branch) Signed-off-by: Qihang Cai --- libra/Cargo.toml | 1 + libra/src/command/lfs.rs | 44 ++++++++++++++++++++++++++++++++++++++++ libra/src/utils/lfs.rs | 11 ++++++++++ 3 files changed, 56 insertions(+) diff --git a/libra/Cargo.toml b/libra/Cargo.toml index 2c4e60a07..0fb2b72fc 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -44,6 +44,7 @@ serde = { workspace = true } serde_json = { workspace = true } async_static = "0.1.3" once_cell = "1.19.0" +byte-unit = "5.1.4" [target.'cfg(unix)'.dependencies] # only on Unix pager = "0.16.0" diff --git a/libra/src/command/lfs.rs b/libra/src/command/lfs.rs index c863ca926..1f5e83616 100644 --- a/libra/src/command/lfs.rs +++ b/libra/src/command/lfs.rs @@ -3,14 +3,17 @@ use std::fs::{File, OpenOptions}; use std::io; use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; use std::path::Path; +use byte_unit::UnitType; use reqwest::StatusCode; use ceres::lfs::lfs_structs::LockListQuery; +use mercury::internal::index::Index; use crate::command::{ask_basic_auth, status}; use crate::internal::head::Head; use crate::internal::protocol::lfs_client::LFS_CLIENT; use crate::utils::{lfs, path, util}; use crate::utils::path_ext::PathExt; +/// [Docs](https://github.com/git-lfs/git-lfs/tree/main/docs/man) #[derive(Subcommand, Debug)] pub enum LfsCmds { /// View or add LFS paths to Libra Attributes (root) @@ -42,6 +45,18 @@ pub enum LfsCmds { force: bool, #[clap(long, short)] id: Option + }, + /// Show information about Git LFS files in the index and working tree (current branch) + LsFiles { + /// Show the entire 64 character OID, instead of just first 10. + #[clap(long, short)] + long: bool, + /// Show the size of the LFS object between parenthesis at the end of a line. + #[clap(long, short)] + size: bool, + /// Show only the lfs tracked file names. + #[clap(long, short)] + name_only: bool, } } @@ -153,6 +168,35 @@ pub async fn execute(cmd: LfsCmds) { break; } } + LfsCmds::LsFiles { long, size, name_only} => { + let idx_file = path::index(); + let index = Index::load(&idx_file).unwrap(); + let entries = index.tracked_entries(0); + let storage = util::objects_storage(); + for entry in entries { + if lfs::is_lfs_tracked(&entry.name) { + let data = storage.get(&entry.hash).unwrap(); + if let Some((oid, lfs_size)) = lfs::parse_pointer_data(&data) { + let path_abs = util::workdir_to_absolute(&entry.name); + let is_pointer = lfs::parse_pointer_file(&path_abs).is_ok(); + let _type = if is_pointer { "-" } else { "*" }; + let oid = if long { oid } else { oid[..10].to_owned() }; + let tail = if size { + let byte = byte_unit::Byte::from(lfs_size); + let byte = byte.get_appropriate_unit(UnitType::Decimal); + format!(" ({byte:.2})") + } else { + "".to_string() + }; + if name_only { + println!("{}{}", entry.name, tail); + } else { + println!("{} {} {}{}", oid, _type, entry.name, tail); + } + } + } + } + } } } diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index f114d77eb..b3d6cec2f 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -187,6 +187,17 @@ pub fn parse_pointer_data(data: &[u8]) -> Option<(String, u64)> { None } +/// Read max LFS_POINTER_MAX_SIZE bytes +pub fn parse_pointer_file(path: impl AsRef) -> io::Result<(String, u64)> { + let mut file = File::open(path)?; + let mut buffer = [0; LFS_POINTER_MAX_SIZE]; + let bytes_read = file.read(&mut buffer)?; + if let Some((oid, size)) = parse_pointer_data(&buffer[..bytes_read]) { + return Ok((oid, size)); + } + Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid LFS pointer file")) +} + /// Extract LFS patterns from `.libra_attributes` file pub fn extract_lfs_patterns(file_path: &str) -> io::Result> { let path = Path::new(file_path); From 1d072798a63082ff55197f575f3a029573ba3c7f Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 9 Sep 2024 13:12:33 +0800 Subject: [PATCH 10/16] fix: `lfs ls-files` (show deleted file with '-') & `is_lfs_tracked()` (only absolute path) TODO: workdir path (all) Signed-off-by: Qihang Cai --- libra/src/command/lfs.rs | 8 +++++--- libra/src/utils/lfs.rs | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libra/src/command/lfs.rs b/libra/src/command/lfs.rs index 1f5e83616..caed9204b 100644 --- a/libra/src/command/lfs.rs +++ b/libra/src/command/lfs.rs @@ -174,12 +174,14 @@ pub async fn execute(cmd: LfsCmds) { let entries = index.tracked_entries(0); let storage = util::objects_storage(); for entry in entries { - if lfs::is_lfs_tracked(&entry.name) { + let path_abs = util::workdir_to_absolute(&entry.name); + if lfs::is_lfs_tracked(&path_abs) { let data = storage.get(&entry.hash).unwrap(); if let Some((oid, lfs_size)) = lfs::parse_pointer_data(&data) { - let path_abs = util::workdir_to_absolute(&entry.name); let is_pointer = lfs::parse_pointer_file(&path_abs).is_ok(); - let _type = if is_pointer { "-" } else { "*" }; + // An asterisk (*) after the OID indicates a full object, a minus (-) indicates an LFS pointer. + // or not exists (-) + let _type = if is_pointer || !path_abs.exists() { "-" } else { "*" }; let oid = if long { oid } else { oid[..10].to_owned() }; let tail = if size { let byte = byte_unit::Byte::from(lfs_size); diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index b3d6cec2f..72f4cd08f 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -30,7 +30,7 @@ lazy_static! { /// Check if a file is LFS tracked /// - support Glob pattern matching (TODO: support .gitignore patterns) /// - only check root attributes file now, should check all attributes files in sub-dirs -/// - absolute path or relative path to workdir +/// - absolute path pub fn is_lfs_tracked

(path: P) -> bool where P: AsRef, From ec236fb337486ee25ac9af131e80a2c68871c8bd Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 9 Sep 2024 16:42:52 +0800 Subject: [PATCH 11/16] `libra` support LFS split (chunk) API Signed-off-by: Qihang Cai --- ceres/src/lfs/lfs_structs.rs | 2 +- libra/src/internal/protocol/lfs_client.rs | 79 ++++++++++++++++++----- libra/src/utils/lfs.rs | 2 +- 3 files changed, 64 insertions(+), 19 deletions(-) diff --git a/ceres/src/lfs/lfs_structs.rs b/ceres/src/lfs/lfs_structs.rs index 7d874fb7f..4a2795ede 100644 --- a/ceres/src/lfs/lfs_structs.rs +++ b/ceres/src/lfs/lfs_structs.rs @@ -104,7 +104,7 @@ pub struct FetchchunkResponse { pub chunks: Vec, } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone)] pub struct Link { pub href: String, pub header: HashMap, diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index 47961592d..ab1f2ba02 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -4,9 +4,10 @@ use async_static::async_static; use futures_util::StreamExt; use reqwest::{Client, StatusCode}; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use tokio::io::AsyncWriteExt; use url::Url; -use ceres::lfs::lfs_structs::{BatchRequest, LockList, LockListQuery, LockRequest, Ref, Representation, RequestVars, UnlockRequest, VerifiableLockList, VerifiableLockRequest}; +use ceres::lfs::lfs_structs::{BatchRequest, FetchchunkResponse, Link, LockList, LockListQuery, LockRequest, Ref, Representation, RequestVars, UnlockRequest, VerifiableLockList, VerifiableLockRequest}; use mercury::internal::object::types::ObjectType; use mercury::internal::pack::entry::Entry; use crate::command; @@ -216,27 +217,71 @@ impl LFSClient { let link = resp.objects[0].actions.as_ref().unwrap().get("download").unwrap(); - let mut request = self.client.get(link.href.clone()); - for (k, v) in &link.header { - request = request.header(k, v); - } + let mut is_chunked = false; + // Chunk API + let links = match self.fetch_chunk_links(&link.href).await { + Ok(chunks) => { + is_chunked = true; + tracing::info!("LFS Chunk API supported."); + chunks + }, + Err(_) => vec![link.clone()], + }; - let response = request.send().await.unwrap(); + let mut file = tokio::fs::File::create(path).await.unwrap(); + let mut checksum = Sha256::new(); + println!("Downloading LFS file: {}", oid); + let mut cnt = 0; + let total = links.len(); + for link in links { + cnt += 1; + if is_chunked { + println!("- part: {}/{}", cnt, total); + } - if !response.status().is_success() { - eprintln!("fatal: LFS download failed. Status: {}, Message: {}", response.status(), response.text().await.unwrap()); - return; - } + let mut request = self.client.get(&link.href); + for (k, v) in &link.header { + request = request.header(k, v); + } - println!("Downloading LFS file: {}", oid); - let mut file = tokio::fs::File::create(path).await.unwrap(); - let mut stream = response.bytes_stream(); + let response = request.send().await.unwrap(); + if !response.status().is_success() { + eprintln!("fatal: LFS download failed. Status: {}, Message: {}", response.status(), response.text().await.unwrap()); + return; + } + + let mut stream = response.bytes_stream(); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.unwrap(); + file.write_all(&chunk).await.unwrap(); + checksum.update(&chunk); + } + } + let checksum = hex::encode(checksum.finalize()); + if checksum == oid { + println!("Downloaded."); + } else { + eprintln!("fatal: LFS download failed. Checksum mismatch: {} != {}", checksum, oid); + } + } - while let Some(chunk) = stream.next().await { - let chunk = chunk.unwrap(); - file.write_all(&chunk).await.unwrap(); + /// Only for MonoRepo (mega) + async fn fetch_chunk_links(&self, obj_link: &str) -> Result, ()> { + let request = self.client.get(obj_link.to_owned() + "/chunks"); + let resp = request.send().await.unwrap(); + let code = resp.status(); + if code == StatusCode::NOT_FOUND { + tracing::info!("Remote LFS Server not support Chunks API"); + return Err(()); + } else if !code.is_success() { + tracing::debug!("fatal: LFS get chunk hrefs failed. Status: {}, Message: {}", code, resp.text().await.unwrap()); + return Err(()); } - println!("Downloaded."); // TODO: checksum + let mut res = resp.json::().await.unwrap(); + // sort by offset + res.chunks.sort_by(|a, b| a.offset.cmp(&b.offset)); + Ok(res.chunks.into_iter().map(|c| c.link).collect()) } } diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index 72f4cd08f..2b3994ff4 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -145,7 +145,7 @@ where } /// SHA256 without type -/// TODO: performance optimization, 200MB 4s now, slower than `sha256sum` +// TODO: performance optimization, 200MB 4s now, slower than `sha256sum` pub fn calc_lfs_file_hash

(path: P) -> io::Result where P: AsRef, From f3dc6296a50f7af86cbd5233ec544851d8ec1dc1 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 9 Sep 2024 17:02:53 +0800 Subject: [PATCH 12/16] Optimization: `ring` is much faster than `sha2` crate in `SHA256` calculation ( > 10 times) Signed-off-by: Qihang Cai --- libra/Cargo.toml | 2 +- libra/src/internal/protocol/lfs_client.rs | 6 +++--- libra/src/utils/lfs.rs | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libra/Cargo.toml b/libra/Cargo.toml index 0fb2b72fc..660bfdd88 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -38,7 +38,7 @@ indicatif = "0.17.8" wax = "0.6.0" lazy_static = { workspace = true } regex = { workspace = true } -sha2 = "0.10.8" +ring = "0.17.8" hex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index ab1f2ba02..ddf1cdd4f 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -3,8 +3,8 @@ use std::path::Path; use async_static::async_static; use futures_util::StreamExt; use reqwest::{Client, StatusCode}; +use ring::digest::{Context, SHA256}; use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; use tokio::io::AsyncWriteExt; use url::Url; use ceres::lfs::lfs_structs::{BatchRequest, FetchchunkResponse, Link, LockList, LockListQuery, LockRequest, Ref, Representation, RequestVars, UnlockRequest, VerifiableLockList, VerifiableLockRequest}; @@ -229,7 +229,7 @@ impl LFSClient { }; let mut file = tokio::fs::File::create(path).await.unwrap(); - let mut checksum = Sha256::new(); + let mut checksum = Context::new(&SHA256); println!("Downloading LFS file: {}", oid); let mut cnt = 0; let total = links.len(); @@ -258,7 +258,7 @@ impl LFSClient { checksum.update(&chunk); } } - let checksum = hex::encode(checksum.finalize()); + let checksum = hex::encode(checksum.finish().as_ref()); if checksum == oid { println!("Downloaded."); } else { diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index 2b3994ff4..6d341ba39 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -6,7 +6,7 @@ use lazy_static::lazy_static; use path_abs::{PathInfo, PathOps}; use regex::Regex; use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_TYPE}; -use sha2::{Digest, Sha256}; +use ring::digest::{Context, SHA256}; use url::Url; use wax::Pattern; use mercury::internal::index::Index; @@ -145,13 +145,13 @@ where } /// SHA256 without type -// TODO: performance optimization, 200MB 4s now, slower than `sha256sum` +// `ring` crate is much faster than `sha2` crate ( > 10 times) pub fn calc_lfs_file_hash

(path: P) -> io::Result where P: AsRef, { let path = path.as_ref(); - let mut hash = Sha256::new(); + let mut hash = Context::new(&SHA256); let file = File::open(path)?; let mut reader = BufReader::new(file); let mut buffer = [0; 65536]; @@ -162,7 +162,7 @@ where } hash.update(&buffer[..n]); } - let file_hash = hex::encode(hash.finalize()); + let file_hash = hex::encode(hash.finish().as_ref()); Ok(file_hash) } From faf3b885bb29cdca5d0acda16df101a15c599f98 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 9 Sep 2024 17:52:00 +0800 Subject: [PATCH 13/16] fix mega tests: set `lfs.url` for monorepo Signed-off-by: Qihang Cai --- mega/tests/lfs_test.rs | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/mega/tests/lfs_test.rs b/mega/tests/lfs_test.rs index b36648743..9e9d72d37 100644 --- a/mega/tests/lfs_test.rs +++ b/mega/tests/lfs_test.rs @@ -11,6 +11,7 @@ use rand::Rng; use tempfile::TempDir; const PORT: u16 = 8000; // mega server port +const LARGE_FILE_SIZE_MB: usize = 60; /// check if git lfs is installed fn check_git_lfs() -> bool { let status = Command::new("git") @@ -22,12 +23,24 @@ fn check_git_lfs() -> bool { } fn run_git_cmd(args: &[&str]) { - let status = Command::new("git") + let output = Command::new("git") .args(args) - .status() + .output() .unwrap(); - assert!(status.success(), "Git command failed: git {}", args.join(" ")); + let status = output.status; + // assert!(status.success(), "Git command failed: git {}", args.join(" ")); + if !status.success() { + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + panic!( + "Git command failed: git {}\nStatus: {}\nStdout: {}\nStderr: {}", + args.join(" "), + status, + stdout, + stderr + ); + } } fn is_port_in_use(port: u16) -> bool { @@ -79,7 +92,7 @@ fn lfs_push(url: &str) -> io::Result<()> { run_git_cmd(&["lfs", "track", "*.bin"]); // create large file - generate_large_file("large_file.bin", 60)?; + generate_large_file("large_file.bin", LARGE_FILE_SIZE_MB)?; // add & commit run_git_cmd(&["add", "."]); @@ -99,7 +112,9 @@ fn lfs_clone(url: &str) -> io::Result<()> { // git clone url run_git_cmd(&["clone", url]); - assert!(Path::new("lfs/large_file.bin").exists(), "Failed to clone large file"); + let file = Path::new("lfs/large_file.bin"); + assert!(file.exists(), "Failed to clone large file"); + assert_eq!(file.metadata()?.len(), LARGE_FILE_SIZE_MB as u64 * 1024 * 1024); Ok(()) } @@ -112,6 +127,10 @@ fn lfs_split_with_git() { // start mega server at background run_mega_server(); + // MonoRepo (mega)'s lfs.url is not compatible with git-lfs + let lfs_url = format!("http://localhost:{}", PORT); + run_git_cmd(&["config", "--global", "lfs.url", &lfs_url]); + let url = &format!("http://localhost:{}/third-part/lfs.git", PORT); lfs_push(url).expect("Failed to push large file to mega server"); lfs_clone(url).expect("Failed to clone large file from mega server"); From 5057499be48a56a2974a3b249d3a1e563b8a1989 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 9 Sep 2024 18:22:54 +0800 Subject: [PATCH 14/16] fix `GET /locks`: Allow parameter omission, for Git compatibility Signed-off-by: Qihang Cai --- ceres/src/lfs/lfs_structs.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ceres/src/lfs/lfs_structs.rs b/ceres/src/lfs/lfs_structs.rs index 4a2795ede..71ad4df80 100644 --- a/ceres/src/lfs/lfs_structs.rs +++ b/ceres/src/lfs/lfs_structs.rs @@ -191,9 +191,14 @@ pub struct VerifiableLockList { #[derive(Serialize, Deserialize, Debug)] pub struct LockListQuery { + #[serde(default)] pub path: String, + #[serde(default)] pub id: String, + #[serde(default)] pub cursor: String, + #[serde(default)] pub limit: String, + #[serde(default)] pub refspec: String, } From 4a3b92fe6a34d209009b05c7e76b9ab440593901 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Mon, 9 Sep 2024 18:31:02 +0800 Subject: [PATCH 15/16] improve `lfs locks` output: align `ID` Signed-off-by: Qihang Cai --- libra/src/command/lfs.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libra/src/command/lfs.rs b/libra/src/command/lfs.rs index caed9204b..ab4eb197a 100644 --- a/libra/src/command/lfs.rs +++ b/libra/src/command/lfs.rs @@ -97,8 +97,9 @@ pub async fn execute(cmd: LfsCmds) { }; let locks = LFS_CLIENT.await.get_locks(query).await.locks; if !locks.is_empty() { + let max_path_len = locks.iter().map(|l| l.path.len()).max().unwrap(); for lock in locks { - println!("{}\tID:{}", lock.path, lock.id); + println!("{: Date: Mon, 9 Sep 2024 18:34:19 +0800 Subject: [PATCH 16/16] fix Mega LFS (`delete_lock` & `lfs_add_lock`): into_active_model first before update field Signed-off-by: Qihang Cai --- ceres/Cargo.toml | 1 + ceres/src/lfs/handler.rs | 14 +++++++++----- jupiter/src/storage/lfs_db_storage.rs | 4 ++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ceres/Cargo.toml b/ceres/Cargo.toml index fa0ee9682..ec830fd97 100644 --- a/ceres/Cargo.toml +++ b/ceres/Cargo.toml @@ -29,3 +29,4 @@ bytes = { workspace = true } async-trait = { workspace = true } rand = { workspace = true } sha256 = { workspace = true } +sea-orm = { workspace = true } diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index 86765d899..f4d29adaa 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -5,7 +5,8 @@ use anyhow::Result; use bytes::Bytes; use chrono::{prelude::*, Duration}; use rand::prelude::*; - +use sea_orm::ActiveValue::Set; +use sea_orm::IntoActiveModel; use callisto::{lfs_locks, lfs_objects, lfs_split_relations}; use common::errors::{GitLFSError, MegaError}; use jupiter::context::Context; @@ -554,7 +555,7 @@ async fn lfs_add_lock( match result { // Update - Some(mut val) => { + Some(val) => { let d = val.data.to_owned(); let mut locks_from_data = if !d.is_empty() { let locks_from_data: Vec = serde_json::from_str(&d).unwrap(); @@ -572,7 +573,9 @@ async fn lfs_add_lock( }); let d = serde_json::to_string(&locks_from_data).unwrap(); - d.clone_into(&mut val.data); // FIXME: must turn into `ActiveModel` before modify, or update failed. + // must turn into `ActiveModel` before modify, or update failed. + let mut val = val.into_active_model(); + val.data = Set(d); let res = storage.update_lock(val).await; match res.is_ok() { true => Ok(()), @@ -675,7 +678,7 @@ async fn delete_lock( let result = storage.get_lock_by_id(repo).await.unwrap(); match result { // Exist, then delete. - Some(mut val) => { + Some(val) => { let d = val.data.to_owned(); let locks_from_data = if !d.is_empty() { let locks_from_data: Vec = serde_json::from_str(&d).unwrap(); @@ -725,7 +728,8 @@ async fn delete_lock( // Update remaining locks. let data = serde_json::to_string(&new_locks).unwrap(); - data.clone_into(&mut val.data); + let mut val = val.into_active_model(); + val.data = Set(data); let res = storage.update_lock(val).await; match res.is_ok() { true => Ok(lock_to_delete), diff --git a/jupiter/src/storage/lfs_db_storage.rs b/jupiter/src/storage/lfs_db_storage.rs index ae858eb0a..7b3f5b0c7 100644 --- a/jupiter/src/storage/lfs_db_storage.rs +++ b/jupiter/src/storage/lfs_db_storage.rs @@ -129,9 +129,9 @@ impl LfsDbStorage { pub async fn update_lock( &self, - lfs_lock: lfs_locks::Model, + lfs_lock: lfs_locks::ActiveModel, ) -> Result { - Ok(lfs_locks::Entity::update(lfs_lock.into_active_model()) + Ok(lfs_locks::Entity::update(lfs_lock) .exec(self.get_connection()) .await .unwrap())