diff --git a/ceres/src/lfs/handler.rs b/ceres/src/lfs/handler.rs index 37fe7f04a..4641a7438 100644 --- a/ceres/src/lfs/handler.rs +++ b/ceres/src/lfs/handler.rs @@ -198,7 +198,7 @@ pub async fn lfs_process_batch( let mut meta = meta.unwrap_or_default(); if found && lfs_file_exist(config, &meta).await { // originla download method, split mode use `` - response_objects.push(represent(object, &meta, true, false, false, &server_url).await); + response_objects.push(represent(object, &meta, batch_vars.operation == "download", false, false, &server_url).await); continue; } // Not found @@ -355,6 +355,11 @@ pub async fn lfs_download_object( Ok(meta) => { // client didn't support split, splice the object and return it. let relations = relation_db.get_lfs_relations(meta.oid).await.unwrap(); + if relations.is_empty() { + return Err(GitLFSError::GeneralError( + "oid didn't have chunks".to_string(), + )); + } let mut bytes = vec![0u8; meta.size as usize]; for relation in relations { let sub_bytes = config @@ -476,6 +481,9 @@ async fn lfs_file_exist(config: &LfsConfig, meta: &MetaObject) -> bool { .get_lfs_relations(meta.oid.clone()) .await .unwrap(); + if relations.is_empty() { + return false; + } relations.iter().all(|relation| { config .lfs_storage diff --git a/jupiter/src/storage/lfs_storage.rs b/jupiter/src/storage/lfs_storage.rs index 0b2cf21ca..4b5ff4b15 100644 --- a/jupiter/src/storage/lfs_storage.rs +++ b/jupiter/src/storage/lfs_storage.rs @@ -71,11 +71,6 @@ impl LfsStorage { .all(self.get_connection()) .await .unwrap(); - if result.is_empty() { - return Err(MegaError::with_message( - "Object relation not found, maybe have not been uploaded yet", - )); - } Ok(result) } diff --git a/libra/Cargo.toml b/libra/Cargo.toml index 3f203f4fc..2c4e60a07 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -24,7 +24,7 @@ sha1 = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } futures = { workspace = true } -reqwest = { workspace = true, features = ["stream"] } +reqwest = { workspace = true, features = ["stream", "json"] } tokio-util = { version = "0.7.11", features = ["io"] } color-backtrace = "0.6.1" colored = "2.1.0" @@ -35,6 +35,15 @@ url = "2.5.0" futures-util = "0.3.30" rpassword = "7.3.1" indicatif = "0.17.8" +wax = "0.6.0" +lazy_static = { workspace = true } +regex = { workspace = true } +sha2 = "0.10.8" +hex = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +async_static = "0.1.3" +once_cell = "1.19.0" [target.'cfg(unix)'.dependencies] # only on Unix pager = "0.16.0" diff --git a/libra/README.md b/libra/README.md index 8dfd4c811..dff111a53 100644 --- a/libra/README.md +++ b/libra/README.md @@ -70,6 +70,7 @@ achieving unified management. - [ ] `rebase` - [x] `index-pack` - [x] `remote` +- [x] `lfs` - [ ] `config` #### Remote - [x] `push` @@ -78,8 +79,9 @@ achieving unified management. - [x] `fetch` ### Others -- [ ] `.gitignore` and `.gitattributes` -- [ ] `lfs` +- [ ] `.gitignore` +- [x] `.gitattributes` (only for `lfs` now) +- [x] `LFS` (embedded) - [ ] `ssh` ## Development diff --git a/libra/src/command/add.rs b/libra/src/command/add.rs index fd032340c..606ab2b17 100644 --- a/libra/src/command/add.rs +++ b/libra/src/command/add.rs @@ -5,7 +5,7 @@ use crate::command::status; use mercury::internal::index::{Index, IndexEntry}; use crate::utils::object_ext::BlobExt; -use crate::utils::{path, util}; +use crate::utils::{lfs, path, util}; #[derive(Parser, Debug)] pub struct AddArgs { @@ -97,7 +97,8 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) { if verbose { println!("removed: {}", file_str); } - } else { + } else { // FIXME: unreachable code! This situation is not included in `status::changes_to_be_staged()` + // FIXME: should check files in original input paths // TODO do this check earlier, once fatal occurs, nothing should be done // file is not tracked && not exists, which means wrong pathspec println!("fatal: pathspec '{}' did not match any files", file.display()); @@ -106,7 +107,7 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) { // file exists if !index.tracked(file_str, 0) { // file is not tracked - let blob = Blob::from_file(&file_abs); + let blob = gen_blob_from_file(&file_abs); blob.save(); index.add(IndexEntry::new_from_file(file, blob.id, &workdir).unwrap()); if verbose { @@ -116,7 +117,7 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) { // file is tracked, maybe modified if index.is_modified(file_str, 0, &workdir) { // file is modified(meta), but content may not change - let blob = Blob::from_file(&file_abs); + let blob = gen_blob_from_file(&file_abs); if !index.verify_hash(file_str, 0, &blob.id) { // content is changed blob.save(); @@ -129,6 +130,17 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) { } } } + +/// Generate a `Blob` from a file +/// - if the file is tracked by LFS, generate a `Blob` with pointer file +fn gen_blob_from_file(path: impl AsRef) -> Blob { + if lfs::is_lfs_tracked(&path) { + Blob::from_lfs_file(&path) + } else { + Blob::from_file(&path) + } +} + #[cfg(test)] mod test { use super::*; diff --git a/libra/src/command/lfs.rs b/libra/src/command/lfs.rs new file mode 100644 index 000000000..04a079072 --- /dev/null +++ b/libra/src/command/lfs.rs @@ -0,0 +1,125 @@ +use clap::Subcommand; +use std::fs::{File, OpenOptions}; +use std::io; +use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write}; +use std::path::Path; +use crate::utils::{lfs, path, util}; +use crate::utils::path_ext::PathExt; + +#[derive(Subcommand, Debug)] +pub enum LfsCmds { + /// View or add LFS paths to Libra Attributes (root) + Track { + pattern: Option>, + }, + /// Remove LFS paths from Libra Attributes + Untrack { + path: Vec, + }, +} + +pub async fn execute(cmd: LfsCmds) { + // TODO: attributes file should be created in current dir, NOT root dir + let attr_path = path::attributes().to_string_or_panic(); + match cmd { + LfsCmds::Track { pattern } => { // TODO: deduplicate + match pattern { + Some(pattern) => { + let pattern = convert_patterns_to_workdir(pattern); // + add_lfs_patterns(&attr_path, pattern).unwrap(); + } + None => { + let lfs_patterns = lfs::extract_lfs_patterns(&attr_path).unwrap(); + if !lfs_patterns.is_empty() { + println!("Listing tracked patterns"); + for p in lfs_patterns { + println!(" {} ({})", p, util::ATTRIBUTES); // '\t' seems to be 8 spaces, :( + } + } + } + } + } + LfsCmds::Untrack { path } => { + let path = convert_patterns_to_workdir(path); // + untrack_lfs_patterns(&attr_path, path).unwrap(); + } + } +} + +/// temp +fn convert_patterns_to_workdir(patterns: Vec) -> Vec { + patterns.into_iter().map(|p| { + util::to_workdir_path(&p).to_string_or_panic() + }).collect() +} + +fn add_lfs_patterns(file_path: &str, patterns: Vec) -> io::Result<()> { + let mut file = OpenOptions::new() + .create(true) + .read(true) + .append(true) + .open(file_path)?; + + if file.metadata()?.len() > 0 { + file.seek(SeekFrom::End(-1))?; + + let mut last_byte = [0; 1]; + file.read_exact(&mut last_byte)?; + + // ensure the last byte is '\n' + if last_byte[0] != b'\n' { + file.write_all(b"\n")?; + } + } + + let lfs_patterns = lfs::extract_lfs_patterns(file_path)?; + for pattern in patterns { + if lfs_patterns.contains(&pattern) { + continue; + } + println!("Tracking \"{}\"", pattern); + let pattern = format!("{} filter=lfs diff=lfs merge=lfs -text\n", pattern.replace(" ", r"\ ")); + file.write_all(pattern.as_bytes())?; + } + + Ok(()) +} + +fn untrack_lfs_patterns(file_path: &str, patterns: Vec) -> io::Result<()> { + if !Path::new(file_path).exists() { + return Ok(()); + } + let file = File::open(file_path)?; + let reader = BufReader::new(file); + + let mut lines: Vec = Vec::new(); + for line in reader.lines() { + let line = line?; + let mut matched_pattern = None; + // delete the specified lfs patterns + for pattern in &patterns { + let pattern = pattern.replace(" ", r"\ "); + if line.trim_start().starts_with(&pattern) && line.contains("filter=lfs") { + matched_pattern = Some(pattern); + break; + } + } + match matched_pattern { + Some(pattern) => println!("Untracking \"{}\"", pattern), + None => lines.push(line), + } + } + + // clear the file + let mut file = OpenOptions::new() + .write(true) + .truncate(true) + .open(file_path)?; + + for line in lines { + file.write_all(line.as_bytes())?; + file.write_all(b"\n")?; + } + + Ok(()) +} \ No newline at end of file diff --git a/libra/src/command/mod.rs b/libra/src/command/mod.rs index 07aae11d7..a98f6101f 100644 --- a/libra/src/command/mod.rs +++ b/libra/src/command/mod.rs @@ -14,6 +14,7 @@ pub mod remove; pub mod restore; pub mod status; pub mod switch; +pub mod lfs; use crate::internal::protocol::https_client::BasicAuth; use crate::utils::util; @@ -21,6 +22,10 @@ use mercury::{errors::GitError, hash::SHA1, internal::object::ObjectTrait}; use rpassword::read_password; use std::io; use std::io::Write; +use std::path::Path; +use mercury::internal::object::blob::Blob; +use crate::utils; +use crate::utils::object_ext::BlobExt; // impl load for all objects fn load_object(hash: &SHA1) -> Result @@ -111,6 +116,18 @@ pub fn parse_commit_msg(msg_gpg: &str) -> (String, Option) { } } +/// Calculate the hash of a file blob +/// - for `lfs` file: calculate hash of the pointer data +pub fn calc_file_blob_hash(path: impl AsRef) -> io::Result { + let blob = if utils::lfs::is_lfs_tracked(&path) { + let (pointer, _) = utils::lfs::generate_pointer_file(&path); + Blob::from_content(&pointer) + } else { + Blob::from_file(&path) + }; + Ok(blob.id) +} + #[cfg(test)] mod test { use mercury::internal::object::commit::Commit; diff --git a/libra/src/command/push.rs b/libra/src/command/push.rs index 7f8aad6f4..4c44f6837 100644 --- a/libra/src/command/push.rs +++ b/libra/src/command/push.rs @@ -20,6 +20,7 @@ use crate::internal::branch::Branch; use crate::internal::config::Config; use crate::internal::head::Head; use crate::internal::protocol::https_client::{BasicAuth, HttpsClient}; +use crate::internal::protocol::lfs_client::LFSClient; use crate::internal::protocol::ProtocolClient; use crate::utils::object_ext::{BlobExt, CommitExt, TreeExt}; @@ -55,7 +56,7 @@ pub async fn execute(args: PushArgs) { Some(repo) => repo, None => { // e.g. [branch "master"].remote = origin - let remote = Config::get("branch", Some(&branch), "remote").await; + let remote = Config::get_remote(&branch).await; if let Some(remote) = remote { remote } else { @@ -64,7 +65,7 @@ pub async fn execute(args: PushArgs) { } } }; - let repo_url = Config::get("remote", Some(&repository), "url").await; + let repo_url = Config::get_remote_url(&repository).await; if repo_url.is_none() { eprintln!("fatal: remote '{}' not found, please use 'libra remote add'", repository); return; @@ -117,6 +118,11 @@ pub async fn execute(args: PushArgs) { SHA1::from_str(&remote_hash).unwrap() ); + { // upload lfs files + let client = LFSClient::from_url(&url); + client.push_objects(&objs, auth.clone()).await; + } + // let (tx, rx) = mpsc::channel::(); let (entry_tx, entry_rx) = mpsc::channel(1_000_000); let (stream_tx, mut stream_rx) = mpsc::channel(1_000_000); @@ -138,7 +144,7 @@ pub async fn execute(args: PushArgs) { data.extend_from_slice(&pack_data); println!("Delta compression done."); - let res = client.send_pack(data.freeze(), auth).await.unwrap(); + let res = client.send_pack(data.freeze(), auth).await.unwrap(); // TODO: send stream if res.status() != 200 { eprintln!("status code: {}", res.status()); diff --git a/libra/src/command/restore.rs b/libra/src/command/restore.rs index 2ebc56604..8cf359a9f 100644 --- a/libra/src/command/restore.rs +++ b/libra/src/command/restore.rs @@ -3,16 +3,18 @@ use crate::internal::head::Head; use mercury::internal::index::{Index, IndexEntry}; use crate::utils::object_ext::{BlobExt, CommitExt, TreeExt}; use crate::utils::path_ext::PathExt; -use crate::utils::{path, util}; +use crate::utils::{lfs, path, util}; use clap::Parser; use std::collections::{HashMap, HashSet}; -use std::fs; +use std::{fs, io}; use std::path::PathBuf; use mercury::hash::SHA1; use mercury::internal::object::blob::Blob; use mercury::internal::object::commit::Commit; use mercury::internal::object::tree::Tree; use mercury::internal::object::types::ObjectType; +use crate::command::calc_file_blob_hash; +use crate::internal::protocol::lfs_client::LFS_CLIENT; #[derive(Parser, Debug)] pub struct RestoreArgs { @@ -117,7 +119,7 @@ pub async fn execute(args: RestoreArgs) { // The order is very important // `restore_worktree` will decide whether to delete the file based on whether it is tracked in the index. if worktree { - restore_worktree(&paths, &target_blobs); + restore_worktree(&paths, &target_blobs).await; } if staged { restore_index(&paths, &target_blobs); @@ -134,12 +136,33 @@ fn preprocess_blobs(blobs: &[(PathBuf, SHA1)]) -> HashMap { .collect() } -/// restore a blob to file +/// Restore a blob to file. +/// If blob is an LFS pointer, download the actual file from LFS server. /// - `path` : to workdir -fn restore_to_file(hash: &SHA1, path: &PathBuf) { +async fn restore_to_file(hash: &SHA1, path: &PathBuf) -> io::Result<()> { let blob = Blob::load(hash); let path_abs = util::workdir_to_absolute(path); - util::write_file(&blob.data, &path_abs).unwrap(); + if let Some(parent) = path_abs.parent() { + fs::create_dir_all(parent)?; + } + match lfs::parse_pointer_data(&blob.data) { + Some((oid, size)) => { + // LFS file + let lfs_obj_path = lfs::lfs_object_path(&oid); + if lfs_obj_path.exists() { + // found in local cache + fs::copy(&lfs_obj_path, &path_abs)?; + } else { + // not exist, download from server + LFS_CLIENT.await.download_object(&oid, size, &path_abs).await; + } + } + None => { + // normal file + util::write_file(&blob.data, &path_abs)?; + } + } + Ok(()) } /// Get the deleted files in the worktree(vs Index), filtered by `filters` @@ -162,7 +185,7 @@ fn get_worktree_deleted_files_in_filters( /// Restore the worktree /// - `filter`: abs or relative to current (user input) /// - `target_blobs`: to workdir path -pub fn restore_worktree(filter: &Vec, target_blobs: &[(PathBuf, SHA1)]) { +pub async fn restore_worktree(filter: &Vec, target_blobs: &[(PathBuf, SHA1)]) { let target_blobs = preprocess_blobs(target_blobs); let deleted_files = get_worktree_deleted_files_in_filters(filter, &target_blobs); @@ -198,7 +221,7 @@ pub fn restore_worktree(filter: &Vec, target_blobs: &[(PathBuf, SHA1)]) // file not exist, deleted or illegal if target_blobs.contains_key(path_wd) { // file in target_blobs (deleted), need to restore - restore_to_file(&target_blobs[path_wd], path_wd); + restore_to_file(&target_blobs[path_wd], path_wd).await.unwrap(); } else { // not in target_commit and workdir (illegal path), user input unreachable!("It should be checked before"); @@ -206,12 +229,12 @@ pub fn restore_worktree(filter: &Vec, target_blobs: &[(PathBuf, SHA1)]) } else { // file exists let path_wd_str = path_wd.to_string_or_panic(); - let hash = util::calc_file_blob_hash(&path_abs).unwrap(); + let hash = calc_file_blob_hash(&path_abs).unwrap(); if target_blobs.contains_key(path_wd) { // both in target & worktree: 1. modified 2. same if hash != target_blobs[path_wd] { // modified - restore_to_file(&target_blobs[path_wd], path_wd); + restore_to_file(&target_blobs[path_wd], path_wd).await.unwrap(); } // else: same, keep } else { // not in target but in worktree: New file diff --git a/libra/src/command/status.rs b/libra/src/command/status.rs index 0a1462850..0aa75e706 100644 --- a/libra/src/command/status.rs +++ b/libra/src/command/status.rs @@ -9,6 +9,7 @@ use mercury::internal::object::tree::Tree; use crate::internal::head::Head; use mercury::internal::index::Index; +use crate::command::calc_file_blob_hash; use crate::utils::object_ext::{CommitExt, TreeExt}; use crate::utils::{path, util}; @@ -159,7 +160,7 @@ pub fn changes_to_be_staged() -> Changes { changes.deleted.push(file.clone()); } else if index.is_modified(file_str, 0, &workdir) { // only calc the hash if the file is modified (metadata), for optimization - let file_hash = util::calc_file_blob_hash(&file_abs).unwrap(); + let file_hash = calc_file_blob_hash(&file_abs).unwrap(); if !index.verify_hash(file_str, 0, &file_hash) { changes.modified.push(file.clone()); } diff --git a/libra/src/internal/config.rs b/libra/src/internal/config.rs index a88e9f770..0929b8e97 100644 --- a/libra/src/internal/config.rs +++ b/libra/src/internal/config.rs @@ -5,6 +5,7 @@ use sea_orm::ActiveValue::Set; use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter}; use crate::internal::db::get_db_conn_instance; +use crate::internal::head::Head; use crate::internal::model::config; use crate::internal::model::config::Model; @@ -54,6 +55,33 @@ impl Config { values.first().map(|c| c.value.to_owned()) } + /// Get remote repo name by branch name + pub async fn get_remote(branch: &str) -> Option { + // e.g. [branch "master"].remote = origin + Config::get("branch", Some(branch), "remote").await + } + + /// Get remote repo name of current branch + pub async fn get_current_remote() -> Option { + match Head::current().await { + Head::Branch(name) => { + Config::get_remote(&name).await + }, + Head::Detached(_) => None, + } + } + + pub async fn get_remote_url(remote: &str) -> Option { + Config::get("remote", Some(remote), "url").await + } + + pub async fn get_current_remote_url() -> Option { + match Config::get_current_remote().await { + Some(remote) => Config::get_remote_url(&remote).await, + None => None, + } + } + /// Get all configuration values /// - e.g. remote.origin.url can be multiple pub async fn get_all(configuration: &str, name: Option<&str>, key: &str) -> Vec { diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs new file mode 100644 index 000000000..77c21a04d --- /dev/null +++ b/libra/src/internal/protocol/lfs_client.rs @@ -0,0 +1,195 @@ +use std::path::Path; +use async_static::async_static; +use futures_util::StreamExt; +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use tokio::io::AsyncWriteExt; +use url::Url; +use ceres::lfs::lfs_structs::{BatchRequest, Representation, RequestVars}; +use mercury::internal::object::types::ObjectType; +use mercury::internal::pack::entry::Entry; +use crate::internal::config::Config; +use crate::internal::protocol::https_client::BasicAuth; +use crate::internal::protocol::ProtocolClient; +use crate::utils::lfs; + +async_static! { + pub static ref LFS_CLIENT: LFSClient = LFSClient::new().await; +} + +pub struct LFSClient { + pub url: Url, + pub client: Client, +} + +/// see [successful-responses](https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md#successful-responses) +#[derive(Serialize, Deserialize)] +struct LfsBatchResponse { + transfer: Option, + objects: Vec, + hash_algo: Option, +} + +impl ProtocolClient for LFSClient { + /// Construct LFSClient from a given Repo URL. + fn from_url(repo_url: &Url) -> Self { + let lfs_server = Url::parse(&lfs::generate_lfs_server_url(repo_url.to_string())).unwrap(); + let client = Client::builder() + .http1_only() + .default_headers(lfs::LFS_HEADERS.clone()) + .build() + .unwrap(); + Self { + url: lfs_server.join("/objects/batch").unwrap(), + client, + } + } +} + +impl LFSClient { + /// Construct LFSClient from current remote URL. + pub async fn new() -> Self { + let url = Config::get_current_remote_url().await; + match url { + Some(url) => LFSClient::from_url(&Url::parse(&url).unwrap()), + None => panic!("fatal: current remote url not found"), + } + } + + /// push LFS objects to remote server + pub async fn push_objects<'a, I>(&self, objs: I, auth: Option) + where + I: IntoIterator + { + // filter pointer file within blobs + let mut lfs_oids = Vec::new(); + for blob in objs.into_iter().filter(|e| e.obj_type == ObjectType::Blob) { + let oid = lfs::parse_pointer_data(&blob.data); + if let Some(oid) = oid { + lfs_oids.push(oid); + } + } + + let mut lfs_objs = Vec::new(); + for (oid, _) in &lfs_oids { + let path = lfs::lfs_object_path(oid); + if !path.exists() { + eprintln!("fatal: LFS object not found: {}", oid); + return; + } + let size = path.metadata().unwrap().len() as i64; + lfs_objs.push(RequestVars { + oid: oid.to_owned(), + size, + ..Default::default() + }) + } + + let batch_request = BatchRequest { + operation: "upload".to_string(), + transfers: vec![lfs::LFS_TRANSFER_API.to_string()], + objects: lfs_objs, + hash_algo: lfs::LFS_HASH_ALGO.to_string(), + enable_split: None, + }; + + let mut request = self.client.post(self.url.clone()).json(&batch_request); + if let Some(auth) = auth { + request = request.basic_auth(auth.username, Some(auth.password)); + } + + let response = request.send().await.unwrap(); + + let resp = response.json::().await.unwrap(); + tracing::debug!("LFS push response:\n {:#?}", serde_json::to_value(&resp).unwrap()); + + // TODO: parallel upload + for obj in resp.objects { + self.upload_object(obj).await; + } + println!("LFS objects push completed."); + } + + /// upload (PUT) one LFS file to remote server + async fn upload_object(&self, object: Representation) { + if let Some(err) = object.error { + eprintln!("fatal: LFS upload failed. Code: {}, Message: {}", err.code, err.message); + return; + } + + if let Some(actions) = object.actions { + let upload_link = actions.get("upload"); + if upload_link.is_none() { + eprintln!("fatal: LFS upload failed. No upload action found"); + return; + } + + let link = upload_link.unwrap(); + let mut request = self.client.put(link.href.clone()); + for (k, v) in &link.header { + request = request.header(k, v); + } + + let file_path = lfs::lfs_object_path(&object.oid); + let file = tokio::fs::File::open(file_path).await.unwrap(); + println!("Uploading LFS file: {}", object.oid); + let resp = request + .body(reqwest::Body::wrap_stream(tokio_util::io::ReaderStream::new(file))) + .send() + .await + .unwrap(); + if !resp.status().is_success() { + eprintln!("fatal: LFS upload failed. Status: {}, Message: {}", resp.status(), resp.text().await.unwrap()); + return; + } + println!("Uploaded."); + } else { + tracing::debug!("LFS file {} already exists on remote server", object.oid); + } + } + + /// download (GET) one LFS file from remote server + pub async fn download_object(&self, oid: &str, size: u64, path: impl AsRef) { + let batch_request = BatchRequest { + operation: "download".to_string(), + transfers: vec![lfs::LFS_TRANSFER_API.to_string()], + objects: vec![RequestVars { + oid: oid.to_owned(), + size: size as i64, + ..Default::default() + }], + hash_algo: lfs::LFS_HASH_ALGO.to_string(), + enable_split: None, + }; + + let request = self.client.post(self.url.clone()).json(&batch_request); + let response = request.send().await.unwrap(); + + let resp = response.json::().await.unwrap(); + tracing::debug!("LFS download response:\n {:#?}", serde_json::to_value(&resp).unwrap()); + + let link = resp.objects[0].actions.as_ref().unwrap().get("download").unwrap(); + + let mut request = self.client.get(link.href.clone()); + for (k, v) in &link.header { + request = request.header(k, v); + } + + let response = request.send().await.unwrap(); + + if !response.status().is_success() { + eprintln!("fatal: LFS download failed. Status: {}, Message: {}", response.status(), response.text().await.unwrap()); + return; + } + + println!("Downloading LFS file: {}", oid); + let mut file = tokio::fs::File::create(path).await.unwrap(); + let mut stream = response.bytes_stream(); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.unwrap(); + file.write_all(&chunk).await.unwrap(); + } + println!("Downloaded."); // TODO: checksum + } +} \ No newline at end of file diff --git a/libra/src/internal/protocol/mod.rs b/libra/src/internal/protocol/mod.rs index 206132edc..c64529366 100644 --- a/libra/src/internal/protocol/mod.rs +++ b/libra/src/internal/protocol/mod.rs @@ -1,6 +1,8 @@ use url::Url; pub mod https_client; +pub mod lfs_client; + #[allow(dead_code)] // todo: unimplemented pub trait ProtocolClient { /// create client from url diff --git a/libra/src/main.rs b/libra/src/main.rs index c711cb4f2..477bf1762 100644 --- a/libra/src/main.rs +++ b/libra/src/main.rs @@ -39,6 +39,8 @@ enum Commands { Restore(command::restore::RestoreArgs), #[command(about = "Show the working tree status")] Status, + #[command(subcommand, about = "Large File Storage")] + Lfs(command::lfs::LfsCmds), #[command(about = "Show commit logs")] Log(command::log::LogArgs), #[command(about = "List, create, or delete branches")] @@ -96,6 +98,7 @@ async fn main() { Commands::Rm(args) => command::remove::execute(args).unwrap(), Commands::Restore(args) => command::restore::execute(args).await, Commands::Status => command::status::execute().await, + Commands::Lfs(cmd) => command::lfs::execute(cmd).await, Commands::Log(args) => command::log::execute(args).await, Commands::Branch(args) => command::branch::execute(args).await, Commands::Commit(args) => command::commit::execute(args).await, diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs new file mode 100644 index 000000000..3f54efb1c --- /dev/null +++ b/libra/src/utils/lfs.rs @@ -0,0 +1,231 @@ +use std::fs::File; +use std::{fs, io}; +use std::io::{BufRead, BufReader, Read}; +use std::path::{Path, PathBuf}; +use lazy_static::lazy_static; +use path_abs::{PathInfo, PathOps}; +use regex::Regex; +use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_TYPE}; +use sha2::{Digest, Sha256}; +use wax::Pattern; +use crate::utils::{path, util}; +use crate::utils::path_ext::PathExt; + +lazy_static! { + static ref LFS_PATTERNS: Vec = { // cache + let attr_path = path::attributes().to_string_or_panic(); + extract_lfs_patterns(&attr_path).unwrap() + }; + + pub static ref LFS_HEADERS: HeaderMap = { + let mut headers = HeaderMap::new(); + headers.insert(ACCEPT, HeaderValue::from_static("application/vnd.git-lfs+json")); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/vnd.git-lfs+json")); + headers + }; +} + +/// Check if a file is LFS tracked +/// - support Glob pattern matching (TODO: support .gitignore patterns) +/// - only check root attributes file now, should check all attributes files in sub-dirs +/// - absolute path or relative path to workdir +pub fn is_lfs_tracked

(path: P) -> bool +where + P: AsRef, +{ + if LFS_PATTERNS.is_empty() { + return false; + } + + let path = util::to_workdir_path(path); + let glob = wax::any(LFS_PATTERNS.iter().map(|s| s.as_str()).collect::>()).unwrap(); + glob.is_match(path.to_str().unwrap()) +} + +const LFS_VERSION: &str = "https://git-lfs.github.com/spec/v1"; +/// This is the original & default transfer adapter. All Git LFS clients and servers SHOULD support it. +pub const LFS_TRANSFER_API: &str = "basic"; +pub const LFS_HASH_ALGO: &str = "sha256"; +const LFS_OID_LEN: usize = 64; +const LFS_POINTER_MAX_SIZE: usize = 300; // bytes + +/// Generate lfs pointer file string +/// - return (pointer content, lfs oid) +/// - absolute path +pub fn generate_pointer_file(path: impl AsRef) -> (String, String) { + let path = path.as_ref(); + // calc file hash without type + let oid = calc_lfs_file_hash(path).unwrap(); + + let pointer = format!("version {}\noid {}:{}\nsize {}\n", + LFS_VERSION, LFS_HASH_ALGO, oid, path.metadata().unwrap().len()); + (pointer, oid) +} + +/// Generate LFS Server Url from repo Url. +/// By default, Git LFS will append `.git/info/lfs` to the end of a Git remote url to build the LFS server URL. +/// [doc: server-discovery](https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md) +/// - like https://git-server.com/foo/bar.git/info/lfs +/// - support ssh & https & git@ format +pub fn generate_lfs_server_url(mut url: String) -> String { + if url.ends_with('/') { + url.pop(); + } + if !url.ends_with(".git") { + url.push_str(".git"); + } + url.push_str("/info/lfs"); + + if url.starts_with("git@") { + // git@git-server.com:foo/bar.git + url = "https://".to_string() + &url[4..].replace(":", "/"); + } else if url.starts_with("ssh://") { + // ssh://git-server.com/foo/bar.git + url = "https://".to_string() + &url[6..]; + } + + url +} + +/// Generate LFS cache path, in `.libra/lfs/objects` +pub fn lfs_object_path(oid: &str) -> PathBuf { + util::storage_path() + .join("lfs/objects") + .join(&oid[..2]) + .join(&oid[2..4]) + .join(oid) +} + +/// Copy LFS file to `.libra/lfs/objects` +/// - absolute path +pub fn backup_lfs_file

(path: P, oid: &str) -> io::Result<()> +where + P: AsRef, +{ + let path = path.as_ref(); + let backup_path = lfs_object_path(oid); + if !backup_path.exists() { + fs::create_dir_all(backup_path.parent().unwrap())?; + fs::copy(path, backup_path)?; + } + Ok(()) +} + +/// SHA256 without type +/// TODO: performance optimization, 200MB 4s now, slower than `sha256sum` +pub fn calc_lfs_file_hash

(path: P) -> io::Result +where + P: AsRef, +{ + let path = path.as_ref(); + let mut hash = Sha256::new(); + let file = File::open(path)?; + let mut reader = BufReader::new(file); + let mut buffer = [0; 65536]; + loop { + let n = reader.read(&mut buffer)?; + if n == 0 { + break; + } + hash.update(&buffer[..n]); + } + let file_hash = hex::encode(hash.finalize()); + Ok(file_hash) +} + +/// Check if `data` is an LFS pointer, return `oid` +pub fn parse_pointer_data(data: &[u8]) -> Option<(String, u64)> { + if data.len() > LFS_POINTER_MAX_SIZE { + return None; + } + // Start with format `version ...` + if let Some(data) = data.strip_prefix(format!("version {}\noid {}:", LFS_VERSION, LFS_HASH_ALGO).as_bytes()) { + if data[LFS_OID_LEN] == b'\n' { + // check `oid` length + let oid = String::from_utf8(data[..LFS_OID_LEN].to_vec()).unwrap(); + if let Some(data) = data.strip_prefix(format!("{}\nsize ", oid).as_bytes()) { + let data = String::from_utf8(data[..].to_vec()).unwrap(); + if let Ok(size) = data.trim_end().parse::() { + return Some((oid, size)); + } + } + } + } + None +} + +/// Extract LFS patterns from `.libra_attributes` file +pub fn extract_lfs_patterns(file_path: &str) -> io::Result> { + let path = Path::new(file_path); + if !path.exists() { + return Ok(Vec::new()); + } + let file = File::open(path)?; + let reader = BufReader::new(file); + + // ' ' needs '\' before it to be escaped + let re = Regex::new(r"^\s*(([^\s#\\]|\\ )+)").unwrap(); + + let mut patterns = Vec::new(); + + for line in reader.lines() { + let line = line?; + if !line.contains("filter=lfs") { + continue; + } + if let Some(cap) = re.captures(&line) { + if let Some(pattern) = cap.get(1) { + let pattern = pattern.as_str().replace(r"\ ", " "); + patterns.push(pattern); + } + } + } + + Ok(patterns) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_pointer_file() { + let path = Path::new("../tests/data/packs/git-2d187177923cd618a75da6c6db45bb89d92bd504.pack"); + let (pointer, _oid) = generate_pointer_file(path); + print!("{}", pointer); + } + + #[test] + fn test_is_pointer_file() { + let data = b"version https://git-lfs.github.com/spec/v1\noid sha256:1234567890abcdef\nsize 1234\n"; + assert!(parse_pointer_data(data).is_some()); + } + + #[test] + fn test_gen_lfs_server_url() { + const LFS_SERVER_URL: &str = "https://github.com/web3infra-foundation/mega.git/info/lfs"; + let url = "https://github.com/web3infra-foundation/mega".to_owned(); + assert_eq!(generate_lfs_server_url(url), LFS_SERVER_URL); + + let url = "https://github.com/web3infra-foundation/mega.git".to_owned(); + assert_eq!(generate_lfs_server_url(url), LFS_SERVER_URL); + + let url = "git@github.com:web3infra-foundation/mega.git".to_owned(); + assert_eq!(generate_lfs_server_url(url), LFS_SERVER_URL); + + let url = "ssh://github.com/web3infra-foundation/mega.git".to_owned(); + assert_eq!(generate_lfs_server_url(url), LFS_SERVER_URL); + } + + #[test] + fn test_parse_pointer_data() { + let data = r#"version https://git-lfs.github.com/spec/v1 +oid sha256:4859402c258b836d02e955d1090e29f586e58b2040504d68afec3d8d43757bba +size 10 +"#; + let res = parse_pointer_data(data.as_bytes()).unwrap(); + println!("{:?}", res); + assert_eq!(res.0, "4859402c258b836d02e955d1090e29f586e58b2040504d68afec3d8d43757bba"); + assert_eq!(res.1, 10); + } +} \ No newline at end of file diff --git a/libra/src/utils/mod.rs b/libra/src/utils/mod.rs index 5035c1a48..5427ad3f1 100644 --- a/libra/src/utils/mod.rs +++ b/libra/src/utils/mod.rs @@ -3,4 +3,5 @@ pub(crate) mod test; pub(crate) mod path; pub(crate) mod object_ext; pub(crate) mod path_ext; -pub(crate) mod client_storage; \ No newline at end of file +pub(crate) mod client_storage; +pub(crate) mod lfs; \ No newline at end of file diff --git a/libra/src/utils/object_ext.rs b/libra/src/utils/object_ext.rs index bba717d3b..6e56f01b5 100644 --- a/libra/src/utils/object_ext.rs +++ b/libra/src/utils/object_ext.rs @@ -1,3 +1,5 @@ +use std::fs; +use std::io::{BufReader, Read}; use std::path::{Path, PathBuf}; use mercury::hash::SHA1; @@ -6,7 +8,7 @@ use mercury::internal::object::commit::Commit; use mercury::internal::object::ObjectTrait; use mercury::internal::object::tree::{Tree, TreeItemMode}; -use crate::utils::util; +use crate::utils::{lfs, util}; pub trait TreeExt { fn load(hash: &SHA1) -> Tree; @@ -20,6 +22,7 @@ pub trait CommitExt { pub trait BlobExt { fn load(hash: &SHA1) -> Blob; fn from_file(path: impl AsRef) -> Blob; + fn from_lfs_file(path: impl AsRef) -> Blob; fn save(&self) -> SHA1; } @@ -71,8 +74,21 @@ impl BlobExt for Blob { /// Create a blob from a file /// - `path`: absolute or relative path to current dir fn from_file(path: impl AsRef) -> Blob { - let file_content = std::fs::read_to_string(path).unwrap(); - Blob::from_content(&file_content) + let mut data = Vec::new(); + let file = fs::File::open(path).unwrap(); + let mut reader = BufReader::new(file); + reader.read_to_end(&mut data).unwrap(); + Blob::from_content_bytes(data) + } + + /// Create a blob from an LFS file + /// - include: create a pointer file & copy the file to `.libra/lfs/objects` + /// - `path`: absolute or relative path to current dir + fn from_lfs_file(path: impl AsRef) -> Blob { + let (pointer, oid) = lfs::generate_pointer_file(&path); + tracing::debug!("\n{}", pointer); + lfs::backup_lfs_file(&path, &oid).unwrap(); + Blob::from_content(&pointer) } fn save(&self) -> SHA1 { diff --git a/libra/src/utils/path.rs b/libra/src/utils/path.rs index dd7c36be1..812f39c97 100644 --- a/libra/src/utils/path.rs +++ b/libra/src/utils/path.rs @@ -4,10 +4,15 @@ use crate::utils::util; pub fn index() -> PathBuf { util::storage_path().join("index") } + pub fn objects() -> PathBuf { util::storage_path().join("objects") } pub fn database() -> PathBuf { util::storage_path().join(util::DATABASE) +} + +pub fn attributes() -> PathBuf { + util::working_dir().join(util::ATTRIBUTES) } \ No newline at end of file diff --git a/libra/src/utils/util.rs b/libra/src/utils/util.rs index 7b64cf427..f520b8928 100644 --- a/libra/src/utils/util.rs +++ b/libra/src/utils/util.rs @@ -1,6 +1,6 @@ use path_abs::{PathAbs, PathInfo}; use std::collections::HashSet; -use std::io::{BufReader, Read, Write}; +use std::io::Write; use std::path::{Path, PathBuf}; use std::{env, fs, io}; @@ -13,6 +13,7 @@ use crate::utils::path_ext::PathExt; pub const ROOT_DIR: &str = ".libra"; pub const DATABASE: &str = "libra.db"; +pub const ATTRIBUTES: &str = ".libra_attributes"; /// Returns the current working directory as a `PathBuf`. /// @@ -192,14 +193,6 @@ where workdir_to_relative(path, cur_dir()) } -pub fn calc_file_blob_hash(path: impl AsRef) -> io::Result { - let file = fs::File::open(path)?; - let mut reader = BufReader::new(file); - let mut data = Vec::new(); - reader.read_to_end(&mut data)?; - Ok(SHA1::from_type_and_data(ObjectType::Blob, &data)) -} - /// List all files in the given dir and its sub_dir, except `.libra` /// - input `path`: absolute path or relative path to the current dir /// - output: to workdir path diff --git a/mercury/src/internal/object/blob.rs b/mercury/src/internal/object/blob.rs index 32c00ca6d..73f37d068 100644 --- a/mercury/src/internal/object/blob.rs +++ b/mercury/src/internal/object/blob.rs @@ -89,6 +89,12 @@ impl Blob { // let mut buf = ReadBoxed::new(blob_content, ObjectType::Blob, content.len()); // Blob::from_buf_read(&mut buf, content.len()) let content = content.as_bytes().to_vec(); + Blob::from_content_bytes(content) + } + + /// Create a new Blob object from the given content bytes. + /// - some file content can't be represented as a string (UTF-8), so we need to use bytes. + pub fn from_content_bytes(content: Vec) -> Self { Blob { id: SHA1::from_type_and_data(ObjectType::Blob, &content), data: content,