Update sync crates script

libra/src/command/add.rs

@@ -1,9 +1,9 @@
-use std::path::{Path, PathBuf};
-use clap::Parser;
-use mercury::internal::object::blob::Blob;
 use crate::command::status;
-use mercury::internal::index::{Index, IndexEntry};
 use crate::utils::object_ext::BlobExt;
+use clap::Parser;
+use mercury::internal::index::{Index, IndexEntry};
+use mercury::internal::object::blob::Blob;
+use std::path::{Path, PathBuf};
 
 use crate::utils::{lfs, path, util};
 
@@ -19,7 +19,7 @@ pub struct AddArgs {
     #[clap(short = 'A', long, group = "mode")]
     pub all: bool,
 
-    /// Update the index just where it already has an entry matching <pathspec>.
+    /// Update the index just where it already has an entry matching **pathspec**.
     /// This removes as well as modifies index entries to match the working tree, but adds no new files.
     #[clap(short, long, group = "mode")]
     pub update: bool,
@@ -49,7 +49,7 @@ pub async fn execute(args: AddArgs) {
 
     // index vs worktree
     let mut changes = status::changes_to_be_staged(); // to workdir
-    // filter paths to fit `pathspec` that user inputs
+                                                      // filter paths to fit `pathspec` that user inputs
     changes.new = util::filter_to_fit_paths(&changes.new, &paths);
     // if `--all` & <pathspec> is given, it will update `index` as well, so no need to filter `deleted` & `modified`
     if args.pathspec.is_empty() || !args.all {
@@ -78,13 +78,21 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) {
     if !util::is_sub_path(file, &workdir) {
         // file is not in the working directory
         // TODO check this earlier, once fatal occurs, nothing should be done
-        println!("fatal: '{}' is outside workdir at '{}'", file.display(), workdir.display());
+        println!(
+            "fatal: '{}' is outside workdir at '{}'",
+            file.display(),
+            workdir.display()
+        );
         return;
     }
     if util::is_sub_path(file, util::storage_path()) {
         // file is in `.libra`
         // Git won't print this
-        println!("warning: '{}' is inside '{}' repo, which will be ignored by `add`", file.display(), util::ROOT_DIR);
+        println!(
+            "warning: '{}' is inside '{}' repo, which will be ignored by `add`",
+            file.display(),
+            util::ROOT_DIR
+        );
         return;
     }
 
@@ -97,11 +105,15 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) {
             if verbose {
                 println!("removed: {}", file_str);
             }
-        } else { // FIXME: unreachable code! This situation is not included in `status::changes_to_be_staged()`
+        } else {
+            // FIXME: unreachable code! This situation is not included in `status::changes_to_be_staged()`
             // FIXME: should check files in original input paths
             // TODO do this check earlier, once fatal occurs, nothing should be done
             // file is not tracked && not exists, which means wrong pathspec
-            println!("fatal: pathspec '{}' did not match any files", file.display());
+            println!(
+                "fatal: pathspec '{}' did not match any files",
+                file.display()
+            );
         }
     } else {
         // file exists

libra/src/internal/protocol/https_client.rs

@@ -5,15 +5,15 @@ use ceres::protocol::ServiceType;
 use ceres::protocol::ServiceType::UploadPack;
 use futures_util::{StreamExt, TryStreamExt};
 use mercury::errors::GitError;
+use mercury::hash::SHA1;
 use reqwest::header::CONTENT_TYPE;
 use reqwest::{Body, Response};
 use std::io::Error as IoError;
 use tokio_util::bytes::BytesMut;
 use url::Url;
-use mercury::hash::SHA1;
 
 /// A Git protocol client that communicates with a Git server over HTTPS.
-/// Only support `SmartProtocol` now, see https://www.git-scm.com/docs/http-protocol for protocol details.
+/// Only support `SmartProtocol` now, see [http-protocol](https://www.git-scm.com/docs/http-protocol) for protocol details.
 pub struct HttpsClient {
     pub(crate) url: Url,
     pub(crate) client: reqwest::Client,

libra/src/utils/lfs.rs

@@ -1,17 +1,17 @@
-use std::fs::File;
-use std::{fs, io};
-use std::io::{BufRead, BufReader, Read};
-use std::path::{Path, PathBuf};
+use crate::utils::path_ext::PathExt;
+use crate::utils::{path, util};
 use lazy_static::lazy_static;
+use mercury::internal::index::Index;
 use path_abs::{PathInfo, PathOps};
 use regex::Regex;
 use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_TYPE};
 use ring::digest::{Context, SHA256};
+use std::fs::File;
+use std::io::{BufRead, BufReader, Read};
+use std::path::{Path, PathBuf};
+use std::{fs, io};
 use url::Url;
 use wax::Pattern;
-use mercury::internal::index::Index;
-use crate::utils::{path, util};
-use crate::utils::path_ext::PathExt;
 
 lazy_static! {
     static ref LFS_PATTERNS: Vec<String> = { // cache
@@ -64,13 +64,16 @@ pub fn generate_pointer_file(path: impl AsRef<Path>) -> (String, String) {
 }
 
 pub fn format_pointer_string(oid: &str, size: u64) -> String {
-    format!("version {}\noid {}:{}\nsize {}\n", LFS_VERSION, LFS_HASH_ALGO, oid, size)
+    format!(
+        "version {}\noid {}:{}\nsize {}\n",
+        LFS_VERSION, LFS_HASH_ALGO, oid, size
+    )
 }
 
 /// Generate LFS Server Url from repo Url.
 /// By default, Git LFS will append `.git/info/lfs` to the end of a Git remote url to build the LFS server URL.
 /// [doc: server-discovery](https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md)
-/// - like https://git-server.com/foo/bar.git/info/lfs
+/// - like `https://git-server.com/foo/bar.git/info/lfs`
 /// - support ssh & https & git@ format
 #[deprecated(note = "It's for git, not monorepo")]
 #[allow(dead_code)]
@@ -96,10 +99,6 @@ pub fn generate_git_lfs_server_url(mut url: String) -> String {
 
 /// Generate Mono LFS Server Url from repo Url.
 /// - Just get domain with port
-/// ### Example
-/// https://github.com/git-lfs/git-lfs/blob/main/docs/api/locking.md -> https://github.com
-///
-/// http://localhost:8000/xxx/yyy -> http://localhost:8000
 pub fn generate_lfs_server_url(url: String) -> String {
     let url = Url::parse(&url).unwrap();
     match url.port() {
@@ -175,7 +174,9 @@ pub fn parse_pointer_data(data: &[u8]) -> Option<(String, u64)> {
         return None;
     }
     // Start with format `version ...`
-    if let Some(data) = data.strip_prefix(format!("version {}\noid {}:", LFS_VERSION, LFS_HASH_ALGO).as_bytes()) {
+    if let Some(data) =
+        data.strip_prefix(format!("version {}\noid {}:", LFS_VERSION, LFS_HASH_ALGO).as_bytes())
+    {
         if data[LFS_OID_LEN] == b'\n' {
             // check `oid` length
             let oid = String::from_utf8(data[..LFS_OID_LEN].to_vec()).unwrap();
@@ -198,7 +199,10 @@ pub fn parse_pointer_file(path: impl AsRef<Path>) -> io::Result<(String, u64)> {
     if let Some((oid, size)) = parse_pointer_data(&buffer[..bytes_read]) {
         return Ok((oid, size));
     }
-    Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid LFS pointer file"))
+    Err(io::Error::new(
+        io::ErrorKind::InvalidData,
+        "Invalid LFS pointer file",
+    ))
 }
 
 /// Extract LFS patterns from `.libra_attributes` file
@@ -237,14 +241,16 @@ mod tests {
 
     #[test]
     fn test_generate_pointer_file() {
-        let path = Path::new("../tests/data/packs/git-2d187177923cd618a75da6c6db45bb89d92bd504.pack");
+        let path =
+            Path::new("../tests/data/packs/git-2d187177923cd618a75da6c6db45bb89d92bd504.pack");
         let (pointer, _oid) = generate_pointer_file(path);
         print!("{}", pointer);
     }
 
     #[test]
     fn test_is_pointer_file() {
-        let data = b"version https://git-lfs.github.com/spec/v1\noid sha256:1234567890abcdef\nsize 1234\n";
+        let data =
+            b"version https://git-lfs.github.com/spec/v1\noid sha256:1234567890abcdef\nsize 1234\n";
         assert!(parse_pointer_data(data).is_some());
     }
 
@@ -267,9 +273,15 @@ mod tests {
     #[test]
     fn test_gen_mono_lfs_server_url() {
         const LFS_SERVER_URL: &str = "https://github.com/web3infra-foundation/mega.git/info/lfs";
-        assert_eq!(generate_lfs_server_url(LFS_SERVER_URL.to_owned()), "https://github.com");
+        assert_eq!(
+            generate_lfs_server_url(LFS_SERVER_URL.to_owned()),
+            "https://github.com"
+        );
         const LOCAL_LFS_SERVER_URL: &str = "http://localhost:8000/xxx/yyy";
-        assert_eq!(generate_lfs_server_url(LOCAL_LFS_SERVER_URL.to_owned()), "http://localhost:8000");
+        assert_eq!(
+            generate_lfs_server_url(LOCAL_LFS_SERVER_URL.to_owned()),
+            "http://localhost:8000"
+        );
     }
 
     #[test]
@@ -280,7 +292,10 @@ size 10
 "#;
         let res = parse_pointer_data(data.as_bytes()).unwrap();
         println!("{:?}", res);
-        assert_eq!(res.0, "4859402c258b836d02e955d1090e29f586e58b2040504d68afec3d8d43757bba");
+        assert_eq!(
+            res.0,
+            "4859402c258b836d02e955d1090e29f586e58b2040504d68afec3d8d43757bba"
+        );
         assert_eq!(res.1, 10);
     }
-}
+}

libra/src/utils/util.rs

@@ -32,7 +32,6 @@ pub fn cur_dir() -> PathBuf {
 /// Try to get the storage path of the repository, which is the path of the `.libra` directory
 /// - if the current directory is not a repository, return an error
 pub fn try_get_storage_path() -> Result<PathBuf, io::Error> {
-    /*递归获取储存库 */
     let mut cur_dir = env::current_dir()?;
     loop {
         let mut libra = cur_dir.clone();
@@ -96,7 +95,9 @@ pub fn workdir_to_absolute(path: impl AsRef<Path>) -> PathBuf {
 /// - Not check existence
 /// - `true` if path == parent
 pub fn is_sub_path<P, B>(path: P, parent: B) -> bool
-where P: AsRef<Path>, B: AsRef<Path>
+where
+    P: AsRef<Path>,
+    B: AsRef<Path>,
 {
     let path_abs = PathAbs::new(path.as_ref()).unwrap(); // prefix: '\\?\' on Windows
     let parent_abs = PathAbs::new(parent.as_ref()).unwrap();
@@ -307,8 +308,8 @@ pub fn get_commit_base(commit_base: &str) -> Result<SHA1, String> {
 }
 
 /// Get the repository name from the url
-/// - e.g. https://github.com/web3infra-foundation/mega.git/ -> mega
-/// - e.g. https://github.com/web3infra-foundation/mega.git -> mega
+/// - e.g. `https://github.com/web3infra-foundation/mega.git/` -> mega
+/// - e.g. `https://github.com/web3infra-foundation/mega.git` -> mega
 pub fn get_repo_name_from_url(mut url: &str) -> Option<&str> {
     if url.ends_with('/') {
         url = &url[..url.len() - 1];

mono/src/server/https_server.rs

@@ -104,6 +104,35 @@ pub async fn start_http(config: Config, options: HttpOptions) {
         .unwrap();
 }
 
+/// This is the main entry for the mono server.
+/// It is responsible for creating the main router and setting up the necessary middleware.
+///
+/// The main router is composed of three nested routers:
+/// 1. The LFS router nested in the `/`:
+///   - GET or PUT `/objects/:object_id`
+///   - GET or PUT `/locks`
+///   - POST       `/locks/verify`
+///   - POST       `/locks/:id/unlock`
+///   - GET        `/objects/:object_id/chunks/:chunk_id`
+///   - POST       `/objects/batch`
+/// 2. The API router nested in the `/api/v1`:
+///   - GET        `/api/v1/status`
+///   - POST       `/api/v1/create-file`
+///   - GET        `/api/v1/latest-commit`
+///   - GET        `/api/v1/tree/commit-info`
+///   - GET        `/api/v1/tree`
+///   - GET        `/api/v1/blob`
+///   - GET        `/api/v1/file/blob/:object_id`
+///   - GET        `/api/v1/file/tree`
+///   - GET        `/api/v1/path-can-clone`
+/// 3. The OAuth router nested in the `/auth`:
+///   - GET        `/auth/github`
+///   - GET        `/auth/authorized`
+///   - GET        `/auth/logout`
+/// 4. The other routers for the git protocol:
+///   - GET        end of `Regex::new(r"/info/refs$")`
+///   - POST       end of `Regex::new(r"/git-upload-pack$")`
+///   - POST       end of `Regex::new(r"/git-receive-pack$")`
 pub async fn app(config: Config, host: String, port: u16, common: CommonOptions) -> Router {
     let context = Context::new(config.clone()).await;
     context.services.mono_storage.init_monorepo().await;

scripts/crates-sync/crates-sync.py

@@ -1,11 +1,12 @@
-import os  # For file and directory operations
-import sys  # For system-specific parameters and functions
-import json  # For JSON parsing
-import urllib.request  # For downloading files from URLs
-import tarfile  # For handling tar archives
-import subprocess  # For running system commands
-import shutil  # For high-level file operations
-from collections import defaultdict  # For creating dictionaries with default values
+import os
+import sys
+import json
+import urllib.request
+import tarfile
+import subprocess
+import shutil
+from collections import defaultdict
+from datetime import datetime, timedelta
 
 # ANSI color codes
 GREEN = '\033[92m'
@@ -29,7 +30,7 @@ def ensure_directory(path):
     # Create a directory if it doesn't exist
     if not os.path.exists(path):
         os.makedirs(path)
-        print_blue(f"Created directory: {path}")
+        print(f"Created directory: {path}")
 
 def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url):
     # Construct the filename and path for the crate
@@ -41,7 +42,7 @@ def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url)
         ensure_directory(os.path.dirname(crate_path))  # Ensure the directory exists
         download_url = f"{dl_base_url}/{crate_name}/{crate_filename}"
         try:
-            print_green(f"Downloading: {download_url}")
+            print_red(f"Downloading: {download_url}")
             urllib.request.urlretrieve(download_url, crate_path)  # Download the file
             print_red(f"Downloaded: {crate_path}")
         except Exception as e:
@@ -58,11 +59,16 @@ def run_git_command(repo_path, command):
         print_red(f"Command output: {e.output}")
         return None
 
-def init_git_repo(repo_path):
+def init_git_repo(repo_path, git_base_url):
     # Initialize a git repository if it doesn't exist
     if not os.path.exists(os.path.join(repo_path, '.git')):
         run_git_command(repo_path, ['git', 'init', '-b', 'main'])
-        print(f"Initialized git repository in {repo_path}")
+        print_blue(f"Initialized git repository in {repo_path}")
+
+        # Set the LFS domain
+        lfs_url = f"{git_base_url}"
+        run_git_command(repo_path, ['git', 'config', 'lfs.url', lfs_url])
+        print_blue(f"Set LFS domain to: {lfs_url}")
 
 def extract_crate(crate_path, extract_path):
     def is_within_directory(directory, target):
@@ -116,6 +122,10 @@ def filter_member(tarinfo, filterpath):
         return False
 
 def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_base_url):
+    # Record start time for the entire crate
+    crate_start_time = datetime.now()
+    print_blue(f"Started processing crate {crate_name} at {crate_start_time}")
+
     # Process a specific version of a crate
     repo_path = os.path.join(git_repos_dir, crate_name, version)
     ensure_directory(repo_path)
@@ -126,7 +136,7 @@ def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_ba
         return
 
     # Initialize git repo
-    init_git_repo(repo_path)
+    init_git_repo(repo_path, git_base_url)
 
     # Add all files to git
     run_git_command(repo_path, ['git', 'add', '.'])
@@ -146,6 +156,15 @@ def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_ba
     else:
         print_green(f"Successfully pushed {crate_name} version {version} to remote repository.")
 
+    # Record end time and calculate duration for the entire crate
+    crate_end_time = datetime.now()
+    crate_duration = crate_end_time - crate_start_time
+    print_blue(f"Finished processing crate {crate_name} at {crate_end_time}")
+    print_blue(f"Total processing time for crate {crate_name}: {crate_duration}")
+
+    # Print separator
+    print("------------------")
+
 def process_crate(crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url):
     # Process all versions of a crate
     for v in versions: