From de81d1990c40814ea4882888b7853bc49eca0628 Mon Sep 17 00:00:00 2001 From: Quanyi Ma Date: Tue, 17 Sep 2024 00:59:32 +0800 Subject: [PATCH] Update sync crates script Signed-off-by: Quanyi Ma --- libra/src/command/add.rs | 32 ++++++++---- libra/src/internal/protocol/https_client.rs | 4 +- libra/src/utils/lfs.rs | 57 +++++++++++++-------- libra/src/utils/util.rs | 9 ++-- mono/src/server/https_server.rs | 29 +++++++++++ scripts/crates-sync/crates-sync.py | 45 +++++++++++----- 6 files changed, 126 insertions(+), 50 deletions(-) diff --git a/libra/src/command/add.rs b/libra/src/command/add.rs index 606ab2b17..9815aa5ae 100644 --- a/libra/src/command/add.rs +++ b/libra/src/command/add.rs @@ -1,9 +1,9 @@ -use std::path::{Path, PathBuf}; -use clap::Parser; -use mercury::internal::object::blob::Blob; use crate::command::status; -use mercury::internal::index::{Index, IndexEntry}; use crate::utils::object_ext::BlobExt; +use clap::Parser; +use mercury::internal::index::{Index, IndexEntry}; +use mercury::internal::object::blob::Blob; +use std::path::{Path, PathBuf}; use crate::utils::{lfs, path, util}; @@ -19,7 +19,7 @@ pub struct AddArgs { #[clap(short = 'A', long, group = "mode")] pub all: bool, - /// Update the index just where it already has an entry matching . + /// Update the index just where it already has an entry matching **pathspec**. /// This removes as well as modifies index entries to match the working tree, but adds no new files. #[clap(short, long, group = "mode")] pub update: bool, @@ -49,7 +49,7 @@ pub async fn execute(args: AddArgs) { // index vs worktree let mut changes = status::changes_to_be_staged(); // to workdir - // filter paths to fit `pathspec` that user inputs + // filter paths to fit `pathspec` that user inputs changes.new = util::filter_to_fit_paths(&changes.new, &paths); // if `--all` & is given, it will update `index` as well, so no need to filter `deleted` & `modified` if args.pathspec.is_empty() || !args.all { @@ -78,13 +78,21 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) { if !util::is_sub_path(file, &workdir) { // file is not in the working directory // TODO check this earlier, once fatal occurs, nothing should be done - println!("fatal: '{}' is outside workdir at '{}'", file.display(), workdir.display()); + println!( + "fatal: '{}' is outside workdir at '{}'", + file.display(), + workdir.display() + ); return; } if util::is_sub_path(file, util::storage_path()) { // file is in `.libra` // Git won't print this - println!("warning: '{}' is inside '{}' repo, which will be ignored by `add`", file.display(), util::ROOT_DIR); + println!( + "warning: '{}' is inside '{}' repo, which will be ignored by `add`", + file.display(), + util::ROOT_DIR + ); return; } @@ -97,11 +105,15 @@ async fn add_a_file(file: &Path, index: &mut Index, verbose: bool) { if verbose { println!("removed: {}", file_str); } - } else { // FIXME: unreachable code! This situation is not included in `status::changes_to_be_staged()` + } else { + // FIXME: unreachable code! This situation is not included in `status::changes_to_be_staged()` // FIXME: should check files in original input paths // TODO do this check earlier, once fatal occurs, nothing should be done // file is not tracked && not exists, which means wrong pathspec - println!("fatal: pathspec '{}' did not match any files", file.display()); + println!( + "fatal: pathspec '{}' did not match any files", + file.display() + ); } } else { // file exists diff --git a/libra/src/internal/protocol/https_client.rs b/libra/src/internal/protocol/https_client.rs index d6341c584..6ac1acaa4 100644 --- a/libra/src/internal/protocol/https_client.rs +++ b/libra/src/internal/protocol/https_client.rs @@ -5,15 +5,15 @@ use ceres::protocol::ServiceType; use ceres::protocol::ServiceType::UploadPack; use futures_util::{StreamExt, TryStreamExt}; use mercury::errors::GitError; +use mercury::hash::SHA1; use reqwest::header::CONTENT_TYPE; use reqwest::{Body, Response}; use std::io::Error as IoError; use tokio_util::bytes::BytesMut; use url::Url; -use mercury::hash::SHA1; /// A Git protocol client that communicates with a Git server over HTTPS. -/// Only support `SmartProtocol` now, see https://www.git-scm.com/docs/http-protocol for protocol details. +/// Only support `SmartProtocol` now, see [http-protocol](https://www.git-scm.com/docs/http-protocol) for protocol details. pub struct HttpsClient { pub(crate) url: Url, pub(crate) client: reqwest::Client, diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index 41164085a..0496cb5d3 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -1,17 +1,17 @@ -use std::fs::File; -use std::{fs, io}; -use std::io::{BufRead, BufReader, Read}; -use std::path::{Path, PathBuf}; +use crate::utils::path_ext::PathExt; +use crate::utils::{path, util}; use lazy_static::lazy_static; +use mercury::internal::index::Index; use path_abs::{PathInfo, PathOps}; use regex::Regex; use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_TYPE}; use ring::digest::{Context, SHA256}; +use std::fs::File; +use std::io::{BufRead, BufReader, Read}; +use std::path::{Path, PathBuf}; +use std::{fs, io}; use url::Url; use wax::Pattern; -use mercury::internal::index::Index; -use crate::utils::{path, util}; -use crate::utils::path_ext::PathExt; lazy_static! { static ref LFS_PATTERNS: Vec = { // cache @@ -64,13 +64,16 @@ pub fn generate_pointer_file(path: impl AsRef) -> (String, String) { } pub fn format_pointer_string(oid: &str, size: u64) -> String { - format!("version {}\noid {}:{}\nsize {}\n", LFS_VERSION, LFS_HASH_ALGO, oid, size) + format!( + "version {}\noid {}:{}\nsize {}\n", + LFS_VERSION, LFS_HASH_ALGO, oid, size + ) } /// Generate LFS Server Url from repo Url. /// By default, Git LFS will append `.git/info/lfs` to the end of a Git remote url to build the LFS server URL. /// [doc: server-discovery](https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md) -/// - like https://git-server.com/foo/bar.git/info/lfs +/// - like `https://git-server.com/foo/bar.git/info/lfs` /// - support ssh & https & git@ format #[deprecated(note = "It's for git, not monorepo")] #[allow(dead_code)] @@ -96,10 +99,6 @@ pub fn generate_git_lfs_server_url(mut url: String) -> String { /// Generate Mono LFS Server Url from repo Url. /// - Just get domain with port -/// ### Example -/// https://github.com/git-lfs/git-lfs/blob/main/docs/api/locking.md -> https://github.com -/// -/// http://localhost:8000/xxx/yyy -> http://localhost:8000 pub fn generate_lfs_server_url(url: String) -> String { let url = Url::parse(&url).unwrap(); match url.port() { @@ -175,7 +174,9 @@ pub fn parse_pointer_data(data: &[u8]) -> Option<(String, u64)> { return None; } // Start with format `version ...` - if let Some(data) = data.strip_prefix(format!("version {}\noid {}:", LFS_VERSION, LFS_HASH_ALGO).as_bytes()) { + if let Some(data) = + data.strip_prefix(format!("version {}\noid {}:", LFS_VERSION, LFS_HASH_ALGO).as_bytes()) + { if data[LFS_OID_LEN] == b'\n' { // check `oid` length let oid = String::from_utf8(data[..LFS_OID_LEN].to_vec()).unwrap(); @@ -198,7 +199,10 @@ pub fn parse_pointer_file(path: impl AsRef) -> io::Result<(String, u64)> { if let Some((oid, size)) = parse_pointer_data(&buffer[..bytes_read]) { return Ok((oid, size)); } - Err(io::Error::new(io::ErrorKind::InvalidData, "Invalid LFS pointer file")) + Err(io::Error::new( + io::ErrorKind::InvalidData, + "Invalid LFS pointer file", + )) } /// Extract LFS patterns from `.libra_attributes` file @@ -237,14 +241,16 @@ mod tests { #[test] fn test_generate_pointer_file() { - let path = Path::new("../tests/data/packs/git-2d187177923cd618a75da6c6db45bb89d92bd504.pack"); + let path = + Path::new("../tests/data/packs/git-2d187177923cd618a75da6c6db45bb89d92bd504.pack"); let (pointer, _oid) = generate_pointer_file(path); print!("{}", pointer); } #[test] fn test_is_pointer_file() { - let data = b"version https://git-lfs.github.com/spec/v1\noid sha256:1234567890abcdef\nsize 1234\n"; + let data = + b"version https://git-lfs.github.com/spec/v1\noid sha256:1234567890abcdef\nsize 1234\n"; assert!(parse_pointer_data(data).is_some()); } @@ -267,9 +273,15 @@ mod tests { #[test] fn test_gen_mono_lfs_server_url() { const LFS_SERVER_URL: &str = "https://github.com/web3infra-foundation/mega.git/info/lfs"; - assert_eq!(generate_lfs_server_url(LFS_SERVER_URL.to_owned()), "https://github.com"); + assert_eq!( + generate_lfs_server_url(LFS_SERVER_URL.to_owned()), + "https://github.com" + ); const LOCAL_LFS_SERVER_URL: &str = "http://localhost:8000/xxx/yyy"; - assert_eq!(generate_lfs_server_url(LOCAL_LFS_SERVER_URL.to_owned()), "http://localhost:8000"); + assert_eq!( + generate_lfs_server_url(LOCAL_LFS_SERVER_URL.to_owned()), + "http://localhost:8000" + ); } #[test] @@ -280,7 +292,10 @@ size 10 "#; let res = parse_pointer_data(data.as_bytes()).unwrap(); println!("{:?}", res); - assert_eq!(res.0, "4859402c258b836d02e955d1090e29f586e58b2040504d68afec3d8d43757bba"); + assert_eq!( + res.0, + "4859402c258b836d02e955d1090e29f586e58b2040504d68afec3d8d43757bba" + ); assert_eq!(res.1, 10); } -} \ No newline at end of file +} diff --git a/libra/src/utils/util.rs b/libra/src/utils/util.rs index 50aea8d82..daae3872e 100644 --- a/libra/src/utils/util.rs +++ b/libra/src/utils/util.rs @@ -32,7 +32,6 @@ pub fn cur_dir() -> PathBuf { /// Try to get the storage path of the repository, which is the path of the `.libra` directory /// - if the current directory is not a repository, return an error pub fn try_get_storage_path() -> Result { - /*递归获取储存库 */ let mut cur_dir = env::current_dir()?; loop { let mut libra = cur_dir.clone(); @@ -96,7 +95,9 @@ pub fn workdir_to_absolute(path: impl AsRef) -> PathBuf { /// - Not check existence /// - `true` if path == parent pub fn is_sub_path(path: P, parent: B) -> bool -where P: AsRef, B: AsRef +where + P: AsRef, + B: AsRef, { let path_abs = PathAbs::new(path.as_ref()).unwrap(); // prefix: '\\?\' on Windows let parent_abs = PathAbs::new(parent.as_ref()).unwrap(); @@ -307,8 +308,8 @@ pub fn get_commit_base(commit_base: &str) -> Result { } /// Get the repository name from the url -/// - e.g. https://github.com/web3infra-foundation/mega.git/ -> mega -/// - e.g. https://github.com/web3infra-foundation/mega.git -> mega +/// - e.g. `https://github.com/web3infra-foundation/mega.git/` -> mega +/// - e.g. `https://github.com/web3infra-foundation/mega.git` -> mega pub fn get_repo_name_from_url(mut url: &str) -> Option<&str> { if url.ends_with('/') { url = &url[..url.len() - 1]; diff --git a/mono/src/server/https_server.rs b/mono/src/server/https_server.rs index cd480bd94..8cfd1f39b 100644 --- a/mono/src/server/https_server.rs +++ b/mono/src/server/https_server.rs @@ -104,6 +104,35 @@ pub async fn start_http(config: Config, options: HttpOptions) { .unwrap(); } +/// This is the main entry for the mono server. +/// It is responsible for creating the main router and setting up the necessary middleware. +/// +/// The main router is composed of three nested routers: +/// 1. The LFS router nested in the `/`: +/// - GET or PUT `/objects/:object_id` +/// - GET or PUT `/locks` +/// - POST `/locks/verify` +/// - POST `/locks/:id/unlock` +/// - GET `/objects/:object_id/chunks/:chunk_id` +/// - POST `/objects/batch` +/// 2. The API router nested in the `/api/v1`: +/// - GET `/api/v1/status` +/// - POST `/api/v1/create-file` +/// - GET `/api/v1/latest-commit` +/// - GET `/api/v1/tree/commit-info` +/// - GET `/api/v1/tree` +/// - GET `/api/v1/blob` +/// - GET `/api/v1/file/blob/:object_id` +/// - GET `/api/v1/file/tree` +/// - GET `/api/v1/path-can-clone` +/// 3. The OAuth router nested in the `/auth`: +/// - GET `/auth/github` +/// - GET `/auth/authorized` +/// - GET `/auth/logout` +/// 4. The other routers for the git protocol: +/// - GET end of `Regex::new(r"/info/refs$")` +/// - POST end of `Regex::new(r"/git-upload-pack$")` +/// - POST end of `Regex::new(r"/git-receive-pack$")` pub async fn app(config: Config, host: String, port: u16, common: CommonOptions) -> Router { let context = Context::new(config.clone()).await; context.services.mono_storage.init_monorepo().await; diff --git a/scripts/crates-sync/crates-sync.py b/scripts/crates-sync/crates-sync.py index 58ef5aa3e..cf746236a 100755 --- a/scripts/crates-sync/crates-sync.py +++ b/scripts/crates-sync/crates-sync.py @@ -1,11 +1,12 @@ -import os # For file and directory operations -import sys # For system-specific parameters and functions -import json # For JSON parsing -import urllib.request # For downloading files from URLs -import tarfile # For handling tar archives -import subprocess # For running system commands -import shutil # For high-level file operations -from collections import defaultdict # For creating dictionaries with default values +import os +import sys +import json +import urllib.request +import tarfile +import subprocess +import shutil +from collections import defaultdict +from datetime import datetime, timedelta # ANSI color codes GREEN = '\033[92m' @@ -29,7 +30,7 @@ def ensure_directory(path): # Create a directory if it doesn't exist if not os.path.exists(path): os.makedirs(path) - print_blue(f"Created directory: {path}") + print(f"Created directory: {path}") def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url): # Construct the filename and path for the crate @@ -41,7 +42,7 @@ def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url) ensure_directory(os.path.dirname(crate_path)) # Ensure the directory exists download_url = f"{dl_base_url}/{crate_name}/{crate_filename}" try: - print_green(f"Downloading: {download_url}") + print_red(f"Downloading: {download_url}") urllib.request.urlretrieve(download_url, crate_path) # Download the file print_red(f"Downloaded: {crate_path}") except Exception as e: @@ -58,11 +59,16 @@ def run_git_command(repo_path, command): print_red(f"Command output: {e.output}") return None -def init_git_repo(repo_path): +def init_git_repo(repo_path, git_base_url): # Initialize a git repository if it doesn't exist if not os.path.exists(os.path.join(repo_path, '.git')): run_git_command(repo_path, ['git', 'init', '-b', 'main']) - print(f"Initialized git repository in {repo_path}") + print_blue(f"Initialized git repository in {repo_path}") + + # Set the LFS domain + lfs_url = f"{git_base_url}" + run_git_command(repo_path, ['git', 'config', 'lfs.url', lfs_url]) + print_blue(f"Set LFS domain to: {lfs_url}") def extract_crate(crate_path, extract_path): def is_within_directory(directory, target): @@ -116,6 +122,10 @@ def filter_member(tarinfo, filterpath): return False def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_base_url): + # Record start time for the entire crate + crate_start_time = datetime.now() + print_blue(f"Started processing crate {crate_name} at {crate_start_time}") + # Process a specific version of a crate repo_path = os.path.join(git_repos_dir, crate_name, version) ensure_directory(repo_path) @@ -126,7 +136,7 @@ def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_ba return # Initialize git repo - init_git_repo(repo_path) + init_git_repo(repo_path, git_base_url) # Add all files to git run_git_command(repo_path, ['git', 'add', '.']) @@ -146,6 +156,15 @@ def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_ba else: print_green(f"Successfully pushed {crate_name} version {version} to remote repository.") + # Record end time and calculate duration for the entire crate + crate_end_time = datetime.now() + crate_duration = crate_end_time - crate_start_time + print_blue(f"Finished processing crate {crate_name} at {crate_end_time}") + print_blue(f"Total processing time for crate {crate_name}: {crate_duration}") + + # Print separator + print("------------------") + def process_crate(crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url): # Process all versions of a crate for v in versions: