From 1fefe068119a5b59d82c81a3eca5ebe75fe57ce6 Mon Sep 17 00:00:00 2001 From: Quanyi Ma Date: Mon, 9 Sep 2024 16:39:00 +0800 Subject: [PATCH] Add crates sync to monorepo script Signed-off-by: Quanyi Ma --- scripts/crates-sync/README.md | 86 ++++++++++++ scripts/crates-sync/crates-sync.py | 217 +++++++++++++++++++++++++++++ 2 files changed, 303 insertions(+) create mode 100644 scripts/crates-sync/README.md create mode 100755 scripts/crates-sync/crates-sync.py diff --git a/scripts/crates-sync/README.md b/scripts/crates-sync/README.md new file mode 100644 index 000000000..e944a87f0 --- /dev/null +++ b/scripts/crates-sync/README.md @@ -0,0 +1,86 @@ +# Crates.io Index Sync to Monorepo Repository + +## Overview + +This Python script is designed to process the crates.io index, download crate files, extract them, and manage Git repositories for each crate version. It's particularly useful for creating a local mirror of crates or for analyzing the crates.io ecosystem. + +## Features + +- Scans the crates.io index directory +- Downloads missing crate files +- Extracts crate contents to version-specific directories +- Creates and manages Git repositories for each crate version +- Pushes changes to a remote Git repository +- Handles invalid versions and empty crate files +- Provides detailed logging of the process +- Allows customization of the Git repository base URL + +## Requirements + +- Python 3.6+ +- `packaging` library (for version parsing) +- Git (installed and accessible from command line) + +## Installation + +1. Clone this repository or download the script. +2. Install the required Python package: + ``` + pip install packaging + # or install python-packaging on some systems + # like Arch: sudo pacman -S python-packaging + ``` + +## Usage + +Run the script from the command line with four required arguments: + +``` +python crates-sync.py +``` + +Where: +- `` is the path to the local copy of the [crates.io index](https://github.com/rust-lang/crates.io-index) +- `` is the directory where downloaded crate files will be stored +- `` is the directory where Git repositories will be created +- `` is the base URL for the remote Git repositories (e.g., "https://git.example.com") host by Mega + +## How It Works + +1. The script scans the crates.io index directory, reading information about each crate and its versions. +2. For each crate version: + - It checks if the crate file exists locally, downloading it if necessary. + - It extracts the crate contents to a version-specific directory. + - It creates a Git repository for the crate version, commits the contents, and pushes to a remote repository. +3. The script handles various edge cases, such as invalid versions and empty crate files. +4. Upon completion, it reports the total number of crates processed. + +## Configuration + +The script expects a `config.json` file in the crates.io index directory, which should contain a `dl` key with the base URL for downloading crates. + +## Error Handling + +- The script provides warnings for invalid versions, empty crate files, and failed Git operations. +- It skips processing of crates that fail to extract or have invalid versions. + +## Customization + +You can modify the script to change: +- The remote Git repository URL format (now customizable via the `` parameter) +- The commit message format +- The logging verbosity + +## Limitations + +- The script assumes a specific structure for the crates.io index. +- It requires Git to be installed and configured on the system. +- Large-scale processing may take a significant amount of time and disk space. + +## Contributing + +Contributions to improve the script are welcome. Please submit pull requests or open issues for any bugs or feature requests. + +## Disclaimer + +This script is not officially associated with crates.io or the Rust project. Use it responsibly and in accordance with crates.io's terms of service. diff --git a/scripts/crates-sync/crates-sync.py b/scripts/crates-sync/crates-sync.py new file mode 100755 index 000000000..c24cfea37 --- /dev/null +++ b/scripts/crates-sync/crates-sync.py @@ -0,0 +1,217 @@ +import os +import sys +import json +import urllib.request +import tarfile +import subprocess +import shutil +from collections import defaultdict +from packaging import version + +def ensure_directory(path): + if not os.path.exists(path): + os.makedirs(path) + print(f"Created directory: {path}") + +def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url): + crate_filename = f"{crate_name}-{crate_version}.crate" + crate_path = os.path.join(crates_dir, crate_name, crate_filename) + if not os.path.exists(crate_path): + download_url = f"{dl_base_url}/{crate_name}/{crate_filename}" + try: + print(f"Downloading: {download_url}") + urllib.request.urlretrieve(download_url, crate_path) + print(f"Downloaded: {crate_path}") + except Exception as e: + print(f"Error downloading {crate_filename}: {str(e)}") + return crate_path + +def run_git_command(repo_path, command): + try: + result = subprocess.run(command, cwd=repo_path, check=True, capture_output=True, text=True) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + print(f"Warning: Git command failed: {e}") + print(f"Command output: {e.output}") + return None + +def init_git_repo(repo_path): + if not os.path.exists(os.path.join(repo_path, '.git')): + run_git_command(repo_path, ['git', 'init', '-b', 'main']) + print(f"Initialized git repository in {repo_path}") + +def extract_crate(crate_path, extract_path): + def is_within_directory(directory, target): + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + prefix = os.path.commonprefix([abs_directory, abs_target]) + return prefix == abs_directory + + def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise Exception("Attempted Path Traversal in Tar File") + + def filter_member(tarinfo, filterpath): + if is_within_directory(path, os.path.join(filterpath, tarinfo.name)): + return tarinfo + else: + return None + + tar.extractall(path, members, numeric_owner=numeric_owner, filter=filter_member) + + try: + with tarfile.open(crate_path, 'r:gz') as tar: + if not tar.getmembers(): + print(f"Warning: Empty crate file {crate_path}. Skipping extraction.") + return False + + # Create a temporary directory for extraction + temp_extract_path = extract_path + "_temp" + ensure_directory(temp_extract_path) + + # Extract to the temporary directory + safe_extract(tar, temp_extract_path) + + # Move contents from the nested directory to the target directory + nested_dir = os.path.join(temp_extract_path, os.listdir(temp_extract_path)[0]) + for item in os.listdir(nested_dir): + shutil.move(os.path.join(nested_dir, item), extract_path) + + # Remove the temporary directory + shutil.rmtree(temp_extract_path) + + print(f"Extracted version to {extract_path}") + return True + except tarfile.ReadError: + print(f"Warning: Failed to read crate file {crate_path}. Skipping extraction.") + return False + +def process_crate_version(crate_name, version, version_path, git_repos_dir, git_base_url): + repo_path = os.path.join(git_repos_dir, crate_name, version) + ensure_directory(repo_path) + + # Copy extracted files to the repo directory + for item in os.listdir(version_path): + s = os.path.join(version_path, item) + d = os.path.join(repo_path, item) + if os.path.isdir(s): + shutil.copytree(s, d, dirs_exist_ok=True) + else: + shutil.copy2(s, d) + + # Initialize git repo + init_git_repo(repo_path) + + # Add all files to git + run_git_command(repo_path, ['git', 'add', '.']) + + # Commit changes + commit_message = f"Add {crate_name} version {version}" + run_git_command(repo_path, ['git', 'commit', '-m', commit_message]) + + # Add remote and push + remote_url = f"{git_base_url}/third-part/rust/crates/{crate_name}/{version}.git" + run_git_command(repo_path, ['git', 'remote', 'add', 'mega', remote_url]) + + # Push to remote + push_result = run_git_command(repo_path, ['git', 'push', '-u', 'mega', 'main']) + if push_result is None: + print(f"Warning: Failed to push {crate_name} version {version} to remote repository.") + else: + print(f"Successfully pushed {crate_name} version {version} to remote repository.") + +def process_crate(crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url): + def version_key(v): + try: + return version.parse(v) + except version.InvalidVersion: + print(f"Warning: Invalid version '{v}' for crate '{crate_name}'. Skipping this version.") + return version.parse("0.0.0") # Use a default low version number + + versions_to_process = sorted(versions, key=version_key) + + for v in versions_to_process: + try: + version.parse(v) # Check again if the version is valid + except version.InvalidVersion: + continue # Skip invalid version + + version_path = os.path.join(crates_dir, crate_name, v) + if os.path.exists(version_path): + print(f"Directory for {crate_name} version {v} already exists. Skipping.") + continue + + crate_path = check_and_download_crate(crates_dir, crate_name, v, dl_base_url) + ensure_directory(version_path) + if extract_crate(crate_path, version_path): + process_crate_version(crate_name, v, version_path, git_repos_dir, git_base_url) + else: + print(f"Skipping processing for {crate_name} version {v} due to extraction failure.") + + print(f"Finished processing {crate_name}") + +def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url): + crates = defaultdict(set) + dl_base_url = None + + # Check if the directories exist + for path in [index_path, crates_dir, git_repos_dir]: + if not os.path.isdir(path): + print(f"Error: The directory {path} does not exist.") + sys.exit(1) + + # Read the config.json to get the dl base URL + config_path = os.path.join(index_path, 'config.json') + try: + with open(config_path, 'r') as config_file: + config = json.load(config_file) + dl_base_url = config.get('dl') + if not dl_base_url: + print("Error: 'dl' key not found in config.json") + sys.exit(1) + except Exception as e: + print(f"Error reading config.json: {str(e)}") + sys.exit(1) + + # Walk through the index directory + for root, dirs, files in os.walk(index_path): + dirs[:] = [d for d in dirs if d not in ['.git', '.github']] + + for file in files: + if (root == index_path and file == 'config.json') or file == 'README.md': + continue + + full_path = os.path.join(root, file) + + try: + with open(full_path, 'r') as f: + for line in f: + line = line.strip() + if line: + crate_info = json.loads(line) + crate_name = crate_info['name'] + crate_version = crate_info['vers'] + crates[crate_name].add(crate_version) + except Exception as e: + print(f"Error processing file {full_path}: {str(e)}") + + # Process each crate + for crate_name, versions in crates.items(): + process_crate(crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url) + + return len(crates) + +def main(): + if len(sys.argv) != 5: + print("Usage: python script.py ") + sys.exit(1) + + index_path, crates_dir, git_repos_dir, git_base_url = sys.argv[1:5] + + total_crates = scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url) + print(f"\nTotal number of crates processed: {total_crates}") + +if __name__ == "__main__": + main()