diff --git a/scripts/crates-sync/crates-sync.py b/scripts/crates-sync/crates-sync.py index c24cfea37..c9e2a5d90 100755 --- a/scripts/crates-sync/crates-sync.py +++ b/scripts/crates-sync/crates-sync.py @@ -1,32 +1,38 @@ -import os -import sys -import json -import urllib.request -import tarfile -import subprocess -import shutil -from collections import defaultdict -from packaging import version +import os # For file and directory operations +import sys # For system-specific parameters and functions +import json # For JSON parsing +import urllib.request # For downloading files from URLs +import tarfile # For handling tar archives +import subprocess # For running system commands +import shutil # For high-level file operations +from collections import defaultdict # For creating dictionaries with default values +from packaging import version # For parsing and comparing version numbers def ensure_directory(path): + # Create a directory if it doesn't exist if not os.path.exists(path): os.makedirs(path) print(f"Created directory: {path}") def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url): + # Construct the filename and path for the crate crate_filename = f"{crate_name}-{crate_version}.crate" crate_path = os.path.join(crates_dir, crate_name, crate_filename) + + # Download the crate if it doesn't exist locally if not os.path.exists(crate_path): + ensure_directory(os.path.dirname(crate_path)) # Ensure the directory exists download_url = f"{dl_base_url}/{crate_name}/{crate_filename}" try: print(f"Downloading: {download_url}") - urllib.request.urlretrieve(download_url, crate_path) + urllib.request.urlretrieve(download_url, crate_path) # Download the file print(f"Downloaded: {crate_path}") except Exception as e: print(f"Error downloading {crate_filename}: {str(e)}") return crate_path def run_git_command(repo_path, command): + # Run a git command in the specified repository try: result = subprocess.run(command, cwd=repo_path, check=True, capture_output=True, text=True) return result.stdout.strip() @@ -36,24 +42,28 @@ def run_git_command(repo_path, command): return None def init_git_repo(repo_path): + # Initialize a git repository if it doesn't exist if not os.path.exists(os.path.join(repo_path, '.git')): run_git_command(repo_path, ['git', 'init', '-b', 'main']) print(f"Initialized git repository in {repo_path}") def extract_crate(crate_path, extract_path): def is_within_directory(directory, target): + # Check if a path is within a directory (for security) abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) prefix = os.path.commonprefix([abs_directory, abs_target]) return prefix == abs_directory def safe_extract(tar, path=".", members=None, *, numeric_owner=False): + # Safely extract files from a tar archive for member in tar.getmembers(): member_path = os.path.join(path, member.name) if not is_within_directory(path, member_path): raise Exception("Attempted Path Traversal in Tar File") def filter_member(tarinfo, filterpath): + # Filter function to ensure extracted files are within the target directory if is_within_directory(path, os.path.join(filterpath, tarinfo.name)): return tarinfo else: @@ -67,20 +77,8 @@ def filter_member(tarinfo, filterpath): print(f"Warning: Empty crate file {crate_path}. Skipping extraction.") return False - # Create a temporary directory for extraction - temp_extract_path = extract_path + "_temp" - ensure_directory(temp_extract_path) - - # Extract to the temporary directory - safe_extract(tar, temp_extract_path) - - # Move contents from the nested directory to the target directory - nested_dir = os.path.join(temp_extract_path, os.listdir(temp_extract_path)[0]) - for item in os.listdir(nested_dir): - shutil.move(os.path.join(nested_dir, item), extract_path) - - # Remove the temporary directory - shutil.rmtree(temp_extract_path) + # Extract directly to the target directory + safe_extract(tar, extract_path) print(f"Extracted version to {extract_path}") return True @@ -88,18 +86,15 @@ def filter_member(tarinfo, filterpath): print(f"Warning: Failed to read crate file {crate_path}. Skipping extraction.") return False -def process_crate_version(crate_name, version, version_path, git_repos_dir, git_base_url): +def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_base_url): + # Process a specific version of a crate repo_path = os.path.join(git_repos_dir, crate_name, version) ensure_directory(repo_path) - # Copy extracted files to the repo directory - for item in os.listdir(version_path): - s = os.path.join(version_path, item) - d = os.path.join(repo_path, item) - if os.path.isdir(s): - shutil.copytree(s, d, dirs_exist_ok=True) - else: - shutil.copy2(s, d) + # Extract crate directly to the repo directory + if not extract_crate(crate_path, repo_path): + print(f"Skipping processing for {crate_name} version {version} due to extraction failure.") + return # Initialize git repo init_git_repo(repo_path) @@ -107,8 +102,8 @@ def process_crate_version(crate_name, version, version_path, git_repos_dir, git_ # Add all files to git run_git_command(repo_path, ['git', 'add', '.']) - # Commit changes - commit_message = f"Add {crate_name} version {version}" + # Commit changes with updated message format + commit_message = f"{crate_name} {version}" run_git_command(repo_path, ['git', 'commit', '-m', commit_message]) # Add remote and push @@ -123,7 +118,9 @@ def process_crate_version(crate_name, version, version_path, git_repos_dir, git_ print(f"Successfully pushed {crate_name} version {version} to remote repository.") def process_crate(crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url): + # Process all versions of a crate def version_key(v): + # Function to parse version for sorting try: return version.parse(v) except version.InvalidVersion: @@ -138,21 +135,18 @@ def version_key(v): except version.InvalidVersion: continue # Skip invalid version - version_path = os.path.join(crates_dir, crate_name, v) - if os.path.exists(version_path): - print(f"Directory for {crate_name} version {v} already exists. Skipping.") + repo_path = os.path.join(git_repos_dir, crate_name, v) + if os.path.exists(repo_path): + print(f"Repository for {crate_name} version {v} already exists. Skipping.") continue crate_path = check_and_download_crate(crates_dir, crate_name, v, dl_base_url) - ensure_directory(version_path) - if extract_crate(crate_path, version_path): - process_crate_version(crate_name, v, version_path, git_repos_dir, git_base_url) - else: - print(f"Skipping processing for {crate_name} version {v} due to extraction failure.") + process_crate_version(crate_name, v, crate_path, git_repos_dir, git_base_url) print(f"Finished processing {crate_name}") def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url): + # Scan the crates.io index and process all crates crates = defaultdict(set) dl_base_url = None @@ -177,11 +171,11 @@ def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url) # Walk through the index directory for root, dirs, files in os.walk(index_path): - dirs[:] = [d for d in dirs if d not in ['.git', '.github']] + dirs[:] = [d for d in dirs if d not in ['.git', '.github']] # Exclude certain directories for file in files: if (root == index_path and file == 'config.json') or file == 'README.md': - continue + continue # Skip config.json and README.md full_path = os.path.join(root, file) @@ -204,6 +198,7 @@ def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url) return len(crates) def main(): + # Main function to run the script if len(sys.argv) != 5: print("Usage: python script.py ") sys.exit(1) @@ -214,4 +209,4 @@ def main(): print(f"\nTotal number of crates processed: {total_crates}") if __name__ == "__main__": - main() + main() # Run the main function if this script is executed directly