From 7c921be11110a06ca853bd828ed9e3dc551c60d8 Mon Sep 17 00:00:00 2001 From: Quanyi Ma Date: Wed, 11 Sep 2024 10:24:57 +0800 Subject: [PATCH] Remove crate version valid function from script Signed-off-by: Quanyi Ma --- scripts/crates-sync/crates-sync.py | 44 ++++++++++-------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/scripts/crates-sync/crates-sync.py b/scripts/crates-sync/crates-sync.py index e710c6568..11c90b205 100755 --- a/scripts/crates-sync/crates-sync.py +++ b/scripts/crates-sync/crates-sync.py @@ -8,14 +8,14 @@ from collections import defaultdict from packaging import version +# Create a directory if it doesn't exist def ensure_directory(path): - # Create a directory if it doesn't exist if not os.path.exists(path): os.makedirs(path) print(f"Created directory: {path}") +# Construct the filename and path for the crate def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url): - # Construct the filename and path for the crate crate_filename = f"{crate_name}-{crate_version}.crate" crate_path = os.path.join(crates_dir, crate_name, crate_filename) @@ -31,8 +31,8 @@ def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url) print(f"Error downloading {crate_filename}: {str(e)}") return crate_path +# Run a git command in the specified repository def run_git_command(repo_path, command): - # Run a git command in the specified repository try: result = subprocess.run(command, cwd=repo_path, check=True, capture_output=True, text=True) return result.stdout.strip() @@ -41,29 +41,29 @@ def run_git_command(repo_path, command): print(f"Command output: {e.output}") return None +# Initialize a git repository if it doesn't exist def init_git_repo(repo_path): - # Initialize a git repository if it doesn't exist if not os.path.exists(os.path.join(repo_path, '.git')): run_git_command(repo_path, ['git', 'init', '-b', 'main']) print(f"Initialized git repository in {repo_path}") def extract_crate(crate_path, extract_path): + # Check if a path is within a directory (for security) def is_within_directory(directory, target): - # Check if a path is within a directory (for security) abs_directory = os.path.abspath(directory) abs_target = os.path.abspath(target) prefix = os.path.commonprefix([abs_directory, abs_target]) return prefix == abs_directory + # Safely extract files from a tar archive def safe_extract(tar, path=".", members=None, *, numeric_owner=False): - # Safely extract files from a tar archive for member in tar.getmembers(): member_path = os.path.join(path, member.name) if not is_within_directory(path, member_path): raise Exception("Attempted Path Traversal in Tar File") + # Filter function to ensure extracted files are within the target directory def filter_member(tarinfo, filterpath): - # Filter function to ensure extracted files are within the target directory if is_within_directory(path, os.path.join(filterpath, tarinfo.name)): return tarinfo else: @@ -98,8 +98,8 @@ def filter_member(tarinfo, filterpath): print(f"Warning: Failed to read crate file {crate_path}. Skipping extraction.") return False +# Process a specific version of a crate def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_base_url): - # Process a specific version of a crate repo_path = os.path.join(git_repos_dir, crate_name, version) ensure_directory(repo_path) @@ -116,7 +116,7 @@ def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_ba # Commit changes with updated message format commit_message = f"{crate_name} {version}" - run_git_command(repo_path, ['git', 'commit', '-m', commit_message]) + run_git_command(repo_path, ['git', 'commit', '-a', '-s', '-S', '-m', commit_message]) # Add remote and push remote_url = f"{git_base_url}/third-part/rust/crates/{crate_name}/{version}.git" @@ -129,24 +129,9 @@ def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_ba else: print(f"Successfully pushed {crate_name} version {version} to remote repository.") +# Process all versions of a crate def process_crate(crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url): - # Process all versions of a crate - def version_key(v): - # Function to parse version for sorting - try: - return version.parse(v) - except version.InvalidVersion: - print(f"Warning: Invalid version '{v}' for crate '{crate_name}'. Skipping this version.") - return version.parse("0.0.0") # Use a default low version number - - versions_to_process = sorted(versions, key=version_key) - - for v in versions_to_process: - try: - version.parse(v) # Check again if the version is valid - except version.InvalidVersion: - continue # Skip invalid version - + for v in versions: repo_path = os.path.join(git_repos_dir, crate_name, v) if os.path.exists(repo_path): print(f"Repository for {crate_name} version {v} already exists. Skipping.") @@ -157,8 +142,8 @@ def version_key(v): print(f"Finished processing {crate_name}") +# Scan the crates.io index and process all crates def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url): - # Scan the crates.io index and process all crates crates = defaultdict(set) dl_base_url = None @@ -209,8 +194,8 @@ def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url) return len(crates) +# Main function to run the script def main(): - # Main function to run the script if len(sys.argv) != 5: print("Usage: python script.py ") sys.exit(1) @@ -220,5 +205,6 @@ def main(): total_crates = scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url) print(f"\nTotal number of crates processed: {total_crates}") +# Run the main function if this script is executed directly if __name__ == "__main__": - main() # Run the main function if this script is executed directly + main()