Update sync crates files to Monorepo script

scripts/crates-sync/crates-sync.py

@@ -1,32 +1,38 @@
-import os
-import sys
-import json
-import urllib.request
-import tarfile
-import subprocess
-import shutil
-from collections import defaultdict
-from packaging import version
+import os  # For file and directory operations
+import sys  # For system-specific parameters and functions
+import json  # For JSON parsing
+import urllib.request  # For downloading files from URLs
+import tarfile  # For handling tar archives
+import subprocess  # For running system commands
+import shutil  # For high-level file operations
+from collections import defaultdict  # For creating dictionaries with default values
+from packaging import version  # For parsing and comparing version numbers
 
 def ensure_directory(path):
+    # Create a directory if it doesn't exist
     if not os.path.exists(path):
         os.makedirs(path)
         print(f"Created directory: {path}")
 
 def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url):
+    # Construct the filename and path for the crate
     crate_filename = f"{crate_name}-{crate_version}.crate"
     crate_path = os.path.join(crates_dir, crate_name, crate_filename)
+
+    # Download the crate if it doesn't exist locally
     if not os.path.exists(crate_path):
+        ensure_directory(os.path.dirname(crate_path))  # Ensure the directory exists
         download_url = f"{dl_base_url}/{crate_name}/{crate_filename}"
         try:
             print(f"Downloading: {download_url}")
-            urllib.request.urlretrieve(download_url, crate_path)
+            urllib.request.urlretrieve(download_url, crate_path)  # Download the file
             print(f"Downloaded: {crate_path}")
         except Exception as e:
             print(f"Error downloading {crate_filename}: {str(e)}")
     return crate_path
 
 def run_git_command(repo_path, command):
+    # Run a git command in the specified repository
     try:
         result = subprocess.run(command, cwd=repo_path, check=True, capture_output=True, text=True)
         return result.stdout.strip()
@@ -36,24 +42,28 @@ def run_git_command(repo_path, command):
         return None
 
 def init_git_repo(repo_path):
+    # Initialize a git repository if it doesn't exist
     if not os.path.exists(os.path.join(repo_path, '.git')):
         run_git_command(repo_path, ['git', 'init', '-b', 'main'])
         print(f"Initialized git repository in {repo_path}")
 
 def extract_crate(crate_path, extract_path):
     def is_within_directory(directory, target):
+        # Check if a path is within a directory (for security)
         abs_directory = os.path.abspath(directory)
         abs_target = os.path.abspath(target)
         prefix = os.path.commonprefix([abs_directory, abs_target])
         return prefix == abs_directory
 
     def safe_extract(tar, path=".", members=None, *, numeric_owner=False):
+        # Safely extract files from a tar archive
         for member in tar.getmembers():
             member_path = os.path.join(path, member.name)
             if not is_within_directory(path, member_path):
                 raise Exception("Attempted Path Traversal in Tar File")
 
         def filter_member(tarinfo, filterpath):
+            # Filter function to ensure extracted files are within the target directory
             if is_within_directory(path, os.path.join(filterpath, tarinfo.name)):
                 return tarinfo
             else:
@@ -67,48 +77,33 @@ def filter_member(tarinfo, filterpath):
                 print(f"Warning: Empty crate file {crate_path}. Skipping extraction.")
                 return False
 
-            # Create a temporary directory for extraction
-            temp_extract_path = extract_path + "_temp"
-            ensure_directory(temp_extract_path)
-
-            # Extract to the temporary directory
-            safe_extract(tar, temp_extract_path)
-
-            # Move contents from the nested directory to the target directory
-            nested_dir = os.path.join(temp_extract_path, os.listdir(temp_extract_path)[0])
-            for item in os.listdir(nested_dir):
-                shutil.move(os.path.join(nested_dir, item), extract_path)
-
-            # Remove the temporary directory
-            shutil.rmtree(temp_extract_path)
+            # Extract directly to the target directory
+            safe_extract(tar, extract_path)
 
         print(f"Extracted version to {extract_path}")
         return True
     except tarfile.ReadError:
         print(f"Warning: Failed to read crate file {crate_path}. Skipping extraction.")
         return False
 
-def process_crate_version(crate_name, version, version_path, git_repos_dir, git_base_url):
+def process_crate_version(crate_name, version, crate_path, git_repos_dir, git_base_url):
+    # Process a specific version of a crate
     repo_path = os.path.join(git_repos_dir, crate_name, version)
     ensure_directory(repo_path)
 
-    # Copy extracted files to the repo directory
-    for item in os.listdir(version_path):
-        s = os.path.join(version_path, item)
-        d = os.path.join(repo_path, item)
-        if os.path.isdir(s):
-            shutil.copytree(s, d, dirs_exist_ok=True)
-        else:
-            shutil.copy2(s, d)
+    # Extract crate directly to the repo directory
+    if not extract_crate(crate_path, repo_path):
+        print(f"Skipping processing for {crate_name} version {version} due to extraction failure.")
+        return
 
     # Initialize git repo
     init_git_repo(repo_path)
 
     # Add all files to git
     run_git_command(repo_path, ['git', 'add', '.'])
 
-    # Commit changes
-    commit_message = f"Add {crate_name} version {version}"
+    # Commit changes with updated message format
+    commit_message = f"{crate_name} {version}"
     run_git_command(repo_path, ['git', 'commit', '-m', commit_message])
 
     # Add remote and push
@@ -123,7 +118,9 @@ def process_crate_version(crate_name, version, version_path, git_repos_dir, git_
         print(f"Successfully pushed {crate_name} version {version} to remote repository.")
 
 def process_crate(crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url):
+    # Process all versions of a crate
     def version_key(v):
+        # Function to parse version for sorting
         try:
             return version.parse(v)
         except version.InvalidVersion:
@@ -138,21 +135,18 @@ def version_key(v):
         except version.InvalidVersion:
             continue  # Skip invalid version
 
-        version_path = os.path.join(crates_dir, crate_name, v)
-        if os.path.exists(version_path):
-            print(f"Directory for {crate_name} version {v} already exists. Skipping.")
+        repo_path = os.path.join(git_repos_dir, crate_name, v)
+        if os.path.exists(repo_path):
+            print(f"Repository for {crate_name} version {v} already exists. Skipping.")
             continue
 
         crate_path = check_and_download_crate(crates_dir, crate_name, v, dl_base_url)
-        ensure_directory(version_path)
-        if extract_crate(crate_path, version_path):
-            process_crate_version(crate_name, v, version_path, git_repos_dir, git_base_url)
-        else:
-            print(f"Skipping processing for {crate_name} version {v} due to extraction failure.")
+        process_crate_version(crate_name, v, crate_path, git_repos_dir, git_base_url)
 
     print(f"Finished processing {crate_name}")
 
 def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url):
+    # Scan the crates.io index and process all crates
     crates = defaultdict(set)
     dl_base_url = None
 
@@ -177,11 +171,11 @@ def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url)
 
     # Walk through the index directory
     for root, dirs, files in os.walk(index_path):
-        dirs[:] = [d for d in dirs if d not in ['.git', '.github']]
+        dirs[:] = [d for d in dirs if d not in ['.git', '.github']]  # Exclude certain directories
 
         for file in files:
             if (root == index_path and file == 'config.json') or file == 'README.md':
-                continue
+                continue  # Skip config.json and README.md
 
             full_path = os.path.join(root, file)
 
@@ -204,6 +198,7 @@ def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url)
     return len(crates)
 
 def main():
+    # Main function to run the script
     if len(sys.argv) != 5:
         print("Usage: python script.py <path_to_crates.io-index> <path_to_crates_directory> <path_to_git_repos_directory> <git_base_url>")
         sys.exit(1)
@@ -214,4 +209,4 @@ def main():
     print(f"\nTotal number of crates processed: {total_crates}")
 
 if __name__ == "__main__":
-    main()
+    main()  # Run the main function if this script is executed directly