diff --git a/.github/workflows/base.yml b/.github/workflows/base.yml index 418d37611..fedf1a934 100644 --- a/.github/workflows/base.yml +++ b/.github/workflows/base.yml @@ -46,7 +46,8 @@ jobs: - uses: actions/checkout@v3 with: submodules: recursive - - run: | + - run: | + echo "deb http://gb.archive.ubuntu.com/ubuntu jammy main" | sudo tee -a /etc/apt/sources.list sudo apt update sudo apt install -y libwebkit2gtk-4.0-dev \ build-essential \ @@ -125,17 +126,17 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v3 with: - node-version: '22' - cache: 'npm' + node-version: "22" + cache: "npm" cache-dependency-path: ./moon/package.json - name: Install dependencies working-directory: ./moon - run: npm install + run: npm install - name: Run ESLint working-directory: ./moon - run: npm run lint + run: npm run lint - name: Build Next.js application working-directory: ./moon diff --git a/docker/mono-pg-dockerfile b/docker/mono-pg-dockerfile index 26fd0bb5f..4e2b545ed 100644 --- a/docker/mono-pg-dockerfile +++ b/docker/mono-pg-dockerfile @@ -1,4 +1,4 @@ -FROM postgres:16 +FROM postgres:17 # When running the `docker buildx build` command, you may encounter the following warning: # diff --git a/scripts/crates-sync/crates-sync.py b/scripts/crates-sync/crates-sync.py index 0b6fe7c13..c9daad573 100755 --- a/scripts/crates-sync/crates-sync.py +++ b/scripts/crates-sync/crates-sync.py @@ -59,14 +59,13 @@ def run_git_command(repo_path, command): print_red(f"Command output: {e.output}") return None -def init_git_repo(repo_path, git_base_url): +def init_git_repo(repo_path, git_base_url, lfs_url): # Initialize a git repository if it doesn't exist if not os.path.exists(os.path.join(repo_path, '.git')): run_git_command(repo_path, ['git', 'init', '-b', 'main']) print_blue(f"Initialized git repository in {repo_path}") # Set the LFS domain - lfs_url = f"{git_base_url}" run_git_command(repo_path, ['git', 'config', 'lfs.url', lfs_url]) print_blue(f"Set LFS domain to: {lfs_url}") @@ -121,7 +120,7 @@ def filter_member(tarinfo, filterpath): print_red(f"Warning: Failed to read crate file {crate_path}. Skipping extraction.") return False -def process_crate_version(num, crate_name, version, crate_path, git_repos_dir, git_base_url): +def process_crate_version(num, crate_name, version, crate_path, git_repos_dir, git_base_url, lfs_url): # Record start time for the entire crate crate_start_time = datetime.now() print_blue(f"Started processing the crate {crate_name} at {crate_start_time}") @@ -135,8 +134,14 @@ def process_crate_version(num, crate_name, version, crate_path, git_repos_dir, g print_red(f"Skipping processing for {crate_name} version {version} due to extraction failure.") return + # Check for .gitattributes file and remove if it exists + gitattributes_path = os.path.join(repo_path, '.gitattributes') + if os.path.exists(gitattributes_path): + os.remove(gitattributes_path) + print_blue(f"Removed .gitattributes file from {repo_path}") + # Initialize git repo - init_git_repo(repo_path, git_base_url) + init_git_repo(repo_path, git_base_url, lfs_url) # Add all files to git run_git_command(repo_path, ['git', 'add', '.']) @@ -165,7 +170,7 @@ def process_crate_version(num, crate_name, version, crate_path, git_repos_dir, g # Print separator print("------------------") -def process_crate(num, crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url): +def process_crate(num, crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url, lfs_url): # Process all versions of a crate for v in versions: repo_path = os.path.join(git_repos_dir, crate_name, v) @@ -174,12 +179,12 @@ def process_crate(num, crate_name, versions, crates_dir, git_repos_dir, dl_base_ continue crate_path = check_and_download_crate(crates_dir, crate_name, v, dl_base_url) - process_crate_version(num, crate_name, v, crate_path, git_repos_dir, git_base_url) + process_crate_version(num, crate_name, v, crate_path, git_repos_dir, git_base_url, lfs_url) num += 1 print_blue(f"Finished processing {crate_name}") -def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url): +def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url, lfs_url): # Scan the crates.io index and process all crates crates = defaultdict(set) dl_base_url = None @@ -229,21 +234,31 @@ def scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url) # Process each crate for crate_name, versions in crates.items(): - process_crate(num, crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url) + process_crate(num, crate_name, versions, crates_dir, git_repos_dir, dl_base_url, git_base_url, lfs_url) num += len(versions) return len(crates) def main(): + # Record start time for the entire process + total_start_time = datetime.now() + print_blue(f"Started entire process at {total_start_time}") + # Main function to run the script - if len(sys.argv) != 5: - print_red("Usage: python script.py ") + if len(sys.argv) != 6: + print_red("Usage: python script.py ") sys.exit(1) - index_path, crates_dir, git_repos_dir, git_base_url = sys.argv[1:5] + index_path, crates_dir, git_repos_dir, git_base_url, lfs_url = sys.argv[1:6] + + total_crates = scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url, lfs_url) - total_crates = scan_and_process_crates(index_path, crates_dir, git_repos_dir, git_base_url) + # Record end time and calculate duration for the entire process + total_end_time = datetime.now() + total_duration = total_end_time - total_start_time print_blue(f"\nTotal number of crates processed: {total_crates}") + print_blue(f"Finished entire process at {total_end_time}") + print_blue(f"Total processing time: {total_duration}") if __name__ == "__main__": main() # Run the main function if this script is executed directly