fix(ci): prefetch trusted pnpm store for coverage #642
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,198 @@ | ||
| #!/usr/bin/env python3 | ||
| """Materialize pnpm locks from a validated pull-request base commit.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import argparse | ||
| import json | ||
| import pathlib | ||
| import re | ||
| import subprocess | ||
| import sys | ||
| from typing import Any | ||
|
|
||
|
|
||
| SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") | ||
| PNPM_SPEC_RE = re.compile(r"^pnpm@[0-9]+\.[0-9]+\.[0-9]+(?:[+-][A-Za-z0-9._+-]+)?$") | ||
| PNPM_BASE_INPUT_NAMES = ("package.json", "pnpm-workspace.yaml", ".pnpmfile.cjs") | ||
|
|
||
|
|
||
| def _git(repo_root: pathlib.Path, *args: str) -> bytes: | ||
| """Run one read-only git command in the materialized repository.""" | ||
| completed = subprocess.run( | ||
| ["git", "-C", str(repo_root), *args], | ||
| check=False, | ||
| stdout=subprocess.PIPE, | ||
| stderr=subprocess.PIPE, | ||
| ) | ||
| if completed.returncode != 0: | ||
| stderr = completed.stderr.decode("utf-8", errors="replace").strip() | ||
| raise RuntimeError(f"git {args[0]} failed: {stderr}") | ||
| return completed.stdout | ||
|
|
||
|
|
||
| def _regular_base_paths(repo_root: pathlib.Path, base_sha: str) -> set[str]: | ||
| """Return regular blob paths from the exact validated base commit.""" | ||
| entries = _git(repo_root, "ls-tree", "-r", "-z", "--full-tree", base_sha) | ||
| paths: set[str] = set() | ||
| for raw_entry in entries.split(b"\0"): | ||
| if not raw_entry: | ||
| continue | ||
| metadata, separator, raw_path = raw_entry.partition(b"\t") | ||
| if not separator: | ||
| raise RuntimeError("git ls-tree returned a malformed entry") | ||
| fields = metadata.split() | ||
| if len(fields) != 3: | ||
| raise RuntimeError("git ls-tree returned malformed metadata") | ||
| mode, object_type, _object_id = ( | ||
| field.decode("ascii", errors="strict") for field in fields | ||
| ) | ||
| path = raw_path.decode("utf-8", errors="surrogateescape") | ||
| candidate = pathlib.PurePosixPath(path) | ||
| if ( | ||
| object_type == "blob" | ||
| and mode.startswith("100") | ||
| and not candidate.is_absolute() | ||
| and ".." not in candidate.parts | ||
| ): | ||
| paths.add(path) | ||
| return paths | ||
|
Comment on lines
+34
to
+59
There was a problem hiding this comment. 📐 Maintainability & Code Quality | 🟡 Minor | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: #!/bin/bash
# Check whether the negative-filter branch (non-blob / non-100* mode) is exercised by any test.
rg -n "120000|160000|object_type|symlink|submodule" tests/test_materialize_base_javascript_packages.pyRepository: ContextualWisdomLab/.github Length of output: 428 🏁 Script executed: #!/bin/bash
set -euo pipefail
# Show the implementation around the branch in question.
sed -n '1,140p' scripts/ci/materialize_base_javascript_packages.py
printf '\n--- TEST FILE ---\n'
# Show the relevant test sections, including any symlink/submodule/tree-entry cases.
sed -n '1,340p' tests/test_materialize_base_javascript_packages.pyRepository: ContextualWisdomLab/.github Length of output: 16622 🏁 Script executed: #!/bin/bash
set -euo pipefail
# Find all references to the helper and tree-entry filtering.
rg -n "_regular_base_paths|ls-tree|120000|160000|submodule|symlink|blob" scripts/ci tests
printf '\n--- TREE OF RELEVANT TESTS ---\n'
fd -a "test_materialize_base_javascript_packages.py|materialize_base_javascript_packages.py" scripts tests
printf '\n--- TARGETED TEST SECTION ---\n'
sed -n '70,170p' tests/test_materialize_base_javascript_packages.pyRepository: ContextualWisdomLab/.github Length of output: 21008
🤖 Prompt for AI AgentsSource: Coding guidelines |
||
|
|
||
|
|
||
| def base_pnpm_projects( | ||
| repo_root: pathlib.Path, base_sha: str | ||
| ) -> list[tuple[str, str, dict[str, bytes]]]: | ||
| """Return exact base pnpm inputs grouped by lockfile directory.""" | ||
| if not SHA_RE.fullmatch(base_sha): | ||
| raise ValueError("base SHA must be exactly 40 hexadecimal characters") | ||
|
|
||
| repo_root = repo_root.resolve() | ||
| regular_paths = _regular_base_paths(repo_root, base_sha) | ||
| projects: list[tuple[str, str, dict[str, bytes]]] = [] | ||
| for lock_path in sorted( | ||
| path | ||
| for path in regular_paths | ||
| if pathlib.PurePosixPath(path).name == "pnpm-lock.yaml" | ||
| ): | ||
| lock = pathlib.PurePosixPath(lock_path) | ||
| project_root = lock.parent | ||
| package_path = str(project_root / "package.json") | ||
| if package_path not in regular_paths: | ||
| raise ValueError( | ||
| f"trusted base pnpm lock {lock_path} has no regular sibling package.json" | ||
| ) | ||
| try: | ||
| package_data: Any = json.loads( | ||
| _git(repo_root, "show", f"{base_sha}:{package_path}").decode("utf-8") | ||
| ) | ||
| except (UnicodeDecodeError, json.JSONDecodeError) as exc: | ||
| raise ValueError( | ||
| f"trusted base package manifest {package_path} is invalid JSON: {exc}" | ||
| ) from exc | ||
| if not isinstance(package_data, dict): | ||
| raise ValueError( | ||
| f"trusted base package manifest {package_path} must be a JSON object" | ||
| ) | ||
| package_manager = package_data.get("packageManager") | ||
| if not isinstance(package_manager, str) or not PNPM_SPEC_RE.fullmatch( | ||
| package_manager | ||
| ): | ||
| raise ValueError( | ||
| f"trusted base package manifest {package_path} must declare an exact pnpm packageManager version" | ||
| ) | ||
| lock_content = _git(repo_root, "show", f"{base_sha}:{lock_path}") | ||
| if not lock_content.strip(): | ||
| raise ValueError(f"trusted base pnpm lock {lock_path} is empty") | ||
|
|
||
| base_inputs = { | ||
| input_name: _git( | ||
| repo_root, | ||
| "show", | ||
| f"{base_sha}:{project_root / input_name}", | ||
| ) | ||
| for input_name in PNPM_BASE_INPUT_NAMES | ||
| if str(project_root / input_name) in regular_paths | ||
| } | ||
| base_inputs["pnpm-lock.yaml"] = lock_content | ||
|
|
||
| patches_root = project_root / "patches" | ||
| for base_path in sorted(regular_paths): | ||
| candidate = pathlib.PurePosixPath(base_path) | ||
| if candidate == patches_root or patches_root not in candidate.parents: | ||
| continue | ||
| relative_path = str(candidate.relative_to(project_root)) | ||
| base_inputs[relative_path] = _git( | ||
| repo_root, "show", f"{base_sha}:{base_path}" | ||
| ) | ||
|
|
||
| projects.append((lock_path, package_manager, base_inputs)) | ||
| return projects | ||
|
|
||
|
|
||
| def materialize( | ||
| repo_root: pathlib.Path, | ||
| base_sha: str, | ||
| output_dir: pathlib.Path, | ||
| ) -> list[dict[str, str]]: | ||
| """Write base pnpm inputs under generated paths safe for a Docker context.""" | ||
| if output_dir.exists() and output_dir.is_symlink(): | ||
| raise ValueError("output directory must not be a symlink") | ||
| output_dir.mkdir(parents=True, exist_ok=True) | ||
|
|
||
| manifest: list[dict[str, str]] = [] | ||
| for index, (source_path, package_manager, base_inputs) in enumerate( | ||
| base_pnpm_projects(repo_root, base_sha) | ||
| ): | ||
| directory = f"project-{index:03d}" | ||
| project_dir = output_dir / directory | ||
| project_dir.mkdir() | ||
| for relative_path, content in sorted(base_inputs.items()): | ||
| destination = project_dir / relative_path | ||
| destination.parent.mkdir(parents=True, exist_ok=True) | ||
| destination.write_bytes(content) | ||
| manifest.append( | ||
| { | ||
| "directory": directory, | ||
| "package_manager": package_manager, | ||
| "source": source_path, | ||
| } | ||
| ) | ||
|
|
||
| (output_dir / "manifest.json").write_text( | ||
| json.dumps(manifest, indent=2, sort_keys=True) + "\n", | ||
| encoding="utf-8", | ||
| ) | ||
| return manifest | ||
|
|
||
|
|
||
| def main(argv: list[str] | None = None) -> int: | ||
| """Materialize base pnpm locks and report the trusted inputs.""" | ||
| parser = argparse.ArgumentParser() | ||
| parser.add_argument("--repo-root", required=True, type=pathlib.Path) | ||
| parser.add_argument("--base-sha", required=True) | ||
| parser.add_argument("--output-dir", required=True, type=pathlib.Path) | ||
| args = parser.parse_args(argv) | ||
|
|
||
| try: | ||
| manifest = materialize(args.repo_root, args.base_sha, args.output_dir) | ||
| except (OSError, RuntimeError, ValueError) as exc: | ||
| print( | ||
| f"::error::Could not materialize base JavaScript package locks: {exc}", | ||
| file=sys.stderr, | ||
| ) | ||
| return 1 | ||
|
|
||
| if manifest: | ||
| for entry in manifest: | ||
| print( | ||
| "Materialized trusted base pnpm lock " | ||
| f"{entry['source']} for {entry['package_manager']} " | ||
| f"as {entry['directory']}/pnpm-lock.yaml." | ||
| ) | ||
| else: | ||
| print("No tracked pnpm-lock.yaml files exist at the validated base SHA.") | ||
| return 0 | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| raise SystemExit(main()) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🩺 Stability & Availability | 🟠 Major | ⚡ Quick win
pnpm 버전이 정확히
11.5.3이 아니면 전체 coverage-tool 이미지 빌드가 실패해, 모든 언어의 coverage-evidence가 함께 막힙니다.if [ "$package_manager" != "pnpm@11.5.3" ]; then ... exit 1; fi은 base 커밋의 어떤 pnpm-lock.yaml/package.json이든packageManager가pnpm@11.5.3이 아니면 즉시exit 1합니다. 이 블록은 트러스티드 네트워크 Docker 이미지 빌드(docker build) 단계에서 실행되며, 이 이미지 빌드는 Python/R/Rust 등 언어에 관계없이 모든 PR에서 공통으로 수행됩니다. 기존에는 pnpm 버전 불일치가 sandbox 내부의 JS 전용ensure_corepack_runner()에서만 감지되어 해당 언어의 coverage만 실패시켰지만(다른 언어 coverage는 계속 진행), 이번 변경으로 인해 image build 단계에서 하드 실패하면서 PR의 coverage-evidence 전체(Python/Rust/R 포함)가 차단됩니다.repository_dispatch로 여러 대상 저장소를 리뷰하는 구조상, pnpm 11.5.3 이외 버전을 쓰는 대상 저장소가 하나라도 있으면 해당 저장소의 모든 PR이 이 시점에서 막힐 수 있습니다.빌드를 중단시키는 대신, 지원되지 않는 pnpm 버전의 프로젝트는 로그만 남기고 prefetch를 건너뛰도록(그래서 이후 JS 단계에서만 개별적으로 실패하도록) 완화하는 것을 고려해주세요.
🤖 Prompt for AI Agents