diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 47205030c..d52b81701 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -347,6 +347,72 @@ jobs: mkdir -p "$(dirname "$COVERAGE_SOURCE_WORKDIR")" mv "$fetch_dir" "$COVERAGE_SOURCE_WORKDIR" git -C "$COVERAGE_SOURCE_WORKDIR" status --short + # Pre-fetch every locked crate graph in the merge tree (root and nested + # workspaces alike, matching the sandbox's nested-manifest coverage + # discovery) so `cargo llvm-cov` can compile Rust coverage offline. + # Prefer the same pinned toolchain the sandbox measures with so + # lockfile parsing and cache layout cannot drift from the runner's + # default cargo. `cargo fetch` downloads content-addressed crates + # without executing any build scripts, and a failed prefetch keeps + # the same fail-closed sandbox behavior: the offline build surfaces + # the missing dependencies for this head. + # + # The fetch itself never trusts pull-request-controlled cargo + # configuration: it runs from a runner-owned temporary HOME and + # CARGO_HOME (cargo discovers config.toml from the working directory + # and CARGO_HOME, so a config committed to the merge tree — including + # an external credential-provider executable — is never loaded on + # this authenticated runner). Only the content-addressed registry/ + # and git/ caches are copied into the tree-local CARGO_HOME the + # network-isolated sandbox uses; configuration and credential files + # are never propagated. + prefetch_manifests="$(git -C "$COVERAGE_SOURCE_WORKDIR" ls-files 'Cargo.toml' '*/Cargo.toml')" + if [ -n "$prefetch_manifests" ]; then + prefetch_cargo=() + if command -v rustup >/dev/null 2>&1 \ + && rustup toolchain install 1.94.1 --profile minimal --no-self-update >/dev/null 2>&1; then + prefetch_cargo=(rustup run 1.94.1 cargo) + elif command -v cargo >/dev/null 2>&1; then + prefetch_cargo=(cargo) + fi + if [ "${#prefetch_cargo[@]}" -eq 0 ]; then + echo "::warning::cargo is unavailable on this runner, so no locked crate graph was pre-fetched; offline Rust coverage for this head will fail with missing dependencies." + else + prefetch_home="$(mktemp -d "${RUNNER_TEMP:-/tmp}/cargo-prefetch.XXXXXX")" + # Redirecting HOME isolates cargo from pull-request-controlled + # configuration, but rustup resolves its toolchain store from + # RUSTUP_HOME (default $HOME/.rustup). Pin it to the runner's + # own runner-owned install resolved before the override so + # `rustup run 1.94.1` keeps finding the toolchain provisioned + # above; the temporary cargo cache behavior is unchanged. + prefetch_rustup_home="${RUSTUP_HOME:-$HOME/.rustup}" + while IFS= read -r prefetch_manifest; do + [ -n "$prefetch_manifest" ] || continue + prefetch_dir="$(dirname "$prefetch_manifest")" + if [ ! -f "$COVERAGE_SOURCE_WORKDIR/$prefetch_dir/Cargo.lock" ]; then + continue + fi + if ! ( + cd "$prefetch_home" \ + && HOME="$prefetch_home" \ + CARGO_HOME="$prefetch_home/.cargo" \ + RUSTUP_HOME="$prefetch_rustup_home" \ + "${prefetch_cargo[@]}" fetch --locked \ + --manifest-path "$COVERAGE_SOURCE_WORKDIR/$prefetch_manifest" + ); then + echo "::warning::Locked cargo prefetch failed for ${prefetch_manifest}; offline Rust coverage will surface the missing dependencies for this head." + fi + done <<<"$prefetch_manifests" + sandbox_cargo_home="$COVERAGE_SOURCE_WORKDIR/.opencode-sandbox-home/.cargo" + for prefetch_cache_dir in registry git; do + if [ -d "$prefetch_home/.cargo/$prefetch_cache_dir" ]; then + mkdir -p "$sandbox_cargo_home" + cp -a "$prefetch_home/.cargo/$prefetch_cache_dir" "$sandbox_cargo_home/" + fi + done + rm -rf "$prefetch_home" + fi + fi tar -cf "$COVERAGE_SOURCE_ARCHIVE" -C "$COVERAGE_SOURCE_WORKDIR" . - name: Upload materialized pull request merge tree @@ -670,6 +736,25 @@ jobs: && tar -xzf /tmp/cargo-llvm-cov.tar.gz -C /usr/local/bin cargo-llvm-cov \ && chmod 0755 /usr/local/bin/cargo-llvm-cov \ && rm -f /tmp/cargo-llvm-cov.tar.gz + # Pinned rustup toolchain with llvm-tools-preview: the distribution + # rustc/cargo cannot parse modern version-4 Cargo.lock files and ships + # no llvm-cov/llvm-profdata, so `cargo llvm-cov` coverage evidence for + # Rust repositories needs this toolchain. The rustup-init binary is + # sha256-pinned and the toolchain channel is version-pinned. Only + # RUSTUP_HOME is exported for the runtime; CARGO_HOME is deliberately + # not baked into the image so the sandbox's tree-local CARGO_HOME + # (which holds the pre-fetched crate cache) is the only cargo home the + # measured commands ever see. + ENV RUSTUP_HOME=/usr/local/rustup + RUN curl --proto '=https' --tlsv1.2 -fsSLo /tmp/rustup-init \ + https://static.rust-lang.org/rustup/archive/1.28.2/x86_64-unknown-linux-gnu/rustup-init \ + && echo '20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c /tmp/rustup-init' | sha256sum -c - \ + && chmod 0755 /tmp/rustup-init \ + && CARGO_HOME=/usr/local/cargo /tmp/rustup-init -y --no-modify-path --profile minimal \ + --default-toolchain 1.94.1 --component llvm-tools-preview \ + && ln -sf /usr/local/cargo/bin/* /usr/local/bin/ \ + && chmod -R a+rX /usr/local/rustup /usr/local/cargo \ + && rm -f /tmp/rustup-init RUN curl --proto '=https' --tlsv1.2 -fsSLo /tmp/pnpm.tgz \ https://registry.npmjs.org/pnpm/-/pnpm-11.5.3.tgz \ && echo '7ac1c919341c213a34dc0d02afb7143c5c26ac26ee8c4782deea821b8ac64d2134a081fd8941dae6e29bbb48f58dfc2b7fbceeccc07cb2f09d219d342a4969ed /tmp/pnpm.tgz' | sha512sum -c - \ @@ -1723,6 +1808,31 @@ jobs: failures=$((failures + 1)) return 1 fi + # The sandbox has no network access; make cargo resolve strictly from + # the crate cache the online coverage-source-tree job pre-fetched into + # the tree-local CARGO_HOME so missing dependencies fail fast with a + # clear offline error instead of a network timeout. + export CARGO_NET_OFFLINE=true + # Coverage always measures with the trusted image toolchain: a + # repository rust-toolchain(.toml) requesting any other channel would + # make the rustup proxy attempt a download the offline sandbox cannot + # perform. The env override takes precedence over directory overrides, + # keeping toolchain selection deterministic and offline-safe, and + # rustup's auto-install is disabled so a missing pinned toolchain + # fails fast as an image-provisioning error instead of a network + # attempt the sandbox can never satisfy. + export RUSTUP_TOOLCHAIN=1.94.1 + export RUSTUP_AUTO_INSTALL=0 + if ! rustc --version >/dev/null 2>&1; then + append "### Rust coverage toolchain" + append "" + append "- Result: FAIL" + append "- Reason: the trusted offline coverage image cannot resolve the pinned Rust 1.94.1 toolchain (rustc --version failed with rustup auto-install disabled)." + append "- Fix: rebuild the trusted coverage image with the pinned toolchain, then rerun the current-head coverage job." + append "" + failures=$((failures + 1)) + return 1 + fi ensure_rust_gpu_adapter ensure_rust_desktop_deps } diff --git a/scripts/ci/materialize_base_python_requirements.py b/scripts/ci/materialize_base_python_requirements.py index 8158372df..715efac9c 100644 --- a/scripts/ci/materialize_base_python_requirements.py +++ b/scripts/ci/materialize_base_python_requirements.py @@ -15,6 +15,20 @@ SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") +RESOLVER_ONLY_EXACT_OPTIONS = frozenset( + {"--require-hashes", "--no-index", "--prefer-binary", "--pre"} +) +RESOLVER_ONLY_OPTION_PREFIXES = ( + "--index-url ", + "--extra-index-url ", + "--find-links ", + "--trusted-host ", + "--only-binary ", + "--only-binary=", + "--no-binary ", + "--no-binary=", +) + UV_EXPORT_TIMEOUT_SECONDS = 120 @@ -26,6 +40,24 @@ def _is_candidate_lock_name(name: str) -> bool: ) +def _is_candidate_lock_path(candidate: pathlib.PurePosixPath) -> bool: + """Return whether a repository path can hold a pip requirements lock. + + Besides ``requirements*.txt`` basenames anywhere in the tree, a repository + may keep its compiled locks inside a ``requirements/`` directory under + other basenames (for example ``requirements/ci.txt`` compiled from + ``requirements/ci.in``). Those are name candidates too; the content-based + hash-pin check remains the safety gate that excludes unpinned inputs. + """ + if _is_candidate_lock_name(candidate.name): + return True + return ( + len(candidate.parts) >= 2 + and candidate.parts[-2] == "requirements" + and candidate.name.endswith(".txt") + ) + + def _requirement_lines(content: bytes) -> list[str]: """Return logical requirement lines, joining backslash line-continuations. @@ -46,24 +78,36 @@ def _requirement_lines(content: bytes) -> list[str]: def _is_hash_pinned(content: bytes) -> bool: - """Return whether content carries hash pins and is safe to preflight. + """Return whether every installable requirement is hash-pinned. Discovery is content-based rather than name-based so hash-pinned locks in any - location (a service subdirectory, ``requirements-dev.txt``, - ``requirements-test.txt``) can be considered for offline coverage, while an - unpinned or PR-mutable requirements file is still excluded from the networked - build context. Hash syntax cannot prove that a file includes every transitive - dependency, so the trusted image installer separately preflights every - candidate as an independent ``--require-hashes`` closure. An empty file - carries no installable dependency and is not materialized. + location can be considered for offline coverage. Resolver-only pip options do + not count as dependency evidence, and ``--require-hashes`` is only a policy + directive: it cannot replace an actual ``--hash=`` on each package line. + Requirement-file includes remain eligible because the trusted installer + independently executes every candidate with pip's fail-closed + ``--require-hashes`` enforcement. Empty, option-only, editable, unknown-option, + and unpinned files are excluded from the networked build context. """ lines = _requirement_lines(content) if not lines: return False - return any(line == "--require-hashes" for line in lines) or all( - "--hash=" in line or line.startswith(("-r ", "--requirement ")) - for line in lines - ) + + has_install_target = False + for line in lines: + if line in RESOLVER_ONLY_EXACT_OPTIONS or line.startswith( + RESOLVER_ONLY_OPTION_PREFIXES + ): + continue + if line.startswith(("-r ", "--requirement ")): + has_install_target = True + continue + if line.startswith("-"): + return False + has_install_target = True + if "--hash=" not in line: + return False + return has_install_target def _git(repo_root: pathlib.Path, *args: str) -> bytes: @@ -184,9 +228,10 @@ def base_hash_locks(repo_root: pathlib.Path, base_sha: str) -> list[tuple[str, b or not mode.startswith("100") or candidate.is_absolute() or ".." in candidate.parts + or not (_is_candidate_lock_path(candidate) or candidate.name == "uv.lock") ): continue - if _is_candidate_lock_name(candidate.name): + if _is_candidate_lock_path(candidate): content = _git(repo_root, "show", f"{base_sha}:{path}") if _is_hash_pinned(content): locks.append((path, content)) diff --git a/tests/test_materialize_base_python_requirements.py b/tests/test_materialize_base_python_requirements.py index 41b86b261..9077164a4 100644 --- a/tests/test_materialize_base_python_requirements.py +++ b/tests/test_materialize_base_python_requirements.py @@ -137,10 +137,57 @@ def test_lock_name_candidates_are_pip_requirements_files() -> None: assert not materializer._is_candidate_lock_name("pyproject.toml") +def test_lock_path_candidates_include_requirements_directory_txt_files() -> None: + """Any .txt inside a requirements/ directory is a name candidate.""" + from pathlib import PurePosixPath + + assert materializer._is_candidate_lock_path(PurePosixPath("requirements/ci.txt")) + assert materializer._is_candidate_lock_path( + PurePosixPath("subproject/requirements/package.txt") + ) + assert materializer._is_candidate_lock_path(PurePosixPath("requirements-dev.txt")) + assert not materializer._is_candidate_lock_path(PurePosixPath("requirements/ci.in")) + assert not materializer._is_candidate_lock_path(PurePosixPath("docs/notes.txt")) + assert not materializer._is_candidate_lock_path(PurePosixPath("ci.txt")) + + +def test_materializes_hash_pinned_locks_inside_a_requirements_directory( + tmp_path: Path, +) -> None: + """A requirements/ci.txt-style compiled lock is discovered and materialized. + + Repositories such as fast-mlsirm keep pip-compile outputs under a + ``requirements/`` directory with non-``requirements*`` basenames; the + hash-pinned compiled lock must be installed for offline coverage while its + unpinned ``.in`` input stays excluded by content. + """ + repo = tmp_path / "repo" + repo.mkdir() + git(repo, "init") + git(repo, "config", "user.name", "Test") + git(repo, "config", "user.email", "test@example.invalid") + + requirements_dir = repo / "requirements" + requirements_dir.mkdir() + (requirements_dir / "ci.txt").write_text( + "numpy==2 --hash=sha256:" + ("a" * 64) + "\n", + encoding="utf-8", + ) + (requirements_dir / "ci.in").write_text("numpy\n", encoding="utf-8") + git(repo, "add", ".") + git(repo, "commit", "-m", "base") + base_sha = git(repo, "rev-parse", "HEAD") + + output = tmp_path / "output" + manifest = materializer.materialize(repo, base_sha, output) + + assert [entry["source"] for entry in manifest] == ["requirements/ci.txt"] + + def test_hash_pin_detection_includes_pinned_and_excludes_unpinned_or_empty() -> None: """Only fully hash-pinned, non-empty lock content is materialized.""" assert not materializer._is_hash_pinned(b"# comment only\n\n") - assert materializer._is_hash_pinned(b"--require-hashes\ndemo==1\n") + assert not materializer._is_hash_pinned(b"--require-hashes\ndemo==1\n") assert materializer._is_hash_pinned(b"demo==1 --hash=sha256:" + b"a" * 64 + b"\n") assert materializer._is_hash_pinned(b"-r other-hashes.txt\n") assert not materializer._is_hash_pinned(b"untrusted==1\n") diff --git a/tests/test_materialize_base_python_requirements_security.py b/tests/test_materialize_base_python_requirements_security.py new file mode 100644 index 000000000..642f5f223 --- /dev/null +++ b/tests/test_materialize_base_python_requirements_security.py @@ -0,0 +1,33 @@ +"""Security regressions for trusted Python requirements lock materialization.""" + +from scripts.ci import materialize_base_python_requirements as materializer + + +def test_require_hashes_directive_does_not_replace_per_requirement_hashes() -> None: + """A directive alone cannot authorize an unpinned package requirement.""" + assert not materializer._is_hash_pinned( + b"--require-hashes\nrequests==2.31.0\n" + ) + + +def test_require_hashes_directive_accepts_an_actually_hashed_requirement() -> None: + """Resolver metadata may accompany a package line carrying an actual hash.""" + assert materializer._is_hash_pinned( + b"--require-hashes\n" + b"--index-url https://pypi.org/simple\n" + b"requests==2.31.0 --hash=sha256:" + + b"a" * 64 + + b"\n" + ) + + +def test_directives_without_an_install_target_are_not_materialized() -> None: + """An option-only file carries no dependency closure and remains excluded.""" + assert not materializer._is_hash_pinned( + b"--require-hashes\n--no-index\n--prefer-binary\n" + ) + + +def test_unhashed_editable_requirement_remains_rejected() -> None: + """Install-target options cannot be mistaken for harmless resolver metadata.""" + assert not materializer._is_hash_pinned(b"--require-hashes\n--editable .\n")