From 3a274ab95bc98690689466d31dbd54e2717bde9a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 19:02:06 +0900 Subject: [PATCH 01/13] test(ci): reproduce hourly GitHub API gate incident --- .../test_hourly_product_incident_contract.py | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 services/account_unification/tests/test_hourly_product_incident_contract.py diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py new file mode 100644 index 0000000..0c944c9 --- /dev/null +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -0,0 +1,193 @@ +"""Incident regressions for the fail-closed hourly product-development workflow.""" +from __future__ import annotations + +from pathlib import Path + +import yaml + + +EXPECTED_ENDPOINTS = { + "develop-product-gap": ( + "api.github.com:443", + "cafe.github.com:443", + "codeload.github.com:443", + "github.com:443", + "integrate.api.nvidia.com:443", + "objects.githubusercontent.com:443", + "raw.githubusercontent.com:443", + "registry.npmjs.org:443", + "release-assets.githubusercontent.com:443", + "releases.astral.sh:443", + "results-receiver.actions.githubusercontent.com:443", + "*.actions.githubusercontent.com:443", + "*.blob.core.windows.net:443", + "files.pythonhosted.org:443", + "pypi.org:443", + ), + "reverify-product-gap": ( + "api.github.com:443", + "cafe.github.com:443", + "github.com:443", + "objects.githubusercontent.com:443", + "raw.githubusercontent.com:443", + "release-assets.githubusercontent.com:443", + "releases.astral.sh:443", + "results-receiver.actions.githubusercontent.com:443", + "*.actions.githubusercontent.com:443", + "*.blob.core.windows.net:443", + "files.pythonhosted.org:443", + "pypi.org:443", + ), + "publish-product-gap": ( + "api.github.com:443", + "cafe.github.com:443", + "github.com:443", + "objects.githubusercontent.com:443", + "results-receiver.actions.githubusercontent.com:443", + "*.actions.githubusercontent.com:443", + "*.blob.core.windows.net:443", + ), +} + + +def _repository_root() -> Path: + """Return the Keyverse repository root from this test module.""" + return Path(__file__).resolve().parents[3] + + +def _workflow_source() -> str: + """Return the hourly product-development workflow as reviewed text.""" + return ( + _repository_root() + / ".github" + / "workflows" + / "hourly-product-development.yml" + ).read_text(encoding="utf-8") + + +def _workflow_document() -> dict[str, object]: + """Parse the hourly workflow into a mapping for structural assertions.""" + document = yaml.safe_load(_workflow_source()) + assert isinstance(document, dict) + return document + + +def _job(job_name: str) -> dict[str, object]: + """Return one named workflow job as a mapping.""" + jobs = _workflow_document().get("jobs") + assert isinstance(jobs, dict) + job = jobs.get(job_name) + assert isinstance(job, dict) + return job + + +def _steps(job_name: str) -> list[dict[str, object]]: + """Return mapping-valued steps for one named job.""" + steps = _job(job_name).get("steps") + assert isinstance(steps, list) + return [step for step in steps if isinstance(step, dict)] + + +def _step_by_id(job_name: str, step_id: str) -> dict[str, object]: + """Return the exact step carrying ``step_id`` in ``job_name``.""" + for step in _steps(job_name): + if step.get("id") == step_id: + return step + raise AssertionError(f"{job_name} has no step id {step_id}") + + +def _step_by_name(job_name: str, step_name: str) -> dict[str, object]: + """Return the exact step named ``step_name`` in ``job_name``.""" + for step in _steps(job_name): + if step.get("name") == step_name: + return step + raise AssertionError(f"{job_name} has no step named {step_name}") + + +def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: + """Return the exact ordered Harden Runner endpoint allowlist for a job.""" + for step in _steps(job_name): + action = step.get("uses") + if not isinstance(action, str) or not action.startswith( + "step-security/harden-runner@" + ): + continue + inputs = step.get("with") + assert isinstance(inputs, dict) + assert inputs.get("egress-policy") == "block" + endpoint_block = inputs.get("allowed-endpoints") + assert isinstance(endpoint_block, str) + return tuple( + line.strip() for line in endpoint_block.splitlines() if line.strip() + ) + raise AssertionError(f"{job_name} has no Harden Runner step") + + +def test_github_api_jobs_use_exact_fail_closed_endpoint_sets() -> None: + """Every GitHub-API phase permits only its reviewed exact endpoint set.""" + for job_name, expected in EXPECTED_ENDPOINTS.items(): + actual = _harden_runner_endpoints(job_name) + assert actual == expected + assert "api.github.com:443.evil" not in actual + assert "*.github.com:443" not in actual + + +def test_deterministic_repository_gates_precede_optional_model_credential() -> None: + """Queue, main, release evidence, and dry-run gates run before model access.""" + gate = _step_by_id("develop-product-gap", "gate") + gate_run = gate.get("run") + gate_env = gate.get("env") + assert isinstance(gate_run, str) + assert isinstance(gate_env, dict) + + ordered_markers = ( + "pulls?state=open&per_page=1", + "commits/${DEFAULT_BRANCH}", + "actions/runs?branch=${DEFAULT_BRANCH}", + "commits/${base_sha}/check-runs?per_page=100", + 'if [ "$DRY_RUN" = "true" ]; then', + ) + positions = tuple(gate_run.index(marker) for marker in ordered_markers) + assert positions == tuple(sorted(positions)) + assert "NIM_UPSTREAM_API_KEY" not in gate_env + assert "NIM_UPSTREAM_API_KEY" not in gate_run + assert "NVIDIA_NIM_API_KEY" not in gate_run + + +def test_github_inventory_transport_failures_are_not_false_green() -> None: + """GitHub inventory transport/shape failures terminate the gate unsuccessfully.""" + gate = _step_by_id("develop-product-gap", "gate") + gate_run = gate.get("run") + assert isinstance(gate_run, str) + + failure_messages = ( + "Unable to list open pull requests", + "Unable to interpret the open pull-request response", + "Unable to resolve the default-branch head", + "The default-branch head was malformed", + "Unable to read default-branch workflow evidence", + "Unable to read default-branch check evidence", + ) + for message in failure_messages: + marker = f'echo "::error::{message}' + start = gate_run.index(marker) + branch_tail = gate_run[start : start + 320] + assert "exit 1" in branch_tail + assert f"::warning::{message}" not in gate_run + + +def test_nvidia_secret_is_required_only_on_the_model_backed_path() -> None: + """The NVIDIA secret is checked only after deterministic gates select development.""" + broker = _step_by_name( + "develop-product-gap", + "Start the loopback-only NIM credential broker", + ) + broker_env = broker.get("env") + broker_run = broker.get("run") + assert isinstance(broker_env, dict) + assert isinstance(broker_run, str) + + assert broker_env.get("NIM_UPSTREAM_API_KEY") == "${{ secrets.NVIDIA_NIM_API_KEY }}" + assert 'if [ -z "${NIM_UPSTREAM_API_KEY:-}" ]; then' in broker_run + assert "NVIDIA_NIM_API_KEY is required only for model-backed development" in broker_run + assert "exit 1" in broker_run From a1481e116519f9857b4fc3595c49787348818314 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 19:04:33 +0900 Subject: [PATCH 02/13] fix(ci): fail closed on GitHub inventory egress --- .../workflows/hourly-product-development.yml | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/.github/workflows/hourly-product-development.yml b/.github/workflows/hourly-product-development.yml index 0699637..2424656 100644 --- a/.github/workflows/hourly-product-development.yml +++ b/.github/workflows/hourly-product-development.yml @@ -55,6 +55,7 @@ jobs: disable-telemetry: true allowed-endpoints: | api.github.com:443 + cafe.github.com:443 codeload.github.com:443 github.com:443 integrate.api.nvidia.com:443 @@ -69,13 +70,11 @@ jobs: files.pythonhosted.org:443 pypi.org:443 - - - name: Enforce the credential, queue, and exact-main gate + - name: Enforce the deterministic queue and exact-main gate id: gate shell: bash env: GH_TOKEN: ${{ github.token }} - NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} CURRENT_RUN_ID: ${{ github.run_id }} DRY_RUN: ${{ inputs.dry_run || false }} run: | @@ -85,19 +84,13 @@ jobs: echo "base_sha=" } >>"$GITHUB_OUTPUT" - if [ -z "${NIM_UPSTREAM_API_KEY:-}" ]; then - echo "NVIDIA_NIM_API_KEY is not configured; autonomous development stopped safely." \ - >>"$GITHUB_STEP_SUMMARY" - exit 0 - fi - open_pr_file="${RUNNER_TEMP}/keyverse-open-pulls.json" if ! gh api \ -H "Accept: application/vnd.github+json" \ "repos/${GITHUB_REPOSITORY}/pulls?state=open&per_page=1" \ >"$open_pr_file"; then - echo "::warning::Unable to list open pull requests; refusing to create work." - exit 0 + echo "::error::Unable to list open pull requests; GitHub inventory is unavailable." + exit 1 fi if ! open_pr_count="$(python3 - "$open_pr_file" <<'PY' import json @@ -110,8 +103,8 @@ jobs: print(len(payload)) PY )"; then - echo "::warning::Unable to interpret the open pull-request response; refusing to create work." - exit 0 + echo "::error::Unable to interpret the open pull-request response; GitHub inventory is malformed." + exit 1 fi if [ "$open_pr_count" -ne 0 ]; then echo "An open pull request exists; the protected PR loop owns this hour." \ @@ -125,12 +118,12 @@ jobs: "repos/${GITHUB_REPOSITORY}/commits/${DEFAULT_BRANCH}" \ --jq '.sha' )"; then - echo "::warning::Unable to resolve the default-branch head; refusing to create work." - exit 0 + echo "::error::Unable to resolve the default-branch head; GitHub inventory is unavailable." + exit 1 fi if ! [[ "$base_sha" =~ ^[0-9a-f]{40}$ ]]; then - echo "::warning::The default-branch head was malformed; refusing to create work." - exit 0 + echo "::error::The default-branch head was malformed; GitHub inventory is invalid." + exit 1 fi workflow_runs_file="${RUNNER_TEMP}/keyverse-main-workflow-runs.json" @@ -140,8 +133,8 @@ jobs: --slurp \ "repos/${GITHUB_REPOSITORY}/actions/runs?branch=${DEFAULT_BRANCH}&head_sha=${base_sha}&per_page=100" \ >"$workflow_runs_file"; then - echo "::warning::Unable to read default-branch workflow evidence; refusing to create work." - exit 0 + echo "::error::Unable to read default-branch workflow evidence; GitHub inventory is unavailable." + exit 1 fi if ! python3 - "$workflow_runs_file" "$CORE_WORKFLOWS" <<'PY' import json @@ -203,8 +196,8 @@ jobs: --slurp \ "repos/${GITHUB_REPOSITORY}/commits/${base_sha}/check-runs?per_page=100" \ >"$check_runs_file"; then - echo "::warning::Unable to read default-branch check evidence; refusing to create work." - exit 0 + echo "::error::Unable to read default-branch check evidence; GitHub inventory is unavailable." + exit 1 fi if ! python3 - "$check_runs_file" "$CURRENT_RUN_ID" <<'PY' import json @@ -261,7 +254,7 @@ jobs: fi if [ "$DRY_RUN" = "true" ]; then - echo "Dry run: the NVIDIA NIM OpenCode development gate is ready." \ + echo "Dry run: deterministic repository gates are healthy; model access was not requested." \ >>"$GITHUB_STEP_SUMMARY" exit 0 fi @@ -396,6 +389,10 @@ jobs: NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} run: | set -euo pipefail + if [ -z "${NIM_UPSTREAM_API_KEY:-}" ]; then + echo "::error::NVIDIA_NIM_API_KEY is required only for model-backed development." + exit 1 + fi umask 077 proxy_log="${RUNNER_TEMP}/keyverse-nim-proxy.log" proxy_pid="${RUNNER_TEMP}/keyverse-nim-proxy.pid" @@ -587,6 +584,7 @@ jobs: disable-telemetry: true allowed-endpoints: | api.github.com:443 + cafe.github.com:443 github.com:443 objects.githubusercontent.com:443 raw.githubusercontent.com:443 @@ -598,7 +596,6 @@ jobs: files.pythonhosted.org:443 pypi.org:443 - - name: Check out a fresh protected branch uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -728,6 +725,7 @@ jobs: disable-telemetry: true allowed-endpoints: | api.github.com:443 + cafe.github.com:443 github.com:443 objects.githubusercontent.com:443 results-receiver.actions.githubusercontent.com:443 From d1802174f73b5b2a3b7f8511e7816c8763d01ec0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 19:06:08 +0900 Subject: [PATCH 03/13] test(ci): align hourly model-path credential contract --- .../tests/test_hourly_product_development.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_development.py b/services/account_unification/tests/test_hourly_product_development.py index 328ed50..8455311 100644 --- a/services/account_unification/tests/test_hourly_product_development.py +++ b/services/account_unification/tests/test_hourly_product_development.py @@ -145,10 +145,10 @@ def test_product_development_does_not_reuse_review_agent_credentials() -> None: def test_product_development_fails_closed_without_queue_ownership() -> None: - """Missing NIM access, unhealthy main, or open work suppresses the agent.""" + """Unhealthy main or open work stops before entering the model-backed path.""" workflow = _workflow_source() - assert "NVIDIA_NIM_API_KEY is not configured" in workflow + assert "NVIDIA_NIM_API_KEY is required only for model-backed development" in workflow assert "pulls?state=open&per_page=1" in workflow assert "An open pull request exists" in workflow assert "CORE_WORKFLOWS" in workflow From da5c8a5722e7e75de3fb82b486a0c1e087c2fb02 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 20:05:53 +0900 Subject: [PATCH 04/13] test(ci): require runtime-safe Harden Runner endpoint scalar --- .../test_hourly_product_incident_contract.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py index 0c944c9..1ece947 100644 --- a/services/account_unification/tests/test_hourly_product_incident_contract.py +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -104,8 +104,8 @@ def _step_by_name(job_name: str, step_name: str) -> dict[str, object]: raise AssertionError(f"{job_name} has no step named {step_name}") -def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: - """Return the exact ordered Harden Runner endpoint allowlist for a job.""" +def _harden_runner_endpoint_scalar(job_name: str) -> str: + """Return the serialized Harden Runner endpoint input for one workflow job.""" for step in _steps(job_name): action = step.get("uses") if not isinstance(action, str) or not action.startswith( @@ -117,12 +117,15 @@ def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: assert inputs.get("egress-policy") == "block" endpoint_block = inputs.get("allowed-endpoints") assert isinstance(endpoint_block, str) - return tuple( - line.strip() for line in endpoint_block.splitlines() if line.strip() - ) + return endpoint_block raise AssertionError(f"{job_name} has no Harden Runner step") +def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: + """Return the exact ordered Harden Runner endpoint allowlist for a job.""" + return tuple(_harden_runner_endpoint_scalar(job_name).split()) + + def test_github_api_jobs_use_exact_fail_closed_endpoint_sets() -> None: """Every GitHub-API phase permits only its reviewed exact endpoint set.""" for job_name, expected in EXPECTED_ENDPOINTS.items(): @@ -132,6 +135,14 @@ def test_github_api_jobs_use_exact_fail_closed_endpoint_sets() -> None: assert "*.github.com:443" not in actual +def test_harden_runner_endpoint_input_is_space_delimited_for_runtime() -> None: + """Harden Runner receives one folded, space-delimited endpoint scalar per job.""" + for job_name, expected in EXPECTED_ENDPOINTS.items(): + endpoint_scalar = _harden_runner_endpoint_scalar(job_name) + assert endpoint_scalar == " ".join(expected) + assert "\n" not in endpoint_scalar + + def test_deterministic_repository_gates_precede_optional_model_credential() -> None: """Queue, main, release evidence, and dry-run gates run before model access.""" gate = _step_by_id("develop-product-gap", "gate") From f0d3bfbb6580926dd81a9f16de48680041b8dce9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 21:10:50 +0900 Subject: [PATCH 05/13] test(ci): bind NVIDIA secret to model-backed step --- .../tests/test_hourly_product_development.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_development.py b/services/account_unification/tests/test_hourly_product_development.py index 8455311..1d4b711 100644 --- a/services/account_unification/tests/test_hourly_product_development.py +++ b/services/account_unification/tests/test_hourly_product_development.py @@ -55,6 +55,21 @@ def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: raise AssertionError(f"{job_name} has no harden-runner endpoint policy") +def _step_by_name(job_name: str, step_name: str) -> dict[str, object]: + """Return one exact named workflow step from ``job_name``.""" + jobs = _workflow_document().get("jobs") + assert isinstance(jobs, dict) + job = jobs.get(job_name) + assert isinstance(job, dict) + steps = job.get("steps") + assert isinstance(steps, list) + + for step in steps: + if isinstance(step, dict) and step.get("name") == step_name: + return step + raise AssertionError(f"{job_name} has no step named {step_name}") + + def _permissions_block(source: str, marker: str, terminator: str) -> str: """Return one indentation-sensitive workflow permissions block.""" block_start = source.index(marker) @@ -147,8 +162,20 @@ def test_product_development_does_not_reuse_review_agent_credentials() -> None: def test_product_development_fails_closed_without_queue_ownership() -> None: """Unhealthy main or open work stops before entering the model-backed path.""" workflow = _workflow_source() - - assert "NVIDIA_NIM_API_KEY is required only for model-backed development" in workflow + broker = _step_by_name( + "develop-product-gap", + "Start the loopback-only NIM credential broker", + ) + broker_env = broker.get("env") + broker_run = broker.get("run") + + assert isinstance(broker_env, dict) + assert isinstance(broker_run, str) + assert broker.get("if") == "steps.gate.outputs.develop == 'true'" + assert broker_env.get("NIM_UPSTREAM_API_KEY") == ( + "${{ secrets.NVIDIA_NIM_API_KEY }}" + ) + assert "NVIDIA_NIM_API_KEY is required only for model-backed development" in broker_run assert "pulls?state=open&per_page=1" in workflow assert "An open pull request exists" in workflow assert "CORE_WORKFLOWS" in workflow From dee4419a4d0857de7f0507f69fa2747606f2ecde Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 21:11:27 +0900 Subject: [PATCH 06/13] test(ci): fail closed on malformed GitHub evidence --- .../test_hourly_product_incident_contract.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py index 1ece947..e6c9187 100644 --- a/services/account_unification/tests/test_hourly_product_incident_contract.py +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -186,6 +186,36 @@ def test_github_inventory_transport_failures_are_not_false_green() -> None: assert "exit 1" in branch_tail assert f"::warning::{message}" not in gate_run + malformed_evidence_contracts = ( + ( + "Unsupported workflow-run response shape", + "raise SystemExit(2)", + "workflow_evidence_status", + "Unable to interpret default-branch workflow evidence", + ), + ( + "Unsupported check-run response shape", + "raise SystemExit(2)", + "check_evidence_status", + "Unable to interpret default-branch check evidence", + ), + ) + for parser_marker, malformed_exit, status_name, error_message in ( + malformed_evidence_contracts + ): + parser_start = gate_run.index(parser_marker) + parser_tail = gate_run[parser_start : parser_start + 180] + assert malformed_exit in parser_tail + assert f"{status_name}=$?" in gate_run + assert f"::error::{error_message}" in gate_run + + assert 'workflow_evidence_status=3' not in gate_run + assert 'check_evidence_status=3' not in gate_run + assert "Missing required default-branch workflow evidence" in gate_run + assert "Default branch has pending or unsuccessful required workflow evidence" in gate_run + assert "Missing latest default-branch check evidence" in gate_run + assert "Default branch has pending or unsuccessful latest check evidence" in gate_run + def test_nvidia_secret_is_required_only_on_the_model_backed_path() -> None: """The NVIDIA secret is checked only after deterministic gates select development.""" From f9f69ddef213a4e4f0566cfb02576426c9f5f48e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 21:13:19 +0900 Subject: [PATCH 07/13] fix(ci): preserve Harden Runner ports and malformed evidence failures --- .../workflows/hourly-product-development.yml | 97 +++++++++++++------ 1 file changed, 68 insertions(+), 29 deletions(-) diff --git a/.github/workflows/hourly-product-development.yml b/.github/workflows/hourly-product-development.yml index 2424656..0493a61 100644 --- a/.github/workflows/hourly-product-development.yml +++ b/.github/workflows/hourly-product-development.yml @@ -53,7 +53,7 @@ jobs: with: egress-policy: block disable-telemetry: true - allowed-endpoints: | + allowed-endpoints: >- api.github.com:443 cafe.github.com:443 codeload.github.com:443 @@ -136,22 +136,27 @@ jobs: echo "::error::Unable to read default-branch workflow evidence; GitHub inventory is unavailable." exit 1 fi - if ! python3 - "$workflow_runs_file" "$CORE_WORKFLOWS" <<'PY' + workflow_evidence_status=0 + python3 - "$workflow_runs_file" "$CORE_WORKFLOWS" <<'PY' || workflow_evidence_status=$? import json import sys - with open(sys.argv[1], encoding="utf-8") as stream: - pages = json.load(stream) - required = set(json.loads(sys.argv[2])) + try: + with open(sys.argv[1], encoding="utf-8") as stream: + pages = json.load(stream) + required = set(json.loads(sys.argv[2])) + except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc: + print(f"Malformed workflow-run evidence: {exc}", file=sys.stderr) + raise SystemExit(2) from exc if not isinstance(pages, list) or not required: - raise SystemExit("Unsupported workflow-run response shape") + raise SystemExit(2) runs = [] for page in pages: if not isinstance(page, dict) or not isinstance( page.get("workflow_runs"), list ): - raise SystemExit("Unsupported workflow-run response shape") + raise SystemExit(2) runs.extend(page["workflow_runs"]) latest = {} @@ -168,10 +173,12 @@ jobs: missing = sorted(required.difference(latest)) if missing: - raise SystemExit( + print( "Missing required default-branch workflow evidence: " - + ", ".join(missing) + + ", ".join(missing), + file=sys.stderr, ) + raise SystemExit(3) unhealthy = sorted( name for name, run in latest.items() @@ -179,15 +186,28 @@ jobs: or run.get("conclusion") != "success" ) if unhealthy: - raise SystemExit( + print( "Default branch has pending or unsuccessful required workflow evidence: " - + ", ".join(unhealthy) + + ", ".join(unhealthy), + file=sys.stderr, ) + raise SystemExit(3) PY - then - echo "::warning::Default-branch core workflow evidence is incomplete or unhealthy; refusing to create work." - exit 0 - fi + case "$workflow_evidence_status" in + 0) ;; + 2) + echo "::error::Unable to interpret default-branch workflow evidence; GitHub inventory is malformed." + exit 1 + ;; + 3) + echo "::warning::Default-branch core workflow evidence is incomplete or unhealthy; refusing to create work." + exit 0 + ;; + *) + echo "::error::Default-branch workflow evidence parser failed unexpectedly." + exit 1 + ;; + esac check_runs_file="${RUNNER_TEMP}/keyverse-main-check-runs.json" if ! gh api \ @@ -199,22 +219,27 @@ jobs: echo "::error::Unable to read default-branch check evidence; GitHub inventory is unavailable." exit 1 fi - if ! python3 - "$check_runs_file" "$CURRENT_RUN_ID" <<'PY' + check_evidence_status=0 + python3 - "$check_runs_file" "$CURRENT_RUN_ID" <<'PY' || check_evidence_status=$? import json import sys - with open(sys.argv[1], encoding="utf-8") as stream: - pages = json.load(stream) + try: + with open(sys.argv[1], encoding="utf-8") as stream: + pages = json.load(stream) + except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc: + print(f"Malformed check-run evidence: {exc}", file=sys.stderr) + raise SystemExit(2) from exc current_run_fragment = f"/actions/runs/{sys.argv[2]}" if not isinstance(pages, list): - raise SystemExit("Unsupported check-run response shape") + raise SystemExit(2) checks = [] for page in pages: if not isinstance(page, dict) or not isinstance( page.get("check_runs"), list ): - raise SystemExit("Unsupported check-run response shape") + raise SystemExit(2) checks.extend(page["check_runs"]) latest = {} @@ -234,7 +259,8 @@ jobs: latest[key] = check if not latest: - raise SystemExit("Missing latest default-branch check evidence") + print("Missing latest default-branch check evidence", file=sys.stderr) + raise SystemExit(3) accepted = {"success", "neutral", "skipped"} unhealthy = sorted( f"{key[0]}/{key[1]}" @@ -243,15 +269,28 @@ jobs: or check.get("conclusion") not in accepted ) if unhealthy: - raise SystemExit( + print( "Default branch has pending or unsuccessful latest check evidence: " - + ", ".join(unhealthy) + + ", ".join(unhealthy), + file=sys.stderr, ) + raise SystemExit(3) PY - then - echo "::warning::Default-branch check evidence is incomplete or unhealthy; refusing to create work." - exit 0 - fi + case "$check_evidence_status" in + 0) ;; + 2) + echo "::error::Unable to interpret default-branch check evidence; GitHub inventory is malformed." + exit 1 + ;; + 3) + echo "::warning::Default-branch check evidence is incomplete or unhealthy; refusing to create work." + exit 0 + ;; + *) + echo "::error::Default-branch check evidence parser failed unexpectedly." + exit 1 + ;; + esac if [ "$DRY_RUN" = "true" ]; then echo "Dry run: deterministic repository gates are healthy; model access was not requested." \ @@ -582,7 +621,7 @@ jobs: with: egress-policy: block disable-telemetry: true - allowed-endpoints: | + allowed-endpoints: >- api.github.com:443 cafe.github.com:443 github.com:443 @@ -723,7 +762,7 @@ jobs: with: egress-policy: block disable-telemetry: true - allowed-endpoints: | + allowed-endpoints: >- api.github.com:443 cafe.github.com:443 github.com:443 From da7e49ffd0d6ee5e3fb24931510018d69f2d568d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 21:15:33 +0900 Subject: [PATCH 08/13] test(ci): parse folded Harden Runner endpoints --- .../tests/test_hourly_product_development.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_development.py b/services/account_unification/tests/test_hourly_product_development.py index 1d4b711..b0d6929 100644 --- a/services/account_unification/tests/test_hourly_product_development.py +++ b/services/account_unification/tests/test_hourly_product_development.py @@ -49,9 +49,7 @@ def _harden_runner_endpoints(job_name: str) -> tuple[str, ...]: assert isinstance(inputs, dict) endpoint_block = inputs.get("allowed-endpoints") assert isinstance(endpoint_block, str) - return tuple( - line.strip() for line in endpoint_block.splitlines() if line.strip() - ) + return tuple(endpoint_block.split()) raise AssertionError(f"{job_name} has no harden-runner endpoint policy") From f24d4b46e344f0f2c375c65fafcb3d9410284fa2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 8 Aug 2026 21:16:04 +0900 Subject: [PATCH 09/13] test(ci): bind malformed evidence assertions to parser structure --- .../tests/test_hourly_product_incident_contract.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py index e6c9187..c4fdd1f 100644 --- a/services/account_unification/tests/test_hourly_product_incident_contract.py +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -188,13 +188,13 @@ def test_github_inventory_transport_failures_are_not_false_green() -> None: malformed_evidence_contracts = ( ( - "Unsupported workflow-run response shape", + "if not isinstance(pages, list) or not required:", "raise SystemExit(2)", "workflow_evidence_status", "Unable to interpret default-branch workflow evidence", ), ( - "Unsupported check-run response shape", + "if not isinstance(pages, list):", "raise SystemExit(2)", "check_evidence_status", "Unable to interpret default-branch check evidence", From d6eea5e47a8d7ac0f6f0f3fce3991c6fd1ed096d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 9 Aug 2026 00:33:16 +0900 Subject: [PATCH 10/13] test(ci): expose hourly feasibility gaps --- .../test_hourly_product_incident_contract.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py index c4fdd1f..22f493c 100644 --- a/services/account_unification/tests/test_hourly_product_incident_contract.py +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -217,6 +217,50 @@ def test_github_inventory_transport_failures_are_not_false_green() -> None: assert "Default branch has pending or unsuccessful latest check evidence" in gate_run +def test_default_branch_check_evidence_requires_success() -> None: + """Neutral or skipped default-main checks never qualify as healthy evidence.""" + gate = _step_by_id("develop-product-gap", "gate") + gate_run = gate.get("run") + assert isinstance(gate_run, str) + + accepted_start = gate_run.index("accepted =") + accepted_block = gate_run[accepted_start : accepted_start + 120] + assert 'accepted = {"success"}' in accepted_block + assert '"neutral"' not in accepted_block + assert '"skipped"' not in accepted_block + + +def test_model_fallback_budget_fits_outer_job_timeout() -> None: + """All sequential model candidates plus setup reserve fit the job deadline.""" + document = _workflow_document() + env = document.get("env") + assert isinstance(env, dict) + candidates = str(env.get("OPENCODE_MODEL_CANDIDATES", "")).split() + per_model_seconds = int(str(env.get("OPENCODE_RUN_TIMEOUT_SECONDS", "0"))) + timeout_minutes = int(str(_job("develop-product-gap").get("timeout-minutes", 0))) + + assert candidates + setup_and_packaging_reserve_seconds = 15 * 60 + assert timeout_minutes * 60 >= ( + len(candidates) * per_model_seconds + setup_and_packaging_reserve_seconds + ) + + +def test_nvidia_secret_is_materialized_only_by_broker() -> None: + """The raw NVIDIA secret exists only in the conditional loopback broker step.""" + secret_expression = "${{ secrets.NVIDIA_NIM_API_KEY }}" + materializing_steps: list[str] = [] + for step in _steps("develop-product-gap"): + env = step.get("env") + if not isinstance(env, dict) or secret_expression not in env.values(): + continue + name = step.get("name") + assert isinstance(name, str) + materializing_steps.append(name) + + assert materializing_steps == ["Start the loopback-only NIM credential broker"] + + def test_nvidia_secret_is_required_only_on_the_model_backed_path() -> None: """The NVIDIA secret is checked only after deterministic gates select development.""" broker = _step_by_name( From 35975b79feef5bd6714013ae38bd584b4d9e18c8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 9 Aug 2026 00:39:10 +0900 Subject: [PATCH 11/13] fix(ci): close hourly feasibility gaps --- .github/workflows/hourly-product-development.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/hourly-product-development.yml b/.github/workflows/hourly-product-development.yml index 0493a61..da09279 100644 --- a/.github/workflows/hourly-product-development.yml +++ b/.github/workflows/hourly-product-development.yml @@ -37,7 +37,7 @@ jobs: develop-product-gap: name: Discover and package one bounded product gap runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 180 permissions: actions: read checks: read @@ -261,7 +261,7 @@ jobs: if not latest: print("Missing latest default-branch check evidence", file=sys.stderr) raise SystemExit(3) - accepted = {"success", "neutral", "skipped"} + accepted = {"success"} unhealthy = sorted( f"{key[0]}/{key[1]}" for key, check in latest.items() @@ -578,8 +578,6 @@ jobs: if: steps.gate.outputs.develop == 'true' id: package shell: bash - env: - KEYVERSE_FORBIDDEN_SECRET: ${{ secrets.NVIDIA_NIM_API_KEY }} run: | set -euo pipefail artifact_dir="${RUNNER_TEMP}/hourly-product-change" From 2b572dd20c45e91853bdb9c20ba4f7b2bc6c625c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 9 Aug 2026 00:41:11 +0900 Subject: [PATCH 12/13] test(ci): align hourly boundary contracts --- .../tests/test_hourly_product_development.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/account_unification/tests/test_hourly_product_development.py b/services/account_unification/tests/test_hourly_product_development.py index b0d6929..eb50768 100644 --- a/services/account_unification/tests/test_hourly_product_development.py +++ b/services/account_unification/tests/test_hourly_product_development.py @@ -82,7 +82,7 @@ def test_product_development_runs_hourly_without_cancelling_a_decision() -> None assert 'cron: "41 * * * *"' in workflow assert "hourly-product-development-${{ github.repository }}" in workflow assert "cancel-in-progress: false" in workflow - assert "timeout-minutes: 45" in workflow + assert "timeout-minutes: 180" in workflow assert "timeout-minutes: 30" in workflow assert "timeout-minutes: 15" in workflow @@ -141,9 +141,9 @@ def test_nim_credential_is_brokered_outside_the_agent_environment() -> None: assert "Start the loopback-only NIM credential broker" in workflow assert "NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}" in workflow + assert workflow.count("${{ secrets.NVIDIA_NIM_API_KEY }}") == 1 assert "NVIDIA_API_KEY=keyverse-local-broker" in workflow assert "env -i" in workflow - assert "KEYVERSE_FORBIDDEN_SECRET: ${{ secrets.NVIDIA_NIM_API_KEY }}" in workflow assert "NVIDIA_API_KEY=${{ secrets.NVIDIA_NIM_API_KEY }}" not in workflow From 30d8d6304e84b48a2710c1185c3f8d3395c36efc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 11 Aug 2026 19:53:18 +0900 Subject: [PATCH 13/13] fix(ci): keep NIM leak scanning credential-free --- .../workflows/hourly-product-development.yml | 24 +++++++++++ .../hourly-opencode-product-development.md | 5 ++- docs/operations/hourly-product-development.md | 16 +++++--- scripts/ci/hourly_product_guard.py | 41 ++++++++++++++++++- .../tests/test_hourly_product_guard.py | 18 ++++++++ .../test_hourly_product_incident_contract.py | 28 +++++++++++++ 6 files changed, 124 insertions(+), 8 deletions(-) diff --git a/.github/workflows/hourly-product-development.yml b/.github/workflows/hourly-product-development.yml index da09279..4351d54 100644 --- a/.github/workflows/hourly-product-development.yml +++ b/.github/workflows/hourly-product-development.yml @@ -423,6 +423,7 @@ jobs: - name: Start the loopback-only NIM credential broker if: steps.gate.outputs.develop == 'true' + id: nim_broker shell: bash env: NIM_UPSTREAM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} @@ -432,6 +433,26 @@ jobs: echo "::error::NVIDIA_NIM_API_KEY is required only for model-backed development." exit 1 fi + secret_fingerprint="$( + python3 - <<'PY' + import base64 + import hashlib + import os + + secret = os.environ["NIM_UPSTREAM_API_KEY"].encode("utf-8") + representations = ( + secret, + base64.b64encode(secret), + base64.urlsafe_b64encode(secret), + secret.hex().encode("ascii"), + ) + print(",".join( + f"{len(value)}:{hashlib.sha256(value).hexdigest()}" + for value in representations + )) + PY + )" + printf 'secret_fingerprint=%s\n' "$secret_fingerprint" >>"$GITHUB_OUTPUT" umask 077 proxy_log="${RUNNER_TEMP}/keyverse-nim-proxy.log" proxy_pid="${RUNNER_TEMP}/keyverse-nim-proxy.pid" @@ -440,6 +461,7 @@ jobs: --port "$NIM_PROXY_PORT" \ >"$proxy_log" 2>&1 & printf '%s\n' "$!" >"$proxy_pid" + unset NIM_UPSTREAM_API_KEY ready=false for _attempt in $(seq 1 30); do @@ -578,6 +600,8 @@ jobs: if: steps.gate.outputs.develop == 'true' id: package shell: bash + env: + KEYVERSE_FORBIDDEN_SECRET_FINGERPRINT: ${{ steps.nim_broker.outputs.secret_fingerprint }} run: | set -euo pipefail artifact_dir="${RUNNER_TEMP}/hourly-product-change" diff --git a/docs/doctoring/hourly-opencode-product-development.md b/docs/doctoring/hourly-opencode-product-development.md index 0546abd..25dbb16 100644 --- a/docs/doctoring/hourly-opencode-product-development.md +++ b/docs/doctoring/hourly-opencode-product-development.md @@ -12,7 +12,7 @@ patch across jobs, and independently re-runs the repository acceptance suite. | Control area | Repository implementation | | --- | --- | -| Least privilege | Read-only default `GITHUB_TOKEN`; upstream NIM and draft-PR publication use separate, step-scoped credentials. | +| Least privilege | Read-only default `GITHUB_TOKEN`; upstream NIM and draft-PR publication use separate, step-scoped credentials, and only broker-derived fingerprints cross the patch-scanning boundary. | | Untrusted AI output | No `.git` or GitHub/OIDC credentials in the model workspace; bounded path and patch validation; secrets and common encodings rejected. | | Supply-chain integrity | OpenCode and GitHub Actions are commit/digest pinned; generated patches are SHA-256 sealed and reverified on fresh checkouts. | | Verification | Realistic regression tests, 100% production docstrings, 100% statement and branch coverage, package/deployment validation, and exact-base race checks. | @@ -44,6 +44,9 @@ require a separately scoped assessment and evidence package. - Hosted Actions availability, provider availability, and organization secret configuration remain operational dependencies. - Scheduling and draft-PR creation are not release evidence. +- The post-model patch scanner intentionally receives only bounded + `length:sha256` fingerprints for the raw/common encoded NIM credential; it + must never be given the credential again merely to perform leak detection. ## References — APA 7th diff --git a/docs/operations/hourly-product-development.md b/docs/operations/hourly-product-development.md index e4fcee4..abb366a 100644 --- a/docs/operations/hourly-product-development.md +++ b/docs/operations/hourly-product-development.md @@ -37,9 +37,11 @@ publication token, or upstream NVIDIA key. ### `NVIDIA_NIM_API_KEY` -This repository secret is available only to the eligibility check, the local -credential broker, and the post-agent secret scanner. It is not placed in the -OpenCode process environment. The model process receives +This repository secret is available only to the local credential broker. The +broker derives one-way fingerprints for the raw and common encoded forms, +publishes only those fingerprints to the later patch scanner, and then removes +the raw value from its process environment. It is not placed in the OpenCode +process environment. The model process receives `NVIDIA_API_KEY=keyverse-local-broker` and sends requests to `http://127.0.0.1:8765/v1`. @@ -57,7 +59,10 @@ The broker: - limits request size, response size, and concurrent upstream requests. The patch guard rejects the raw key and common Base64, URL-safe Base64, and hex -representations from changed files, the generated patch, and PR metadata. +representations from changed files, the generated patch, and PR metadata when +the trusted broker can hold the raw key. The post-model scanner receives only +the broker-derived `length:sha256` fingerprints and hashes candidate +non-whitespace tokens; it never receives the raw key. ### `OPENCODE_PRODUCT_DEVELOPMENT_TOKEN` @@ -86,7 +91,8 @@ A run proceeds only when all of these statements are true. - The current `main` SHA can be resolved unambiguously. - The exact `main` SHA has completed successful `ci` and `CodeQL` push runs. - The latest check run for every observed app/name pair is complete with a - successful, neutral, or skipped conclusion. + `success` conclusion. Optional neutral/skipped checks are not treated as + required evidence by this gate. - The workflow is not a `dry_run` invocation. Failure to list or parse any required GitHub response stops development. The diff --git a/scripts/ci/hourly_product_guard.py b/scripts/ci/hourly_product_guard.py index 43b1409..c0ab167 100644 --- a/scripts/ci/hourly_product_guard.py +++ b/scripts/ci/hourly_product_guard.py @@ -155,7 +155,7 @@ def _changed_paths( def _forbidden_tokens() -> tuple[bytes, ...]: - """Return raw and common encoded forms of the model credential, if present.""" + """Return raw and common encoded forms when the raw credential is available.""" secret = os.environ.get("KEYVERSE_FORBIDDEN_SECRET", "").encode("utf-8") if not secret: return () @@ -171,10 +171,47 @@ def _forbidden_tokens() -> tuple[bytes, ...]: ) +def _forbidden_fingerprints() -> tuple[tuple[int, bytes], ...]: + """Parse broker-derived ``length:sha256`` credential fingerprints.""" + specification = os.environ.get("KEYVERSE_FORBIDDEN_SECRET_FINGERPRINT", "") + if not specification: + return () + fingerprints: list[tuple[int, bytes]] = [] + for item in specification.split(","): + parts = item.split(":") + if len(parts) != 2 or not parts[0].isdigit() or not re.fullmatch( + r"[0-9a-f]{64}", parts[1] + ): + raise BoundaryError("Malformed protected-credential fingerprint") + length = int(parts[0]) + if length < 1 or length > MAX_FILE_BYTES: + raise BoundaryError("Protected-credential fingerprint length is unsafe") + fingerprints.append((length, bytes.fromhex(parts[1]))) + return tuple(fingerprints) + + +def _contains_fingerprinted_token( + data: bytes, *, length: int, digest: bytes +) -> bool: + """Find an exact non-whitespace token using only its one-way fingerprint.""" + for chunk in data.split(): + if len(chunk) < length: + continue + for offset in range(len(chunk) - length + 1): + if hashlib.sha256(chunk[offset : offset + length]).digest() == digest: + return True + return False + + def _reject_forbidden_tokens(data: bytes, *, label: str) -> None: - """Reject a proposal containing the model credential or a common encoding.""" + """Reject raw, encoded, or broker-fingerprinted protected credentials.""" if any(token in data for token in _forbidden_tokens()): raise BoundaryError(f"Autonomous proposal exposed a protected credential in {label}") + if any( + _contains_fingerprinted_token(data, length=length, digest=digest) + for length, digest in _forbidden_fingerprints() + ): + raise BoundaryError(f"Autonomous proposal exposed a protected credential in {label}") def _read_proposal(workspace: Path) -> tuple[str, str]: diff --git a/services/account_unification/tests/test_hourly_product_guard.py b/services/account_unification/tests/test_hourly_product_guard.py index 5685402..8467fbb 100644 --- a/services/account_unification/tests/test_hourly_product_guard.py +++ b/services/account_unification/tests/test_hourly_product_guard.py @@ -1,6 +1,7 @@ """Behavior tests for the autonomous product patch boundary.""" from __future__ import annotations +import hashlib import importlib.util from pathlib import Path from types import ModuleType @@ -142,6 +143,23 @@ def test_guard_rejects_model_secret_in_raw_and_encoded_forms( guard._read_proposal(workspace) +def test_guard_rejects_model_secret_from_one_way_fingerprints( + monkeypatch, tmp_path +) -> None: + """Post-model validation detects a leaked token without receiving the secret.""" + guard = _load_guard() + secret = b"nim-sensitive-value" + fingerprint = f"{len(secret)}:{hashlib.sha256(secret).hexdigest()}" + monkeypatch.delenv("KEYVERSE_FORBIDDEN_SECRET", raising=False) + monkeypatch.setenv("KEYVERSE_FORBIDDEN_SECRET_FINGERPRINT", fingerprint) + + patch_path = tmp_path / "fingerprinted-secret.patch" + patch_path.write_bytes(b"safe prefix nim-sensitive-value safe suffix") + + with pytest.raises(guard.BoundaryError): + guard._reject_forbidden_tokens(patch_path.read_bytes(), label="patch") + + def test_guard_sanitizes_bounded_pull_request_metadata(tmp_path) -> None: """A model-authored title and body are strict UTF-8, bounded, and removed.""" guard = _load_guard() diff --git a/services/account_unification/tests/test_hourly_product_incident_contract.py b/services/account_unification/tests/test_hourly_product_incident_contract.py index 22f493c..bb1bb2f 100644 --- a/services/account_unification/tests/test_hourly_product_incident_contract.py +++ b/services/account_unification/tests/test_hourly_product_incident_contract.py @@ -261,6 +261,34 @@ def test_nvidia_secret_is_materialized_only_by_broker() -> None: assert materializing_steps == ["Start the loopback-only NIM credential broker"] +def test_nvidia_secret_fingerprint_crosses_the_broker_boundary() -> None: + """Packaging receives only broker-derived fingerprints for leak scanning.""" + broker = _step_by_name( + "develop-product-gap", + "Start the loopback-only NIM credential broker", + ) + package = _step_by_name( + "develop-product-gap", + "Capture the bounded credential-free patch", + ) + broker_run = broker.get("run") + package_env = package.get("env") + assert isinstance(broker_run, str) + assert isinstance(package_env, dict) + assert broker.get("id") == "nim_broker" + assert "sha256" in broker_run + assert "GITHUB_OUTPUT" in broker_run + assert broker_run.index("unset NIM_UPSTREAM_API_KEY") > broker_run.index( + "python scripts/ci/nim_proxy.py" + ) + assert package_env.get("KEYVERSE_FORBIDDEN_SECRET_FINGERPRINT") == ( + "${{ steps.nim_broker.outputs.secret_fingerprint }}" + ) + assert "KEYVERSE_FORBIDDEN_SECRET: ${{ secrets.NVIDIA_NIM_API_KEY }}" not in ( + _workflow_source() + ) + + def test_nvidia_secret_is_required_only_on_the_model_backed_path() -> None: """The NVIDIA secret is checked only after deterministic gates select development.""" broker = _step_by_name(