From f2bc08ae64cffe8845608f02f7962d3a24382a78 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 30 Jul 2026 02:59:03 +0900 Subject: [PATCH 1/5] test(review): allow staged Zen free rollout --- tests/test_opencode_agent_contract.py | 245 ++++++++++++++++++-------- 1 file changed, 173 insertions(+), 72 deletions(-) diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index b9dbb106a..901ef1377 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -85,19 +85,42 @@ def test_code_reviewer_subagent_contract_is_configured(): def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): """Guard every review-pool candidate against silent reasoning-effort drift.""" config = json.loads(Path("opencode.jsonc").read_text(encoding="utf-8")) - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) github_models = config["provider"]["github-models"]["models"] candidates_match = re.search(r'OPENCODE_MODEL_CANDIDATES: "([^"]+)"', workflow) assert candidates_match is not None - conditional_public_candidate = ( - "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " - "&& 'opencode-free/north-mini-code-free ' || '' }}" + public_candidate_options = ( + ( + "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " + "&& 'opencode-free/north-mini-code-free ' || '' }}", + ["opencode-free/north-mini-code-free"], + ), + ( + "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " + "&& 'opencode-free/deepseek-v4-flash-free " + "opencode-free/north-mini-code-free ' || '' }}", + [ + "opencode-free/deepseek-v4-flash-free", + "opencode-free/north-mini-code-free", + ], + ), ) candidates_text = candidates_match.group(1) - assert candidates_text.startswith(conditional_public_candidate) + public_candidate = next( + ( + (prefix, candidates) + for prefix, candidates in public_candidate_options + if candidates_text.startswith(prefix) + ), + None, + ) + assert public_candidate is not None + conditional_public_candidate, public_candidates = public_candidate candidates = [ - "opencode-free/north-mini-code-free", + *public_candidates, *candidates_text.removeprefix(conditional_public_candidate).split(), ] candidate_pairs = [candidate.split("/", 1) for candidate in candidates] @@ -105,7 +128,9 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): model_name for provider, model_name in candidate_pairs if provider == "openai" ] openrouter_models = [ - model_name for provider, model_name in candidate_pairs if provider == "openrouter" + model_name + for provider, model_name in candidate_pairs + if provider == "openrouter" ] github_candidate_models = [ model_name @@ -115,7 +140,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidate_pairs assert candidate_pairs == [ - ["opencode-free", "north-mini-code-free"], + *(candidate.split("/", 1) for candidate in public_candidates), ["github-models", "deepseek/deepseek-v3-0324"], ["openai", "gpt-5.6-luna"], ["openrouter", "deepseek/deepseek-v3.2"], @@ -135,6 +160,21 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert set(github_candidate_models).issubset(set(github_models)) assert '"context": 256000' in workflow assert '"output": 64000' in workflow + generated_config_match = re.search( + r"jq -n '(\{.*?\})' >\"\$\{OPENCODE_REVIEW_WORKDIR\}/opencode\.jsonc\"", + workflow, + re.DOTALL, + ) + assert generated_config_match is not None + generated_config = json.loads(generated_config_match.group(1)) + free_models = generated_config["provider"]["opencode-free"]["models"] + north_model = free_models["north-mini-code-free"] + assert north_model["tool_call"] is True + assert "response_format" not in north_model["options"] + if "deepseek-v4-flash-free" in free_models: + deepseek_model = free_models["deepseek-v4-flash-free"] + assert deepseek_model["tool_call"] is True + assert "response_format" not in deepseek_model.get("options", {}) assert github_candidate_models == [ "deepseek/deepseek-v3-0324", "openai/gpt-4.1", @@ -150,13 +190,17 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): "openai/o3-mini", } assert banned_review_candidates.isdisjoint( - set(direct_openai_models) | set(openrouter_models) | set(github_candidate_models) + set(direct_openai_models) + | set(openrouter_models) + | set(github_candidate_models) ) assert '"openai": {' in workflow assert '"apiKey": "{env:OPENAI_API_KEY}"' in workflow assert '"openrouter": {' in workflow assert '"apiKey": "{env:OPENROUTER_API_KEY}"' in workflow - for model_name in direct_openai_models + openrouter_models + github_candidate_models: + for model_name in ( + direct_openai_models + openrouter_models + github_candidate_models + ): assert f'"{model_name}": {{' in workflow def is_reasoning_capable(model_name: str) -> bool: @@ -198,7 +242,9 @@ def test_model_pool_cannot_synthesize_approval_after_provider_exhaustion(): def test_opencode_trusted_source_ref_is_not_controlled_by_workflow_inputs(): """Check out only the validated workflow-identity source ref output.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert "canonical_ref:" not in workflow assert "INPUT_CANONICAL_REF" not in workflow @@ -223,7 +269,9 @@ def test_opencode_trusted_source_ref_is_not_controlled_by_workflow_inputs(): def test_opencode_bounded_evidence_context_is_resolved_from_event_payload(): """Avoid putting untrusted PR metadata directly into shell environment keys.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) start = workflow.index(" - name: Prepare bounded OpenCode review evidence\n") end = workflow.index("\n - name:", start + 1) step = workflow[start:end] @@ -243,7 +291,9 @@ def test_opencode_bounded_evidence_context_is_resolved_from_event_payload(): def test_opencode_ignores_superseded_cancelled_rollup_checks(): """Do not fail approval on stale cancelled queue entries after same-head success.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) function = workflow.split("filter_superseded_cancelled_rollup_checks() {", 1)[ 1 ].split("collect_current_head_commit_check_runs() {", 1)[0] @@ -264,7 +314,9 @@ def test_opencode_ignores_superseded_cancelled_rollup_checks(): def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): """Keep PR-controlled test execution off the pull_request_target path.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert "required-workflow-bootstrap:" in workflow assert "OpenCode repository-dispatch review run materialized." in workflow bootstrap_start = workflow.index(" required-workflow-bootstrap:\n") @@ -364,17 +416,14 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): ) in measure_step assert 'test "$(/usr/local/bin/node --version)" = "v24.18.0"' in measure_step assert "/usr/local/bin/npm --version >/dev/null" in measure_step - assert ( - "https://registry.npmjs.org/pnpm/-/pnpm-11.5.3.tgz" - ) in measure_step + assert ("https://registry.npmjs.org/pnpm/-/pnpm-11.5.3.tgz") in measure_step assert ( "7ac1c919341c213a34dc0d02afb7143c5c26ac26ee8c4782deea821b8ac64d2134" "a081fd8941dae6e29bbb48f58dfc2b7fbceeccc07cb2f09d219d342a4969ed" " /tmp/pnpm.tgz" ) in measure_step assert ( - "tar --no-same-owner -xzf /tmp/pnpm.tgz -C /opt/pnpm " - "--strip-components=1" + "tar --no-same-owner -xzf /tmp/pnpm.tgz -C /opt/pnpm --strip-components=1" ) in measure_step assert "ln -s /opt/pnpm/bin/pnpm.cjs /usr/local/bin/pnpm" in measure_step assert 'test "$(/usr/local/bin/pnpm --version)" = "11.5.3"' in measure_step @@ -392,17 +441,16 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): in measure_step ) assert ( - 'trusted_git hash-object --no-filters -- \\\n' - ' "$COVERAGE_SOURCE_WORKDIR/$relative_lock"' - in measure_step + "trusted_git hash-object --no-filters -- \\\n" + ' "$COVERAGE_SOURCE_WORKDIR/$relative_lock"' in measure_step ) assert 'hash-object --no-filters -- "$relative_lock"' not in measure_step - assert "refusing --trust-lockfile for PR-controlled dependency resolution" in measure_step - assert "prepare_writable_pnpm_store()" in measure_step assert ( - 'destination="$(mktemp -d /tmp/opencode-pnpm-store.XXXXXX)"' + "refusing --trust-lockfile for PR-controlled dependency resolution" in measure_step ) + assert "prepare_writable_pnpm_store()" in measure_step + assert 'destination="$(mktemp -d /tmp/opencode-pnpm-store.XXXXXX)"' in measure_step assert 'cp -R /opt/pnpm-store/. "$destination/"' in measure_step assert 'chmod -R u+rwX,go-rwx "$destination"' in measure_step assert '--store-dir "$writable_pnpm_store_dir"' in measure_step @@ -426,7 +474,10 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): assert "javascript_coverage_ran_any=1" in measure_step assert measure_step.count("check_javascript_coverage_thresholds") == 2 assert "--require-hashes" in measure_step - assert 'coverage_tool_image="opencode-coverage-tools:${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"' in measure_step + assert ( + 'coverage_tool_image="opencode-coverage-tools:${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"' + in measure_step + ) assert "The networked build context contains only this" in measure_step assert 'install -m 0644 "$trusted_ci_requirements"' in measure_step assert "docker build --pull --no-cache --network=default" in measure_step @@ -503,12 +554,12 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): assert "uv run --no-project" not in measure_step assert "uv run --no-build" not in measure_step assert "python3 -m coverage run -m pytest tests" in measure_step - trusted_requirements = Path( - "requirements-opencode-review-ci-hashes.txt" - ).read_text(encoding="utf-8") - compile_script = Path( - "scripts/ci/compile_opencode_review_lock.sh" - ).read_text(encoding="utf-8") + trusted_requirements = Path("requirements-opencode-review-ci-hashes.txt").read_text( + encoding="utf-8" + ) + compile_script = Path("scripts/ci/compile_opencode_review_lock.sh").read_text( + encoding="utf-8" + ) normalized_compile_script = " ".join(compile_script.replace("\\\n", " ").split()) assert "pytest-cov==7.1.0" in trusted_requirements assert ( @@ -537,7 +588,9 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): def test_opencode_repository_dispatch_authorization_is_fail_closed(): """Reject an untrusted dispatcher or a target outside the exact allowlist.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) validate_step = workflow.split( " - name: Bind workflow inputs to live organization pull request metadata\n", 1, @@ -600,7 +653,9 @@ def test_opencode_repository_dispatch_authorization_is_fail_closed(): def test_opencode_model_exhaustion_retry_stays_owned_by_central_scheduler(): """Do not broaden workflow permissions for a recursive review dispatch.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert "opencode-exhausted-retry:" not in workflow assert "RETRY_DISPATCH_TOKEN" not in workflow assert "contents: write" not in workflow @@ -608,10 +663,12 @@ def test_opencode_model_exhaustion_retry_stays_owned_by_central_scheduler(): def test_opencode_python_coverage_never_resolves_pr_dependency_manifests(): """Use only the trusted image toolchain during networkless PR execution.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") - measure = workflow.split( - " - name: Measure test and docstring evidence\n", 1 - )[1].split("\n - name:", 1)[0] + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) + measure = workflow.split(" - name: Measure test and docstring evidence\n", 1)[ + 1 + ].split("\n - name:", 1)[0] assert "verify_trusted_python_test_toolchain()" in measure assert "PR-selected dependency manifests are never resolved" in measure @@ -626,7 +683,9 @@ def test_opencode_python_coverage_never_resolves_pr_dependency_manifests(): def test_opencode_coverage_prefers_preinstalled_declared_pnpm_before_npm(): """pnpm workspaces must not activate PR-selected tooling or fall back to npm.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) measure_start = workflow.index( " - name: Measure test and docstring evidence\n" ) @@ -660,7 +719,9 @@ def test_opencode_coverage_prefers_preinstalled_declared_pnpm_before_npm(): def test_opencode_coverage_does_not_duplicate_existing_javascript_coverage(): """An existing coverage flag/tool must run once instead of receiving a duplicate flag.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) measure_start = workflow.index( " - name: Measure test and docstring evidence\n" ) @@ -701,7 +762,9 @@ def test_opencode_coverage_discovers_changed_nested_javascript_package(tmp_path) except (OSError, subprocess.SubprocessError) as exc: pytest.skip(f"bash is not usable for this regression test: {exc}") - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) measure_start = workflow.index( " - name: Measure test and docstring evidence\n" ) @@ -904,7 +967,9 @@ def test_code_reviewer_prompt_preserves_review_only_policy(): def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): """Guard the isolated runtime OpenCode workspace and reviewer agent.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert "code-reviewer-prompt.md" in workflow assert "review_execution_contracts.py" in workflow @@ -1210,10 +1275,17 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): "Skipping publish-step failed-check OpenCode diagnosis for central review-process self-repair" in workflow ) - assert ( - "needs.validate-pr-metadata.outputs.is_private == 'false' && " - "'opencode-free/north-mini-code-free ' || ''" - ) in workflow + assert any( + ( + "needs.validate-pr-metadata.outputs.is_private == 'false' && " + f"'{candidates} ' || ''" + ) + in workflow + for candidates in ( + "opencode-free/north-mini-code-free", + "opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free", + ) + ) assert ( "github-models/deepseek/deepseek-v3-0324 " "openai/gpt-5.6-luna " @@ -1452,7 +1524,9 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): def test_opencode_job_timeout_contains_full_sequential_review_budget(): """Keep the outer job alive through evidence, review, and publication.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) def timeout_minutes(pattern: str) -> int: match = re.search(pattern, workflow, re.MULTILINE) @@ -1531,7 +1605,9 @@ def test_opencode_approval_gate_shell_is_parseable(): def test_opencode_review_body_printf_blocks_close_on_separate_line(): """Guard approval-gate review body builders against runner bash parse failures.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) risky_suffixes = ( 'source finding.")"', 'has no blockers.")"', @@ -1545,7 +1621,9 @@ def test_opencode_review_body_printf_blocks_close_on_separate_line(): def test_opencode_review_jq_blocks_do_not_embed_shell_single_quotes(): """Guard jq snippets wrapped in shell single quotes against bash parse failures.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert 'gsub("`"; "\'")' not in workflow assert 'gsub("`"; "'")' in workflow @@ -1567,8 +1645,7 @@ def test_merge_scheduler_uses_escalating_mutation_credentials(): "&& github.event.client_payload.target_repository != '' && " "(secrets.PR_REVIEW_MERGE_TOKEN || " "secrets.OPENCODE_APPROVE_TOKEN || " - "steps.scheduler_app_token.outputs.token) || github.token }}" - in workflow + "steps.scheduler_app_token.outputs.token) || github.token }}" in workflow ) assert "SCHEDULER_MUTATION_TOKEN_SOURCE" in workflow assert 'default: "1"' in workflow @@ -1606,7 +1683,9 @@ def test_merge_scheduler_uses_escalating_mutation_credentials(): def test_opencode_runs_merge_scheduler_after_review_without_repo_local_dispatch(): """Guard immediate post-review merge/update follow-up from OpenCode.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert "Run merge scheduler after approval" in workflow assert "Publish repository_dispatch OpenCode status" in workflow @@ -1628,8 +1707,7 @@ def test_opencode_runs_merge_scheduler_after_review_without_repo_local_dispatch( assert "steps.opencode_app_token.outputs" not in status_step assert "continue-on-error: true" not in status_step assert ( - "same-repository github.token can access cross-repository target" - in status_step + "same-repository github.token can access cross-repository target" in status_step ) assert "status publication failed because pr_head_sha was empty" in status_step assert "exit 1" in status_step @@ -1683,7 +1761,9 @@ def test_opencode_adversarial_prompt_requires_independent_proof(): def test_opencode_privileged_review_security_boundaries_are_fail_closed(): """Guard the Strix-proven command, fork, package, and output-file boundaries.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) coverage_start = workflow.index(" coverage-evidence:\n") coverage_end = workflow.index("\n opencode-review-target:", coverage_start) coverage_job = workflow[coverage_start:coverage_end] @@ -1778,12 +1858,9 @@ def test_opencode_privileged_review_security_boundaries_are_fail_closed(): assert 'live_head_sha="$(jq -r' in trust_step assert '[ "$live_head_sha" != "$EXPECTED_HEAD_SHA" ]' in trust_step assert ( - "EXPECTED_IS_PRIVATE: " - "${{ needs.validate-pr-metadata.outputs.is_private }}" - ) in trust_step - assert ( - 'live_is_private="$(jq -r \'.base.repo.private | tostring\'' + "EXPECTED_IS_PRIVATE: ${{ needs.validate-pr-metadata.outputs.is_private }}" ) in trust_step + assert ("live_is_private=\"$(jq -r '.base.repo.private | tostring'") in trust_step assert '! [[ "$EXPECTED_IS_PRIVATE" =~ ^(true|false)$ ]]' in trust_step assert '! [[ "$live_is_private" =~ ^(true|false)$ ]]' in trust_step assert '[ "$live_is_private" != "$EXPECTED_IS_PRIVATE" ]' in trust_step @@ -1846,7 +1923,9 @@ def test_opencode_privileged_review_security_boundaries_are_fail_closed(): def test_opencode_pending_peer_checks_hold_blocks_required_workflow_until_approval(): """Pending peer checks cannot satisfy the required gate without a review.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert "hold_approval_without_review()" in workflow assert "OpenCode review state unchanged; approval pending" in workflow @@ -1875,7 +1954,9 @@ def test_opencode_pending_peer_checks_hold_blocks_required_workflow_until_approv def test_opencode_strix_security_regressions_are_closed(): """Bind the nine current-head Strix findings to fail-closed contracts.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) config = json.loads(Path("opencode.jsonc").read_text(encoding="utf-8")) assert " validate-pr-metadata:\n" in workflow @@ -1917,7 +1998,9 @@ def test_opencode_strix_security_regressions_are_closed(): def test_opencode_review_publication_prefers_app_token_for_review_writes(): """OpenCode review writes must use the OIDC-backed app token before workflow tokens.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert ("GH_TOKEN: ${{ steps.opencode_app_token.outputs.token }}") in workflow assert ( @@ -1961,7 +2044,9 @@ def test_opencode_review_publication_prefers_app_token_for_review_writes(): def test_opencode_approve_review_publication_failure_fails_closed(): """A rejected APPROVE review write must not leave a successful review gate.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert "APPROVE_PUBLICATION_FAILED" in workflow assert "APPROVE_PUBLICATION_SKIPPED" not in workflow @@ -2005,7 +2090,9 @@ def test_opencode_gate_reads_tolerate_shared_token_throttle(): installation token must degrade the same way on a detected throttle instead of hard-failing the required check under ``set -euo pipefail``. """ - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) # The unguarded top-level reads are now guarded and skip on throttle # rather than tripping set -e. @@ -2060,7 +2147,9 @@ def test_opencode_review_language_signal_is_throttle_proof(): review. Sourcing the signal from the GitHub event payload (no API call) keeps the marker present even when ``gh pr view`` is rate-limited. """ - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) # Event-payload primary source, resolved from the event JSON by the context # script (shlex-quoted, never ${{ }}-inlined) so untrusted PR text is only @@ -2094,7 +2183,9 @@ def test_opencode_changed_file_syntax_gate_is_wired_into_coverage_evidence(): changed files runs in the coverage-evidence job (whose result gates approval) and fails the job on any syntax error. """ - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert "- name: Enforce changed-file syntax gate" in workflow assert "scripts/ci/changed_file_syntax_gate.py" in workflow @@ -2111,7 +2202,9 @@ def test_opencode_changed_file_syntax_gate_is_wired_into_coverage_evidence(): def test_opencode_jq_filters_do_not_embed_literal_expression_openers(): """Literal '${{' inside run scripts is parsed as a GitHub expression opener.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert 'contains("${{")' not in workflow assert 'contains("$" + "{{")' in workflow @@ -2119,7 +2212,9 @@ def test_opencode_jq_filters_do_not_embed_literal_expression_openers(): def test_opencode_model_pool_failure_uses_only_existing_real_model_approval(): """A model-pool failure may not publish a generic deterministic APPROVE review.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert ( "OPENCODE_MODEL_POOL_OUTCOME: ${{ steps.opencode_review_model_pool.outputs.review_status }}" @@ -2168,7 +2263,9 @@ def test_opencode_model_pool_failure_uses_only_existing_real_model_approval(): def test_opencode_review_thread_jq_filters_preserve_bash_single_quotes(): """Guard jq filters embedded in single-quoted shell strings.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) assert 'gsub("`"; "\'")' not in workflow assert workflow.count('gsub("`"; "'")') == 4 @@ -2176,7 +2273,9 @@ def test_opencode_review_thread_jq_filters_preserve_bash_single_quotes(): def test_peer_check_wait_budget_fits_publication_step_timeouts(): """Keep slow-check cadence bounded inside both publication step caps.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) normal_attempts = [ int(value) @@ -2220,7 +2319,9 @@ def test_peer_check_wait_budget_fits_publication_step_timeouts(): def test_slow_peer_wait_matches_only_image_validation_checks(): """Reject lookalike labels when selecting the extended peer-check budget.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) fast_pattern = r"^- validate [^:/]+ image:" general_pattern = r"^- (Build and Publish Docker Images/)?validate [^:/]+ image:" From 64554036bcb80ae32782f68f22a96efae90f74f4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 30 Jul 2026 02:59:20 +0900 Subject: [PATCH 2/5] Revert "test(review): allow staged Zen free rollout" This reverts commit f2bc08ae64cffe8845608f02f7962d3a24382a78. --- tests/test_opencode_agent_contract.py | 245 ++++++++------------------ 1 file changed, 72 insertions(+), 173 deletions(-) diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 901ef1377..b9dbb106a 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -85,42 +85,19 @@ def test_code_reviewer_subagent_contract_is_configured(): def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): """Guard every review-pool candidate against silent reasoning-effort drift.""" config = json.loads(Path("opencode.jsonc").read_text(encoding="utf-8")) - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") github_models = config["provider"]["github-models"]["models"] candidates_match = re.search(r'OPENCODE_MODEL_CANDIDATES: "([^"]+)"', workflow) assert candidates_match is not None - public_candidate_options = ( - ( - "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " - "&& 'opencode-free/north-mini-code-free ' || '' }}", - ["opencode-free/north-mini-code-free"], - ), - ( - "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " - "&& 'opencode-free/deepseek-v4-flash-free " - "opencode-free/north-mini-code-free ' || '' }}", - [ - "opencode-free/deepseek-v4-flash-free", - "opencode-free/north-mini-code-free", - ], - ), + conditional_public_candidate = ( + "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " + "&& 'opencode-free/north-mini-code-free ' || '' }}" ) candidates_text = candidates_match.group(1) - public_candidate = next( - ( - (prefix, candidates) - for prefix, candidates in public_candidate_options - if candidates_text.startswith(prefix) - ), - None, - ) - assert public_candidate is not None - conditional_public_candidate, public_candidates = public_candidate + assert candidates_text.startswith(conditional_public_candidate) candidates = [ - *public_candidates, + "opencode-free/north-mini-code-free", *candidates_text.removeprefix(conditional_public_candidate).split(), ] candidate_pairs = [candidate.split("/", 1) for candidate in candidates] @@ -128,9 +105,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): model_name for provider, model_name in candidate_pairs if provider == "openai" ] openrouter_models = [ - model_name - for provider, model_name in candidate_pairs - if provider == "openrouter" + model_name for provider, model_name in candidate_pairs if provider == "openrouter" ] github_candidate_models = [ model_name @@ -140,7 +115,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidate_pairs assert candidate_pairs == [ - *(candidate.split("/", 1) for candidate in public_candidates), + ["opencode-free", "north-mini-code-free"], ["github-models", "deepseek/deepseek-v3-0324"], ["openai", "gpt-5.6-luna"], ["openrouter", "deepseek/deepseek-v3.2"], @@ -160,21 +135,6 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert set(github_candidate_models).issubset(set(github_models)) assert '"context": 256000' in workflow assert '"output": 64000' in workflow - generated_config_match = re.search( - r"jq -n '(\{.*?\})' >\"\$\{OPENCODE_REVIEW_WORKDIR\}/opencode\.jsonc\"", - workflow, - re.DOTALL, - ) - assert generated_config_match is not None - generated_config = json.loads(generated_config_match.group(1)) - free_models = generated_config["provider"]["opencode-free"]["models"] - north_model = free_models["north-mini-code-free"] - assert north_model["tool_call"] is True - assert "response_format" not in north_model["options"] - if "deepseek-v4-flash-free" in free_models: - deepseek_model = free_models["deepseek-v4-flash-free"] - assert deepseek_model["tool_call"] is True - assert "response_format" not in deepseek_model.get("options", {}) assert github_candidate_models == [ "deepseek/deepseek-v3-0324", "openai/gpt-4.1", @@ -190,17 +150,13 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): "openai/o3-mini", } assert banned_review_candidates.isdisjoint( - set(direct_openai_models) - | set(openrouter_models) - | set(github_candidate_models) + set(direct_openai_models) | set(openrouter_models) | set(github_candidate_models) ) assert '"openai": {' in workflow assert '"apiKey": "{env:OPENAI_API_KEY}"' in workflow assert '"openrouter": {' in workflow assert '"apiKey": "{env:OPENROUTER_API_KEY}"' in workflow - for model_name in ( - direct_openai_models + openrouter_models + github_candidate_models - ): + for model_name in direct_openai_models + openrouter_models + github_candidate_models: assert f'"{model_name}": {{' in workflow def is_reasoning_capable(model_name: str) -> bool: @@ -242,9 +198,7 @@ def test_model_pool_cannot_synthesize_approval_after_provider_exhaustion(): def test_opencode_trusted_source_ref_is_not_controlled_by_workflow_inputs(): """Check out only the validated workflow-identity source ref output.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert "canonical_ref:" not in workflow assert "INPUT_CANONICAL_REF" not in workflow @@ -269,9 +223,7 @@ def test_opencode_trusted_source_ref_is_not_controlled_by_workflow_inputs(): def test_opencode_bounded_evidence_context_is_resolved_from_event_payload(): """Avoid putting untrusted PR metadata directly into shell environment keys.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") start = workflow.index(" - name: Prepare bounded OpenCode review evidence\n") end = workflow.index("\n - name:", start + 1) step = workflow[start:end] @@ -291,9 +243,7 @@ def test_opencode_bounded_evidence_context_is_resolved_from_event_payload(): def test_opencode_ignores_superseded_cancelled_rollup_checks(): """Do not fail approval on stale cancelled queue entries after same-head success.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") function = workflow.split("filter_superseded_cancelled_rollup_checks() {", 1)[ 1 ].split("collect_current_head_commit_check_runs() {", 1)[0] @@ -314,9 +264,7 @@ def test_opencode_ignores_superseded_cancelled_rollup_checks(): def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): """Keep PR-controlled test execution off the pull_request_target path.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert "required-workflow-bootstrap:" in workflow assert "OpenCode repository-dispatch review run materialized." in workflow bootstrap_start = workflow.index(" required-workflow-bootstrap:\n") @@ -416,14 +364,17 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): ) in measure_step assert 'test "$(/usr/local/bin/node --version)" = "v24.18.0"' in measure_step assert "/usr/local/bin/npm --version >/dev/null" in measure_step - assert ("https://registry.npmjs.org/pnpm/-/pnpm-11.5.3.tgz") in measure_step + assert ( + "https://registry.npmjs.org/pnpm/-/pnpm-11.5.3.tgz" + ) in measure_step assert ( "7ac1c919341c213a34dc0d02afb7143c5c26ac26ee8c4782deea821b8ac64d2134" "a081fd8941dae6e29bbb48f58dfc2b7fbceeccc07cb2f09d219d342a4969ed" " /tmp/pnpm.tgz" ) in measure_step assert ( - "tar --no-same-owner -xzf /tmp/pnpm.tgz -C /opt/pnpm --strip-components=1" + "tar --no-same-owner -xzf /tmp/pnpm.tgz -C /opt/pnpm " + "--strip-components=1" ) in measure_step assert "ln -s /opt/pnpm/bin/pnpm.cjs /usr/local/bin/pnpm" in measure_step assert 'test "$(/usr/local/bin/pnpm --version)" = "11.5.3"' in measure_step @@ -441,16 +392,17 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): in measure_step ) assert ( - "trusted_git hash-object --no-filters -- \\\n" - ' "$COVERAGE_SOURCE_WORKDIR/$relative_lock"' in measure_step + 'trusted_git hash-object --no-filters -- \\\n' + ' "$COVERAGE_SOURCE_WORKDIR/$relative_lock"' + in measure_step ) assert 'hash-object --no-filters -- "$relative_lock"' not in measure_step + assert "refusing --trust-lockfile for PR-controlled dependency resolution" in measure_step + assert "prepare_writable_pnpm_store()" in measure_step assert ( - "refusing --trust-lockfile for PR-controlled dependency resolution" + 'destination="$(mktemp -d /tmp/opencode-pnpm-store.XXXXXX)"' in measure_step ) - assert "prepare_writable_pnpm_store()" in measure_step - assert 'destination="$(mktemp -d /tmp/opencode-pnpm-store.XXXXXX)"' in measure_step assert 'cp -R /opt/pnpm-store/. "$destination/"' in measure_step assert 'chmod -R u+rwX,go-rwx "$destination"' in measure_step assert '--store-dir "$writable_pnpm_store_dir"' in measure_step @@ -474,10 +426,7 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): assert "javascript_coverage_ran_any=1" in measure_step assert measure_step.count("check_javascript_coverage_thresholds") == 2 assert "--require-hashes" in measure_step - assert ( - 'coverage_tool_image="opencode-coverage-tools:${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"' - in measure_step - ) + assert 'coverage_tool_image="opencode-coverage-tools:${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"' in measure_step assert "The networked build context contains only this" in measure_step assert 'install -m 0644 "$trusted_ci_requirements"' in measure_step assert "docker build --pull --no-cache --network=default" in measure_step @@ -554,12 +503,12 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): assert "uv run --no-project" not in measure_step assert "uv run --no-build" not in measure_step assert "python3 -m coverage run -m pytest tests" in measure_step - trusted_requirements = Path("requirements-opencode-review-ci-hashes.txt").read_text( - encoding="utf-8" - ) - compile_script = Path("scripts/ci/compile_opencode_review_lock.sh").read_text( - encoding="utf-8" - ) + trusted_requirements = Path( + "requirements-opencode-review-ci-hashes.txt" + ).read_text(encoding="utf-8") + compile_script = Path( + "scripts/ci/compile_opencode_review_lock.sh" + ).read_text(encoding="utf-8") normalized_compile_script = " ".join(compile_script.replace("\\\n", " ").split()) assert "pytest-cov==7.1.0" in trusted_requirements assert ( @@ -588,9 +537,7 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): def test_opencode_repository_dispatch_authorization_is_fail_closed(): """Reject an untrusted dispatcher or a target outside the exact allowlist.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") validate_step = workflow.split( " - name: Bind workflow inputs to live organization pull request metadata\n", 1, @@ -653,9 +600,7 @@ def test_opencode_repository_dispatch_authorization_is_fail_closed(): def test_opencode_model_exhaustion_retry_stays_owned_by_central_scheduler(): """Do not broaden workflow permissions for a recursive review dispatch.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert "opencode-exhausted-retry:" not in workflow assert "RETRY_DISPATCH_TOKEN" not in workflow assert "contents: write" not in workflow @@ -663,12 +608,10 @@ def test_opencode_model_exhaustion_retry_stays_owned_by_central_scheduler(): def test_opencode_python_coverage_never_resolves_pr_dependency_manifests(): """Use only the trusted image toolchain during networkless PR execution.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) - measure = workflow.split(" - name: Measure test and docstring evidence\n", 1)[ - 1 - ].split("\n - name:", 1)[0] + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") + measure = workflow.split( + " - name: Measure test and docstring evidence\n", 1 + )[1].split("\n - name:", 1)[0] assert "verify_trusted_python_test_toolchain()" in measure assert "PR-selected dependency manifests are never resolved" in measure @@ -683,9 +626,7 @@ def test_opencode_python_coverage_never_resolves_pr_dependency_manifests(): def test_opencode_coverage_prefers_preinstalled_declared_pnpm_before_npm(): """pnpm workspaces must not activate PR-selected tooling or fall back to npm.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") measure_start = workflow.index( " - name: Measure test and docstring evidence\n" ) @@ -719,9 +660,7 @@ def test_opencode_coverage_prefers_preinstalled_declared_pnpm_before_npm(): def test_opencode_coverage_does_not_duplicate_existing_javascript_coverage(): """An existing coverage flag/tool must run once instead of receiving a duplicate flag.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") measure_start = workflow.index( " - name: Measure test and docstring evidence\n" ) @@ -762,9 +701,7 @@ def test_opencode_coverage_discovers_changed_nested_javascript_package(tmp_path) except (OSError, subprocess.SubprocessError) as exc: pytest.skip(f"bash is not usable for this regression test: {exc}") - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") measure_start = workflow.index( " - name: Measure test and docstring evidence\n" ) @@ -967,9 +904,7 @@ def test_code_reviewer_prompt_preserves_review_only_policy(): def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): """Guard the isolated runtime OpenCode workspace and reviewer agent.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert "code-reviewer-prompt.md" in workflow assert "review_execution_contracts.py" in workflow @@ -1275,17 +1210,10 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): "Skipping publish-step failed-check OpenCode diagnosis for central review-process self-repair" in workflow ) - assert any( - ( - "needs.validate-pr-metadata.outputs.is_private == 'false' && " - f"'{candidates} ' || ''" - ) - in workflow - for candidates in ( - "opencode-free/north-mini-code-free", - "opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free", - ) - ) + assert ( + "needs.validate-pr-metadata.outputs.is_private == 'false' && " + "'opencode-free/north-mini-code-free ' || ''" + ) in workflow assert ( "github-models/deepseek/deepseek-v3-0324 " "openai/gpt-5.6-luna " @@ -1524,9 +1452,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): def test_opencode_job_timeout_contains_full_sequential_review_budget(): """Keep the outer job alive through evidence, review, and publication.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") def timeout_minutes(pattern: str) -> int: match = re.search(pattern, workflow, re.MULTILINE) @@ -1605,9 +1531,7 @@ def test_opencode_approval_gate_shell_is_parseable(): def test_opencode_review_body_printf_blocks_close_on_separate_line(): """Guard approval-gate review body builders against runner bash parse failures.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") risky_suffixes = ( 'source finding.")"', 'has no blockers.")"', @@ -1621,9 +1545,7 @@ def test_opencode_review_body_printf_blocks_close_on_separate_line(): def test_opencode_review_jq_blocks_do_not_embed_shell_single_quotes(): """Guard jq snippets wrapped in shell single quotes against bash parse failures.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert 'gsub("`"; "\'")' not in workflow assert 'gsub("`"; "'")' in workflow @@ -1645,7 +1567,8 @@ def test_merge_scheduler_uses_escalating_mutation_credentials(): "&& github.event.client_payload.target_repository != '' && " "(secrets.PR_REVIEW_MERGE_TOKEN || " "secrets.OPENCODE_APPROVE_TOKEN || " - "steps.scheduler_app_token.outputs.token) || github.token }}" in workflow + "steps.scheduler_app_token.outputs.token) || github.token }}" + in workflow ) assert "SCHEDULER_MUTATION_TOKEN_SOURCE" in workflow assert 'default: "1"' in workflow @@ -1683,9 +1606,7 @@ def test_merge_scheduler_uses_escalating_mutation_credentials(): def test_opencode_runs_merge_scheduler_after_review_without_repo_local_dispatch(): """Guard immediate post-review merge/update follow-up from OpenCode.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert "Run merge scheduler after approval" in workflow assert "Publish repository_dispatch OpenCode status" in workflow @@ -1707,7 +1628,8 @@ def test_opencode_runs_merge_scheduler_after_review_without_repo_local_dispatch( assert "steps.opencode_app_token.outputs" not in status_step assert "continue-on-error: true" not in status_step assert ( - "same-repository github.token can access cross-repository target" in status_step + "same-repository github.token can access cross-repository target" + in status_step ) assert "status publication failed because pr_head_sha was empty" in status_step assert "exit 1" in status_step @@ -1761,9 +1683,7 @@ def test_opencode_adversarial_prompt_requires_independent_proof(): def test_opencode_privileged_review_security_boundaries_are_fail_closed(): """Guard the Strix-proven command, fork, package, and output-file boundaries.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") coverage_start = workflow.index(" coverage-evidence:\n") coverage_end = workflow.index("\n opencode-review-target:", coverage_start) coverage_job = workflow[coverage_start:coverage_end] @@ -1858,9 +1778,12 @@ def test_opencode_privileged_review_security_boundaries_are_fail_closed(): assert 'live_head_sha="$(jq -r' in trust_step assert '[ "$live_head_sha" != "$EXPECTED_HEAD_SHA" ]' in trust_step assert ( - "EXPECTED_IS_PRIVATE: ${{ needs.validate-pr-metadata.outputs.is_private }}" + "EXPECTED_IS_PRIVATE: " + "${{ needs.validate-pr-metadata.outputs.is_private }}" + ) in trust_step + assert ( + 'live_is_private="$(jq -r \'.base.repo.private | tostring\'' ) in trust_step - assert ("live_is_private=\"$(jq -r '.base.repo.private | tostring'") in trust_step assert '! [[ "$EXPECTED_IS_PRIVATE" =~ ^(true|false)$ ]]' in trust_step assert '! [[ "$live_is_private" =~ ^(true|false)$ ]]' in trust_step assert '[ "$live_is_private" != "$EXPECTED_IS_PRIVATE" ]' in trust_step @@ -1923,9 +1846,7 @@ def test_opencode_privileged_review_security_boundaries_are_fail_closed(): def test_opencode_pending_peer_checks_hold_blocks_required_workflow_until_approval(): """Pending peer checks cannot satisfy the required gate without a review.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert "hold_approval_without_review()" in workflow assert "OpenCode review state unchanged; approval pending" in workflow @@ -1954,9 +1875,7 @@ def test_opencode_pending_peer_checks_hold_blocks_required_workflow_until_approv def test_opencode_strix_security_regressions_are_closed(): """Bind the nine current-head Strix findings to fail-closed contracts.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") config = json.loads(Path("opencode.jsonc").read_text(encoding="utf-8")) assert " validate-pr-metadata:\n" in workflow @@ -1998,9 +1917,7 @@ def test_opencode_strix_security_regressions_are_closed(): def test_opencode_review_publication_prefers_app_token_for_review_writes(): """OpenCode review writes must use the OIDC-backed app token before workflow tokens.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert ("GH_TOKEN: ${{ steps.opencode_app_token.outputs.token }}") in workflow assert ( @@ -2044,9 +1961,7 @@ def test_opencode_review_publication_prefers_app_token_for_review_writes(): def test_opencode_approve_review_publication_failure_fails_closed(): """A rejected APPROVE review write must not leave a successful review gate.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert "APPROVE_PUBLICATION_FAILED" in workflow assert "APPROVE_PUBLICATION_SKIPPED" not in workflow @@ -2090,9 +2005,7 @@ def test_opencode_gate_reads_tolerate_shared_token_throttle(): installation token must degrade the same way on a detected throttle instead of hard-failing the required check under ``set -euo pipefail``. """ - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") # The unguarded top-level reads are now guarded and skip on throttle # rather than tripping set -e. @@ -2147,9 +2060,7 @@ def test_opencode_review_language_signal_is_throttle_proof(): review. Sourcing the signal from the GitHub event payload (no API call) keeps the marker present even when ``gh pr view`` is rate-limited. """ - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") # Event-payload primary source, resolved from the event JSON by the context # script (shlex-quoted, never ${{ }}-inlined) so untrusted PR text is only @@ -2183,9 +2094,7 @@ def test_opencode_changed_file_syntax_gate_is_wired_into_coverage_evidence(): changed files runs in the coverage-evidence job (whose result gates approval) and fails the job on any syntax error. """ - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert "- name: Enforce changed-file syntax gate" in workflow assert "scripts/ci/changed_file_syntax_gate.py" in workflow @@ -2202,9 +2111,7 @@ def test_opencode_changed_file_syntax_gate_is_wired_into_coverage_evidence(): def test_opencode_jq_filters_do_not_embed_literal_expression_openers(): """Literal '${{' inside run scripts is parsed as a GitHub expression opener.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert 'contains("${{")' not in workflow assert 'contains("$" + "{{")' in workflow @@ -2212,9 +2119,7 @@ def test_opencode_jq_filters_do_not_embed_literal_expression_openers(): def test_opencode_model_pool_failure_uses_only_existing_real_model_approval(): """A model-pool failure may not publish a generic deterministic APPROVE review.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert ( "OPENCODE_MODEL_POOL_OUTCOME: ${{ steps.opencode_review_model_pool.outputs.review_status }}" @@ -2263,9 +2168,7 @@ def test_opencode_model_pool_failure_uses_only_existing_real_model_approval(): def test_opencode_review_thread_jq_filters_preserve_bash_single_quotes(): """Guard jq filters embedded in single-quoted shell strings.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") assert 'gsub("`"; "\'")' not in workflow assert workflow.count('gsub("`"; "'")') == 4 @@ -2273,9 +2176,7 @@ def test_opencode_review_thread_jq_filters_preserve_bash_single_quotes(): def test_peer_check_wait_budget_fits_publication_step_timeouts(): """Keep slow-check cadence bounded inside both publication step caps.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") normal_attempts = [ int(value) @@ -2319,9 +2220,7 @@ def test_peer_check_wait_budget_fits_publication_step_timeouts(): def test_slow_peer_wait_matches_only_image_validation_checks(): """Reject lookalike labels when selecting the extended peer-check budget.""" - workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( - encoding="utf-8" - ) + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text(encoding="utf-8") fast_pattern = r"^- validate [^:/]+ image:" general_pattern = r"^- (Build and Publish Docker Images/)?validate [^:/]+ image:" From 3f05cb30dec75d6aa3ed2c8a2a45ee55a2ba832a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 30 Jul 2026 03:00:02 +0900 Subject: [PATCH 3/5] test(review): allow staged Zen free rollout --- tests/test_opencode_agent_contract.py | 64 ++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index b9dbb106a..4dc7dd55a 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -90,14 +90,35 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): candidates_match = re.search(r'OPENCODE_MODEL_CANDIDATES: "([^"]+)"', workflow) assert candidates_match is not None - conditional_public_candidate = ( - "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " - "&& 'opencode-free/north-mini-code-free ' || '' }}" + public_candidate_options = ( + ( + "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " + "&& 'opencode-free/north-mini-code-free ' || '' }}", + ["opencode-free/north-mini-code-free"], + ), + ( + "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " + "&& 'opencode-free/deepseek-v4-flash-free " + "opencode-free/north-mini-code-free ' || '' }}", + [ + "opencode-free/deepseek-v4-flash-free", + "opencode-free/north-mini-code-free", + ], + ), ) candidates_text = candidates_match.group(1) - assert candidates_text.startswith(conditional_public_candidate) + public_candidate = next( + ( + (prefix, candidates) + for prefix, candidates in public_candidate_options + if candidates_text.startswith(prefix) + ), + None, + ) + assert public_candidate is not None + conditional_public_candidate, public_candidates = public_candidate candidates = [ - "opencode-free/north-mini-code-free", + *public_candidates, *candidates_text.removeprefix(conditional_public_candidate).split(), ] candidate_pairs = [candidate.split("/", 1) for candidate in candidates] @@ -115,7 +136,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidate_pairs assert candidate_pairs == [ - ["opencode-free", "north-mini-code-free"], + *(candidate.split("/", 1) for candidate in public_candidates), ["github-models", "deepseek/deepseek-v3-0324"], ["openai", "gpt-5.6-luna"], ["openrouter", "deepseek/deepseek-v3.2"], @@ -135,6 +156,21 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert set(github_candidate_models).issubset(set(github_models)) assert '"context": 256000' in workflow assert '"output": 64000' in workflow + generated_config_match = re.search( + r"jq -n '(\{.*?\})' >\"\$\{OPENCODE_REVIEW_WORKDIR\}/opencode\.jsonc\"", + workflow, + re.DOTALL, + ) + assert generated_config_match is not None + generated_config = json.loads(generated_config_match.group(1)) + free_models = generated_config["provider"]["opencode-free"]["models"] + north_model = free_models["north-mini-code-free"] + assert north_model["tool_call"] is True + assert "response_format" not in north_model["options"] + if "deepseek-v4-flash-free" in free_models: + deepseek_model = free_models["deepseek-v4-flash-free"] + assert deepseek_model["tool_call"] is True + assert "response_format" not in deepseek_model.get("options", {}) assert github_candidate_models == [ "deepseek/deepseek-v3-0324", "openai/gpt-4.1", @@ -1210,10 +1246,18 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): "Skipping publish-step failed-check OpenCode diagnosis for central review-process self-repair" in workflow ) - assert ( - "needs.validate-pr-metadata.outputs.is_private == 'false' && " - "'opencode-free/north-mini-code-free ' || ''" - ) in workflow + assert any( + ( + "needs.validate-pr-metadata.outputs.is_private == 'false' && " + f"'{candidates} ' || ''" + ) + in workflow + for candidates in ( + "opencode-free/north-mini-code-free", + "opencode-free/deepseek-v4-flash-free " + "opencode-free/north-mini-code-free", + ) + ) assert ( "github-models/deepseek/deepseek-v3-0324 " "openai/gpt-5.6-luna " From f7dd1385e3b46001555ce7e556021c7ee0bbcd22 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 30 Jul 2026 03:28:20 +0900 Subject: [PATCH 4/5] fix(review): add DeepSeek Zen free reviewer --- .github/workflows/opencode-review-dispatch.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 14fcf2666..cdc986cc0 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -3361,6 +3361,14 @@ jobs: "baseURL": "https://opencode.ai/zen/v1" }, "models": { + "deepseek-v4-flash-free": { + "name": "DeepSeek V4 Flash Free", + "tool_call": true, + "limit": { + "context": 256000, + "output": 64000 + } + }, "north-mini-code-free": { "name": "North Mini Code Free", "tool_call": true, @@ -3694,9 +3702,11 @@ jobs: NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" # High-sensitivity review candidates only. Public repositories first - # use OpenCode Zen's anonymous North Mini Code free endpoint. Zen's - # free-period privacy terms permit retention, so private repositories - # never include this candidate. The pool then falls through to + # use OpenCode Zen's anonymous DeepSeek V4 Flash Free endpoint, then + # North Mini Code Free as fallback. During their free periods, + # DeepSeek data may be used for improvement and North data may be + # retained and used for improvement, so private repositories never + # include either candidate. The pool then falls through to # DeepSeek V3, the direct GPT-5.6 Luna slot, pinned PAID # OpenRouter coder models (free-tier candidates hit the shared # free-models-per-day cap and hung for the full candidate timeout, @@ -3707,7 +3717,7 @@ jobs: # cost-efficient tier, cheaper than the legacy gpt-5 it replaced # ($1/$6 vs $1.25/$10 per 1M tokens) so the org OpenAI budget # stretches further between top-ups. - OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'opencode-free/north-mini-code-free ' || '' }}github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" + OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free ' || '' }}github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" # One attempt per model, then fall through to the next model. Retrying # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. From 471fae2d056a53c8c91754dcfbb60f41aa55c89a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 30 Jul 2026 04:17:02 +0900 Subject: [PATCH 5/5] test(review): require DeepSeek Zen free pool --- tests/test_opencode_agent_contract.py | 60 ++++++++------------------- 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 4dc7dd55a..9e8399cc4 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -90,35 +90,16 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): candidates_match = re.search(r'OPENCODE_MODEL_CANDIDATES: "([^"]+)"', workflow) assert candidates_match is not None - public_candidate_options = ( - ( - "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " - "&& 'opencode-free/north-mini-code-free ' || '' }}", - ["opencode-free/north-mini-code-free"], - ), - ( - "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " - "&& 'opencode-free/deepseek-v4-flash-free " - "opencode-free/north-mini-code-free ' || '' }}", - [ - "opencode-free/deepseek-v4-flash-free", - "opencode-free/north-mini-code-free", - ], - ), + conditional_public_candidate = ( + "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " + "&& 'opencode-free/deepseek-v4-flash-free " + "opencode-free/north-mini-code-free ' || '' }}" ) candidates_text = candidates_match.group(1) - public_candidate = next( - ( - (prefix, candidates) - for prefix, candidates in public_candidate_options - if candidates_text.startswith(prefix) - ), - None, - ) - assert public_candidate is not None - conditional_public_candidate, public_candidates = public_candidate + assert candidates_text.startswith(conditional_public_candidate) candidates = [ - *public_candidates, + "opencode-free/deepseek-v4-flash-free", + "opencode-free/north-mini-code-free", *candidates_text.removeprefix(conditional_public_candidate).split(), ] candidate_pairs = [candidate.split("/", 1) for candidate in candidates] @@ -136,7 +117,8 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidate_pairs assert candidate_pairs == [ - *(candidate.split("/", 1) for candidate in public_candidates), + ["opencode-free", "deepseek-v4-flash-free"], + ["opencode-free", "north-mini-code-free"], ["github-models", "deepseek/deepseek-v3-0324"], ["openai", "gpt-5.6-luna"], ["openrouter", "deepseek/deepseek-v3.2"], @@ -164,13 +146,12 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert generated_config_match is not None generated_config = json.loads(generated_config_match.group(1)) free_models = generated_config["provider"]["opencode-free"]["models"] + deepseek_model = free_models["deepseek-v4-flash-free"] north_model = free_models["north-mini-code-free"] + assert deepseek_model["tool_call"] is True + assert "response_format" not in deepseek_model.get("options", {}) assert north_model["tool_call"] is True assert "response_format" not in north_model["options"] - if "deepseek-v4-flash-free" in free_models: - deepseek_model = free_models["deepseek-v4-flash-free"] - assert deepseek_model["tool_call"] is True - assert "response_format" not in deepseek_model.get("options", {}) assert github_candidate_models == [ "deepseek/deepseek-v3-0324", "openai/gpt-4.1", @@ -1246,18 +1227,11 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): "Skipping publish-step failed-check OpenCode diagnosis for central review-process self-repair" in workflow ) - assert any( - ( - "needs.validate-pr-metadata.outputs.is_private == 'false' && " - f"'{candidates} ' || ''" - ) - in workflow - for candidates in ( - "opencode-free/north-mini-code-free", - "opencode-free/deepseek-v4-flash-free " - "opencode-free/north-mini-code-free", - ) - ) + assert ( + "needs.validate-pr-metadata.outputs.is_private == 'false' && " + "'opencode-free/deepseek-v4-flash-free " + "opencode-free/north-mini-code-free ' || ''" + ) in workflow assert ( "github-models/deepseek/deepseek-v3-0324 " "openai/gpt-5.6-luna "