From d29c68f58469b817e8ec7546ab3d8852790327ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 1 Jul 2026 03:58:43 +0900 Subject: [PATCH 1/2] Remove workflow-only deterministic approval fallback --- .github/workflows/opencode-review.yml | 46 --------------------------- scripts/ci/test_strix_quick_gate.sh | 10 +++--- 2 files changed, 5 insertions(+), 51 deletions(-) diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index fb69a8c69..9cc5d93da 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -4325,32 +4325,6 @@ jobs: changed_files_summary="bounded current-head evidence" fi - if low_risk_model_exhaustion_fallback_applies; then - body="$(printf '%s\n' \ - "## Pull request overview" \ - "" \ - "OpenCode exhausted the configured model pool without a usable current-head review conclusion, but this PR only changes workflow files and all current-head peer gates passed." \ - "" \ - "## Findings" \ - "" \ - "No blocking findings." \ - "" \ - "## Summary" \ - "" \ - "- Result: APPROVE" \ - "- Reason: model output was unavailable, but deterministic current-head gates passed for a workflow-only change." \ - "- Deterministic evidence: current-head changed-file evidence (${changed_files_summary}); coverage-evidence result ${COVERAGE_EVIDENCE_RESULT:-unknown}; peer checks from statusCheckRollup excluding this OpenCode check; no unresolved human review threads; mergeability was not dirty." \ - "- Model outcome: model_pool=${OPENCODE_MODEL_POOL_OUTCOME:-unknown}; selected_model=${OPENCODE_MODEL_POOL_MODEL:-none}." \ - "- Head SHA: \`${HEAD_SHA}\`" \ - "- Workflow run: ${RUN_ID}" \ - "- Workflow attempt: ${RUN_ATTEMPT}" \ - "" \ - "This bounded workflow-only fallback does not apply to source, test, lockfile, dependency manifest, generated artifact, or documentation changes.")" - body="$(ensure_review_body_has_change_graph "$body")" - create_pull_review "APPROVE" "$body" - return 0 - fi - first_line_file="$(mktemp)" if gh pr diff "$PR_NUMBER" --repo "$GH_REPOSITORY" --patch 2>/dev/null | awk ' @@ -4434,26 +4408,6 @@ jobs: return 0 } - low_risk_model_exhaustion_fallback_applies() { - local changed_count=0 - local changed_path - - if [ ! -s "${OPENCODE_CHANGED_FILES_FILE:-}" ]; then - return 1 - fi - - while IFS= read -r changed_path; do - [ -n "$changed_path" ] || continue - changed_count=$((changed_count + 1)) - case "$changed_path" in - .github/workflows/*.yml|.github/workflows/*.yaml) ;; - *) return 1 ;; - esac - done <"$OPENCODE_CHANGED_FILES_FILE" - - [ "$changed_count" -gt 0 ] && [ "$changed_count" -le 2 ] - } - request_changes_for_merge_conflict_if_present() { local pr_json merge_state mergeable base_ref head_ref body change_graph diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 70b30d8c8..3d22a7736 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -578,8 +578,8 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_not_contains "$workflow_file" 'approve_review_tooling_bootstrap_after_model_failure' "opencode approval must not use deterministic review-tooling bootstrap approval after model-output failures" assert_file_not_contains "$workflow_file" 'Deterministic review-tooling bootstrap fallback approval was used' "opencode approval must not publish model-exhaustion approvals" assert_file_not_contains "$workflow_file" 'model-exhaustion approval did not apply' "opencode approval failure text should describe retry exhaustion, not model-exhaustion criteria" - assert_file_contains "$workflow_file" "low_risk_model_exhaustion_fallback_applies" "opencode approval has a narrow workflow-only fallback after green current-head gates" - assert_file_contains "$workflow_file" "This bounded workflow-only fallback does not apply to source, test, lockfile, dependency manifest, generated artifact, or documentation changes." "opencode workflow-only fallback documents its excluded surfaces" + assert_file_not_contains "$workflow_file" "low_risk_model_exhaustion_fallback_applies" "opencode approval must not approve from workflow-only deterministic fallback" + assert_file_not_contains "$workflow_file" "This bounded workflow-only fallback does not apply to source, test, lockfile, dependency manifest, generated artifact, or documentation changes." "opencode approval must not publish model-exhaustion approvals" assert_file_contains "$workflow_file" 'wait_for_peer_github_checks "$pending_file"' "model-failure hold waits for peer checks before changing review state" assert_file_contains "$workflow_file" 'pending_checks_file="$(mktemp)"' "model-failure hold writes pending-check evidence to a real temp file" assert_file_contains "$workflow_file" 'collect_github_checks_with_retry collect_failed_github_checks "$failed_file"' "model-failure hold rejects current-head failed peer checks" @@ -602,8 +602,8 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'scripts/ci/test_strix_quick_gate.sh)' "opencode central review fallback allowlist includes only the central gate self-test" assert_file_contains "$workflow_file" 'wait_for_peer_github_checks "$pending_checks_file"' "opencode model-failure path waits for peer checks before failing closed" assert_file_contains "$workflow_file" 'collect_unresolved_human_review_threads "$unresolved_human_threads_file"' "opencode model-failure path re-queries human threads before failing closed" - assert_file_contains "$workflow_file" ".github/workflows/*.yml|.github/workflows/*.yaml" "opencode model-exhaustion fallback is limited to workflow files" - assert_file_contains "$workflow_file" '[ "$changed_count" -gt 0 ] && [ "$changed_count" -le 2 ]' "opencode workflow-only fallback caps changed-file count" + assert_file_not_contains "$workflow_file" ".github/workflows/*.yml|.github/workflows/*.yaml" "opencode model-exhaustion fallback must not allow workflow-only deterministic approval" + assert_file_not_contains "$workflow_file" '[ "$changed_count" -gt 0 ] && [ "$changed_count" -le 2 ]' "opencode model-exhaustion fallback must not cap deterministic approval scope" assert_file_contains "$workflow_file" "all configured OpenCode model attempts failed to produce a usable current-head control block" "opencode model-output failures fail the check without publishing a review" assert_file_contains "$workflow_file" "no model produced a valid review control block" "opencode model-failure path documents why approval is withheld" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "2"' "opencode primary and deepseek review paths retry model execution" @@ -857,7 +857,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/emit_opencode_failed_check_fallback_findings.sh" '^CMD \["/app/scripts/docker_entrypoint\.sh"\]' "opencode failed-check fallback maps missing Docker entrypoint reports to the Dockerfile CMD line" assert_file_contains "$workflow_file" "Unrelated speculative findings are invalid when failed-check evidence is present." "opencode review prompt forbids unrelated failed-check findings" assert_file_contains "$workflow_file" "run_failed_check_diagnosis" "opencode approval gate reruns OpenCode diagnosis when checks fail after the initial review" - assert_file_contains "$workflow_file" "deterministic current-head gates passed for a workflow-only change" "opencode approval gate records the narrow workflow-only model-failure fallback" + assert_file_not_contains "$workflow_file" "deterministic current-head gates passed for a workflow-only change" "opencode approval gate must not record deterministic model-failure approval" assert_file_contains "$workflow_file" "request_changes_after_model_exhaustion" "opencode model-failure path reuses green current-head gates only to decide fail-closed diagnostics" assert_file_contains "$workflow_file" "request_changes_for_merge_conflict_if_present" "opencode approval gate checks mergeability before approving model or fallback output" assert_file_contains "$workflow_file" "Merge Conflict Guidance" "opencode approval gate emits explicit conflict guidance when mergeability is dirty" From df4e61053255be6bf1b7549c5026adde639a8224 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 1 Jul 2026 04:11:17 +0900 Subject: [PATCH 2/2] Enforce high effort for reasoning model candidates --- scripts/ci/run_opencode_review_model_pool.sh | 34 ++++++++++++++++++++ scripts/ci/test_strix_quick_gate.sh | 2 ++ tests/test_opencode_agent_contract.py | 4 +++ 3 files changed, 40 insertions(+) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 1d76111b2..0f3481daf 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -52,6 +52,39 @@ write_prompt() { python3 "$GITHUB_WORKSPACE/scripts/ci/render_opencode_prompt_template.py" "$prompt_file" } +reasoning_capable_model() { + case "$1" in + openai/gpt-5* | openai/o3* | openai/o4* | deepseek/deepseek-r1*) + return 0 + ;; + *) + return 1 + ;; + esac +} + +assert_reasoning_effort_for_candidate() { + local model_candidate="$1" + local provider="${model_candidate%%/*}" + local model_name="${model_candidate#*/}" + + if ! reasoning_capable_model "$model_name"; then + return 0 + fi + if [ "$provider" = "$model_candidate" ] || [ -z "$model_name" ]; then + printf 'OpenCode candidate %s is not provider-qualified.\n' "$model_candidate" + return 1 + fi + if ! jq -e --arg provider "$provider" --arg model "$model_name" ' + .provider[$provider].models[$model].reasoning == true + and .provider[$provider].models[$model].options.reasoningEffort == "high" + and .provider[$provider].models[$model].variants.high.reasoningEffort == "high" + ' opencode.jsonc >/dev/null; then + printf 'OpenCode reasoning-capable candidate %s must set reasoningEffort=high in opencode.jsonc.\n' "$model_candidate" + return 1 + fi +} + run_one_model_attempt() { local model_candidate="$1" local attempt="$2" @@ -115,6 +148,7 @@ main() { cd "$OPENCODE_REVIEW_WORKDIR" for model_candidate in $OPENCODE_MODEL_CANDIDATES; do + assert_reasoning_effort_for_candidate "$model_candidate" safe_model="${model_candidate//\//-}" prompt_file="${RUNNER_TEMP}/opencode-review-${safe_model}-prompt.md" candidate_output_file="${RUNNER_TEMP}/opencode-review-${safe_model}.md" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 3d22a7736..0e4a902ee 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -502,6 +502,8 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" "Do not spend the session listing every changed path before reviewing" "opencode review prompt prevents fallback sessions from exhausting steps on file listing" assert_file_contains "$workflow_file" "Always return a final control block instead of a progress summary" "opencode review prompt requires a gate conclusion instead of a progress summary" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'timeout --kill-after=30s "${run_timeout_seconds}s" opencode run' "opencode review model pool has a kill-after bounded timeout" + assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "assert_reasoning_effort_for_candidate" "opencode review validates high reasoning effort before running capable model candidates" + assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'reasoningEffort == "high"' "opencode review requires high reasoning effort in opencode.jsonc for capable models" assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "180"' "opencode primary review is bounded tightly enough to reach fallback models promptly" assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "2400"' "opencode model pool has a merge-safe total retry budget" assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed" diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 9f7282c8f..f92e3ced8 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -156,6 +156,10 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "opencode_review_model_pool" in workflow assert "run_opencode_review_model_pool.sh" in workflow assert "OPENCODE_MODEL_CANDIDATES" in workflow + model_pool_runner = Path("scripts/ci/run_opencode_review_model_pool.sh").read_text(encoding="utf-8") + assert "assert_reasoning_effort_for_candidate" in model_pool_runner + assert 'reasoningEffort == "high"' in model_pool_runner + assert "OpenCode reasoning-capable candidate %s must set reasoningEffort=high" in model_pool_runner assert 'timeout-minutes: 45' in workflow assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "180"' in workflow assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "2400"' in workflow