From 5efb41226a40137196a601191b4873f4894a7546 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 24 Jun 2026 13:56:58 +0900 Subject: [PATCH 1/4] Ignore transient governance helper cancellations --- scripts/ci/collect_failed_check_evidence.sh | 1 + scripts/ci/test_strix_quick_gate.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/scripts/ci/collect_failed_check_evidence.sh b/scripts/ci/collect_failed_check_evidence.sh index 5fba5b3bd..8814e3ece 100755 --- a/scripts/ci/collect_failed_check_evidence.sh +++ b/scripts/ci/collect_failed_check_evidence.sh @@ -292,6 +292,7 @@ gh api graphql \ if .__typename == "CheckRun" then select((.status // "") == "COMPLETED") | select((.conclusion // "" | ascii_upcase) as $c | ["FAILURE","TIMED_OUT","ACTION_REQUIRED","CANCELLED","STARTUP_FAILURE"] | index($c)) + | select(((.conclusion // "" | ascii_downcase) == "cancelled" and (.name // "") == "metadata-only gate evaluation" and (.checkSuite.workflowRun.workflow.name // "") == "PR Governance") | not) | [ "check_run", (((.checkSuite.workflowRun.workflow.name // "") + "/" + (.name // "check")) | gsub("^/"; "")), diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 0b26c4307..0ea2894bb 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -572,6 +572,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" 'select((.workflowName // "") == "Strix Security Scan" or (.workflowName // "") == "Strix")' "failed-check evidence only appends Strix workflow runs" assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" 'group_by(.__context_key)' "failed-check evidence groups manual Strix statuses by context before accepting superseding success" assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" 'map(last)' "failed-check evidence accepts only the latest status per context" + assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" 'metadata-only gate evaluation' "failed-check evidence ignores cancelled metadata-only PR Governance helper gates" assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" '"strix security scan/"*' "failed-check evidence maps stale Strix workflow helper checks to the manual strix evidence status" assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" '[ "$failed_run_id" -ge "$success_run_id" ]' "failed-check evidence only supersedes Strix helper checks older than the manual success run" assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" 'redact_sensitive_log()' "failed-check evidence redacts sensitive values before emitting logs" From 7428ca7e04ead1befbb4ee75d1a9f0fe9e652ad9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 24 Jun 2026 14:09:01 +0900 Subject: [PATCH 2/4] Ignore governance helper cancellations in approval gate --- .github/workflows/opencode-review.yml | 1 + scripts/ci/test_strix_quick_gate.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index 69548e2c8..e9fedce98 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -3132,6 +3132,7 @@ jobs: | select((.name // "") != "opencode-review") | select((.checkSuite.workflowRun.workflow.name // "") != "OpenCode Review") | select((.conclusion // "" | ascii_upcase) as $c | ["FAILURE","TIMED_OUT","ACTION_REQUIRED","CANCELLED","STARTUP_FAILURE"] | index($c)) + | select(((.conclusion // "" | ascii_downcase) == "cancelled" and (.name // "") == "metadata-only gate evaluation" and (.checkSuite.workflowRun.workflow.name // "") == "PR Governance") | not) | "- " + ((.checkSuite.workflowRun.workflow.name // "") + "/" + (.name // "check") | gsub("^/"; "")) + ": " + (.conclusion // "unknown") + (if (.detailsUrl // "") != "" then " (" + .detailsUrl + ")" else "" end) elif .__typename == "StatusContext" then select(((.context // "") | ascii_downcase | contains("opencode-review")) | not) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 0ea2894bb..9ff03cd75 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -573,6 +573,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" 'group_by(.__context_key)' "failed-check evidence groups manual Strix statuses by context before accepting superseding success" assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" 'map(last)' "failed-check evidence accepts only the latest status per context" assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" 'metadata-only gate evaluation' "failed-check evidence ignores cancelled metadata-only PR Governance helper gates" + assert_file_contains "$workflow_file" 'metadata-only gate evaluation' "opencode approval gate ignores cancelled metadata-only PR Governance helper gates" assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" '"strix security scan/"*' "failed-check evidence maps stale Strix workflow helper checks to the manual strix evidence status" assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" '[ "$failed_run_id" -ge "$success_run_id" ]' "failed-check evidence only supersedes Strix helper checks older than the manual success run" assert_file_contains "$REPO_ROOT/scripts/ci/collect_failed_check_evidence.sh" 'redact_sensitive_log()' "failed-check evidence redacts sensitive values before emitting logs" From faa6b51e4f29541f58c5589e124d9b4e5f1a641e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 24 Jun 2026 14:43:35 +0900 Subject: [PATCH 3/4] Prevent generic failed-check review fallbacks --- .github/workflows/opencode-review.yml | 48 ++++++++++++++----- ...opencode_failed_check_fallback_findings.sh | 3 +- scripts/ci/test_strix_quick_gate.sh | 6 ++- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index e9fedce98..eea5068bc 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -2447,7 +2447,8 @@ jobs: if "${repo_root%/}/scripts/ci/emit_opencode_failed_check_fallback_findings.sh" "$evidence_file" "$repo_root"; then return 0 fi - printf 'OpenCode failed-check fallback helper exited non-zero; using inline fallback.\n' >&2 + printf 'OpenCode failed-check fallback helper did not produce source-backed findings; leaving the PR review unchanged for rerun.\n' >&2 + return 1 fi extract_strix_failed_check_block() { @@ -2615,7 +2616,8 @@ jobs: rm -f "$strix_evidence_file" if [ "$finding_index" -eq 0 ]; then - printf 'No automated line-specific fallback pattern matched this failed check. Do not approve or post a URL-only review; inspect the failed-check evidence below, identify the exact failing source line, explain the root cause, and provide the focused rerun command before approval.\n\n' + printf 'No automated source-backed fallback pattern matched this failed check; leaving the PR review unchanged for rerun.\n' >&2 + return 1 fi } @@ -2623,12 +2625,19 @@ jobs: local failed_checks_file="$1" local evidence_file="$2" local body_file="$3" + local findings_file + + findings_file="$(mktemp)" + if ! emit_line_specific_fallback_findings "$evidence_file" >"$findings_file"; then + rm -f "$findings_file" + return 1 + fi { printf '## Pull request overview\n\n' - printf 'OpenCode reviewed the current-head bounded evidence and found failing GitHub Checks that need source-backed diagnosis before merge.\n\n' + printf 'OpenCode reviewed the current-head bounded evidence and found source-backed failed-check findings that must be addressed before merge.\n\n' printf -- '- Result: REQUEST_CHANGES\n' - printf -- "- Reason: one or more GitHub Checks failed on current head \`%s\`.\n" "$HEAD_SHA" + printf -- "- Reason: failed current-head checks were mapped to line-specific findings below for \`%s\`.\n" "$HEAD_SHA" printf -- "- Head SHA: \`%s\`\n" "$HEAD_SHA" printf -- '- Workflow run: %s\n' "$RUN_ID" printf -- '- Workflow attempt: %s\n\n' "$RUN_ATTEMPT" @@ -2636,7 +2645,7 @@ jobs: cat "$failed_checks_file" printf '\n\n\n' printf '## Findings\n\n' - emit_line_specific_fallback_findings "$evidence_file" + cat "$findings_file" printf '
\nFailed check evidence for line-specific fixes\n\n' if [ -s "$evidence_file" ]; then sed -n '1,900p' "$evidence_file" @@ -2645,6 +2654,7 @@ jobs: fi printf '\n
\n' } >"$body_file" + rm -f "$findings_file" } is_github_billing_lock_evidence() { @@ -3400,9 +3410,12 @@ jobs: if run_failed_check_diagnosis "$failed_checks_file" "$failed_check_evidence_file" "$failed_check_review_body_file" "$failed_check_review_payload_file" "$failed_check_inline_failure_body_file"; then create_pull_review_with_payload "REQUEST_CHANGES" "$(cat "$failed_check_review_body_file")" "$failed_check_review_payload_file" "$failed_check_inline_failure_body_file" - else - build_failed_check_fallback_body "$failed_checks_file" "$failed_check_evidence_file" "$failed_check_review_body_file" + elif build_failed_check_fallback_body "$failed_checks_file" "$failed_check_evidence_file" "$failed_check_review_body_file"; then create_pull_review "REQUEST_CHANGES" "$(cat "$failed_check_review_body_file")" + else + echo "::error::Failed GitHub Checks were present, but OpenCode could not derive source-backed line-specific findings after retries. Leaving the PR review unchanged so the current-head review can be rerun with better evidence." + echo "::endgroup::" + exit 1 fi echo "::endgroup::" exit 0 @@ -3587,11 +3600,14 @@ jobs: create_pull_review_with_payload "REQUEST_CHANGES" "$(cat "$failed_check_review_body_file")" "$failed_check_review_payload_file" "$failed_check_inline_failure_body_file" echo "::endgroup::" exit 0 - else - build_failed_check_fallback_body "$failed_checks_file" "$failed_check_evidence_file" "$failed_check_review_body_file" + elif build_failed_check_fallback_body "$failed_checks_file" "$failed_check_evidence_file" "$failed_check_review_body_file"; then create_pull_review "REQUEST_CHANGES" "$(cat "$failed_check_review_body_file")" echo "::endgroup::" exit 0 + else + echo "::error::Failed GitHub Checks were present, but OpenCode could not derive source-backed line-specific findings after retries. Leaving the PR review unchanged so the current-head review can be rerun with better evidence." + echo "::endgroup::" + exit 1 fi fi unresolved_human_threads_file="$(mktemp)" @@ -3658,9 +3674,12 @@ jobs: publish_request_changes_from_control "$control_json" elif run_failed_check_diagnosis "$failed_checks_file" "$failed_check_evidence_file" "$failed_check_review_body_file" "$failed_check_review_payload_file" "$failed_check_inline_failure_body_file"; then create_pull_review_with_payload "REQUEST_CHANGES" "$(cat "$failed_check_review_body_file")" "$failed_check_review_payload_file" "$failed_check_inline_failure_body_file" - else - build_failed_check_fallback_body "$failed_checks_file" "$failed_check_evidence_file" "$failed_check_review_body_file" + elif build_failed_check_fallback_body "$failed_checks_file" "$failed_check_evidence_file" "$failed_check_review_body_file"; then create_pull_review "REQUEST_CHANGES" "$(cat "$failed_check_review_body_file")" + else + echo "::error::Failed GitHub Checks were present, but OpenCode could not derive source-backed line-specific findings after retries. Leaving the PR review unchanged so the current-head review can be rerun with better evidence." + echo "::endgroup::" + exit 1 fi else publish_request_changes_from_control "$control_json" @@ -3692,9 +3711,12 @@ jobs: fi if run_failed_check_diagnosis "$failed_checks_file" "$failed_check_evidence_file" "$failed_check_review_body_file" "$failed_check_review_payload_file" "$failed_check_inline_failure_body_file"; then create_pull_review_with_payload "REQUEST_CHANGES" "$(cat "$failed_check_review_body_file")" "$failed_check_review_payload_file" "$failed_check_inline_failure_body_file" - else - build_failed_check_fallback_body "$failed_checks_file" "$failed_check_evidence_file" "$failed_check_review_body_file" + elif build_failed_check_fallback_body "$failed_checks_file" "$failed_check_evidence_file" "$failed_check_review_body_file"; then create_pull_review "REQUEST_CHANGES" "$(cat "$failed_check_review_body_file")" + else + echo "::error::Failed GitHub Checks were present, but OpenCode could not derive source-backed line-specific findings after retries. Leaving the PR review unchanged so the current-head review can be rerun with better evidence." + echo "::endgroup::" + exit 1 fi else if request_changes_for_merge_conflict_if_present; then diff --git a/scripts/ci/emit_opencode_failed_check_fallback_findings.sh b/scripts/ci/emit_opencode_failed_check_fallback_findings.sh index cdd432031..f8d13e8c8 100755 --- a/scripts/ci/emit_opencode_failed_check_fallback_findings.sh +++ b/scripts/ci/emit_opencode_failed_check_fallback_findings.sh @@ -745,5 +745,6 @@ emit_strix_provider_failure_finding "$strix_evidence_file" emit_strix_cancelled_without_log_finding "$strix_evidence_file" if [ "$finding_index" -eq 0 ]; then - printf 'No automated line-specific fallback pattern matched this failed check. Do not approve or post a URL-only review; inspect the failed-check evidence below, identify the exact failing source line, explain the root cause, and provide the focused rerun command before approval.\n\n' + printf 'No source-backed failed-check fallback finding matched the available evidence; leaving the PR review unchanged so the current-head review can be rerun with better evidence.\n' >&2 + exit 1 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 9ff03cd75..8f4505e11 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -724,7 +724,9 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" '"lsp": "allow"' "opencode generated config enables LSP" assert_file_contains "$workflow_file" '"lsp": true' "opencode generated config starts built-in LSP servers when available" assert_file_contains "$workflow_file" "OpenCode runtime tools are enabled: bash, task, webfetch, websearch, and lsp." "opencode review prompt names the enabled runtime tools" - assert_file_contains "$workflow_file" "OpenCode failed-check fallback helper exited non-zero; using inline fallback." "opencode failed-check fallback handles helper failures without aborting under set -e" + assert_file_contains "$workflow_file" "OpenCode failed-check fallback helper did not produce source-backed findings; leaving the PR review unchanged for rerun." "opencode failed-check fallback avoids generic review comments when helper output is not source-backed" + assert_file_contains "$workflow_file" "could not derive source-backed line-specific findings after retries" "opencode failed-check fallback fails the check instead of posting URL-only request-changes reviews" + assert_file_not_contains "$workflow_file" "OpenCode failed-check fallback helper exited non-zero; using inline fallback." "opencode failed-check fallback must not silently downgrade helper failures to generic inline fallback reviews" assert_file_contains "$workflow_file" "Do not depend on Copilot Review, CodeRabbitAI, or any human reviewer" "opencode review format is independent of other review agents" assert_file_contains "$REPO_ROOT/scripts/ci/emit_opencode_failed_check_fallback_findings.sh" "emit_strix_report_findings" "failed-check fallback emits every Strix vulnerability report as a separate finding" assert_file_contains "$REPO_ROOT/scripts/ci/emit_opencode_failed_check_fallback_findings.sh" "Strix provider signal left current-head security evidence incomplete" "failed-check fallback does not claim reports are absent after Strix emitted vulnerabilities" @@ -748,7 +750,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" "- Root cause:" "opencode review request-changes body includes root cause per finding" assert_file_contains "$workflow_file" "- Regression test:" "opencode review request-changes body includes regression test direction per finding" assert_file_contains "$workflow_file" "- Suggested diff:" "opencode review request-changes body includes suggested diff per finding" - assert_file_contains "$workflow_file" "OpenCode reviewed the current-head bounded evidence and found failing GitHub Checks that need source-backed diagnosis before merge." "opencode review workflow requests changes when current-head GitHub Checks failed" + assert_file_contains "$workflow_file" "OpenCode reviewed the current-head bounded evidence and found source-backed failed-check findings that must be addressed before merge." "opencode review workflow requests changes only when current-head failed checks are mapped to source-backed findings" assert_file_contains "$workflow_file" "OpenCode reviewed the current-head evidence but could not verify peer GitHub Checks before approval." "opencode review workflow explains check lookup failures instead of approving" assert_file_contains "$workflow_file" '["FAILURE","TIMED_OUT","ACTION_REQUIRED","CANCELLED","STARTUP_FAILURE"]' "opencode review workflow treats failed check-run conclusions as request-changes blockers" assert_file_contains "$workflow_file" '["FAILURE","ERROR"]' "opencode review workflow treats failed status contexts as request-changes blockers" From 18f9b7a43712e60b80de1cb55332facc533cc6fa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 24 Jun 2026 15:12:15 +0900 Subject: [PATCH 4/4] Reject generic fallback helper output --- .github/workflows/opencode-review.yml | 13 ++++++++++++- scripts/ci/test_strix_quick_gate.sh | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index eea5068bc..2a207bc50 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -2444,9 +2444,20 @@ jobs: local strix_evidence_file if [ -x "${repo_root%/}/scripts/ci/emit_opencode_failed_check_fallback_findings.sh" ]; then - if "${repo_root%/}/scripts/ci/emit_opencode_failed_check_fallback_findings.sh" "$evidence_file" "$repo_root"; then + local helper_findings_file + helper_findings_file="$(mktemp)" + if "${repo_root%/}/scripts/ci/emit_opencode_failed_check_fallback_findings.sh" "$evidence_file" "$repo_root" >"$helper_findings_file"; then + if grep -Fq "No deterministic missing-string markers" "$helper_findings_file" || + ! grep -Eq '^### [0-9]+\. ' "$helper_findings_file"; then + printf 'OpenCode failed-check fallback helper returned non-source-backed generic output; leaving the PR review unchanged for rerun.\n' >&2 + rm -f "$helper_findings_file" + return 1 + fi + cat "$helper_findings_file" + rm -f "$helper_findings_file" return 0 fi + rm -f "$helper_findings_file" printf 'OpenCode failed-check fallback helper did not produce source-backed findings; leaving the PR review unchanged for rerun.\n' >&2 return 1 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 8f4505e11..bd550fd78 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -725,6 +725,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" '"lsp": true' "opencode generated config starts built-in LSP servers when available" assert_file_contains "$workflow_file" "OpenCode runtime tools are enabled: bash, task, webfetch, websearch, and lsp." "opencode review prompt names the enabled runtime tools" assert_file_contains "$workflow_file" "OpenCode failed-check fallback helper did not produce source-backed findings; leaving the PR review unchanged for rerun." "opencode failed-check fallback avoids generic review comments when helper output is not source-backed" + assert_file_contains "$workflow_file" "OpenCode failed-check fallback helper returned non-source-backed generic output; leaving the PR review unchanged for rerun." "opencode failed-check fallback rejects stale helper scripts that exit zero with generic no-evidence text" assert_file_contains "$workflow_file" "could not derive source-backed line-specific findings after retries" "opencode failed-check fallback fails the check instead of posting URL-only request-changes reviews" assert_file_not_contains "$workflow_file" "OpenCode failed-check fallback helper exited non-zero; using inline fallback." "opencode failed-check fallback must not silently downgrade helper failures to generic inline fallback reviews" assert_file_contains "$workflow_file" "Do not depend on Copilot Review, CodeRabbitAI, or any human reviewer" "opencode review format is independent of other review agents"