[codex] Fail OpenCode approve when review publish fails

.github/workflows/opencode-review.yml

@@ -37,6 +37,7 @@ concurrency:
     opencode-review-${{
     github.event.pull_request.base.repo.full_name || github.event.inputs.target_repository || github.repository }}-${{
     github.event_name == 'pull_request_target' && format('pr-{0}', github.event.pull_request.number) ||
+    github.event.inputs.pr_number && format('pr-{0}', github.event.inputs.pr_number) ||
     github.event.inputs.pr_number || github.run_id }}
   cancel-in-progress: true
 
@@ -2592,7 +2593,7 @@ jobs:
       - name: Run OpenCode PR Review model pool
         id: opencode_review_model_pool
         if: needs.coverage-evidence.result == 'success'
-        timeout-minutes: 45
+        timeout-minutes: 350
         continue-on-error: true
         env:
           STRIX_GITHUB_MODELS_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }}
@@ -2621,11 +2622,11 @@ jobs:
           # the SAME model 5x let a rate-limited/hung leader consume the whole
           # step, so the pool never reached a healthy fallback model.
           OPENCODE_MODEL_ATTEMPTS: "1"
-          # Bound provider stalls so the org queue gets an actionable log line
-          # and reaches the next candidate instead of pinning the PR for hours.
-          OPENCODE_RUN_TIMEOUT_SECONDS: "900"
+          # 90 min per model is intentionally deep enough for tool-using reviews
+          # while still forcing stale providers to yield to the next candidate.
+          OPENCODE_RUN_TIMEOUT_SECONDS: "5400"
           OPENCODE_EXPORT_TIMEOUT_SECONDS: "120"
-          OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "2400"
+          OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "18000"
           # Stop after one catalog pass; the long per-model timeout and total
           # budget provide room for deep reviews without looping to job timeout.
           OPENCODE_POOL_MAX_CYCLES: "1"
@@ -2762,12 +2763,6 @@ jobs:
             fi
           }
 
-          gh_error_is_rate_limited() {
-            local error_file="$1"
-            [ -s "$error_file" ] || return 1
-            grep -Eiq '(API rate limit exceeded|rate limit exceeded|secondary rate limit)' "$error_file"
-          }
-
                     emit_change_flow_mermaid_graph() {
                       local merge_state="${1:-UNKNOWN}"
                       local changed_files_file surfaces_file idx next_node
@@ -3023,6 +3018,18 @@ jobs:
           APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "30"
           CHECK_LOOKUP_RETRY_ATTEMPTS: "5"
           CHECK_LOOKUP_RETRY_SLEEP_SECONDS: "5"
+          OPENCODE_MODEL_CANDIDATES: "github-models/openai/gpt-5-nano"
+          OPENCODE_MODEL_ATTEMPTS: "1"
+          OPENCODE_RUN_TIMEOUT_SECONDS: "240"
+          OPENCODE_EXPORT_TIMEOUT_SECONDS: "120"
+          OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "360"
+          OPENCODE_BACKOFF_INITIAL_SECONDS: "30"
+          OPENCODE_BACKOFF_MAX_SECONDS: "30"
+          OPENCODE_FIRST_ATTEMPT_AGENT: ci-review
+          OPENCODE_AGENT: ci-review-fallback
+          OPENCODE_EXHAUSTED_REKICK_INITIAL_SLEEP_SECONDS: "15"
+          OPENCODE_EXHAUSTED_REKICK_MAX_SLEEP_SECONDS: "300"
+          OPENCODE_EXHAUSTED_REKICK_MAX_TOTAL_SECONDS: "4200"
         run: |
           set -euo pipefail
           echo "::group::OpenCode Review Approval Gate"
@@ -3071,12 +3078,6 @@ jobs:
             fi
           }
 
-          gh_error_is_rate_limited() {
-            local error_file="$1"
-            [ -s "$error_file" ] || return 1
-            grep -Eiq '(API rate limit exceeded|rate limit exceeded|secondary rate limit)' "$error_file"
-          }
-
                     emit_change_flow_mermaid_graph() {
                       local merge_state="${1:-UNKNOWN}"
                       local changed_files_file surfaces_file idx next_node
@@ -3298,26 +3299,12 @@ jobs:
                 fi
                 warn_gh_publication_failure "pull review with fallback review token" "$gh_error_file"
               fi
-              if [ "$event" = "APPROVE" ] && gh_error_is_rate_limited "$gh_error_file"; then
-                rm -f "$gh_error_file" "$review_payload_file"
-                update_review_overview "$event" "$body" || true
-                if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
-                  {
-                    printf '## OpenCode approve review publication skipped\n\n'
-                    printf -- '- Head SHA: `%s`\n' "$HEAD_SHA"
-                    printf -- '- Workflow run: %s\n' "$RUN_ID"
-                    printf -- '- Workflow attempt: %s\n\n' "$RUN_ATTEMPT"
-                    printf 'OpenCode completed the approval gate, but GitHub rejected the pull-review write due to API rate limiting. The required workflow remains successful because failed checks, mergeability, and unresolved review threads were already gated before approval.\n\n'
-                    printf '%s\n' "$body"
-                  } >>"$GITHUB_STEP_SUMMARY"
-                fi
-                printf '::warning::OpenCode could not publish the APPROVE pull review for head %s because the GitHub API rate limit was exceeded; keeping the successful approval gate result because pre-approval source, check, mergeability, and review-thread gates passed.\n' "$HEAD_SHA"
-                return 0
-              fi
               rm -f "$gh_error_file" "$review_payload_file"
               update_review_overview "$event" "$body" || true
               printf '::error::OpenCode could not publish the pull review for head %s, so the review state was not changed.\n' "$HEAD_SHA"
-              echo "::endgroup::"
+              case "$event" in
+                REQUEST_CHANGES | INLINE_COMMENT_PUBLISH_FAILED) echo "::endgroup::" ;;
+              esac
               exit 1
             fi
             rm -f "$gh_error_file" "$review_payload_file"
@@ -5009,6 +4996,64 @@ jobs:
             scripts/ci/collect_failed_check_evidence.sh "$evidence_file"
           }
 
+          rekick_model_pool_on_exhaustion() {
+            local rekick_attempt=1
+            local rekick_output outcome model rekick_status
+            local sleep_seconds="${OPENCODE_EXHAUSTED_REKICK_INITIAL_SLEEP_SECONDS:-15}"
+            local max_sleep_seconds="${OPENCODE_EXHAUSTED_REKICK_MAX_SLEEP_SECONDS:-300}"
+            local max_total_seconds="${OPENCODE_EXHAUSTED_REKICK_MAX_TOTAL_SECONDS:-4200}"
+            local started_at now elapsed
+
+            started_at="$(date +%s)"
+
+            while [ "${OPENCODE_MODEL_POOL_OUTCOME:-}" = "exhausted" ]; do
+              if [ "$max_total_seconds" -gt 0 ]; then
+                now="$(date +%s)"
+                elapsed="$((now - started_at))"
+                if [ "$elapsed" -ge "$max_total_seconds" ]; then
+                  printf 'OpenCode model pool remained exhausted for %s seconds; stopping re-kicks and continuing with fail-closed handling.\n' "$elapsed" >&2
+                  break
+                fi
+              fi
+              printf 'OpenCode model pool exhausted; re-kicking model pool (attempt %s).\n' "$rekick_attempt"
+              rekick_output="$(mktemp)"
+              rekick_status=0
+              GITHUB_OUTPUT="$rekick_output" OPENCODE_OUTPUT_FILE="$OPENCODE_MODEL_POOL_OUTPUT_FILE" \
+                bash "$GITHUB_WORKSPACE/scripts/ci/run_opencode_review_model_pool.sh" || rekick_status=$?
+              if [ "$rekick_status" -ne 0 ]; then
+                printf 'OpenCode model pool re-kick command exited with status %s; continuing with fail-closed handling.\n' "$rekick_status" >&2
+              fi
+              outcome="$(awk -F= '/^review_status=/{v=$2} END{print v}' "$rekick_output")"
+              model="$(awk -F= '/^review_model=/{v=$2} END{print v}' "$rekick_output")"
+              rm -f "$rekick_output"
+
+              if [ -z "$outcome" ]; then
+                printf 'OpenCode model pool re-kick produced no review_status output; treating outcome as exhausted.\n' >&2
+                outcome="exhausted"
+              fi
+              OPENCODE_MODEL_POOL_OUTCOME="$outcome"
+              OPENCODE_MODEL_POOL_MODEL="$model"
+              if [ "$outcome" = "success" ]; then
+                if [ -z "$model" ]; then
+                  printf 'OpenCode model pool re-kick succeeded but published an empty review_model.\n' >&2
+                fi
+                printf 'OpenCode model pool re-kick recovered with model: %s\n' "${model:-unknown}"
+                break
+              fi
+              if [ "$sleep_seconds" -gt 0 ]; then
+                printf 'OpenCode model pool still exhausted after re-kick attempt %s; retrying in %s seconds.\n' "$rekick_attempt" "$sleep_seconds"
+                sleep "$sleep_seconds"
+              fi
+              if [ "$sleep_seconds" -lt "$max_sleep_seconds" ]; then
+                sleep_seconds=$((sleep_seconds * 2))
+                if [ "$sleep_seconds" -gt "$max_sleep_seconds" ]; then
+                  sleep_seconds="$max_sleep_seconds"
+                fi
+              fi
+              rekick_attempt=$((rekick_attempt + 1))
+            done
+          }
+
           live_head_sha="$(gh api -X GET "repos/${GH_REPOSITORY}/pulls/${PR_NUMBER}" --jq '.head.sha')"
           if [ "$live_head_sha" != "$HEAD_SHA" ]; then
             echo "stale OpenCode run: event head=${HEAD_SHA}, live head=${live_head_sha}; skipping review side effects."
@@ -5020,6 +5065,7 @@ jobs:
             request_changes_for_coverage_evidence_failure
           fi
 
+          rekick_model_pool_on_exhaustion
           opencode_review_outcome="${OPENCODE_MODEL_POOL_OUTCOME:-unknown}"
 
           # The model pool uses continue-on-error so this final step can publish

scripts/ci/noema_review_gate.py

@@ -277,6 +277,11 @@ def call_llm(repo: str, number: int, pr: dict[str, Any], diff: str, truncated: b
     if not api_url or not api_key:
         print("Noema LLM review unavailable: NOEMA_LLM_API_URL or NOEMA_LLM_API_KEY is not configured.")
         return None
+    if not (api_url.lower().startswith("http://") or api_url.lower().startswith("https://")):
+        raise ValueError(
+            "URL scheme must be http or https; NOEMA_LLM_API_URL must start "
+            "with http:// or https:// to prevent SSRF vulnerabilities"
+        )
     parsed = urllib.parse.urlparse(api_url)
     if parsed.scheme.lower() not in {"http", "https"}:
         raise ValueError("URL scheme must be http or https; NOEMA_LLM_API_URL must start with http:// or https://")
@@ -299,12 +304,6 @@ def call_llm(repo: str, number: int, pr: dict[str, Any], diff: str, truncated: b
             if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_unspecified:
                 raise ValueError("URL cannot target internal IP addresses")
 
-    if not (api_url.lower().startswith("http://") or api_url.lower().startswith("https://")):
-        raise ValueError(
-            "NOEMA_LLM_API_URL must start with http:// or https:// "
-            "to prevent SSRF vulnerabilities"
-        )
-
     prompt = {
         "role": "user",
         "content": "\n".join(

scripts/ci/run_opencode_review_model_pool.sh

@@ -134,7 +134,7 @@ run_one_model_attempt() {
 	local opencode_export_file="$8"
 	local run_timeout_seconds export_timeout_seconds opencode_status session_id
 
-	run_timeout_seconds="${OPENCODE_RUN_TIMEOUT_SECONDS:-180}"
+	run_timeout_seconds="${OPENCODE_RUN_TIMEOUT_SECONDS:-5400}"
 	export_timeout_seconds="${OPENCODE_EXPORT_TIMEOUT_SECONDS:-60}"
 
 	rm -f "$opencode_json_file" "$opencode_export_file" "$candidate_output_file"
@@ -193,8 +193,8 @@ main() {
 	local -a model_candidates
 
 	attempts="${OPENCODE_MODEL_ATTEMPTS:-3}"
-	original_run_timeout="${OPENCODE_RUN_TIMEOUT_SECONDS:-900}"
-	budget_seconds="${OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-2400}"
+	original_run_timeout="${OPENCODE_RUN_TIMEOUT_SECONDS:-5400}"
+	budget_seconds="${OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-18000}"
 	max_cycles="${OPENCODE_POOL_MAX_CYCLES:-0}"
 	deadline=0
 	if [ "$budget_seconds" -gt 0 ]; then

scripts/ci/test_strix_quick_gate.sh

@@ -533,10 +533,10 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() {
 	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "tokens_limit_reached" "opencode review detects provider context-window overflow"
 	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow"
 	assert_file_contains "$workflow_file" 'timeout-minutes: 360' "opencode review target uses the maximum GitHub-hosted runner timeout"
-	assert_file_contains "$workflow_file" 'timeout-minutes: 45' "opencode model pool has a queue-friendly runner budget while the script deadline leaves approval headroom"
+	assert_file_contains "$workflow_file" 'timeout-minutes: 350' "opencode model pool has a deep-review runner budget while the script deadline leaves approval headroom"
 	assert_file_contains "$workflow_file" 'continue-on-error: true' "opencode approval gate still runs after model-pool failure to publish a reason"
-	assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "900"' "opencode primary review has a bounded per-model timeout before trying fallback models"
-	assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "2400"' "opencode model pool exits before the job timeout so the approval gate can publish a reason"
+	assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode primary review has a deep but bounded per-model timeout before trying fallback models"
+	assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "18000"' "opencode model pool exits before the job timeout so the approval gate can publish a reason"
 	assert_file_contains "$workflow_file" 'OPENCODE_POOL_MAX_CYCLES: "1"' "opencode model pool stops after one full candidate pass instead of looping to the job timeout"
 	assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed"
 	assert_file_contains "$workflow_file" "id: opencode_review_model_pool" "opencode DeepSeek V3 fallback still runs after a primary model timeout or step failure when coverage evidence passed"
@@ -644,11 +644,11 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() {
 	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode model pool has no configured model candidates." "opencode model pool fails fast when no candidates are configured"
 	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OPENAI_API_KEY is not configured" "opencode model pool skips native OpenAI candidates when the org secret is absent"
 	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "configured max cycle count" "opencode model pool exits before the job timeout after configured cycles"
-	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-2400' "opencode model pool keeps a safe default retry budget unless the workflow explicitly disables it"
+	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-18000' "opencode model pool keeps a deep-review default retry budget unless the workflow explicitly disables it"
 	assert_file_not_contains "$workflow_file" "no model produced a valid review control block" "opencode model-failure path no longer documents a final exhausted state"
 	assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode primary and fallback paths avoid multi-attempt stalls on one model"
 	assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode catalog fallback tries each model once before moving on"
-	assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "900"' "opencode catalog fallback has a bounded per-model review timeout before step timeout"
+	assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode catalog fallback has a deep but bounded per-model review timeout before step timeout"
 	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode %s attempt %s/%s failed" "opencode catalog fallback records per-model retry failures"
 	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "exponential backoff" "opencode model retry paths use exponential backoff instead of fixed sleeps"
 	assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5-mini openai/gpt-5 github-models/openai/o4-mini" "opencode review tries the observed high-success DeepSeek V3 path before native OpenAI and compact OpenAI reasoning fallbacks"
@@ -847,25 +847,11 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() {
 	assert_file_contains "$workflow_file" 'warn_gh_publication_failure "initial review overview comment"' "opencode initial overview comment soft-fails permission-denied publication errors"
 	assert_file_contains "$workflow_file" 'warn_gh_publication_failure "pull review with primary review token"' "opencode approval explains primary review publication failures"
 	assert_file_contains "$workflow_file" 'warn_gh_publication_failure "pull review with fallback review token"' "opencode approval explains fallback review publication failures"
-	assert_file_contains "$workflow_file" 'gh_error_is_rate_limited()' "opencode approval detects rate-limited publication failures"
-	assert_file_contains "$workflow_file" '[ "$event" = "APPROVE" ] && gh_error_is_rate_limited "$gh_error_file"' "opencode approval only soft-fails rate-limited approve publication failures"
-	local approval_step
-	local helper_line
-	local approve_call_line
-	approval_step="$(
-		awk '
-			/^[[:space:]]*- name: Approve PR if OpenCode review passed/ { in_step = 1 }
-			in_step { print }
-			in_step && /^[[:space:]]*- name: Run merge scheduler after approval/ { exit }
-		' "$workflow_file"
-	)"
-	helper_line="$(grep -nF 'gh_error_is_rate_limited()' <<<"$approval_step" | head -n 1 | cut -d: -f1 || true)"
-	approve_call_line="$(grep -nF '[ "$event" = "APPROVE" ] && gh_error_is_rate_limited "$gh_error_file"' <<<"$approval_step" | head -n 1 | cut -d: -f1 || true)"
-	if [ -z "$helper_line" ] || [ -z "$approve_call_line" ] || [ "$helper_line" -ge "$approve_call_line" ]; then
-		record_failure "opencode approval step must define gh_error_is_rate_limited before create_pull_review uses it"
-	fi
-	assert_file_contains "$workflow_file" 'OpenCode could not publish the APPROVE pull review for head %s because the GitHub API rate limit was exceeded' "opencode approval keeps successful gate results for rate-limited approval review publication"
-	assert_file_contains "$workflow_file" 'OpenCode could not publish the pull review for head %s, so the review state was not changed.' "opencode approval fails when review publication fails"
+	assert_file_contains "$workflow_file" 'OpenCode could not publish the pull review for head %s, so the review state was not changed.' "opencode approval fails closed when review publication fails"
+	assert_file_contains "$workflow_file" 'REQUEST_CHANGES | INLINE_COMMENT_PUBLISH_FAILED) echo "::endgroup::" ;;' "opencode only closes a review-body log group for events that opened one"
+	assert_file_not_contains "$workflow_file" 'OpenCode approve review publication skipped' "opencode approval does not report skipped approval publication as success"
+	assert_file_not_contains "$workflow_file" 'keeping the successful approval gate result' "opencode approval does not soft-pass without publishing an approving review"
+	assert_file_not_contains "$workflow_file" 'gh_error_is_rate_limited()' "opencode approval does not soft-pass rate-limited approval publication failures"
 	assert_file_contains "$workflow_file" 'warn_gh_publication_failure "review overview comment"' "opencode approval soft-fails permission-denied overview publication"
 	assert_file_not_contains "$workflow_file" 'gh api -X DELETE "repos/${GH_REPOSITORY}/issues/comments/${comment_id}"' "opencode review must not delete Review Overview gate evidence"
 	assert_file_not_contains "$workflow_file" '--file "$OPENCODE_EVIDENCE_FILE"' "opencode review must not attach evidence content to GitHub Models requests"