From 4b4d0a4a6a3ce3c9f68c8aee331a537f9a5a9761 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 8 Jul 2026 20:18:39 +0900 Subject: [PATCH 1/4] fix(opencode-review): reach a healthy fallback model within the step budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Required OpenCode Review check timed out red on every gyeot PR (ContextualWisdomLab/gyeot #4, #5, #6 — including a one-file docs PR), so the model-pool step is structurally, not transiently, broken. Root cause (from the job logs): the 3-4 front-loaded mini models (o4-mini, gpt-5-mini, gpt-5-nano) each hang/rate-limit and are killed at the 90-min (5400s) per-model `timeout` (exit 124). Three hangs burn ~270 min before the pool can reach any high-quota fallback (deepseek/mistral/llama, candidates #6-11), so the 350-min hard step timeout kills the job first — opaquely, with no recorded review_status. Fix: - OPENCODE_RUN_TIMEOUT_SECONDS 5400 -> 1800 (30 min): still ample for a deep tool-using review (5x the 600s that was too short), but low enough that the full 13-candidate list is traversable inside the 350-min step even if every leading model hangs, so a responsive fallback is reached. - OPENCODE_TOTAL_RETRY_BUDGET_SECONDS 0 -> 20400 (340 min): a clean internal deadline 10 min under the step timeout. The pool already caps each run at min(run_timeout, deadline-now) and exits with an explicit "retry deadline elapsed" reason + recorded status, so a genuine exhaustion now prints its cause instead of GitHub's opaque "timed out after 350 minutes". Also updates the strix quick-gate assertions that pin these two values. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01RTAMs4bpSZS77Xe3RQjv9P --- .github/workflows/opencode-review.yml | 23 +++++++++++++++++------ scripts/ci/test_strix_quick_gate.sh | 6 +++--- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index b0f387961..48842d594 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -2357,13 +2357,24 @@ jobs: # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. OPENCODE_MODEL_ATTEMPTS: "1" - # 90 min per model — generous for a deep tool-using review, but bounded - # so a rate-limited model yields to the next one instead of eating the - # 350-min step. (20400s = 340min gave one model the entire budget with - # no fallback; 600s was too short for a proper review.) - OPENCODE_RUN_TIMEOUT_SECONDS: "5400" + # 30 min per model — ample for a deep tool-using review (5x the 600s + # that was too short), but low enough that the 3-4 front-loaded mini + # models can each hang/rate-limit and STILL leave budget to reach a + # healthy high-quota fallback (deepseek/mistral/llama, candidates + # #6-11) inside the 350-min step. At 90 min/model the pool burned + # ~270 min on the first 3 hanging models (each killed at exit 124) + # and never reached any fallback, so every review timed out red on + # PRs of every size (a one-file docs PR failed identically). At 30 + # min/model the full 13-candidate list is traversable within budget. + OPENCODE_RUN_TIMEOUT_SECONDS: "1800" OPENCODE_EXPORT_TIMEOUT_SECONDS: "120" - OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "0" + # Clean internal deadline 10 min under the 350-min step timeout. The + # pool caps each per-model run at min(run_timeout, deadline-now) and + # exits with an explicit "retry deadline elapsed" reason + recorded + # review_status when the budget is spent, instead of being hard-killed + # by GitHub's opaque "##[error] timed out after 350 minutes" with no + # recorded status. Keeps every failure's cause visible in the log. + OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "20400" OPENCODE_BACKOFF_INITIAL_SECONDS: "30" OPENCODE_BACKOFF_MAX_SECONDS: "30" OPENCODE_FIRST_ATTEMPT_AGENT: ci-review diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index f51cfff59..b00c2e67c 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -522,8 +522,8 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow" assert_file_contains "$workflow_file" 'timeout-minutes: 360' "opencode review target uses the maximum GitHub-hosted runner timeout" assert_file_contains "$workflow_file" 'timeout-minutes: 285' "opencode model pool keeps retrying for most of the job budget while leaving approval headroom" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode primary review has a bounded per-model timeout before trying fallback models" - assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "0"' "opencode model pool has no script-level retry budget" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "1800"' "opencode primary review has a bounded per-model timeout before trying fallback models" + assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "20400"' "opencode model pool bounds its retry budget just under the step timeout so it exits with a recorded reason instead of an opaque hard kill" assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed" assert_file_contains "$workflow_file" "id: opencode_review_model_pool" "opencode DeepSeek V3 fallback still runs after a primary model timeout or step failure when coverage evidence passed" assert_file_contains "$workflow_file" "always()" "opencode fallback chain uses always() so failed model steps cannot skip every fallback" @@ -632,7 +632,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_not_contains "$workflow_file" "no model produced a valid review control block" "opencode model-failure path no longer documents a final exhausted state" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode primary and fallback paths avoid multi-attempt stalls on one model" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode catalog fallback tries each model once before moving on" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode catalog fallback has a bounded model review timeout before step timeout" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "1800"' "opencode catalog fallback has a bounded model review timeout before step timeout" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode %s attempt %s/%s failed" "opencode catalog fallback records per-model retry failures" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "exponential backoff" "opencode model retry paths use exponential backoff instead of fixed sleeps" assert_file_contains "$workflow_file" "github-models/openai/o4-mini github-models/openai/o3-mini" "opencode review tries compact OpenAI reasoning model fallbacks early" From b79cd181eb0ebaca1d845fa7e2bceccb92276b5e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 9 Jul 2026 13:07:09 +0900 Subject: [PATCH 2/4] test: align OpenCode model pool timeout contract --- scripts/ci/test_strix_quick_gate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index b00c2e67c..6ca4e5399 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -521,7 +521,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "tokens_limit_reached" "opencode review detects provider context-window overflow" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow" assert_file_contains "$workflow_file" 'timeout-minutes: 360' "opencode review target uses the maximum GitHub-hosted runner timeout" - assert_file_contains "$workflow_file" 'timeout-minutes: 285' "opencode model pool keeps retrying for most of the job budget while leaving approval headroom" + assert_file_contains "$workflow_file" 'timeout-minutes: 350' "opencode model pool keeps retrying for most of the job budget while leaving approval headroom" assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "1800"' "opencode primary review has a bounded per-model timeout before trying fallback models" assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "20400"' "opencode model pool bounds its retry budget just under the step timeout so it exits with a recorded reason instead of an opaque hard kill" assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed" From 984fa13fb06c943876d200bd60baeaa7b336ffd2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 9 Jul 2026 13:39:11 +0900 Subject: [PATCH 3/4] chore: retrigger checks after runner config fix From d355b1a6f06069d2d813629eef51fdb95794d2e9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 9 Jul 2026 13:40:53 +0900 Subject: [PATCH 4/4] chore: retrigger checks on standard hosted runners