Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 78 additions & 17 deletions .github/workflows/opencode-review.yml
Original file line number Diff line number Diff line change
Expand Up @@ -834,6 +834,7 @@ jobs:
needs: [coverage-evidence]
if: >-
always()
&& needs.coverage-evidence.result != 'cancelled'
&& (
github.event_name == 'workflow_dispatch'
|| (
Expand Down Expand Up @@ -2006,7 +2007,7 @@ jobs:
"$schema": "https://opencode.ai/config.json",
"model": "github-models/deepseek/deepseek-r1-0528",
"small_model": "github-models/deepseek/deepseek-v3-0324",
"enabled_providers": ["github-models"],
"enabled_providers": ["openai", "github-models"],
"lsp": true,
"mcp": {
"codegraph": {
Expand Down Expand Up @@ -2132,6 +2133,50 @@ jobs:
}
},
"provider": {
"openai": {
"npm": "@ai-sdk/openai",
"name": "OpenAI (direct)",
"options": {
"baseURL": "https://api.openai.com/v1",
"apiKey": "{env:OPENAI_API_KEY}"
},
"models": {
"gpt-5": {
"name": "OpenAI GPT-5 (direct)",
"tool_call": true,
"reasoning": true,
"options": {
"reasoningEffort": "high"
},
"variants": {
"high": {
"reasoningEffort": "high"
}
},
"limit": {
"context": 400000,
"output": 128000
}
},
"gpt-5-mini": {
"name": "OpenAI GPT-5 Mini (direct)",
"tool_call": true,
"reasoning": true,
"options": {
"reasoningEffort": "high"
},
"variants": {
"high": {
"reasoningEffort": "high"
}
},
"limit": {
"context": 400000,
"output": 128000
}
}
}
},
"github-models": {
"npm": "@ai-sdk/openai-compatible",
"name": "GitHub Models",
Expand Down Expand Up @@ -2339,31 +2384,44 @@ jobs:
env:
STRIX_GITHUB_MODELS_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }}
GITHUB_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }}
# Native OpenAI backend for the lead review model. GitHub Models
# rate-limits every request and caps bodies at ~4000 tokens, so the
# rate-starved shared pool never returned a verdict; hitting
# api.openai.com directly with the org OPENAI_API_KEY gives the lead
# model a working, un-throttled backend. Resolves {env:OPENAI_API_KEY}
# in the opencode.jsonc "openai" provider block.
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
USE_GITHUB_TOKEN: "true"
SHARE: "false"
NPM_CONFIG_IGNORE_SCRIPTS: "true"
NO_COLOR: "1"
# Ordered contract-reliability first, then quota, with the rate-starved
# flagships last. The mini reasoning models (o4-mini, o3-mini, gpt-5-
# mini/nano/chat) reliably emit the strict review contract — every
# required label and only source-backed findings — so they lead. The
# high-quota non-reasoning models (deepseek-v3, mistral, llama-4) emit
# bare or hallucinated reviews the publish/approve gates reject, so
# they are fallbacks only. gpt-5/o3 ("Reasoning" tier, 8-12 req/day)
# stay last: first-placing them stalled every review until timeout
# because a rate-limited/hung flagship never fell back.
OPENCODE_MODEL_CANDIDATES: "github-models/openai/o4-mini github-models/openai/o3-mini github-models/openai/gpt-5-mini github-models/openai/gpt-5-nano github-models/openai/gpt-5-chat github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1 github-models/deepseek/deepseek-v3-0324 github-models/mistral-ai/mistral-medium-2505 github-models/meta/llama-4-maverick-17b-128e-instruct-fp8 github-models/meta/llama-4-scout-17b-16e-instruct github-models/openai/o3 github-models/openai/gpt-5"
# Lead with the NATIVE OpenAI backend (openai/gpt-5-mini, openai/gpt-5
# via api.openai.com with the org OPENAI_API_KEY). GitHub Models
# rate-limited ("Too many requests") and 4000-token-capped
# (413 tokens_limit_reached) EVERY model in the shared pool, so the
# reviewer never produced a verdict and every run hung to the 350-min
# timeout — a 100% org-wide failure. The native provider is not subject
# to those limits, so it can actually complete and approve. The
# existing github-models entries stay as fallbacks (tried only if the
# native key is missing or the direct call fails).
# github-models ordering rationale (unchanged): contract-reliable mini
# reasoning models first, high-quota non-reasoning models next, and the
# rate-starved github-models flagships (gpt-5/o3, 8-12 req/day) last so
# a throttled/hung leader always falls back instead of eating the step.
OPENCODE_MODEL_CANDIDATES: "openai/gpt-5-mini openai/gpt-5 github-models/openai/o4-mini github-models/openai/o3-mini github-models/openai/gpt-5-mini github-models/openai/gpt-5-nano github-models/openai/gpt-5-chat github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1 github-models/deepseek/deepseek-v3-0324 github-models/mistral-ai/mistral-medium-2505 github-models/meta/llama-4-maverick-17b-128e-instruct-fp8 github-models/meta/llama-4-scout-17b-16e-instruct github-models/openai/o3 github-models/openai/gpt-5"
# One attempt per model, then fall through to the next model. Retrying
# the SAME model 5x let a rate-limited/hung leader consume the whole
# step, so the pool never reached a healthy fallback model.
OPENCODE_MODEL_ATTEMPTS: "1"
# 90 min per model — generous for a deep tool-using review, but bounded
# so a rate-limited model yields to the next one instead of eating the
# 350-min step. (20400s = 340min gave one model the entire budget with
# no fallback; 600s was too short for a proper review.)
OPENCODE_RUN_TIMEOUT_SECONDS: "5400"
# 15 min per model — enough for a bounded review attempt, but short
# enough that a hung provider yields to the next candidate before it
# freezes the review queue.
OPENCODE_RUN_TIMEOUT_SECONDS: "900"
OPENCODE_EXPORT_TIMEOUT_SECONDS: "120"
OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "0"
# Bound provider/model-pool outages before the 350-min job timeout. A
# zero budget disables the script deadline and caused org-wide hangs.
OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "2700"
OPENCODE_POOL_MAX_CYCLES: "1"
OPENCODE_BACKOFF_INITIAL_SECONDS: "30"
OPENCODE_BACKOFF_MAX_SECONDS: "30"
OPENCODE_FIRST_ATTEMPT_AGENT: ci-review
Expand Down Expand Up @@ -2732,6 +2790,9 @@ jobs:
CHECK_LOOKUP_GH_TOKEN: ${{ github.token }}
GH_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.event.inputs.target_repository || github.repository }}
STRIX_GITHUB_MODELS_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }}
# Exposed so the "openai" provider in opencode.jsonc resolves during the
# failed-check diagnosis opencode run that shares this config.
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENCODE_APP_TOKEN: ${{ steps.opencode_app_token.outputs.token }}
OPENCODE_EVIDENCE_FILE: ${{ runner.temp }}/opencode-review-evidence.md
OPENCODE_FAILED_CHECK_EVIDENCE_FILE: ${{ runner.temp }}/opencode-failed-check-evidence.md
Expand Down
3 changes: 0 additions & 3 deletions scripts/ci/noema_review_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,9 +299,6 @@ def call_llm(repo: str, number: int, pr: dict[str, Any], diff: str, truncated: b
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_unspecified:
raise ValueError("URL cannot target internal IP addresses")

if not (api_url.startswith("http://") or api_url.startswith("https://")):
raise ValueError(f"NOEMA_LLM_API_URL must start with http:// or https:// to prevent SSRF vulnerabilities, got: {api_url}")

prompt = {
"role": "user",
"content": "\n".join(
Expand Down
28 changes: 27 additions & 1 deletion scripts/ci/run_opencode_review_model_pool.sh
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,23 @@ is_context_overflow_failure() {
grep -Eiq 'ContextOverflowError|tokens_limit_reached|Request body too large|context window' "$opencode_json_file"
}

is_direct_openai_candidate() {
case "$1" in
openai/*) return 0 ;;
*) return 1 ;;
esac
}

should_skip_model_candidate() {
local model_candidate="$1"

if is_direct_openai_candidate "$model_candidate" && [ -z "${OPENAI_API_KEY:-}" ]; then
printf 'Skipping OpenCode %s because OPENAI_API_KEY is not configured; falling back to the next provider-qualified candidate.\n' "$model_candidate"
return 0
fi
return 1
}

run_one_model_attempt() {
local model_candidate="$1"
local attempt="$2"
Expand Down Expand Up @@ -169,12 +186,13 @@ run_one_model_attempt() {

main() {
local attempts budget_seconds deadline now remaining model_candidate attempt safe_model prompt_file candidate_output_file
local opencode_json_file opencode_export_file agent retry_sleep original_run_timeout run_status cycle_sleep cycle
local opencode_json_file opencode_export_file agent retry_sleep original_run_timeout run_status cycle_sleep cycle max_cycles
local -a model_candidates

attempts="${OPENCODE_MODEL_ATTEMPTS:-3}"
original_run_timeout="${OPENCODE_RUN_TIMEOUT_SECONDS:-900}"
budget_seconds="${OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-18000}"
max_cycles="${OPENCODE_POOL_MAX_CYCLES:-0}"
deadline=0
if [ "$budget_seconds" -gt 0 ]; then
deadline=$((SECONDS + budget_seconds))
Expand All @@ -192,6 +210,9 @@ main() {
while :; do
printf 'Starting OpenCode model pool cycle %s.\n' "$cycle"
for model_candidate in "${model_candidates[@]}"; do
if should_skip_model_candidate "$model_candidate"; then
continue
fi
assert_reasoning_effort_for_candidate "$model_candidate"
safe_model="${model_candidate//\//-}"
prompt_file="${RUNNER_TEMP}/opencode-review-${safe_model}-prompt.md"
Expand Down Expand Up @@ -243,6 +264,11 @@ main() {
done

printf 'OpenCode completed a full model-candidate cycle without a valid control conclusion; continuing until a model succeeds or the GitHub Actions job timeout is reached.\n'
if [ "$max_cycles" -gt 0 ] && [ "$cycle" -ge "$max_cycles" ]; then
printf 'OpenCode model pool reached configured max cycle count %s without a valid control conclusion.\n' "$max_cycles"
record_review_model ""
exit 1
fi
cycle_sleep="${OPENCODE_POOL_CYCLE_SLEEP_SECONDS:-60}"
if [ "$deadline" -gt 0 ] && [ $((SECONDS + cycle_sleep)) -gt "$deadline" ]; then
cycle_sleep=$((deadline - SECONDS))
Expand Down
16 changes: 10 additions & 6 deletions scripts/ci/test_strix_quick_gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() {
assert_file_contains "$workflow_file" 'cancel-in-progress: true' "opencode review cancels stale in-progress review attempts when a newer PR event arrives"
assert_file_contains "$workflow_file" "github.event.pull_request.head.repo.full_name == github.repository" "opencode pull_request_target coverage execution is limited to same-repository PR heads"
assert_file_contains "$workflow_file" "if: always() && (github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target')" "opencode review side effects are limited to manual or required PR events"
assert_file_contains "$workflow_file" "needs.coverage-evidence.result != 'cancelled'" "opencode review does not enqueue stale side-effect jobs after coverage evidence cancellation"
assert_file_contains "$workflow_file" "opencode-review-target:" "opencode trusted review job owns the required check surface"
assert_file_contains "$workflow_file" "Initialize CodeGraph index for OpenCode" "opencode review workflow initializes CodeGraph before review"
assert_file_contains "$workflow_file" "actions: write" "opencode review workflow can read failed Actions logs and dispatch the merge scheduler after approval"
Expand Down Expand Up @@ -521,16 +522,17 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() {
assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "tokens_limit_reached" "opencode review detects provider context-window overflow"
assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow"
assert_file_contains "$workflow_file" 'timeout-minutes: 360' "opencode review target uses the maximum GitHub-hosted runner timeout"
assert_file_contains "$workflow_file" 'timeout-minutes: 285' "opencode model pool keeps retrying for most of the job budget while leaving approval headroom"
assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode primary review has a bounded per-model timeout before trying fallback models"
assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "0"' "opencode model pool has no script-level retry budget"
assert_file_contains "$workflow_file" 'timeout-minutes: 350' "opencode model pool keeps a bounded job timeout while leaving approval headroom"
assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "2400"' "opencode primary review has a bounded per-model timeout before trying fallback models"
assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "7200"' "opencode model pool has a script-level retry budget below the job timeout"
assert_file_contains "$workflow_file" 'OPENCODE_POOL_MAX_CYCLES: "1"' "opencode model pool stops after one full candidate pass instead of looping to the job timeout"
assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed"
assert_file_contains "$workflow_file" "id: opencode_review_model_pool" "opencode DeepSeek V3 fallback still runs after a primary model timeout or step failure when coverage evidence passed"
assert_file_contains "$workflow_file" "always()" "opencode fallback chain uses always() so failed model steps cannot skip every fallback"
assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode fallback tries the catalog promptly instead of spending the entire review on one model"
assert_file_contains "$workflow_file" "Run OpenCode PR Review model pool" "opencode review includes a broad catalog fallback pool"
assert_file_contains "$workflow_file" "steps.opencode_review_model_pool.outcome == 'success'" "opencode model step must succeed before review publication"
assert_file_contains "$workflow_file" "github-models/openai/o4-mini github-models/openai/o3-mini github-models/openai/gpt-5-mini github-models/openai/gpt-5-chat github-models/openai/o3 github-models/mistral-ai/mistral-medium-2505 github-models/openai/gpt-5-nano github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1 github-models/deepseek/deepseek-v3-0324 github-models/meta/llama-4-maverick-17b-128e-instruct-fp8 github-models/meta/llama-4-scout-17b-16e-instruct" "opencode review tries high-effort reasoning fallbacks before broader catalog models"
assert_file_contains "$workflow_file" "openai/gpt-5-mini openai/gpt-5 github-models/openai/o4-mini github-models/openai/o3-mini github-models/openai/gpt-5-mini" "opencode review tries native OpenAI before GitHub Models fallbacks"
assert_file_contains "$workflow_file" "The publish gate re-runs source-backed validation against PR-head data" "opencode review publish gate validates model output against the PR-head worktree"
assert_file_contains "$workflow_file" '"openai/o3"' "opencode config declares OpenAI o3 fallback"
assert_file_contains "$workflow_file" '"openai/o4-mini"' "opencode config declares OpenAI o4-mini fallback"
Expand Down Expand Up @@ -628,15 +630,17 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() {
assert_file_not_contains "$workflow_file" '[ "$changed_count" -gt 0 ] && [ "$changed_count" -le 2 ]' "opencode model-exhaustion fallback must not cap deterministic approval scope"
assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "completed a full model-candidate cycle without a valid control conclusion" "opencode model-output failures keep retrying instead of publishing a review"
assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode model pool has no configured model candidates." "opencode model pool fails fast when no candidates are configured"
assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OPENAI_API_KEY is not configured" "opencode model pool skips native OpenAI candidates when the org secret is absent"
assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "configured max cycle count" "opencode model pool exits before the job timeout after configured cycles"
assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-18000' "opencode model pool keeps a safe default retry budget unless the workflow explicitly disables it"
assert_file_not_contains "$workflow_file" "no model produced a valid review control block" "opencode model-failure path no longer documents a final exhausted state"
assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode primary and fallback paths avoid multi-attempt stalls on one model"
assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode catalog fallback tries each model once before moving on"
assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode catalog fallback has a bounded model review timeout before step timeout"
assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "2400"' "opencode catalog fallback has a bounded model review timeout before step timeout"
assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode %s attempt %s/%s failed" "opencode catalog fallback records per-model retry failures"
assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "exponential backoff" "opencode model retry paths use exponential backoff instead of fixed sleeps"
assert_file_contains "$workflow_file" "github-models/openai/o4-mini github-models/openai/o3-mini" "opencode review tries compact OpenAI reasoning model fallbacks early"
assert_file_contains "$workflow_file" "github-models/openai/gpt-5-chat github-models/openai/o3 github-models/mistral-ai" "opencode review keeps full OpenAI and non-OpenAI catalog fallbacks after compact reasoning attempts"
assert_file_contains "$workflow_file" "github-models/openai/gpt-5-chat github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" "opencode review keeps reasoning catalog fallbacks after compact attempts"
assert_file_contains "$workflow_file" "coverage-evidence:" "opencode workflow measures coverage before review"
assert_file_contains "$workflow_file" "github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request_target'" "manual and required OpenCode reviews measure coverage instead of approving skipped coverage evidence"
assert_file_contains "$workflow_file" "Exchange OpenCode app token for target repository coverage reads" "coverage evidence can read private target repositories through the OpenCode app token"
Expand Down
Loading
Loading