From 08c4f5498525aaeb6302666977149e0295d28825 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 29 Jul 2026 20:43:25 +0900 Subject: [PATCH 01/12] fix(review): force JSON from Zen free model --- .github/workflows/opencode-review-dispatch.yml | 3 ++- tests/test_opencode_agent_contract.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 74cc90ef9..8f83b48ec 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -3365,7 +3365,8 @@ jobs: "tool_call": true, "reasoning": true, "options": { - "reasoningEffort": "high" + "reasoningEffort": "high", + "response_format": {"type": "json_object"} }, "variants": { "high": { diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index f0f0839f3..29804187f 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -135,6 +135,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert set(github_candidate_models).issubset(set(github_models)) assert '"context": 256000' in workflow assert '"output": 64000' in workflow + assert '"response_format": {"type": "json_object"}' in workflow assert github_candidate_models == [ "deepseek/deepseek-v3-0324", "openai/gpt-4.1", From 7e5825b1214cdaea728a238fdd04e0a765f5f594 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 29 Jul 2026 20:57:54 +0900 Subject: [PATCH 02/12] test(review): scope Zen JSON contract assertion --- tests/test_opencode_agent_contract.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index fac054a2d..502a50d4a 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -135,7 +135,19 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert set(github_candidate_models).issubset(set(github_models)) assert '"context": 256000' in workflow assert '"output": 64000' in workflow - assert '"response_format": {"type": "json_object"}' in workflow + generated_config_match = re.search( + r"jq -n '(\{.*?\})' >\"\$\{OPENCODE_REVIEW_WORKDIR\}/opencode\.jsonc\"", + workflow, + re.DOTALL, + ) + assert generated_config_match is not None + generated_config = json.loads(generated_config_match.group(1)) + assert ( + generated_config["provider"]["opencode-free"]["models"][ + "north-mini-code-free" + ]["options"]["response_format"]["type"] + == "json_object" + ) assert github_candidate_models == [ "deepseek/deepseek-v3-0324", "openai/gpt-4.1", From 3dde7e9250ba191e2a3870a20623c5a4ba68c9ba Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 29 Jul 2026 21:21:12 +0900 Subject: [PATCH 03/12] fix(review): validate only final assistant response --- scripts/ci/run_opencode_review_model_pool.sh | 11 ++++- tests/test_opencode_model_pool_runner.py | 45 ++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index b709ca807..7c611fceb 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -477,7 +477,16 @@ run_one_model_attempt() { printf 'OpenCode %s attempt %s/%s session export did not complete within %ss.\n' "$model_candidate" "$attempt" "$attempts" "$export_timeout_seconds" return 1 fi - jq -r '.messages[] | select(.info.role == "assistant") | .parts[]? | select(.type == "text") | .text' "$opencode_export_file" >"$candidate_output_file" + jq -r ' + [ + .messages[]? + | select(.info.role == "assistant") + | [.parts[]? | select(.type == "text") | .text] + | join("\n") + | select(length > 0) + ] + | last // empty + ' "$opencode_export_file" >"$candidate_output_file" if [ ! -s "$candidate_output_file" ]; then printf 'OpenCode %s attempt %s/%s session export did not include assistant text.\n' "$model_candidate" "$attempt" "$attempts" emit_rejected_opencode_artifact_metadata "assistant-empty-export" "$opencode_export_file" diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 5e8233487..9c53bf603 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -513,6 +513,51 @@ def test_invalid_control_output_suppresses_assistant_content(tmp_path: Path) -> assert_secret_absent(result, secret) +def test_only_final_assistant_message_reaches_control_validation( + tmp_path: Path, +) -> None: + """Intermediate assistant turns cannot corrupt the final control candidate.""" + result = run_failed_model( + tmp_path, + json_line='{"type":"step_start","sessionID":"session-1"}', + extra_env={ + "FAKE_OPENCODE_RUN_EXIT": "0", + "FAKE_OPENCODE_EXPORT": json.dumps( + { + "messages": [ + { + "info": {"role": "assistant"}, + "parts": [ + { + "type": "text", + "text": "intermediate analysis must be ignored", + } + ], + }, + { + "info": {"role": "assistant"}, + "parts": [ + { + "type": "text", + "text": "final control candidate", + } + ], + }, + ] + } + ), + }, + ) + + candidate_output = ( + tmp_path + / "runner-temp" + / "opencode-review-github-models-openai-gpt-5.md" + ) + assert result.returncode == 1 + assert candidate_output.read_text(encoding="utf-8") == "final control candidate\n" + + def test_runner_never_cats_rejected_provider_artifacts() -> None: """Provider-controlled rejection files are never replayed with direct cat calls.""" runner = RUNNER.read_text(encoding="utf-8") From 3eac54c1bfbdcfac85dc27ea5eebb3f71884ddd1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 29 Jul 2026 21:52:11 +0900 Subject: [PATCH 04/12] fix(review): constrain Zen output with JSON schema --- scripts/ci/run_opencode_review_model_pool.sh | 309 +++++++++++++++++++ tests/test_opencode_model_pool_runner.py | 108 +++++++ 2 files changed, 417 insertions(+) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 7c611fceb..2c37e5c4e 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -345,6 +345,13 @@ is_openrouter_candidate() { esac } +is_structured_output_candidate() { + case "$1" in + opencode-free/*) return 0 ;; + *) return 1 ;; + esac +} + is_low_sensitivity_candidate() { case "$1" in openai/*-mini | openai/*-nano | \ @@ -396,6 +403,301 @@ cap_model_run_timeout() { fi } +reserve_loopback_port() { + python3 - <<'PY' +import socket + +with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + print(sock.getsockname()[1]) +PY +} + +stop_opencode_server() { + local server_pid="$1" + + if ! kill -0 "$server_pid" 2>/dev/null; then + wait "$server_pid" 2>/dev/null || true + return 0 + fi + kill "$server_pid" 2>/dev/null || true + for _ in $(seq 1 20); do + kill -0 "$server_pid" 2>/dev/null || break + sleep 0.1 + done + kill -9 "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true +} + +write_structured_review_request() { + local model_candidate="$1" + local agent="$2" + local prompt_file="$3" + local request_file="$4" + local provider_id="${model_candidate%%/*}" + local model_id="${model_candidate#*/}" + + jq -n \ + --arg provider_id "$provider_id" \ + --arg model_id "$model_id" \ + --arg agent "$agent" \ + --rawfile prompt "$prompt_file" \ + --arg head_sha "$HEAD_SHA" \ + --arg run_id "$RUN_ID" \ + --arg run_attempt "$RUN_ATTEMPT" ' + { + model: { + providerID: $provider_id, + modelID: $model_id + }, + agent: $agent, + parts: [ + { + type: "text", + text: $prompt + } + ], + format: { + type: "json_schema", + retryCount: 2, + schema: { + type: "object", + additionalProperties: false, + properties: { + head_sha: {type: "string", enum: [$head_sha]}, + run_id: {type: "string", enum: [$run_id]}, + run_attempt: {type: "string", enum: [$run_attempt]}, + result: { + type: "string", + enum: ["APPROVE", "REQUEST_CHANGES"] + }, + reason: {type: "string", minLength: 1}, + summary: {type: "string", minLength: 1}, + adversarial_validation: { + type: "object", + additionalProperties: false, + properties: { + status: { + type: "string", + enum: ["passed", "failed"] + }, + probes: { + type: "array", + items: { + type: "object", + additionalProperties: false, + properties: { + path: {type: "string", minLength: 1}, + line: {type: "integer", minimum: 1}, + hypothesis: {type: "string", minLength: 1}, + attack_or_counterexample: { + type: "string", + minLength: 1 + }, + evidence: {type: "string", minLength: 1}, + outcome: { + type: "string", + enum: ["falsified", "confirmed"] + } + }, + required: [ + "path", + "line", + "hypothesis", + "attack_or_counterexample", + "evidence", + "outcome" + ] + } + }, + residual_risk: {type: "string", minLength: 1} + }, + required: ["status", "probes", "residual_risk"] + }, + findings: { + type: "array", + items: { + type: "object", + additionalProperties: false, + properties: { + path: {type: "string", minLength: 1}, + line: {type: "integer", minimum: 1}, + severity: {type: "string", minLength: 1}, + title: {type: "string", minLength: 1}, + problem: {type: "string", minLength: 1}, + root_cause: {type: "string", minLength: 1}, + fix_direction: {type: "string", minLength: 1}, + regression_test_direction: { + type: "string", + minLength: 1 + }, + suggested_diff: {type: "string", minLength: 1} + }, + required: [ + "path", + "line", + "severity", + "title", + "problem", + "root_cause", + "fix_direction", + "regression_test_direction", + "suggested_diff" + ] + } + } + }, + required: [ + "head_sha", + "run_id", + "run_attempt", + "result", + "reason", + "summary", + "adversarial_validation", + "findings" + ] + } + } + } + ' >"$request_file" +} + +run_structured_output_attempt() { + local model_candidate="$1" + local attempt="$2" + local attempts="$3" + local agent="$4" + local prompt_file="$5" + local candidate_output_file="$6" + local opencode_json_file="$7" + local opencode_export_file="$8" + local opencode_stderr_file="$9" + local run_timeout_seconds="${10}" + local server_port server_url server_pid startup_timeout_seconds startup_deadline + local server_stdout_file session_file request_file http_status curl_status session_id + local server_ready=false + + server_port="$(reserve_loopback_port)" + server_url="http://127.0.0.1:${server_port}" + server_stdout_file="${opencode_json_file}.server.stdout" + session_file="${opencode_export_file}.session-create.json" + request_file="${opencode_export_file}.request.json" + startup_timeout_seconds="$(env_integer_or_default OPENCODE_SERVER_STARTUP_TIMEOUT_SECONDS 30)" + rm -f "$server_stdout_file" "$session_file" "$request_file" + + env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ + opencode serve --hostname 127.0.0.1 --port "$server_port" \ + >"$server_stdout_file" 2>"$opencode_stderr_file" & + server_pid=$! + startup_deadline=$((SECONDS + startup_timeout_seconds)) + while [ "$SECONDS" -lt "$startup_deadline" ]; do + if curl --silent --show-error --fail --max-time 2 \ + "${server_url}/global/health" >/dev/null 2>>"$opencode_stderr_file"; then + server_ready=true + break + fi + kill -0 "$server_pid" 2>/dev/null || break + sleep 1 + done + if [ "$server_ready" != "true" ]; then + stop_opencode_server "$server_pid" + printf 'OpenCode %s attempt %s/%s could not start the loopback structured-output server within %ss.\n' \ + "$model_candidate" "$attempt" "$attempts" "$startup_timeout_seconds" + emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" + rm -f "$server_stdout_file" "$session_file" "$request_file" + return 1 + fi + + jq -n \ + --arg title "PR #${PR_NUMBER} OpenCode bounded review ${model_candidate} attempt ${attempt}/${attempts}" \ + '{title: $title}' >"$request_file" + set +e + http_status="$(curl --silent --show-error --max-time 15 \ + --output "$session_file" --write-out '%{http_code}' \ + --header 'Content-Type: application/json' \ + --request POST --data-binary "@${request_file}" \ + "${server_url}/session" 2>>"$opencode_stderr_file")" + curl_status=$? + set -e + if [ "$curl_status" -ne 0 ] || [[ "$http_status" != 2?? ]]; then + [ ! -f "$session_file" ] || cp "$session_file" "$opencode_json_file" + stop_opencode_server "$server_pid" + printf 'OpenCode %s attempt %s/%s session creation failed with transport status %s and HTTP status %s.\n' \ + "$model_candidate" "$attempt" "$attempts" "$curl_status" "${http_status:-unavailable}" + emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" + rm -f "$server_stdout_file" "$session_file" "$request_file" + if is_fatal_provider_failure "$opencode_json_file"; then + return 2 + fi + return 1 + fi + session_id="$(jq -r '.id // empty' "$session_file")" + if [ -z "$session_id" ]; then + stop_opencode_server "$server_pid" + printf 'OpenCode %s attempt %s/%s session creation response did not include a session id.\n' \ + "$model_candidate" "$attempt" "$attempts" + emit_rejected_opencode_artifact_metadata "sessionless-structured-response" "$session_file" + rm -f "$server_stdout_file" "$session_file" "$request_file" + return 1 + fi + + if ! write_structured_review_request "$model_candidate" "$agent" "$prompt_file" "$request_file"; then + stop_opencode_server "$server_pid" + printf 'OpenCode %s attempt %s/%s could not build the structured review request.\n' \ + "$model_candidate" "$attempt" "$attempts" + rm -f "$server_stdout_file" "$session_file" "$request_file" + return 1 + fi + set +e + http_status="$(curl --silent --show-error --max-time "$run_timeout_seconds" \ + --output "$opencode_export_file" --write-out '%{http_code}' \ + --header 'Content-Type: application/json' \ + --request POST --data-binary "@${request_file}" \ + "${server_url}/session/${session_id}/message" 2>>"$opencode_stderr_file")" + curl_status=$? + set -e + stop_opencode_server "$server_pid" + rm -f "$server_stdout_file" "$session_file" "$request_file" + + if [ "$curl_status" -ne 0 ] || [[ "$http_status" != 2?? ]]; then + [ ! -f "$opencode_export_file" ] || cp "$opencode_export_file" "$opencode_json_file" + printf 'OpenCode %s attempt %s/%s structured review failed with transport status %s and HTTP status %s.\n' \ + "$model_candidate" "$attempt" "$attempts" "$curl_status" "${http_status:-unavailable}" + emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" + if is_fatal_provider_failure "$opencode_json_file"; then + return 2 + fi + return 1 + fi + if jq -e '.info.error != null' "$opencode_export_file" >/dev/null 2>&1; then + cp "$opencode_export_file" "$opencode_json_file" + printf 'OpenCode %s attempt %s/%s structured review returned a provider error.\n' \ + "$model_candidate" "$attempt" "$attempts" + emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" + if is_fatal_provider_failure "$opencode_json_file"; then + return 2 + fi + return 1 + fi + jq -c '(.info.structured // .info.structured_output) // empty' \ + "$opencode_export_file" >"$candidate_output_file" + if [ ! -s "$candidate_output_file" ]; then + printf 'OpenCode %s attempt %s/%s response did not include structured review output.\n' \ + "$model_candidate" "$attempt" "$attempts" + emit_rejected_opencode_artifact_metadata "missing-structured-output" "$opencode_export_file" + return 3 + fi + if ! normalize_opencode_output "$candidate_output_file"; then + printf 'OpenCode %s attempt %s/%s structured output did not include a valid control conclusion.\n' \ + "$model_candidate" "$attempt" "$attempts" + emit_rejected_opencode_artifact_metadata "invalid-structured-control-output" "$candidate_output_file" + return 3 + fi + return 0 +} + run_one_model_attempt() { local model_candidate="$1" local attempt="$2" @@ -414,6 +716,13 @@ run_one_model_attempt() { opencode_stderr_file="${opencode_json_file}.stderr" rm -f "$opencode_json_file" "$opencode_stderr_file" "$opencode_export_file" "$candidate_output_file" + if is_structured_output_candidate "$model_candidate"; then + run_structured_output_attempt \ + "$model_candidate" "$attempt" "$attempts" "$agent" "$prompt_file" \ + "$candidate_output_file" "$opencode_json_file" "$opencode_export_file" \ + "$opencode_stderr_file" "$run_timeout_seconds" + return $? + fi set +e timeout --kill-after=30s "${run_timeout_seconds}s" \ env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 9c53bf603..3d9cf34e0 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -152,7 +152,13 @@ def run_failed_model( fake_opencode = fake_bin / "opencode" fake_opencode.write_text( "#!/usr/bin/env bash\n" + 'if [ "${1:-}" = serve ]; then\n' + ' [ -z "${FAKE_OPENCODE_COMMAND_LOG:-}" ] || printf \'serve\\n\' >> "$FAKE_OPENCODE_COMMAND_LOG"\n' + " trap 'exit 0' TERM INT\n" + " while :; do sleep 1; done\n" + "fi\n" 'if [ "${1:-}" = run ]; then\n' + ' [ -z "${FAKE_OPENCODE_COMMAND_LOG:-}" ] || printf \'run\\n\' >> "$FAKE_OPENCODE_COMMAND_LOG"\n' ' [ -z "${FAKE_OPENCODE_PROMPT_CAPTURE:-}" ] || printf \'%s\\n\' "$2" > "$FAKE_OPENCODE_PROMPT_CAPTURE"\n' ' [ -z "${FAKE_OPENCODE_JSON:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_JSON"\n' ' [ -z "${FAKE_OPENCODE_STDERR:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_STDERR" >&2\n' @@ -168,6 +174,70 @@ def run_failed_model( encoding="utf-8", ) fake_opencode.chmod(0o755) + fake_curl = fake_bin / "curl" + fake_curl.write_text( + """#!/usr/bin/env bash +set -euo pipefail + +output_file="" +data_argument="" +url="" +while [ "$#" -gt 0 ]; do + case "$1" in + --output|-o) + output_file="$2" + shift 2 + ;; + --data-binary) + data_argument="$2" + shift 2 + ;; + --header|--request|--max-time|--write-out|-H|-X|-w) + shift 2 + ;; + --silent|--show-error|--fail|-s|-S|-f) + shift + ;; + http://*) + url="$1" + shift + ;; + *) + shift + ;; + esac +done + +case "$url" in + */global/health) + if [ -n "${FAKE_OPENCODE_COMMAND_LOG:-}" ]; then + for _ in $(seq 1 20); do + [ ! -s "$FAKE_OPENCODE_COMMAND_LOG" ] || break + sleep 0.05 + done + fi + exit "${FAKE_CURL_HEALTH_EXIT:-0}" + ;; + */session) + printf '{"id":"session-fake"}\n' > "$output_file" + printf '%s' "${FAKE_CURL_SESSION_HTTP_STATUS:-200}" + ;; + */message) + if [ -n "${FAKE_CURL_REQUEST_CAPTURE:-}" ] && [[ "$data_argument" = @* ]]; then + cp "${data_argument#@}" "$FAKE_CURL_REQUEST_CAPTURE" + fi + printf '%s\n' "${FAKE_OPENCODE_STRUCTURED_RESPONSE:-}" > "$output_file" + printf '%s' "${FAKE_CURL_MESSAGE_HTTP_STATUS:-200}" + ;; + *) + printf 'unexpected fake curl URL: %s\n' "$url" >&2 + exit 2 + ;; +esac +""", + encoding="utf-8", + ) + fake_curl.chmod(0o755) github_output = tmp_path / "github-output.txt" env = os.environ.copy() for name in CENTRAL_FALLBACK_ENV: @@ -558,6 +628,44 @@ def test_only_final_assistant_message_reaches_control_validation( assert candidate_output.read_text(encoding="utf-8") == "final control candidate\n" +def test_zen_candidate_uses_json_schema_session_transport(tmp_path: Path) -> None: + """Zen review output is constrained before the existing semantic validator.""" + request_capture = tmp_path / "structured-request.json" + command_log = tmp_path / "opencode-command.log" + structured = {"head_sha": "not-the-current-head"} + result = run_failed_model( + tmp_path, + model_candidates="opencode-free/north-mini-code-free", + extra_env={ + "FAKE_CURL_REQUEST_CAPTURE": bash_path(request_capture), + "FAKE_OPENCODE_COMMAND_LOG": bash_path(command_log), + "FAKE_OPENCODE_STRUCTURED_RESPONSE": json.dumps( + {"info": {"structured": structured}} + ), + }, + ) + + candidate_output = ( + tmp_path + / "runner-temp" + / "opencode-review-opencode-free-north-mini-code-free.md" + ) + request = json.loads(request_capture.read_text(encoding="utf-8")) + assert result.returncode == 1 + assert command_log.read_text(encoding="utf-8").splitlines() == ["serve"] + assert json.loads(candidate_output.read_text(encoding="utf-8")) == structured + assert request["model"] == { + "providerID": "opencode-free", + "modelID": "north-mini-code-free", + } + assert request["agent"] == "ci-review-fallback" + assert request["format"]["type"] == "json_schema" + assert request["format"]["schema"]["properties"]["head_sha"]["enum"] == [ + "1" * 40 + ] + assert "kind=invalid-structured-control-output" in result.stdout + + def test_runner_never_cats_rejected_provider_artifacts() -> None: """Provider-controlled rejection files are never replayed with direct cat calls.""" runner = RUNNER.read_text(encoding="utf-8") From 4b9422db26c1b7590ddc7f8f1bfd0c2ed06b9e00 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 29 Jul 2026 21:56:48 +0900 Subject: [PATCH 05/12] Revert "fix(review): constrain Zen output with JSON schema" This reverts commit 3eac54c1bfbdcfac85dc27ea5eebb3f71884ddd1. --- scripts/ci/run_opencode_review_model_pool.sh | 309 ------------------- tests/test_opencode_model_pool_runner.py | 108 ------- 2 files changed, 417 deletions(-) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 2c37e5c4e..7c611fceb 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -345,13 +345,6 @@ is_openrouter_candidate() { esac } -is_structured_output_candidate() { - case "$1" in - opencode-free/*) return 0 ;; - *) return 1 ;; - esac -} - is_low_sensitivity_candidate() { case "$1" in openai/*-mini | openai/*-nano | \ @@ -403,301 +396,6 @@ cap_model_run_timeout() { fi } -reserve_loopback_port() { - python3 - <<'PY' -import socket - -with socket.socket() as sock: - sock.bind(("127.0.0.1", 0)) - print(sock.getsockname()[1]) -PY -} - -stop_opencode_server() { - local server_pid="$1" - - if ! kill -0 "$server_pid" 2>/dev/null; then - wait "$server_pid" 2>/dev/null || true - return 0 - fi - kill "$server_pid" 2>/dev/null || true - for _ in $(seq 1 20); do - kill -0 "$server_pid" 2>/dev/null || break - sleep 0.1 - done - kill -9 "$server_pid" 2>/dev/null || true - wait "$server_pid" 2>/dev/null || true -} - -write_structured_review_request() { - local model_candidate="$1" - local agent="$2" - local prompt_file="$3" - local request_file="$4" - local provider_id="${model_candidate%%/*}" - local model_id="${model_candidate#*/}" - - jq -n \ - --arg provider_id "$provider_id" \ - --arg model_id "$model_id" \ - --arg agent "$agent" \ - --rawfile prompt "$prompt_file" \ - --arg head_sha "$HEAD_SHA" \ - --arg run_id "$RUN_ID" \ - --arg run_attempt "$RUN_ATTEMPT" ' - { - model: { - providerID: $provider_id, - modelID: $model_id - }, - agent: $agent, - parts: [ - { - type: "text", - text: $prompt - } - ], - format: { - type: "json_schema", - retryCount: 2, - schema: { - type: "object", - additionalProperties: false, - properties: { - head_sha: {type: "string", enum: [$head_sha]}, - run_id: {type: "string", enum: [$run_id]}, - run_attempt: {type: "string", enum: [$run_attempt]}, - result: { - type: "string", - enum: ["APPROVE", "REQUEST_CHANGES"] - }, - reason: {type: "string", minLength: 1}, - summary: {type: "string", minLength: 1}, - adversarial_validation: { - type: "object", - additionalProperties: false, - properties: { - status: { - type: "string", - enum: ["passed", "failed"] - }, - probes: { - type: "array", - items: { - type: "object", - additionalProperties: false, - properties: { - path: {type: "string", minLength: 1}, - line: {type: "integer", minimum: 1}, - hypothesis: {type: "string", minLength: 1}, - attack_or_counterexample: { - type: "string", - minLength: 1 - }, - evidence: {type: "string", minLength: 1}, - outcome: { - type: "string", - enum: ["falsified", "confirmed"] - } - }, - required: [ - "path", - "line", - "hypothesis", - "attack_or_counterexample", - "evidence", - "outcome" - ] - } - }, - residual_risk: {type: "string", minLength: 1} - }, - required: ["status", "probes", "residual_risk"] - }, - findings: { - type: "array", - items: { - type: "object", - additionalProperties: false, - properties: { - path: {type: "string", minLength: 1}, - line: {type: "integer", minimum: 1}, - severity: {type: "string", minLength: 1}, - title: {type: "string", minLength: 1}, - problem: {type: "string", minLength: 1}, - root_cause: {type: "string", minLength: 1}, - fix_direction: {type: "string", minLength: 1}, - regression_test_direction: { - type: "string", - minLength: 1 - }, - suggested_diff: {type: "string", minLength: 1} - }, - required: [ - "path", - "line", - "severity", - "title", - "problem", - "root_cause", - "fix_direction", - "regression_test_direction", - "suggested_diff" - ] - } - } - }, - required: [ - "head_sha", - "run_id", - "run_attempt", - "result", - "reason", - "summary", - "adversarial_validation", - "findings" - ] - } - } - } - ' >"$request_file" -} - -run_structured_output_attempt() { - local model_candidate="$1" - local attempt="$2" - local attempts="$3" - local agent="$4" - local prompt_file="$5" - local candidate_output_file="$6" - local opencode_json_file="$7" - local opencode_export_file="$8" - local opencode_stderr_file="$9" - local run_timeout_seconds="${10}" - local server_port server_url server_pid startup_timeout_seconds startup_deadline - local server_stdout_file session_file request_file http_status curl_status session_id - local server_ready=false - - server_port="$(reserve_loopback_port)" - server_url="http://127.0.0.1:${server_port}" - server_stdout_file="${opencode_json_file}.server.stdout" - session_file="${opencode_export_file}.session-create.json" - request_file="${opencode_export_file}.request.json" - startup_timeout_seconds="$(env_integer_or_default OPENCODE_SERVER_STARTUP_TIMEOUT_SECONDS 30)" - rm -f "$server_stdout_file" "$session_file" "$request_file" - - env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ - -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ - opencode serve --hostname 127.0.0.1 --port "$server_port" \ - >"$server_stdout_file" 2>"$opencode_stderr_file" & - server_pid=$! - startup_deadline=$((SECONDS + startup_timeout_seconds)) - while [ "$SECONDS" -lt "$startup_deadline" ]; do - if curl --silent --show-error --fail --max-time 2 \ - "${server_url}/global/health" >/dev/null 2>>"$opencode_stderr_file"; then - server_ready=true - break - fi - kill -0 "$server_pid" 2>/dev/null || break - sleep 1 - done - if [ "$server_ready" != "true" ]; then - stop_opencode_server "$server_pid" - printf 'OpenCode %s attempt %s/%s could not start the loopback structured-output server within %ss.\n' \ - "$model_candidate" "$attempt" "$attempts" "$startup_timeout_seconds" - emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" - rm -f "$server_stdout_file" "$session_file" "$request_file" - return 1 - fi - - jq -n \ - --arg title "PR #${PR_NUMBER} OpenCode bounded review ${model_candidate} attempt ${attempt}/${attempts}" \ - '{title: $title}' >"$request_file" - set +e - http_status="$(curl --silent --show-error --max-time 15 \ - --output "$session_file" --write-out '%{http_code}' \ - --header 'Content-Type: application/json' \ - --request POST --data-binary "@${request_file}" \ - "${server_url}/session" 2>>"$opencode_stderr_file")" - curl_status=$? - set -e - if [ "$curl_status" -ne 0 ] || [[ "$http_status" != 2?? ]]; then - [ ! -f "$session_file" ] || cp "$session_file" "$opencode_json_file" - stop_opencode_server "$server_pid" - printf 'OpenCode %s attempt %s/%s session creation failed with transport status %s and HTTP status %s.\n' \ - "$model_candidate" "$attempt" "$attempts" "$curl_status" "${http_status:-unavailable}" - emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" - rm -f "$server_stdout_file" "$session_file" "$request_file" - if is_fatal_provider_failure "$opencode_json_file"; then - return 2 - fi - return 1 - fi - session_id="$(jq -r '.id // empty' "$session_file")" - if [ -z "$session_id" ]; then - stop_opencode_server "$server_pid" - printf 'OpenCode %s attempt %s/%s session creation response did not include a session id.\n' \ - "$model_candidate" "$attempt" "$attempts" - emit_rejected_opencode_artifact_metadata "sessionless-structured-response" "$session_file" - rm -f "$server_stdout_file" "$session_file" "$request_file" - return 1 - fi - - if ! write_structured_review_request "$model_candidate" "$agent" "$prompt_file" "$request_file"; then - stop_opencode_server "$server_pid" - printf 'OpenCode %s attempt %s/%s could not build the structured review request.\n' \ - "$model_candidate" "$attempt" "$attempts" - rm -f "$server_stdout_file" "$session_file" "$request_file" - return 1 - fi - set +e - http_status="$(curl --silent --show-error --max-time "$run_timeout_seconds" \ - --output "$opencode_export_file" --write-out '%{http_code}' \ - --header 'Content-Type: application/json' \ - --request POST --data-binary "@${request_file}" \ - "${server_url}/session/${session_id}/message" 2>>"$opencode_stderr_file")" - curl_status=$? - set -e - stop_opencode_server "$server_pid" - rm -f "$server_stdout_file" "$session_file" "$request_file" - - if [ "$curl_status" -ne 0 ] || [[ "$http_status" != 2?? ]]; then - [ ! -f "$opencode_export_file" ] || cp "$opencode_export_file" "$opencode_json_file" - printf 'OpenCode %s attempt %s/%s structured review failed with transport status %s and HTTP status %s.\n' \ - "$model_candidate" "$attempt" "$attempts" "$curl_status" "${http_status:-unavailable}" - emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" - if is_fatal_provider_failure "$opencode_json_file"; then - return 2 - fi - return 1 - fi - if jq -e '.info.error != null' "$opencode_export_file" >/dev/null 2>&1; then - cp "$opencode_export_file" "$opencode_json_file" - printf 'OpenCode %s attempt %s/%s structured review returned a provider error.\n' \ - "$model_candidate" "$attempt" "$attempts" - emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" - if is_fatal_provider_failure "$opencode_json_file"; then - return 2 - fi - return 1 - fi - jq -c '(.info.structured // .info.structured_output) // empty' \ - "$opencode_export_file" >"$candidate_output_file" - if [ ! -s "$candidate_output_file" ]; then - printf 'OpenCode %s attempt %s/%s response did not include structured review output.\n' \ - "$model_candidate" "$attempt" "$attempts" - emit_rejected_opencode_artifact_metadata "missing-structured-output" "$opencode_export_file" - return 3 - fi - if ! normalize_opencode_output "$candidate_output_file"; then - printf 'OpenCode %s attempt %s/%s structured output did not include a valid control conclusion.\n' \ - "$model_candidate" "$attempt" "$attempts" - emit_rejected_opencode_artifact_metadata "invalid-structured-control-output" "$candidate_output_file" - return 3 - fi - return 0 -} - run_one_model_attempt() { local model_candidate="$1" local attempt="$2" @@ -716,13 +414,6 @@ run_one_model_attempt() { opencode_stderr_file="${opencode_json_file}.stderr" rm -f "$opencode_json_file" "$opencode_stderr_file" "$opencode_export_file" "$candidate_output_file" - if is_structured_output_candidate "$model_candidate"; then - run_structured_output_attempt \ - "$model_candidate" "$attempt" "$attempts" "$agent" "$prompt_file" \ - "$candidate_output_file" "$opencode_json_file" "$opencode_export_file" \ - "$opencode_stderr_file" "$run_timeout_seconds" - return $? - fi set +e timeout --kill-after=30s "${run_timeout_seconds}s" \ env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 3d9cf34e0..9c53bf603 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -152,13 +152,7 @@ def run_failed_model( fake_opencode = fake_bin / "opencode" fake_opencode.write_text( "#!/usr/bin/env bash\n" - 'if [ "${1:-}" = serve ]; then\n' - ' [ -z "${FAKE_OPENCODE_COMMAND_LOG:-}" ] || printf \'serve\\n\' >> "$FAKE_OPENCODE_COMMAND_LOG"\n' - " trap 'exit 0' TERM INT\n" - " while :; do sleep 1; done\n" - "fi\n" 'if [ "${1:-}" = run ]; then\n' - ' [ -z "${FAKE_OPENCODE_COMMAND_LOG:-}" ] || printf \'run\\n\' >> "$FAKE_OPENCODE_COMMAND_LOG"\n' ' [ -z "${FAKE_OPENCODE_PROMPT_CAPTURE:-}" ] || printf \'%s\\n\' "$2" > "$FAKE_OPENCODE_PROMPT_CAPTURE"\n' ' [ -z "${FAKE_OPENCODE_JSON:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_JSON"\n' ' [ -z "${FAKE_OPENCODE_STDERR:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_STDERR" >&2\n' @@ -174,70 +168,6 @@ def run_failed_model( encoding="utf-8", ) fake_opencode.chmod(0o755) - fake_curl = fake_bin / "curl" - fake_curl.write_text( - """#!/usr/bin/env bash -set -euo pipefail - -output_file="" -data_argument="" -url="" -while [ "$#" -gt 0 ]; do - case "$1" in - --output|-o) - output_file="$2" - shift 2 - ;; - --data-binary) - data_argument="$2" - shift 2 - ;; - --header|--request|--max-time|--write-out|-H|-X|-w) - shift 2 - ;; - --silent|--show-error|--fail|-s|-S|-f) - shift - ;; - http://*) - url="$1" - shift - ;; - *) - shift - ;; - esac -done - -case "$url" in - */global/health) - if [ -n "${FAKE_OPENCODE_COMMAND_LOG:-}" ]; then - for _ in $(seq 1 20); do - [ ! -s "$FAKE_OPENCODE_COMMAND_LOG" ] || break - sleep 0.05 - done - fi - exit "${FAKE_CURL_HEALTH_EXIT:-0}" - ;; - */session) - printf '{"id":"session-fake"}\n' > "$output_file" - printf '%s' "${FAKE_CURL_SESSION_HTTP_STATUS:-200}" - ;; - */message) - if [ -n "${FAKE_CURL_REQUEST_CAPTURE:-}" ] && [[ "$data_argument" = @* ]]; then - cp "${data_argument#@}" "$FAKE_CURL_REQUEST_CAPTURE" - fi - printf '%s\n' "${FAKE_OPENCODE_STRUCTURED_RESPONSE:-}" > "$output_file" - printf '%s' "${FAKE_CURL_MESSAGE_HTTP_STATUS:-200}" - ;; - *) - printf 'unexpected fake curl URL: %s\n' "$url" >&2 - exit 2 - ;; -esac -""", - encoding="utf-8", - ) - fake_curl.chmod(0o755) github_output = tmp_path / "github-output.txt" env = os.environ.copy() for name in CENTRAL_FALLBACK_ENV: @@ -628,44 +558,6 @@ def test_only_final_assistant_message_reaches_control_validation( assert candidate_output.read_text(encoding="utf-8") == "final control candidate\n" -def test_zen_candidate_uses_json_schema_session_transport(tmp_path: Path) -> None: - """Zen review output is constrained before the existing semantic validator.""" - request_capture = tmp_path / "structured-request.json" - command_log = tmp_path / "opencode-command.log" - structured = {"head_sha": "not-the-current-head"} - result = run_failed_model( - tmp_path, - model_candidates="opencode-free/north-mini-code-free", - extra_env={ - "FAKE_CURL_REQUEST_CAPTURE": bash_path(request_capture), - "FAKE_OPENCODE_COMMAND_LOG": bash_path(command_log), - "FAKE_OPENCODE_STRUCTURED_RESPONSE": json.dumps( - {"info": {"structured": structured}} - ), - }, - ) - - candidate_output = ( - tmp_path - / "runner-temp" - / "opencode-review-opencode-free-north-mini-code-free.md" - ) - request = json.loads(request_capture.read_text(encoding="utf-8")) - assert result.returncode == 1 - assert command_log.read_text(encoding="utf-8").splitlines() == ["serve"] - assert json.loads(candidate_output.read_text(encoding="utf-8")) == structured - assert request["model"] == { - "providerID": "opencode-free", - "modelID": "north-mini-code-free", - } - assert request["agent"] == "ci-review-fallback" - assert request["format"]["type"] == "json_schema" - assert request["format"]["schema"]["properties"]["head_sha"]["enum"] == [ - "1" * 40 - ] - assert "kind=invalid-structured-control-output" in result.stdout - - def test_runner_never_cats_rejected_provider_artifacts() -> None: """Provider-controlled rejection files are never replayed with direct cat calls.""" runner = RUNNER.read_text(encoding="utf-8") From b11ee1792ffae0ea1eecaff49b6b362b1e299e37 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 29 Jul 2026 22:37:12 +0900 Subject: [PATCH 06/12] fix(opencode): repair free model control output --- scripts/ci/run_opencode_review_model_pool.sh | 23 +++++- tests/test_opencode_model_pool_runner.py | 75 ++++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 7c611fceb..36d6e4406 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -406,7 +406,7 @@ run_one_model_attempt() { local opencode_json_file="$7" local opencode_export_file="$8" local run_timeout_seconds export_timeout_seconds opencode_status session_id opencode_stderr_file - local opencode_pid fatal_poll_seconds + local opencode_pid fatal_poll_seconds repair_json_file repair_status repair_timeout_seconds run_timeout_seconds="${OPENCODE_RUN_TIMEOUT_SECONDS:-600}" export_timeout_seconds="${OPENCODE_EXPORT_TIMEOUT_SECONDS:-120}" @@ -493,6 +493,27 @@ run_one_model_attempt() { return 1 fi if ! normalize_opencode_output "$candidate_output_file"; then + if [[ "$model_candidate" == opencode-free/* ]]; then + repair_json_file="${opencode_json_file}.control-repair" + repair_timeout_seconds="$(env_integer_or_default OPENCODE_CONTROL_REPAIR_TIMEOUT_SECONDS 120)" + set +e + timeout --kill-after=30s "${repair_timeout_seconds}s" \ + env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ + opencode run --session "$session_id" \ + "Rewrite your preceding review as exactly one JSON control object using the schema already provided. Preserve its conclusion and findings; emit no prose or Markdown. Use head_sha=${HEAD_SHA}, run_id=${RUN_ID}, and run_attempt=${RUN_ATTEMPT}." \ + --pure --agent "$agent" --model "$model_candidate" --format json \ + >"$repair_json_file" 2>>"$opencode_stderr_file" + repair_status=$? + set -e + if [ "$repair_status" -eq 0 ]; then + jq -rs '[.[] | select(.type == "text") | .part.text] | last // empty' \ + "$repair_json_file" >"$candidate_output_file" + if [ -s "$candidate_output_file" ] && normalize_opencode_output "$candidate_output_file"; then + return 0 + fi + fi + fi printf 'OpenCode %s attempt %s/%s output did not include a valid control conclusion.\n' "$model_candidate" "$attempt" "$attempts" emit_rejected_opencode_artifact_metadata "invalid-control-output" "$candidate_output_file" return 3 diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 9c53bf603..9a4fe14c6 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -153,6 +153,10 @@ def run_failed_model( fake_opencode.write_text( "#!/usr/bin/env bash\n" 'if [ "${1:-}" = run ]; then\n' + ' if [[ " $* " == *" --session "* ]] && [ -n "${FAKE_OPENCODE_REPAIR_JSON:-}" ]; then\n' + ' printf \'%s\\n\' "$FAKE_OPENCODE_REPAIR_JSON"\n' + ' exit "${FAKE_OPENCODE_REPAIR_EXIT:-0}"\n' + " fi\n" ' [ -z "${FAKE_OPENCODE_PROMPT_CAPTURE:-}" ] || printf \'%s\\n\' "$2" > "$FAKE_OPENCODE_PROMPT_CAPTURE"\n' ' [ -z "${FAKE_OPENCODE_JSON:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_JSON"\n' ' [ -z "${FAKE_OPENCODE_STDERR:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_STDERR" >&2\n' @@ -513,6 +517,77 @@ def test_invalid_control_output_suppresses_assistant_content(tmp_path: Path) -> assert_secret_absent(result, secret) +def test_free_model_repairs_invalid_control_in_the_same_session(tmp_path: Path) -> None: + """A free review can repair formatting without repeating repository analysis.""" + changed_file = "scripts/ci/run_opencode_review_model_pool.sh" + posture = "\n".join( + [ + f"Approval sufficiency: affirmative evidence from {changed_file}.", + f"Verification posture: CodeGraph inspected {changed_file}.", + "Linter/static: passed.", + "TDD/regression: passed.", + "Coverage: coverage execution evidence proves 100%.", + "Docstring coverage: coverage execution evidence proves 100%.", + "DAG: checked.", + "PoC/execution: checked.", + "DDD/domain: checked.", + "CDD/context: checked.", + "Similar issues: checked.", + "Claim/concept check: checked.", + "Standards search: checked.", + "Compatibility/convention: checked.", + "Breaking-change/backcompat: checked.", + "Performance: checked.", + "Developer experience: checked.", + "User experience: checked.", + "Visual/DOM: no web surface.", + "Accessibility/i18n: checked.", + "Supply-chain/license: checked.", + "Packaging: checked.", + "Security/privacy: checked.", + ] + ) + control = { + "head_sha": "1" * 40, + "run_id": "29189945378", + "run_attempt": "1", + "result": "APPROVE", + "reason": f"Reviewed current-head changed-file evidence in {changed_file}.", + "summary": posture, + "findings": [], + } + result = run_failed_model( + tmp_path, + json_line='{"type":"step_start","sessionID":"session-1"}', + model_candidates="opencode-free/north-mini-code-free", + changed_files=[changed_file], + extra_env={ + "FAKE_OPENCODE_RUN_EXIT": "0", + "FAKE_OPENCODE_EXPORT": json.dumps( + { + "messages": [ + { + "info": {"role": "assistant"}, + "parts": [{"type": "text", "text": "verbose review"}], + } + ] + } + ), + "FAKE_OPENCODE_REPAIR_JSON": json.dumps( + { + "type": "text", + "part": {"type": "text", "text": json.dumps(control)}, + } + ), + }, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert json.loads( + (tmp_path / "selected-output.md").read_text(encoding="utf-8") + ) == control + + def test_only_final_assistant_message_reaches_control_validation( tmp_path: Path, ) -> None: From 1bf7ce4f577001eb37a29aa55da20d97732be96a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 29 Jul 2026 22:49:48 +0900 Subject: [PATCH 07/12] fix(opencode): require actionable repaired controls --- scripts/ci/run_opencode_review_model_pool.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 36d6e4406..5b8a871f4 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -501,7 +501,7 @@ run_one_model_attempt() { env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ opencode run --session "$session_id" \ - "Rewrite your preceding review as exactly one JSON control object using the schema already provided. Preserve its conclusion and findings; emit no prose or Markdown. Use head_sha=${HEAD_SHA}, run_id=${RUN_ID}, and run_attempt=${RUN_ATTEMPT}." \ + "Rewrite your preceding review as exactly one JSON control object using the schema already provided; emit no prose or Markdown. Preserve every source-backed finding. REQUEST_CHANGES requires at least one complete source-backed finding; if none exists, use APPROVE with findings []. Use head_sha=${HEAD_SHA}, run_id=${RUN_ID}, and run_attempt=${RUN_ATTEMPT}." \ --pure --agent "$agent" --model "$model_candidate" --format json \ >"$repair_json_file" 2>>"$opencode_stderr_file" repair_status=$? From 6553d40648b208b859fa5602895cc0e769bb8f0e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 29 Jul 2026 23:07:04 +0900 Subject: [PATCH 08/12] fix(opencode): return validator feedback to free model --- scripts/ci/run_opencode_review_model_pool.sh | 10 ++++++++-- tests/test_opencode_model_pool_runner.py | 7 +++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 5b8a871f4..f97a7152d 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -407,6 +407,7 @@ run_one_model_attempt() { local opencode_export_file="$8" local run_timeout_seconds export_timeout_seconds opencode_status session_id opencode_stderr_file local opencode_pid fatal_poll_seconds repair_json_file repair_status repair_timeout_seconds + local validation_output validation_error run_timeout_seconds="${OPENCODE_RUN_TIMEOUT_SECONDS:-600}" export_timeout_seconds="${OPENCODE_EXPORT_TIMEOUT_SECONDS:-120}" @@ -492,8 +493,13 @@ run_one_model_attempt() { emit_rejected_opencode_artifact_metadata "assistant-empty-export" "$opencode_export_file" return 1 fi - if ! normalize_opencode_output "$candidate_output_file"; then + if ! validation_output="$(normalize_opencode_output "$candidate_output_file" 2>&1)"; then + printf '%s\n' "$validation_output" if [[ "$model_candidate" == opencode-free/* ]]; then + validation_error="$(printf '%s\n' "$validation_output" | sed -n '/CONTROL_REJECTED/p' | tail -n 1)" + if [ -z "$validation_error" ]; then + validation_error="$(printf '%s\n' "$validation_output" | tail -n 1)" + fi repair_json_file="${opencode_json_file}.control-repair" repair_timeout_seconds="$(env_integer_or_default OPENCODE_CONTROL_REPAIR_TIMEOUT_SECONDS 120)" set +e @@ -501,7 +507,7 @@ run_one_model_attempt() { env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ opencode run --session "$session_id" \ - "Rewrite your preceding review as exactly one JSON control object using the schema already provided; emit no prose or Markdown. Preserve every source-backed finding. REQUEST_CHANGES requires at least one complete source-backed finding; if none exists, use APPROVE with findings []. Use head_sha=${HEAD_SHA}, run_id=${RUN_ID}, and run_attempt=${RUN_ATTEMPT}." \ + "Rewrite your preceding review as exactly one JSON control object using the schema already provided; emit no prose or Markdown. Correct this validator rejection: ${validation_error}. Preserve every source-backed finding. REQUEST_CHANGES requires at least one complete source-backed finding; if none exists, use APPROVE with findings []. Use head_sha=${HEAD_SHA}, run_id=${RUN_ID}, and run_attempt=${RUN_ATTEMPT}." \ --pure --agent "$agent" --model "$model_candidate" --format json \ >"$repair_json_file" 2>>"$opencode_stderr_file" repair_status=$? diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 9a4fe14c6..5e002348f 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -154,6 +154,7 @@ def run_failed_model( "#!/usr/bin/env bash\n" 'if [ "${1:-}" = run ]; then\n' ' if [[ " $* " == *" --session "* ]] && [ -n "${FAKE_OPENCODE_REPAIR_JSON:-}" ]; then\n' + ' [ -z "${FAKE_OPENCODE_REPAIR_PROMPT_CAPTURE:-}" ] || printf \'%s\\n\' "$4" > "$FAKE_OPENCODE_REPAIR_PROMPT_CAPTURE"\n' ' printf \'%s\\n\' "$FAKE_OPENCODE_REPAIR_JSON"\n' ' exit "${FAKE_OPENCODE_REPAIR_EXIT:-0}"\n' " fi\n" @@ -520,6 +521,7 @@ def test_invalid_control_output_suppresses_assistant_content(tmp_path: Path) -> def test_free_model_repairs_invalid_control_in_the_same_session(tmp_path: Path) -> None: """A free review can repair formatting without repeating repository analysis.""" changed_file = "scripts/ci/run_opencode_review_model_pool.sh" + repair_prompt = tmp_path / "repair-prompt.txt" posture = "\n".join( [ f"Approval sufficiency: affirmative evidence from {changed_file}.", @@ -563,6 +565,7 @@ def test_free_model_repairs_invalid_control_in_the_same_session(tmp_path: Path) changed_files=[changed_file], extra_env={ "FAKE_OPENCODE_RUN_EXIT": "0", + "FAKE_OPENCODE_REPAIR_PROMPT_CAPTURE": bash_path(repair_prompt), "FAKE_OPENCODE_EXPORT": json.dumps( { "messages": [ @@ -583,6 +586,10 @@ def test_free_model_repairs_invalid_control_in_the_same_session(tmp_path: Path) ) assert result.returncode == 0, result.stdout + result.stderr + assert ( + "Correct this validator rejection: CONTROL_REJECTED:" + in repair_prompt.read_text(encoding="utf-8") + ) assert json.loads( (tmp_path / "selected-output.md").read_text(encoding="utf-8") ) == control From 207f91fe52b2eaa04d7979b787c1d21fe7ce624d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 29 Jul 2026 23:22:01 +0900 Subject: [PATCH 09/12] fix(opencode): export repaired session conclusion --- scripts/ci/run_opencode_review_model_pool.sh | 32 ++++++++++++-------- tests/test_opencode_model_pool_runner.py | 21 +++++++++++-- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index f97a7152d..a2f89e872 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -17,6 +17,19 @@ record_pool_exhausted() { record_review_status "exhausted" } +extract_final_assistant_text() { + jq -r ' + [ + .messages[]? + | select(.info.role == "assistant") + | [.parts[]? | select(.type == "text") | .text] + | join("\n") + | select(length > 0) + ] + | last // empty + ' "$1" >"$2" +} + finish_pool_without_model() { record_pool_exhausted return 1 @@ -478,16 +491,7 @@ run_one_model_attempt() { printf 'OpenCode %s attempt %s/%s session export did not complete within %ss.\n' "$model_candidate" "$attempt" "$attempts" "$export_timeout_seconds" return 1 fi - jq -r ' - [ - .messages[]? - | select(.info.role == "assistant") - | [.parts[]? | select(.type == "text") | .text] - | join("\n") - | select(length > 0) - ] - | last // empty - ' "$opencode_export_file" >"$candidate_output_file" + extract_final_assistant_text "$opencode_export_file" "$candidate_output_file" if [ ! -s "$candidate_output_file" ]; then printf 'OpenCode %s attempt %s/%s session export did not include assistant text.\n' "$model_candidate" "$attempt" "$attempts" emit_rejected_opencode_artifact_metadata "assistant-empty-export" "$opencode_export_file" @@ -513,8 +517,12 @@ run_one_model_attempt() { repair_status=$? set -e if [ "$repair_status" -eq 0 ]; then - jq -rs '[.[] | select(.type == "text") | .part.text] | last // empty' \ - "$repair_json_file" >"$candidate_output_file" + if timeout --kill-after=15s "${export_timeout_seconds}s" \ + env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ + opencode export "$session_id" --pure >"$opencode_export_file"; then + extract_final_assistant_text "$opencode_export_file" "$candidate_output_file" + fi if [ -s "$candidate_output_file" ] && normalize_opencode_output "$candidate_output_file"; then return 0 fi diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 5e002348f..2d62ec20b 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -155,6 +155,7 @@ def run_failed_model( 'if [ "${1:-}" = run ]; then\n' ' if [[ " $* " == *" --session "* ]] && [ -n "${FAKE_OPENCODE_REPAIR_JSON:-}" ]; then\n' ' [ -z "${FAKE_OPENCODE_REPAIR_PROMPT_CAPTURE:-}" ] || printf \'%s\\n\' "$4" > "$FAKE_OPENCODE_REPAIR_PROMPT_CAPTURE"\n' + ' [ -z "${FAKE_OPENCODE_REPAIR_STATE_FILE:-}" ] || : > "$FAKE_OPENCODE_REPAIR_STATE_FILE"\n' ' printf \'%s\\n\' "$FAKE_OPENCODE_REPAIR_JSON"\n' ' exit "${FAKE_OPENCODE_REPAIR_EXIT:-0}"\n' " fi\n" @@ -165,7 +166,11 @@ def run_failed_model( ' exit "${FAKE_OPENCODE_RUN_EXIT:-1}"\n' "fi\n" 'if [ "${1:-}" = export ]; then\n' - ' [ -z "${FAKE_OPENCODE_EXPORT:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_EXPORT"\n' + ' if [ -n "${FAKE_OPENCODE_REPAIR_STATE_FILE:-}" ] && [ -f "$FAKE_OPENCODE_REPAIR_STATE_FILE" ] && [ -n "${FAKE_OPENCODE_REPAIR_EXPORT:-}" ]; then\n' + ' printf \'%s\\n\' "$FAKE_OPENCODE_REPAIR_EXPORT"\n' + " else\n" + ' [ -z "${FAKE_OPENCODE_EXPORT:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_EXPORT"\n' + " fi\n" ' exit "${FAKE_OPENCODE_EXPORT_EXIT:-0}"\n' "fi\n" "printf 'unexpected fake opencode command: %s\\n' \"$*\" >&2\n" @@ -522,6 +527,7 @@ def test_free_model_repairs_invalid_control_in_the_same_session(tmp_path: Path) """A free review can repair formatting without repeating repository analysis.""" changed_file = "scripts/ci/run_opencode_review_model_pool.sh" repair_prompt = tmp_path / "repair-prompt.txt" + repair_state = tmp_path / "repair-state" posture = "\n".join( [ f"Approval sufficiency: affirmative evidence from {changed_file}.", @@ -566,6 +572,7 @@ def test_free_model_repairs_invalid_control_in_the_same_session(tmp_path: Path) extra_env={ "FAKE_OPENCODE_RUN_EXIT": "0", "FAKE_OPENCODE_REPAIR_PROMPT_CAPTURE": bash_path(repair_prompt), + "FAKE_OPENCODE_REPAIR_STATE_FILE": bash_path(repair_state), "FAKE_OPENCODE_EXPORT": json.dumps( { "messages": [ @@ -579,7 +586,17 @@ def test_free_model_repairs_invalid_control_in_the_same_session(tmp_path: Path) "FAKE_OPENCODE_REPAIR_JSON": json.dumps( { "type": "text", - "part": {"type": "text", "text": json.dumps(control)}, + "part": {"type": "text", "text": "streamed repair event"}, + } + ), + "FAKE_OPENCODE_REPAIR_EXPORT": json.dumps( + { + "messages": [ + { + "info": {"role": "assistant"}, + "parts": [{"type": "text", "text": json.dumps(control)}], + } + ] } ), }, From 56aba120d7f1a214d5b617433d4709736aecfb15 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 29 Jul 2026 23:43:45 +0900 Subject: [PATCH 10/12] fix(opencode): prefer JSON-capable free reviewer --- .github/workflows/opencode-review-dispatch.yml | 13 ++++++++++--- tests/test_opencode_agent_contract.py | 17 +++++++++-------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 7cec39de3..c2b2f4fcf 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -3361,13 +3361,20 @@ jobs: "baseURL": "https://opencode.ai/zen/v1" }, "models": { + "deepseek-v4-flash-free": { + "name": "DeepSeek V4 Flash Free", + "tool_call": true, + "limit": { + "context": 256000, + "output": 64000 + } + }, "north-mini-code-free": { "name": "North Mini Code Free", "tool_call": true, "reasoning": true, "options": { - "reasoningEffort": "high", - "response_format": {"type": "json_object"} + "reasoningEffort": "high" }, "variants": { "high": { @@ -3708,7 +3715,7 @@ jobs: # cost-efficient tier, cheaper than the legacy gpt-5 it replaced # ($1/$6 vs $1.25/$10 per 1M tokens) so the org OpenAI budget # stretches further between top-ups. - OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'opencode-free/north-mini-code-free ' || '' }}github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" + OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free ' || '' }}github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" # One attempt per model, then fall through to the next model. Retrying # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 502a50d4a..809aa1886 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -92,11 +92,13 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidates_match is not None conditional_public_candidate = ( "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " - "&& 'opencode-free/north-mini-code-free ' || '' }}" + "&& 'opencode-free/deepseek-v4-flash-free " + "opencode-free/north-mini-code-free ' || '' }}" ) candidates_text = candidates_match.group(1) assert candidates_text.startswith(conditional_public_candidate) candidates = [ + "opencode-free/deepseek-v4-flash-free", "opencode-free/north-mini-code-free", *candidates_text.removeprefix(conditional_public_candidate).split(), ] @@ -115,6 +117,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert candidate_pairs assert candidate_pairs == [ + ["opencode-free", "deepseek-v4-flash-free"], ["opencode-free", "north-mini-code-free"], ["github-models", "deepseek/deepseek-v3-0324"], ["openai", "gpt-5.6-luna"], @@ -142,12 +145,9 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): ) assert generated_config_match is not None generated_config = json.loads(generated_config_match.group(1)) - assert ( - generated_config["provider"]["opencode-free"]["models"][ - "north-mini-code-free" - ]["options"]["response_format"]["type"] - == "json_object" - ) + free_models = generated_config["provider"]["opencode-free"]["models"] + assert free_models["deepseek-v4-flash-free"]["tool_call"] is True + assert "response_format" not in free_models["north-mini-code-free"]["options"] assert github_candidate_models == [ "deepseek/deepseek-v3-0324", "openai/gpt-4.1", @@ -1225,7 +1225,8 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): ) assert ( "needs.validate-pr-metadata.outputs.is_private == 'false' && " - "'opencode-free/north-mini-code-free ' || ''" + "'opencode-free/deepseek-v4-flash-free " + "opencode-free/north-mini-code-free ' || ''" ) in workflow assert ( "github-models/deepseek/deepseek-v3-0324 " From ae72d8b19aa483bdce0c9a21fea823e1f287cca6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 30 Jul 2026 00:17:07 +0900 Subject: [PATCH 11/12] refactor(review): keep Zen free fallback minimal --- scripts/ci/run_opencode_review_model_pool.sh | 50 +------ tests/test_opencode_model_pool_runner.py | 146 +------------------ 2 files changed, 4 insertions(+), 192 deletions(-) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index a2f89e872..b709ca807 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -17,19 +17,6 @@ record_pool_exhausted() { record_review_status "exhausted" } -extract_final_assistant_text() { - jq -r ' - [ - .messages[]? - | select(.info.role == "assistant") - | [.parts[]? | select(.type == "text") | .text] - | join("\n") - | select(length > 0) - ] - | last // empty - ' "$1" >"$2" -} - finish_pool_without_model() { record_pool_exhausted return 1 @@ -419,8 +406,7 @@ run_one_model_attempt() { local opencode_json_file="$7" local opencode_export_file="$8" local run_timeout_seconds export_timeout_seconds opencode_status session_id opencode_stderr_file - local opencode_pid fatal_poll_seconds repair_json_file repair_status repair_timeout_seconds - local validation_output validation_error + local opencode_pid fatal_poll_seconds run_timeout_seconds="${OPENCODE_RUN_TIMEOUT_SECONDS:-600}" export_timeout_seconds="${OPENCODE_EXPORT_TIMEOUT_SECONDS:-120}" @@ -491,43 +477,13 @@ run_one_model_attempt() { printf 'OpenCode %s attempt %s/%s session export did not complete within %ss.\n' "$model_candidate" "$attempt" "$attempts" "$export_timeout_seconds" return 1 fi - extract_final_assistant_text "$opencode_export_file" "$candidate_output_file" + jq -r '.messages[] | select(.info.role == "assistant") | .parts[]? | select(.type == "text") | .text' "$opencode_export_file" >"$candidate_output_file" if [ ! -s "$candidate_output_file" ]; then printf 'OpenCode %s attempt %s/%s session export did not include assistant text.\n' "$model_candidate" "$attempt" "$attempts" emit_rejected_opencode_artifact_metadata "assistant-empty-export" "$opencode_export_file" return 1 fi - if ! validation_output="$(normalize_opencode_output "$candidate_output_file" 2>&1)"; then - printf '%s\n' "$validation_output" - if [[ "$model_candidate" == opencode-free/* ]]; then - validation_error="$(printf '%s\n' "$validation_output" | sed -n '/CONTROL_REJECTED/p' | tail -n 1)" - if [ -z "$validation_error" ]; then - validation_error="$(printf '%s\n' "$validation_output" | tail -n 1)" - fi - repair_json_file="${opencode_json_file}.control-repair" - repair_timeout_seconds="$(env_integer_or_default OPENCODE_CONTROL_REPAIR_TIMEOUT_SECONDS 120)" - set +e - timeout --kill-after=30s "${repair_timeout_seconds}s" \ - env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ - -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ - opencode run --session "$session_id" \ - "Rewrite your preceding review as exactly one JSON control object using the schema already provided; emit no prose or Markdown. Correct this validator rejection: ${validation_error}. Preserve every source-backed finding. REQUEST_CHANGES requires at least one complete source-backed finding; if none exists, use APPROVE with findings []. Use head_sha=${HEAD_SHA}, run_id=${RUN_ID}, and run_attempt=${RUN_ATTEMPT}." \ - --pure --agent "$agent" --model "$model_candidate" --format json \ - >"$repair_json_file" 2>>"$opencode_stderr_file" - repair_status=$? - set -e - if [ "$repair_status" -eq 0 ]; then - if timeout --kill-after=15s "${export_timeout_seconds}s" \ - env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ - -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ - opencode export "$session_id" --pure >"$opencode_export_file"; then - extract_final_assistant_text "$opencode_export_file" "$candidate_output_file" - fi - if [ -s "$candidate_output_file" ] && normalize_opencode_output "$candidate_output_file"; then - return 0 - fi - fi - fi + if ! normalize_opencode_output "$candidate_output_file"; then printf 'OpenCode %s attempt %s/%s output did not include a valid control conclusion.\n' "$model_candidate" "$attempt" "$attempts" emit_rejected_opencode_artifact_metadata "invalid-control-output" "$candidate_output_file" return 3 diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 2d62ec20b..5e8233487 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -153,12 +153,6 @@ def run_failed_model( fake_opencode.write_text( "#!/usr/bin/env bash\n" 'if [ "${1:-}" = run ]; then\n' - ' if [[ " $* " == *" --session "* ]] && [ -n "${FAKE_OPENCODE_REPAIR_JSON:-}" ]; then\n' - ' [ -z "${FAKE_OPENCODE_REPAIR_PROMPT_CAPTURE:-}" ] || printf \'%s\\n\' "$4" > "$FAKE_OPENCODE_REPAIR_PROMPT_CAPTURE"\n' - ' [ -z "${FAKE_OPENCODE_REPAIR_STATE_FILE:-}" ] || : > "$FAKE_OPENCODE_REPAIR_STATE_FILE"\n' - ' printf \'%s\\n\' "$FAKE_OPENCODE_REPAIR_JSON"\n' - ' exit "${FAKE_OPENCODE_REPAIR_EXIT:-0}"\n' - " fi\n" ' [ -z "${FAKE_OPENCODE_PROMPT_CAPTURE:-}" ] || printf \'%s\\n\' "$2" > "$FAKE_OPENCODE_PROMPT_CAPTURE"\n' ' [ -z "${FAKE_OPENCODE_JSON:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_JSON"\n' ' [ -z "${FAKE_OPENCODE_STDERR:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_STDERR" >&2\n' @@ -166,11 +160,7 @@ def run_failed_model( ' exit "${FAKE_OPENCODE_RUN_EXIT:-1}"\n' "fi\n" 'if [ "${1:-}" = export ]; then\n' - ' if [ -n "${FAKE_OPENCODE_REPAIR_STATE_FILE:-}" ] && [ -f "$FAKE_OPENCODE_REPAIR_STATE_FILE" ] && [ -n "${FAKE_OPENCODE_REPAIR_EXPORT:-}" ]; then\n' - ' printf \'%s\\n\' "$FAKE_OPENCODE_REPAIR_EXPORT"\n' - " else\n" - ' [ -z "${FAKE_OPENCODE_EXPORT:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_EXPORT"\n' - " fi\n" + ' [ -z "${FAKE_OPENCODE_EXPORT:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_EXPORT"\n' ' exit "${FAKE_OPENCODE_EXPORT_EXIT:-0}"\n' "fi\n" "printf 'unexpected fake opencode command: %s\\n' \"$*\" >&2\n" @@ -523,140 +513,6 @@ def test_invalid_control_output_suppresses_assistant_content(tmp_path: Path) -> assert_secret_absent(result, secret) -def test_free_model_repairs_invalid_control_in_the_same_session(tmp_path: Path) -> None: - """A free review can repair formatting without repeating repository analysis.""" - changed_file = "scripts/ci/run_opencode_review_model_pool.sh" - repair_prompt = tmp_path / "repair-prompt.txt" - repair_state = tmp_path / "repair-state" - posture = "\n".join( - [ - f"Approval sufficiency: affirmative evidence from {changed_file}.", - f"Verification posture: CodeGraph inspected {changed_file}.", - "Linter/static: passed.", - "TDD/regression: passed.", - "Coverage: coverage execution evidence proves 100%.", - "Docstring coverage: coverage execution evidence proves 100%.", - "DAG: checked.", - "PoC/execution: checked.", - "DDD/domain: checked.", - "CDD/context: checked.", - "Similar issues: checked.", - "Claim/concept check: checked.", - "Standards search: checked.", - "Compatibility/convention: checked.", - "Breaking-change/backcompat: checked.", - "Performance: checked.", - "Developer experience: checked.", - "User experience: checked.", - "Visual/DOM: no web surface.", - "Accessibility/i18n: checked.", - "Supply-chain/license: checked.", - "Packaging: checked.", - "Security/privacy: checked.", - ] - ) - control = { - "head_sha": "1" * 40, - "run_id": "29189945378", - "run_attempt": "1", - "result": "APPROVE", - "reason": f"Reviewed current-head changed-file evidence in {changed_file}.", - "summary": posture, - "findings": [], - } - result = run_failed_model( - tmp_path, - json_line='{"type":"step_start","sessionID":"session-1"}', - model_candidates="opencode-free/north-mini-code-free", - changed_files=[changed_file], - extra_env={ - "FAKE_OPENCODE_RUN_EXIT": "0", - "FAKE_OPENCODE_REPAIR_PROMPT_CAPTURE": bash_path(repair_prompt), - "FAKE_OPENCODE_REPAIR_STATE_FILE": bash_path(repair_state), - "FAKE_OPENCODE_EXPORT": json.dumps( - { - "messages": [ - { - "info": {"role": "assistant"}, - "parts": [{"type": "text", "text": "verbose review"}], - } - ] - } - ), - "FAKE_OPENCODE_REPAIR_JSON": json.dumps( - { - "type": "text", - "part": {"type": "text", "text": "streamed repair event"}, - } - ), - "FAKE_OPENCODE_REPAIR_EXPORT": json.dumps( - { - "messages": [ - { - "info": {"role": "assistant"}, - "parts": [{"type": "text", "text": json.dumps(control)}], - } - ] - } - ), - }, - ) - - assert result.returncode == 0, result.stdout + result.stderr - assert ( - "Correct this validator rejection: CONTROL_REJECTED:" - in repair_prompt.read_text(encoding="utf-8") - ) - assert json.loads( - (tmp_path / "selected-output.md").read_text(encoding="utf-8") - ) == control - - -def test_only_final_assistant_message_reaches_control_validation( - tmp_path: Path, -) -> None: - """Intermediate assistant turns cannot corrupt the final control candidate.""" - result = run_failed_model( - tmp_path, - json_line='{"type":"step_start","sessionID":"session-1"}', - extra_env={ - "FAKE_OPENCODE_RUN_EXIT": "0", - "FAKE_OPENCODE_EXPORT": json.dumps( - { - "messages": [ - { - "info": {"role": "assistant"}, - "parts": [ - { - "type": "text", - "text": "intermediate analysis must be ignored", - } - ], - }, - { - "info": {"role": "assistant"}, - "parts": [ - { - "type": "text", - "text": "final control candidate", - } - ], - }, - ] - } - ), - }, - ) - - candidate_output = ( - tmp_path - / "runner-temp" - / "opencode-review-github-models-openai-gpt-5.md" - ) - assert result.returncode == 1 - assert candidate_output.read_text(encoding="utf-8") == "final control candidate\n" - - def test_runner_never_cats_rejected_provider_artifacts() -> None: """Provider-controlled rejection files are never replayed with direct cat calls.""" runner = RUNNER.read_text(encoding="utf-8") From f019213367bc0ee45b2ab701c6706489ecb3f20a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 30 Jul 2026 00:24:16 +0900 Subject: [PATCH 12/12] test(review): pin free reviewer tool contracts --- tests/test_opencode_agent_contract.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 809aa1886..9e8399cc4 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -146,8 +146,12 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert generated_config_match is not None generated_config = json.loads(generated_config_match.group(1)) free_models = generated_config["provider"]["opencode-free"]["models"] - assert free_models["deepseek-v4-flash-free"]["tool_call"] is True - assert "response_format" not in free_models["north-mini-code-free"]["options"] + deepseek_model = free_models["deepseek-v4-flash-free"] + north_model = free_models["north-mini-code-free"] + assert deepseek_model["tool_call"] is True + assert "response_format" not in deepseek_model.get("options", {}) + assert north_model["tool_call"] is True + assert "response_format" not in north_model["options"] assert github_candidate_models == [ "deepseek/deepseek-v3-0324", "openai/gpt-4.1",