diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 94f5cbb7c..91ecded91 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -730,6 +730,7 @@ jobs: summary_file="${RUNNER_TEMP}/coverage-evidence.md" summary_output_file="${RUNNER_TEMP}/coverage-evidence-output.md" failures=0 + r_peer_check_required=0 append() { printf '%s\n' "$*" >>"$summary_file" @@ -803,6 +804,61 @@ jobs: rm -f "$log_file" } + run_r_package_testthat() { + local package_name="$1" + local log_file rc classification + log_file="$(mktemp)" + append "### R package testthat suite" + append "" + append '```text' + append_command \ + Rscript -e 'lib <- Sys.getenv("R_LIBS_USER"); .libPaths(c(lib, .libPaths())); testthat::test_dir("tests/testthat")' + set +e + timeout --kill-after=20 900 setpriv \ + --reuid "$OPENCODE_SANDBOX_UID" \ + --regid "$OPENCODE_SANDBOX_GID" \ + --clear-groups \ + env \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_URL \ + -u ACTIONS_RUNTIME_TOKEN \ + -u GH_TOKEN \ + -u GITHUB_TOKEN \ + GITHUB_ENV=/dev/null \ + GITHUB_PATH=/dev/null \ + GITHUB_OUTPUT=/dev/null \ + GITHUB_STEP_SUMMARY=/dev/null \ + BASH_ENV=/dev/null \ + UV_NO_BUILD=1 \ + HOME=/work/.opencode-sandbox-home \ + XDG_CACHE_HOME=/work/.opencode-sandbox-cache \ + PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ + Rscript -e 'lib <- Sys.getenv("R_LIBS_USER"); .libPaths(c(lib, .libPaths())); testthat::test_dir("tests/testthat")' \ + >"$log_file" 2>&1 + rc=$? + set -e + emit_captured_log "$log_file" + append '```' + append "" + if [ "$rc" -eq 0 ]; then + append "- Result: PASS" + elif classification="$( + python3 -I "$GITHUB_WORKSPACE/scripts/ci/r_coverage_peer_gate.py" \ + classify-testthat \ + --log "$log_file" \ + --package "$package_name" 2>/dev/null + )"; then + append "- Result: PASS" + append "- Reason: ${classification}; direct sandbox failures are deferred only to a successful current-head peer R CMD check." + r_peer_check_required=1 + else + append "- Result: FAIL (exit ${rc})" + failures=$((failures + 1)) + fi + append "" + rm -f "$log_file" + } + run_and_capture_advisory() { local label="$1" shift @@ -1326,6 +1382,7 @@ jobs: } run_r_test_coverage() { + local package_name ensure_r_runtime if ! command -v Rscript >/dev/null 2>&1; then append "### R test coverage" @@ -1344,8 +1401,10 @@ jobs: Rscript -e 'required <- c("covr", "testthat"); missing <- required[!vapply(required, requireNamespace, logical(1), quietly = TRUE)]; if (length(missing)) stop("signed distribution coverage packages unavailable: ", paste(missing, collapse = ", "))' if [ -f DESCRIPTION ]; then if [ -d tests/testthat ]; then - run_and_capture "R package testthat suite" \ - Rscript -e 'lib <- Sys.getenv("R_LIBS_USER"); .libPaths(c(lib, .libPaths())); if (!requireNamespace("testthat", quietly = TRUE)) { message("testthat unavailable in coverage runner; deferring to required peer R CMD check evidence."); quit(status = 0) }; testthat::test_dir("tests/testthat")' + package_name="$( + Rscript -e 'pkg <- tryCatch(read.dcf("DESCRIPTION")[1, "Package"], error = function(e) ""); cat(pkg)' + )" + run_r_package_testthat "$package_name" else append "### R package testthat suite" append "" @@ -1706,6 +1765,9 @@ jobs: else append "- Test evidence: supported repository test suites passed" append "- Docstring evidence: configured repository docstring gates passed or docstring coverage was advisory" + if [ "$r_peer_check_required" -eq 1 ]; then + append "- R test evidence: deferred package-load failures require a successful current-head peer R CMD check" + fi fi else append "- Result: FAIL" @@ -3374,8 +3436,8 @@ jobs: "name": "DeepSeek V4 Flash Free", "tool_call": true, "limit": { - "context": 256000, - "output": 64000 + "context": 200000, + "output": 128000 } }, "north-mini-code-free": { @@ -3394,6 +3456,42 @@ jobs: "context": 256000, "output": 64000 } + }, + "big-pickle": { + "name": "Big Pickle", + "tool_call": true, + "reasoning": true, + "limit": { + "context": 200000, + "output": 32000 + } + }, + "laguna-s-2.1-free": { + "name": "Laguna S 2.1 Free", + "tool_call": true, + "reasoning": true, + "limit": { + "context": 256000, + "output": 32000 + } + }, + "ling-3.0-flash-free": { + "name": "Ling-3.0-flash Free", + "tool_call": true, + "reasoning": true, + "limit": { + "context": 262144, + "output": 32768 + } + }, + "mimo-v2.5-free": { + "name": "MiMo V2.5 Free", + "tool_call": true, + "reasoning": true, + "limit": { + "context": 200000, + "output": 32000 + } } } }, @@ -3711,8 +3809,8 @@ jobs: NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" # High-sensitivity review candidates only. Public repositories first - # use OpenCode Zen's anonymous Nemotron 3 Ultra, DeepSeek V4 Flash, - # and North Mini Code free endpoints. Their trial/free-period data + # use OpenCode Zen's anonymous active, zero-cost model endpoints. + # Their trial/free-period data # may be logged, retained, or used for product/model improvement, so # private repositories never include them. The pool then falls # through to @@ -3726,7 +3824,7 @@ jobs: # cost-efficient tier, cheaper than the legacy gpt-5 it replaced # ($1/$6 vs $1.25/$10 per 1M tokens) so the org OpenAI budget # stretches further between top-ups. - OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'opencode-free/nemotron-3-ultra-free opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free ' || '' }}github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" + OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'opencode-free/nemotron-3-ultra-free opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free opencode-free/laguna-s-2.1-free opencode-free/ling-3.0-flash-free opencode-free/big-pickle opencode-free/mimo-v2.5-free ' || '' }}github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" # One attempt per model, then fall through to the next model. Retrying # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. @@ -3739,11 +3837,11 @@ jobs: OPENCODE_EXPORT_TIMEOUT_SECONDS: "180" OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700" OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "12000" - # Keep cycling through the high-sensitivity candidate catalog until - # the retry budget or step timeout is exhausted; a single invalid - # cycle can be all provider formatting noise rather than review - # evidence. - OPENCODE_POOL_MAX_CYCLES: "0" + # A second pass through the same provider catalog repeats the same + # quota/format failures and can occupy the required check for hours. + # Exhaust each distinct candidate once, then publish the bounded + # model-unavailable fallback with current-head evidence. + OPENCODE_POOL_MAX_CYCLES: "1" OPENCODE_DYNAMIC_REVIEW_CADENCE: "true" OPENCODE_SMALL_CHANGE_FILE_THRESHOLD: "3" OPENCODE_MEDIUM_CHANGE_FILE_THRESHOLD: "20" @@ -3757,14 +3855,14 @@ jobs: OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "11700" OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "5400" OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "11700" - OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "0" + OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "1" OPENCODE_FREE_RUN_TIMEOUT_SECONDS: "600" # This installation currently reports a 4k request-body limit for # GitHub Models GPT-5 endpoints even though the public catalog is # larger. Keep the exact runtime failure visible without spending a # full medium/large cadence slot after the long-context candidate. OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS: "45" - OPENCODE_DYNAMIC_MAX_CYCLES: "0" + OPENCODE_DYNAMIC_MAX_CYCLES: "1" CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE: ${{ steps.central_review_process_fallback_scope.outputs.eligible || 'false' }} CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL: ${{ steps.central_review_process_fallback_scope.outputs.scope_label || 'unsupported' }} OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "5400" @@ -6527,6 +6625,40 @@ jobs: return 2 } + coverage_defers_to_r_cmd_check() { + printf '%s\n' "${COVERAGE_EVIDENCE_SUMMARY:-}" | + grep -Fq -- "- R test evidence: deferred package-load failures require a successful current-head peer R CMD check" + } + + collect_successful_r_cmd_check_evidence() { + local output_file="$1" + if ! gh pr checks "$PR_NUMBER" \ + --repo "$GH_REPOSITORY" \ + --json name,state,workflow >"$output_file"; then + return 1 + fi + python3 "$GITHUB_WORKSPACE/scripts/ci/r_coverage_peer_gate.py" \ + require-check \ + --checks-json "$output_file" >/dev/null + } + + require_r_cmd_check_for_deferred_coverage() { + local checks_file + if ! coverage_defers_to_r_cmd_check; then + return 0 + fi + checks_file="$(mktemp)" + if collect_github_checks_with_retry \ + collect_successful_r_cmd_check_evidence "$checks_file"; then + rm -f "$checks_file" + printf 'Verified successful current-head R CMD check after bounded R coverage deferral.\n' + return 0 + fi + rm -f "$checks_file" + printf '::notice::R package-load coverage deferral cannot authorize approval without a successful current-head R CMD check.\n' + return 1 + } + stop_without_review_after_model_unavailable() { local body body="$(printf '%s\n' \ @@ -6643,6 +6775,10 @@ jobs: return 0 fi + if ! require_r_cmd_check_for_deferred_coverage; then + return 1 + fi + failed_check_evidence_file="$(mktemp)" if ! collect_open_code_scanning_alerts "$failed_check_evidence_file"; then printf '::notice::Current-head model-unavailable evidence fallback skipped because open code-scanning alerts could not be read.\n' @@ -7027,6 +7163,27 @@ jobs: stop_failed_check_fallback_unavailable fi fi + if ! require_r_cmd_check_for_deferred_coverage; then + body="$(printf '%s\n' \ + "## Pull request overview" \ + "" \ + "OpenCode reviewed the current-head source evidence but R package tests were deferred after bounded package-load-only failures." \ + "" \ + "## Approval hold" \ + "" \ + "### Successful current-head R CMD check evidence is required" \ + "- Problem: coverage-evidence deferred package-load-only testthat failures, but no successful current-head R CMD check was found." \ + "- Root cause: deterministic coverage deferral is safe only when the repository's peer R CMD check installs dependencies and passes on this exact head." \ + "- Fix: add or repair the R CMD check workflow and rerun it successfully on the current head." \ + "- Regression test: Keep deferred R coverage fail-closed unless a successful R CMD check is present." \ + "" \ + "- Result: WAITING_FOR_R_CMD_CHECK" \ + "- Head SHA: \`${HEAD_SHA}\`" \ + "- Workflow run: ${RUN_ID}" \ + "- Workflow attempt: ${RUN_ATTEMPT}" + )" + hold_approval_without_review "WAITING_FOR_R_CMD_CHECK" "$body" + fi unresolved_reviewer_threads_file="$(mktemp)" reviewer_thread_review_body_file="$(mktemp)" if ! collect_unresolved_reviewer_threads "$unresolved_reviewer_threads_file"; then diff --git a/scripts/ci/r_coverage_peer_gate.py b/scripts/ci/r_coverage_peer_gate.py new file mode 100644 index 000000000..af19b02bc --- /dev/null +++ b/scripts/ci/r_coverage_peer_gate.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Gate R coverage deferral on bounded testthat and peer-check evidence.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import re +import sys +from typing import Any, Sequence + + +MAX_LOG_BYTES = 2_000_000 +PACKAGE_NAME_RE = re.compile(r"[A-Za-z][A-Za-z0-9.]*\Z") +FAIL_SUMMARY_RE = re.compile(r"^\[\s*FAIL\s+(\d+)\s*\|", re.MULTILINE) +ERROR_BLOCK_RE = re.compile(r"^Error \(", re.MULTILINE) +PACKAGE_NOT_FOUND_CONDITION_RE = re.compile( + r"^$", + re.MULTILINE, +) +MISSING_PACKAGE_RE = re.compile(r"there is no package called ['\"]([^'\"]+)['\"]") +R_CMD_CHECK_RE = re.compile(r"\br[\s_-]*cmd[\s_-]*check\b", re.IGNORECASE) + + +def classify_testthat_failure(text: str, package: str) -> bool: + """Return whether every testthat failure is the uninstalled package under test.""" + if not PACKAGE_NAME_RE.fullmatch(package): + return False + summaries = FAIL_SUMMARY_RE.findall(text) + if not summaries or "Error: Test failures" not in text: + return False + failure_count = int(summaries[-1]) + if failure_count <= 0: + return False + error_count = len(ERROR_BLOCK_RE.findall(text)) + condition_count = len(PACKAGE_NOT_FOUND_CONDITION_RE.findall(text)) + missing_packages = MISSING_PACKAGE_RE.findall(text) + return ( + error_count == failure_count + and condition_count == failure_count + and len(missing_packages) == failure_count + and all(name == package for name in missing_packages) + ) + + +def has_successful_r_cmd_check(checks: Any) -> bool: + """Return whether check JSON contains a successful R CMD check workflow.""" + if not isinstance(checks, list): + return False + for check in checks: + if not isinstance(check, dict): + continue + if str(check.get("state") or "").upper() != "SUCCESS": + continue + label = f"{check.get('workflow') or ''} {check.get('name') or ''}" + if R_CMD_CHECK_RE.search(label): + return True + return False + + +def _read_bounded_text(path: Path) -> str | None: + """Read a regular bounded log, returning None for unsafe or unreadable input.""" + try: + if not path.is_file() or path.is_symlink() or path.stat().st_size > MAX_LOG_BYTES: + return None + return path.read_text(encoding="utf-8", errors="replace") + except OSError: + return None + + +def _read_json(path: Path) -> Any: + """Read JSON from a regular bounded file, returning None on invalid input.""" + text = _read_bounded_text(path) + if text is None: + return None + try: + return json.loads(text) + except json.JSONDecodeError: + return None + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse the requested peer-gate operation.""" + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + classify = subparsers.add_parser("classify-testthat") + classify.add_argument("--log", type=Path, required=True) + classify.add_argument("--package", required=True) + + require_check = subparsers.add_parser("require-check") + require_check.add_argument("--checks-json", type=Path, required=True) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the selected R coverage peer-evidence gate.""" + args = parse_args(argv) + if args.command == "classify-testthat": + text = _read_bounded_text(args.log) + if text is not None and classify_testthat_failure(text, args.package): + print( + "testthat failures were exclusively packageNotFoundError " + f"conditions for package {args.package}" + ) + return 0 + print("testthat failure is not safely deferrable", file=sys.stderr) + return 1 + + checks = _read_json(args.checks_json) + if has_successful_r_cmd_check(checks): + print("successful current-head R CMD check evidence found") + return 0 + print("successful current-head R CMD check evidence was not found", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 497999e91..c336c20da 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -547,7 +547,11 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" "r-cran-covr" "opencode R coverage uses the signed distribution covr package instead of mutable CRAN resolution" assert_file_contains "$workflow_file" "r-cran-testthat" "opencode R coverage uses the signed distribution testthat package instead of mutable CRAN resolution" assert_file_contains "$workflow_file" "R package testthat suite" "opencode R package coverage requires package testthat evidence" - assert_file_contains "$workflow_file" "testthat unavailable in coverage runner; deferring to required peer R CMD check evidence." "opencode R package tests defer only when testthat cannot be installed in the coverage runner" + assert_file_contains "$workflow_file" "r_coverage_peer_gate.py" "opencode R package coverage classifies bounded package-load-only failures with trusted code" + assert_file_contains "$workflow_file" "- R test evidence: deferred package-load failures require a successful current-head peer R CMD check" "opencode R package coverage records explicit peer-check deferral evidence" + assert_file_contains "$workflow_file" "require_r_cmd_check_for_deferred_coverage" "opencode approval verifies deferred R evidence against current-head peer checks" + assert_file_contains "$workflow_file" "WAITING_FOR_R_CMD_CHECK" "opencode approval fails closed when deferred R coverage lacks successful peer evidence" + assert_file_not_contains "$workflow_file" 'if (!is.na(pkg) && !requireNamespace(pkg, quietly = TRUE))' "opencode R coverage does not skip the entire test suite merely because the source package is not preinstalled" assert_file_contains "$workflow_file" "covr package_coverage unavailable after package tests; treating missing-line report as advisory." "opencode R package coverage does not block on covr installation reproduction after tests pass" assert_file_contains "$workflow_file" "signed distribution coverage packages unavailable" "opencode R coverage verifies distribution-provided covr/testthat are loadable" assert_file_contains "$workflow_file" "repository: ContextualWisdomLab/.github" "opencode required workflow checks out the central source repository" @@ -705,7 +709,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'continue-on-error: true' "opencode approval gate still runs after model-pool failure to publish a reason" assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode primary review preserves legitimate full-hour provider sessions" assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700"' "opencode model pool exits before the step timeout so the approval gate can publish a reason" - assert_file_contains "$workflow_file" 'OPENCODE_POOL_MAX_CYCLES: "0"' "opencode model pool keeps cycling until the bounded retry budget or step timeout is exhausted" + assert_file_contains "$workflow_file" 'OPENCODE_POOL_MAX_CYCLES: "1"' "opencode model pool exhausts each candidate only once before bounded fallback" assert_file_not_contains "$workflow_file" 'opencode-exhausted-retry:' "opencode model exhaustion retries stay owned by the least-privilege central scheduler" assert_file_not_contains "$workflow_file" 'RETRY_DISPATCH_TOKEN' "opencode does not retain a recursive write-token dispatch path" assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed" diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 0e791907f..12ffcb5ef 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -94,7 +94,11 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): "${{ needs.validate-pr-metadata.outputs.is_private == 'false' " "&& 'opencode-free/nemotron-3-ultra-free " "opencode-free/deepseek-v4-flash-free " - "opencode-free/north-mini-code-free ' || '' }}" + "opencode-free/north-mini-code-free " + "opencode-free/laguna-s-2.1-free " + "opencode-free/ling-3.0-flash-free " + "opencode-free/big-pickle " + "opencode-free/mimo-v2.5-free ' || '' }}" ) candidates_text = candidates_match.group(1) assert candidates_text.startswith(conditional_public_candidate) @@ -102,6 +106,10 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): "opencode-free/nemotron-3-ultra-free", "opencode-free/deepseek-v4-flash-free", "opencode-free/north-mini-code-free", + "opencode-free/laguna-s-2.1-free", + "opencode-free/ling-3.0-flash-free", + "opencode-free/big-pickle", + "opencode-free/mimo-v2.5-free", *candidates_text.removeprefix(conditional_public_candidate).split(), ] candidate_pairs = [candidate.split("/", 1) for candidate in candidates] @@ -122,6 +130,10 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): ["opencode-free", "nemotron-3-ultra-free"], ["opencode-free", "deepseek-v4-flash-free"], ["opencode-free", "north-mini-code-free"], + ["opencode-free", "laguna-s-2.1-free"], + ["opencode-free", "ling-3.0-flash-free"], + ["opencode-free", "big-pickle"], + ["opencode-free", "mimo-v2.5-free"], ["github-models", "deepseek/deepseek-v3-0324"], ["openai", "gpt-5.6-luna"], ["openrouter", "deepseek/deepseek-v3.2"], @@ -149,6 +161,15 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert generated_config_match is not None generated_config = json.loads(generated_config_match.group(1)) free_models = generated_config["provider"]["opencode-free"]["models"] + assert set(free_models) == { + "nemotron-3-ultra-free", + "deepseek-v4-flash-free", + "north-mini-code-free", + "laguna-s-2.1-free", + "ling-3.0-flash-free", + "big-pickle", + "mimo-v2.5-free", + } nemotron_model = free_models["nemotron-3-ultra-free"] deepseek_model = free_models["deepseek-v4-flash-free"] north_model = free_models["north-mini-code-free"] @@ -156,9 +177,26 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): assert nemotron_model["limit"] == {"context": 1000000, "output": 128000} assert "response_format" not in nemotron_model.get("options", {}) assert deepseek_model["tool_call"] is True + assert deepseek_model["limit"] == {"context": 200000, "output": 128000} assert "response_format" not in deepseek_model.get("options", {}) assert north_model["tool_call"] is True assert "response_format" not in north_model["options"] + assert free_models["laguna-s-2.1-free"]["limit"] == { + "context": 256000, + "output": 32000, + } + assert free_models["ling-3.0-flash-free"]["limit"] == { + "context": 262144, + "output": 32768, + } + assert free_models["big-pickle"]["limit"] == { + "context": 200000, + "output": 32000, + } + assert free_models["mimo-v2.5-free"]["limit"] == { + "context": 200000, + "output": 32000, + } assert github_candidate_models == [ "deepseek/deepseek-v3-0324", "openai/gpt-4.1", @@ -1227,7 +1265,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): in workflow ) assert "OpenCode model pool exceeded the outer" in workflow - assert 'OPENCODE_POOL_MAX_CYCLES: "0"' in workflow + assert 'OPENCODE_POOL_MAX_CYCLES: "1"' in workflow assert re.search( r"Run OpenCode PR Review model pool[\s\S]{0,280}continue-on-error: true", workflow, @@ -1257,7 +1295,11 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): "needs.validate-pr-metadata.outputs.is_private == 'false' && " "'opencode-free/nemotron-3-ultra-free " "opencode-free/deepseek-v4-flash-free " - "opencode-free/north-mini-code-free ' || ''" + "opencode-free/north-mini-code-free " + "opencode-free/laguna-s-2.1-free " + "opencode-free/ling-3.0-flash-free " + "opencode-free/big-pickle " + "opencode-free/mimo-v2.5-free ' || ''" ) in workflow assert ( "github-models/deepseek/deepseek-v3-0324 " @@ -1276,7 +1318,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert 'OPENCODE_EXPORT_TIMEOUT_SECONDS: "180"' in workflow assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700"' in workflow assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "12000"' in workflow - assert 'OPENCODE_POOL_MAX_CYCLES: "0"' in workflow + assert 'OPENCODE_POOL_MAX_CYCLES: "1"' in workflow assert 'OPENCODE_DYNAMIC_REVIEW_CADENCE: "true"' in workflow assert ( "OPENCODE_CHANGED_FILES_FILE: ${{ runner.temp }}/opencode-changed-files.txt" @@ -1292,10 +1334,10 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow assert 'OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "5400"' in workflow assert 'OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "11700"' in workflow - assert 'OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "0"' in workflow + assert 'OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "1"' in workflow assert 'OPENCODE_FREE_RUN_TIMEOUT_SECONDS: "600"' in workflow assert 'OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS: "45"' in workflow - assert 'OPENCODE_DYNAMIC_MAX_CYCLES: "0"' in workflow + assert 'OPENCODE_DYNAMIC_MAX_CYCLES: "1"' in workflow assert 'OPENCODE_BACKOFF_MAX_SECONDS: "30"' in workflow publish_step = workflow.split(" - name: Publish OpenCode review outcome", 1)[ 1 @@ -1760,11 +1802,11 @@ def test_opencode_privileged_review_security_boundaries_are_fail_closed(): assert syntax_step < measure_step assert "\n - name:" not in measure.split("\n run: |", 1)[1] assert 'UV_NO_BUILD: "1"' in measure - assert measure.count("GITHUB_ENV=/dev/null") == 2 - assert measure.count("GITHUB_PATH=/dev/null") == 2 - assert measure.count("GITHUB_OUTPUT=/dev/null") == 2 - assert measure.count("GITHUB_STEP_SUMMARY=/dev/null") == 2 - assert measure.count("BASH_ENV=/dev/null") == 2 + assert measure.count("GITHUB_ENV=/dev/null") == 3 + assert measure.count("GITHUB_PATH=/dev/null") == 3 + assert measure.count("GITHUB_OUTPUT=/dev/null") == 3 + assert measure.count("GITHUB_STEP_SUMMARY=/dev/null") == 3 + assert measure.count("BASH_ENV=/dev/null") == 3 assert "uv sync --project" not in measure assert "uv run --no-project" not in measure assert "uv run --no-build" not in measure @@ -2316,3 +2358,28 @@ def test_slow_peer_wait_matches_only_image_validation_checks(): or re.search(package_build_pattern, candidate, re.IGNORECASE) is not None ) assert slow_build_match is slow_build_expected, candidate + + +def test_r_package_load_deferral_requires_current_head_r_cmd_check(): + """R package-load-only failures may defer only to explicit peer evidence.""" + workflow = Path(".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) + marker = ( + "- R test evidence: deferred package-load failures require a successful " + "current-head peer R CMD check" + ) + + assert "run_r_package_testthat" in workflow + assert "r_coverage_peer_gate.py" in workflow + assert marker in workflow + assert "require_r_cmd_check_for_deferred_coverage" in workflow + assert workflow.count("require_r_cmd_check_for_deferred_coverage") == 3 + assert "WAITING_FOR_R_CMD_CHECK" in workflow + assert "testthat unavailable in coverage runner" not in workflow + assert ( + "pkg <- tryCatch(read.dcf(\"DESCRIPTION\")[1, \"Package\"]" in workflow + ) + assert ( + "if (!is.na(pkg) && !requireNamespace(pkg, quietly = TRUE))" not in workflow + ) diff --git a/tests/test_r_coverage_peer_gate.py b/tests/test_r_coverage_peer_gate.py new file mode 100644 index 000000000..e0be03e5f --- /dev/null +++ b/tests/test_r_coverage_peer_gate.py @@ -0,0 +1,147 @@ +"""Tests for fail-closed R coverage deferral and peer-check evidence.""" + +from __future__ import annotations + +import json +from pathlib import Path +import runpy +import sys + +import pytest + +from scripts.ci import r_coverage_peer_gate as gate + + +def test_classifies_only_own_package_not_found_failures() -> None: + """Every reported failure must be packageNotFoundError for the package under test.""" + text = """\ +Error ('test-one.R:1:1'): first + +Error in `loadNamespace(x)`: there is no package called 'aFIPC' +Error ('test-two.R:2:1'): second + +Error in `loadNamespace(x)`: there is no package called 'aFIPC' +[ FAIL 2 | WARN 0 | SKIP 1 | PASS 3 ] +Error: Test failures +""" + assert gate.classify_testthat_failure(text, "aFIPC") + + +def test_rejects_invalid_or_mixed_test_failures() -> None: + """Malformed names, assertions, other packages, and count drift fail closed.""" + assertion = "[ FAIL 1 | WARN 0 | SKIP 0 | PASS 0 ]\nError: Test failures\n" + other_package = """\ +Error ('test-one.R:1:1'): first + +Error in `loadNamespace(x)`: there is no package called 'mirt' +[ FAIL 1 | WARN 0 | SKIP 0 | PASS 0 ] +Error: Test failures +""" + mismatched = other_package.replace("FAIL 1", "FAIL 2").replace("mirt", "aFIPC") + zero_failures = "[ FAIL 0 | WARN 0 | SKIP 0 | PASS 1 ]\nError: Test failures\n" + + assert not gate.classify_testthat_failure("", "aFIPC") + assert not gate.classify_testthat_failure( + "[ FAIL 1 | WARN 0 | SKIP 0 | PASS 0 ]", "aFIPC" + ) + assert not gate.classify_testthat_failure(zero_failures, "aFIPC") + assert not gate.classify_testthat_failure(assertion, "aFIPC") + assert not gate.classify_testthat_failure(other_package, "aFIPC") + assert not gate.classify_testthat_failure(mismatched, "aFIPC") + assert not gate.classify_testthat_failure(other_package, "../aFIPC") + + +def test_requires_successful_r_cmd_check_workflow() -> None: + """Only a successful check whose workflow/name identifies R CMD check qualifies.""" + checks = [ + {"workflow": "R CMD check", "name": "check", "state": "SUCCESS"}, + {"workflow": "Other", "name": "test", "state": "FAILURE"}, + ] + assert gate.has_successful_r_cmd_check(checks) + assert gate.has_successful_r_cmd_check( + [{"workflow": "", "name": "R-CMD-check", "state": "success"}] + ) + assert not gate.has_successful_r_cmd_check( + [{"workflow": "R CMD check", "name": "check", "state": "FAILURE"}] + ) + assert not gate.has_successful_r_cmd_check( + [{"workflow": "CI", "name": "tests", "state": "SUCCESS"}] + ) + assert not gate.has_successful_r_cmd_check({"checks": checks}) + assert not gate.has_successful_r_cmd_check(["invalid"]) + + +def test_cli_classifies_log_and_check_json(tmp_path: Path, capsys) -> None: + """Both CLI modes accept bounded valid evidence and reject invalid JSON.""" + log = tmp_path / "testthat.log" + log.write_text( + "Error ('x.R:1:1'): x\n" + "\n" + "Error in `loadNamespace(x)`: there is no package called 'pkg'\n" + "[ FAIL 1 | WARN 0 | SKIP 0 | PASS 0 ]\n" + "Error: Test failures\n", + encoding="utf-8", + ) + checks = tmp_path / "checks.json" + checks.write_text( + json.dumps([{"workflow": "R CMD check", "name": "check", "state": "SUCCESS"}]), + encoding="utf-8", + ) + + assert gate.main(["classify-testthat", "--log", str(log), "--package", "pkg"]) == 0 + assert gate.main(["require-check", "--checks-json", str(checks)]) == 0 + + checks.write_text("{", encoding="utf-8") + assert gate.main(["require-check", "--checks-json", str(checks)]) == 1 + assert ( + gate.main( + [ + "require-check", + "--checks-json", + str(tmp_path / "missing-checks.json"), + ] + ) + == 1 + ) + assert "not found" in capsys.readouterr().err + + +def test_cli_rejects_unsafe_or_oversized_logs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Missing, symlinked, and oversized artifacts never authorize deferral.""" + missing = tmp_path / "missing.log" + assert ( + gate.main(["classify-testthat", "--log", str(missing), "--package", "pkg"]) + == 1 + ) + + target = tmp_path / "target.log" + target.write_text("small", encoding="utf-8") + link = tmp_path / "link.log" + link.symlink_to(target) + assert gate.main(["classify-testthat", "--log", str(link), "--package", "pkg"]) == 1 + + target.write_bytes(b"x" * (gate.MAX_LOG_BYTES + 1)) + assert gate.main(["classify-testthat", "--log", str(target), "--package", "pkg"]) == 1 + + monkeypatch.setattr(Path, "is_file", lambda _path: (_ for _ in ()).throw(OSError())) + assert gate._read_bounded_text(target) is None + + +def test_script_entrypoint_returns_cli_status( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The executable entrypoint propagates the fail-closed CLI status.""" + missing = tmp_path / "missing.log" + script = Path("scripts/ci/r_coverage_peer_gate.py") + monkeypatch.setattr( + sys, + "argv", + [str(script), "classify-testthat", "--log", str(missing), "--package", "pkg"], + ) + + with pytest.raises(SystemExit) as raised: + runpy.run_path(str(script), run_name="__main__") + + assert raised.value.code == 1