From e66c2a9fc601bdcb67b0634df564e4ee815bad43 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:11:35 +0800 Subject: [PATCH 1/2] ci: streamline risk-scoped PR verification Add conservative change classification, an always-reporting required aggregate, production-backed offline RAG contracts, and a fast critical UI gate with advisory regression coverage. Verified with verify:cheap, verify:pr-local, 124 Chromium tests, and CI workflow guards. --- .github/pull_request_template.md | 9 +- .github/workflows/ci.yml | 303 ++++++++++++++--- AGENTS.md | 2 +- docs/branch-review-ledger.md | 8 +- docs/process-hardening.md | 14 +- package.json | 5 + scripts/check-codex-autofix-workflow.mjs | 27 ++ scripts/ci-change-scope.mjs | 405 +++++++++++++++++++++++ scripts/eval-rag-offline.mjs | 107 ++++++ scripts/verify-pr-local.mjs | 40 +++ tests/rag-offline-answer.test.ts | 54 ++- tests/ui-smoke.spec.ts | 33 +- 12 files changed, 937 insertions(+), 70 deletions(-) create mode 100644 scripts/ci-change-scope.mjs create mode 100644 scripts/eval-rag-offline.mjs create mode 100644 scripts/verify-pr-local.mjs diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index db23b23aa..8dda12c6e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,10 +4,15 @@ ## Verification -- [ ] `npm run verify:cheap` +- [ ] `npm run verify:pr-local` + +During development, use `npm run verify:cheap` as the faster iteration gate before the final PR-local preflight. + - [ ] `npm run verify:ui` when UI, routing, styling, browser behavior, reduced-motion, or forced-colors behavior changed - [ ] `npm run verify:release` before release or handoff confidence claims -- [ ] `npm run format:check` + +For retrieval, ranking, selection, chunking, source/citation rendering, or answer-contract changes, `verify:pr-local` runs `eval:rag:offline` automatically. Run the offline command directly during iteration before spending a live eval. + - [ ] **`npm run eval:retrieval:quality` (must stay 36/36) when retrieval, ranking, selection, chunking, or scoring behavior changed** — CI cannot run it (needs live keys), so run it locally and paste the summary. A metadata/governance-weighting change once buried correct docs (recall 1.0→0.76) and only this eval caught it. - [ ] `npm run eval:rag -- --limit 15` + `npm run eval:quality -- --rag-only` when answer generation, the synthesis prompt, or answer post-processing changed (grounded-supported must not drop; citation-failure 0) - [ ] `npm run check:production-readiness` when clinical workflow, privacy, environment, Supabase, source governance, or deployment behavior changed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 330a76bc1..fad714b66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,13 +22,46 @@ env: NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY: placeholder-ci-anon-key jobs: - # Fast deterministic merge gate: static checks, unit tests, and build. - # Intended as a required status check on main together with ui-smoke, - # Gitleaks, and one migration replay gate. - verify: + changes: + name: Change scope runs-on: ubuntu-24.04 - timeout-minutes: 25 + timeout-minutes: 5 + outputs: + docs_only: ${{ steps.scope.outputs.docs_only }} + source_changed: ${{ steps.scope.outputs.source_changed }} + coverage_changed: ${{ steps.scope.outputs.coverage_changed }} + ui_changed: ${{ steps.scope.outputs.ui_changed }} + db_changed: ${{ steps.scope.outputs.db_changed }} + container_changed: ${{ steps.scope.outputs.container_changed }} + rag_eval_changed: ${{ steps.scope.outputs.rag_eval_changed }} + workflow_changed: ${{ steps.scope.outputs.workflow_changed }} + codex_autofix_changed: ${{ steps.scope.outputs.codex_autofix_changed }} + build_changed: ${{ steps.scope.outputs.build_changed }} + changed_files: ${{ steps.scope.outputs.changed_files }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: ".nvmrc" + + - name: Classify changed files + id: scope + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha || github.sha }} + run: node scripts/ci-change-scope.mjs + + static-pr: + name: Static PR checks + needs: changes + runs-on: ubuntu-24.04 + timeout-minutes: 20 steps: - name: Checkout uses: actions/checkout@v6 @@ -42,9 +75,49 @@ jobs: cache: npm cache-dependency-path: package-lock.json - - name: Workflow action pin check + - name: Install dependencies + run: npm ci + + - name: Runtime alignment + run: npm run check:runtime + + - name: GitHub Actions pin check run: npm run check:github-actions + - name: CI scope self-test + run: npm run check:ci-scope + + - name: Format check + run: npm run format:check + + - name: Lint + run: npm run lint + + - name: Typecheck + run: npm run typecheck + + - name: Unit tests + run: npm run test + + safety: + name: Safety and config checks + needs: changes + if: needs.changes.outputs.docs_only != 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: package-lock.json + - name: Setup Deno uses: denoland/setup-deno@v2 with: @@ -53,16 +126,7 @@ jobs: - name: Install dependencies run: npm ci - - name: Runtime alignment - run: npm run check:runtime - - - name: Codex auto-resolve workflow guard - run: npm run check:codex-autofix-workflow - - name: Dependency vulnerability audit - # Non-mutating gate. The document parsers (pdf-parse, pdfjs-dist, mammoth, - # exceljs, jszip) process untrusted uploads, so high/critical advisories in - # the production tree must block the merge rather than wait for Dependabot. run: npm audit --omit=dev --audit-level=high - name: Edge function typecheck @@ -71,40 +135,78 @@ jobs: - name: Production readiness (CI-safe) run: npm run check:production-readiness:ci - - name: Format check - run: npm run format:check + - name: Codex auto-resolve workflow guard + if: needs.changes.outputs.codex_autofix_changed == 'true' + run: npm run check:codex-autofix-workflow - - name: Lint - run: npm run lint + - name: Offline RAG preflight + if: needs.changes.outputs.rag_eval_changed == 'true' + run: npm run eval:rag:offline - - name: Typecheck - run: npm run typecheck + coverage: + name: Unit coverage + needs: changes + if: needs.changes.outputs.coverage_changed == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci - name: Unit tests (with coverage gate) run: npm run test:coverage + build: + name: Build + needs: changes + if: needs.changes.outputs.build_changed == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 25 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + - name: Build run: npm run build - name: Deployment boot smoke if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') - # No placeholder fallbacks: when the repository secrets are missing the - # smoke must fail with its explicit missing-env error rather than - # "pass" against a server that never saw real production env. env: SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} RAG_QUERY_HASH_SECRET: ${{ secrets.RAG_QUERY_HASH_SECRET }} run: npm run check:deployment-readiness - # Chromium browser smoke in parallel with verify, so the slowest check no - # longer serializes behind lint/typecheck/tests/build and a flaky smoke - # rerun does not repeat them. The runner boots its own dev server, so no - # build step is needed here. - ui-smoke: + ui-critical: + name: Critical UI smoke + needs: changes + if: needs.changes.outputs.ui_changed == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 25 - + timeout-minutes: 15 steps: - name: Checkout uses: actions/checkout@v6 @@ -130,28 +232,70 @@ jobs: - name: Install Chromium browser run: npx playwright install --with-deps chromium - - name: Chromium UI smoke - id: chromium-smoke - run: npm run test:e2e:chromium + - name: Chromium critical UI smoke + run: npm run test:e2e:critical - name: Upload UI diagnostics if: failure() uses: actions/upload-artifact@v7 with: - name: verify-ui-diagnostics-${{ github.run_id }} + name: critical-ui-diagnostics-${{ github.run_id }} + path: | + test-results/ + playwright-report/ + if-no-files-found: ignore + + ui-regression: + name: Advisory UI regression + needs: changes + if: github.event_name == 'pull_request' && needs.changes.outputs.ui_changed == 'true' + continue-on-error: true + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Restore Chromium browser cache + uses: actions/cache@v6 + with: + path: ~/.cache/ms-playwright + key: playwright-chromium-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + + - name: Install Chromium browser + run: npx playwright install --with-deps chromium + + - name: Chromium advisory regression + run: npm run test:e2e:advisory + + - name: Upload advisory UI diagnostics + if: failure() + uses: actions/upload-artifact@v7 + with: + name: advisory-ui-diagnostics-${{ github.run_id }} path: | test-results/ playwright-report/ if-no-files-found: ignore - # Database migration replay in parallel with verify and ui-smoke. This is the - # repo-owned replay gate; if the external Supabase Preview check is already - # required and reliable, keep this advisory or path-filtered rather than - # requiring both replay gates on every PR. db-reset-verify: + name: Migration replay + needs: changes + if: needs.changes.outputs.db_changed == 'true' runs-on: ubuntu-24.04 timeout-minutes: 15 - steps: - name: Checkout uses: actions/checkout@v6 @@ -193,9 +337,86 @@ jobs: | sort -u \ | xargs -r docker save -o /tmp/supabase-docker-cache/images.tar + pr-required: + name: PR required + needs: [changes, static-pr, safety, coverage, build, ui-critical, db-reset-verify] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Verify required in-scope jobs + env: + DOCS_ONLY: ${{ needs.changes.outputs.docs_only }} + COVERAGE_CHANGED: ${{ needs.changes.outputs.coverage_changed }} + UI_CHANGED: ${{ needs.changes.outputs.ui_changed }} + DB_CHANGED: ${{ needs.changes.outputs.db_changed }} + BUILD_CHANGED: ${{ needs.changes.outputs.build_changed }} + CHANGES_RESULT: ${{ needs.changes.result }} + STATIC_RESULT: ${{ needs.static-pr.result }} + SAFETY_RESULT: ${{ needs.safety.result }} + COVERAGE_RESULT: ${{ needs.coverage.result }} + BUILD_RESULT: ${{ needs.build.result }} + UI_RESULT: ${{ needs.ui-critical.result }} + DB_RESULT: ${{ needs.db-reset-verify.result }} + run: | + set -euo pipefail + + require_success() { + local name="$1" + local result="$2" + if [ "$result" != "success" ]; then + echo "::error::$name result was $result" + exit 1 + fi + } + + require_skipped_or_success() { + local name="$1" + local result="$2" + if [ "$result" != "success" ] && [ "$result" != "skipped" ]; then + echo "::error::$name result was $result" + exit 1 + fi + } + + require_success "changes" "$CHANGES_RESULT" + require_success "static-pr" "$STATIC_RESULT" + + if [ "$DOCS_ONLY" = "true" ]; then + require_skipped_or_success "safety" "$SAFETY_RESULT" + else + require_success "safety" "$SAFETY_RESULT" + fi + + if [ "$COVERAGE_CHANGED" = "true" ]; then + require_success "coverage" "$COVERAGE_RESULT" + else + require_skipped_or_success "coverage" "$COVERAGE_RESULT" + fi + + if [ "$BUILD_CHANGED" = "true" ]; then + require_success "build" "$BUILD_RESULT" + else + require_skipped_or_success "build" "$BUILD_RESULT" + fi + + if [ "$UI_CHANGED" = "true" ]; then + require_success "ui-critical" "$UI_RESULT" + else + require_skipped_or_success "ui-critical" "$UI_RESULT" + fi + + if [ "$DB_CHANGED" = "true" ]; then + require_success "db-reset-verify" "$DB_RESULT" + else + require_skipped_or_success "db-reset-verify" "$DB_RESULT" + fi + + echo "Required in-scope PR checks passed." + release-browser-matrix: if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') - needs: [verify, ui-smoke, db-reset-verify] + needs: [pr-required] runs-on: ubuntu-24.04 timeout-minutes: 70 diff --git a/AGENTS.md b/AGENTS.md index d08affc1d..8ae338b87 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -390,7 +390,7 @@ Automatic Codex review is review-only by default. This repository includes `.git - The workflow must only trigger from Codex review bot reviews or comments on open pull requests. - The workflow must skip review-thread replies and auto-resolve request comments so Codex fix summaries do not re-trigger the workflow. -- The workflow must ask Codex to resolve all review comments using these repository instructions. +- The workflow must ask Codex to resolve only actionable Codex review findings for the triggering pull request and current head using these repository instructions. - The workflow must avoid duplicate requests for the same pull request, even after follow-up commits change the head SHA. - The workflow must not run Codex directly with API credentials. - P0 and P1 findings should always be fixed. diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index e4d61b8c5..c088a6ac0 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -18,6 +18,8 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD ## Review Records -| Date | Branch or ref | Reviewed HEAD | Scope | Outcome | Checks | -| ---------- | -------------- | ------------- | -------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| 2026-07-09 | example/branch | abc1234 | branch-cleanup | Example: already merged into `main`; no unique patch content. | `git log --right-only --cherry-pick main...example/branch`; `git diff --name-status main...example/branch` | +| Date | Branch or ref | Reviewed HEAD | Scope | Outcome | Checks | +| ---------- | ----------------------------- | ---------------------------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-07-09 | example/branch | abc1234 | branch-cleanup | Example: already merged into `main`; no unique patch content. | `git log --right-only --cherry-pick main...example/branch`; `git diff --name-status main...example/branch` | +| 2026-07-10 | codex/pr-testing-streamlining | 155c801cd58f797037d8aaa8b885405a1c599249 | working-tree diff: PR testing streamlining | Changes requested: 2 P1 clinical-gate defects and 7 P2/P3 scope, local parity, and UI-process defects. | `npm run check:ci-scope`; `npm run check:github-actions`; `npm run check:codex-autofix-workflow`; `npm run eval:rag:offline`; `npm run test:e2e:critical`; targeted scope classifications; `git diff --check` | +| 2026-07-10 | codex/pr-testing-streamlining | 155c801cd58f797037d8aaa8b885405a1c599249 | working-tree remediation review | All recorded P1-P3 findings fixed; no remaining high-confidence issue in the changed scope. | `npm run verify:cheap`; `npm run verify:pr-local`; `npm run eval:rag:offline`; `npm run test:e2e:critical`; `npm run test:e2e:advisory`; CI YAML parse; scope/action/Codex guards; `git diff --check` | diff --git a/docs/process-hardening.md b/docs/process-hardening.md index 1043f28e3..f9d9cd98a 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -5,9 +5,10 @@ This document turns the current process review into phased, durable repo practic ## Phase 1 - Active now - `npm run verify:cheap` is the default broad local gate for source/config/test changes: `check:runtime`, `sitemap:check`, lint, typecheck, and unit tests. +- `npm run verify:pr-local` is the closest local mirror of the normal PR gate: runtime, format, lint, typecheck, unit tests, conditional build, and offline RAG preflight when changed-file scope requires it. Local scope resolves against the repository default base rather than a feature-branch upstream; set `PR_BASE_REF` explicitly for release-targeted PRs. - `npm run verify:ui` is the default UI gate: `check:runtime` plus Chromium Playwright smoke, stress, and accessibility media checks (`test:e2e:chromium`). - `npm run verify:release` is the release-confidence gate: `check:runtime`, lint, typecheck, unit tests, build, full Playwright browser matrix, `check:production-readiness`, `governance:release`, and `eval:quality:release` (the last step needs live Supabase and OpenAI keys). -- CI runs three parallel PR jobs in the `CI` workflow: `verify` (runtime alignment, dependency vulnerability audit, edge-function typecheck, CI-safe production readiness, `format:check`, lint, typecheck, unit tests with coverage, build), `ui-smoke` (Chromium Playwright against its own dev server), and `db-reset-verify` (Supabase local start plus `supabase db reset` migration replay). The external `Supabase Preview` check also replays the migration chain on branch databases when enabled. A gated `release-browser-matrix` job runs the full Playwright browser set on `main`, `release/*`, manual dispatch, and the weekly schedule. +- CI uses a risk-scoped PR gate: `changes` classifies paths, `static-pr` always runs runtime/action/scope/format/lint/typecheck/unit checks, and `pr-required` is the single always-reporting required aggregate. Coverage, build, critical UI smoke, migration replay, safety/config, Codex workflow validation, and the production-backed offline RAG preflight run only when their file scopes apply. UI PRs also run a non-blocking advisory Chromium regression job. The external `Supabase Preview` check may still replay migrations on branch databases when enabled. A gated `release-browser-matrix` job runs the full Playwright browser set on `main`, `release/*`, manual dispatch, and the weekly schedule. - `tests/ui-accessibility.spec.ts` covers reduced-motion and forced-colors dashboard usability so those modes are no longer only reviewed by inspection. - `tests/ui-tools.spec.ts` covers the Applications dashboard mode at mobile and desktop sizes, including the `/applications` compatibility redirect. - `AGENTS.md` now points future agents to these gates and to this document. @@ -128,12 +129,13 @@ ingestion worker from current `main` until `20260708130000` is live** — `worke passes `p_worker_id`. Ordered apply steps, R17 manual `CONCURRENTLY` index, and post-apply probes: [`docs/operator-apply-july8-batch.md`](operator-apply-july8-batch.md) · `npm run check:july8-live-batch`. -## PR merge gate: tiered CI + required checks (2026-07-02) +## PR merge gate: risk-scoped CI + required aggregate (2026-07-10) -- CI is now three parallel PR jobs instead of one serial 6-7 minute job: `verify` (runtime alignment, production dependency audit, edge typecheck, CI-safe production readiness, format, lint, typecheck, unit tests with coverage gate, build), `ui-smoke` (Chromium Playwright smoke against its own dev server), and `db-reset-verify` (local Supabase migration replay). Wall-clock PR feedback drops to the slowest independent gate, and a flaky smoke rerun no longer repeats lint/typecheck/tests/build. -- The deployment boot smoke, full browser matrix, live drift check, live eval canary, and Docker image builds remain gated, scheduled, manual, or path-filtered — they are deliberately not normal required checks for every source-only PR. -- **Required-check debt: RESOLVED 2026-07-02; updated 2026-07-09.** Branch protection for `main` should require the exact active check contexts for `CI / verify`, `CI / ui-smoke`, `Secret Scan / Gitleaks`, and one migration replay gate. Prefer the external `Supabase Preview` check when it is enabled and reliable; otherwise require `CI / db-reset-verify`. Requiring both replay gates is conservative but usually redundant. Keep `SAST / Semgrep` required only if the repository owner accepts its external-rule/network dependency as part of the normal merge gate. Do not require path-filtered or scheduled/manual contexts such as `Docker image build / app-image`, `Docker image build / worker-image`, `CI / release-browser-matrix`, `Eval Canary`, or `Live drift check`; they can be absent on ordinary PRs and would leave branches stuck at "Expected - Waiting for status to be reported." Keep "require branches up to date" OFF unless the branch flow changes, keep `enforce_admins` OFF only as an emergency bypass decision, and avoid requiring duplicate legacy contexts named only `verify` or `ui-smoke` when GitHub already reports the active `CI / ...` contexts. -- If `ui-smoke` proves flaky as a required check, demote it to advisory (remove from required contexts) rather than tolerating red merges — the deterministic `verify` gate stays required regardless. If `db-reset-verify` becomes the bottleneck, path-filter it to Supabase migrations/schema/config and database-access code before demoting it entirely. +- CI now has one always-reporting required aggregate: `CI / PR required`. The aggregate depends on `changes`, `static-pr`, `safety`, `coverage`, `build`, `ui-critical`, and `db-reset-verify`, then enforces only the jobs whose scopes apply. +- `static-pr` is the deterministic baseline for every PR: runtime, action pin check, CI scope self-test, format, lint, typecheck, and unit tests. Coverage, build, safety/config, critical UI, production-backed offline RAG preflight, and migration replay are independent jobs so reruns stay focused. Coverage is limited to `src`/test changes; process-only changes do not trigger builds. +- `db-reset-verify` is path-scoped to Supabase migrations/schema/config and database-access code. Do not also require an external Supabase Preview replay unless the repo owner intentionally wants duplicate migration replay. +- `ui-critical` is path-scoped to UI/routing/styling/browser-facing changes and runs only the `@critical` Chromium smoke subset. `ui-regression` runs the remaining Chromium cases on UI pull requests as advisory feedback and is deliberately excluded from `pr-required`. The full browser matrix remains main/release/manual/scheduled. +- Branch protection for `main` should require `CI / PR required` and `Secret Scan / Gitleaks`. Keep `SAST / Semgrep` required only if the repository owner accepts its external-rule/network dependency as part of the normal merge gate. Do not require path-filtered or scheduled/manual contexts such as `CI / Unit coverage`, `CI / Critical UI smoke`, `CI / Migration replay`, `Docker image build / app-image`, `Docker image build / worker-image`, `CI / release-browser-matrix`, `Eval Canary`, or `Live drift check`; they can be skipped on ordinary PRs and would leave branches stuck at "Expected - Waiting for status to be reported." ## CSS cascade layering (2026-07-02) diff --git a/package.json b/package.json index 58afcbc87..acddbd6d5 100644 --- a/package.json +++ b/package.json @@ -20,13 +20,17 @@ "test:e2e": "node scripts/run-playwright.mjs", "test:e2e:all": "node scripts/run-playwright.mjs", "test:e2e:accessibility": "node scripts/run-playwright.mjs tests/ui-accessibility.spec.ts --project=chromium", + "test:e2e:critical": "node scripts/run-playwright.mjs --project=chromium --grep @critical", + "test:e2e:advisory": "node scripts/run-playwright.mjs --project=chromium --grep-invert @critical", "test:e2e:chromium": "node scripts/run-playwright.mjs --project=chromium", "test:e2e:visual": "node scripts/run-playwright.mjs --config=playwright.visual.config.ts", "verify:cheap": "npm run check:runtime && npm run check:github-actions && npm run sitemap:check && npm run lint && npm run typecheck && npm run test", + "verify:pr-local": "node scripts/verify-pr-local.mjs", "verify:ui": "npm run check:runtime && npm run test:e2e:chromium", "verify:release": "npm run check:runtime && npm run lint && npm run typecheck && npm run test && npm run build && npm run test:e2e && npm run check:production-readiness && npm run governance:release && npm run eval:quality:release", "ci:env-check": "node scripts/check-ci-env.mjs", "check:github-actions": "node scripts/check-github-action-pins.mjs", + "check:ci-scope": "node scripts/ci-change-scope.mjs --self-test", "sitemap:update": "tsx scripts/generate-site-map.ts", "sitemap:check": "tsx scripts/generate-site-map.ts --check", "check:runtime": "tsx scripts/check-runtime.ts", @@ -81,6 +85,7 @@ "promote:public-documents": "tsx scripts/promote-public-documents.ts", "promote:public-documents:batch": "tsx scripts/promote-public-documents-batch.ts", "eval:rag": "node scripts/run-eval-safe.mjs scripts/eval-rag.ts", + "eval:rag:offline": "node scripts/eval-rag-offline.mjs", "eval:answer-quality": "node scripts/run-eval-safe.mjs scripts/eval-answer-quality.ts", "eval:quality": "node scripts/run-eval-safe.mjs scripts/eval-quality.ts", "eval:quality:release": "npm run eval:quality -- --fail-on-threshold --source-metadata-debt docs/release-source-metadata-debt-2026-06-30.json", diff --git a/scripts/check-codex-autofix-workflow.mjs b/scripts/check-codex-autofix-workflow.mjs index d7e5ffa20..e2c168ba3 100644 --- a/scripts/check-codex-autofix-workflow.mjs +++ b/scripts/check-codex-autofix-workflow.mjs @@ -1,7 +1,11 @@ import fs from "node:fs"; const workflowPath = ".github/workflows/codex-autofix-review-comments.yml"; +const agentInstructionsPath = "AGENTS.md"; +const reviewProtocolPath = "docs/codex-review-protocol.md"; const workflow = fs.readFileSync(workflowPath, "utf8"); +const agentInstructions = fs.readFileSync(agentInstructionsPath, "utf8"); +const reviewProtocol = fs.readFileSync(reviewProtocolPath, "utf8"); const failures = []; const scopedResolveCommand = "@codex resolve actionable Codex review findings for this pull request and current head"; @@ -36,6 +40,29 @@ for (const { pattern, message } of forbiddenPatterns) { } } +for (const [path, contents] of [ + [agentInstructionsPath, agentInstructions], + [reviewProtocolPath, reviewProtocol], +]) { + if (/resolve all review comments/i.test(contents)) { + failures.push(`${path} must not instruct Codex to resolve all review comments.`); + } +} + +if (!agentInstructions.includes(scopedResolveCommand)) { + failures.push(`${agentInstructionsPath} must contain the scoped actionable-findings command.`); +} + +if (!reviewProtocol.includes("If the user clearly asks to fix confirmed findings, make the smallest safe change")) { + failures.push(`${reviewProtocolPath} must preserve the scoped fix boundary.`); +} + +if ( + !reviewProtocol.includes("Ask before any OpenAI, Supabase, GitHub/GitLab, hosted CI, or provider-backed workflow") +) { + failures.push(`${reviewProtocolPath} must preserve the provider confirmation boundary.`); +} + if (!workflow.includes("group: codex-autoresolve-${{ github.event.pull_request.number }}")) { failures.push("Codex auto-resolve concurrency must be scoped to the pull request number only."); } diff --git a/scripts/ci-change-scope.mjs b/scripts/ci-change-scope.mjs new file mode 100644 index 000000000..2f3fe30b5 --- /dev/null +++ b/scripts/ci-change-scope.mjs @@ -0,0 +1,405 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { appendFileSync } from "node:fs"; + +const zeroSha = /^0{40}$/; + +const outputs = [ + "docs_only", + "source_changed", + "coverage_changed", + "ui_changed", + "db_changed", + "container_changed", + "rag_eval_changed", + "workflow_changed", + "codex_autofix_changed", + "build_changed", +]; + +function normalizePath(filePath) { + return filePath + .replaceAll("\\", "/") + .replace(/^\.\/+/, "") + .replace(/^github\//, ".github/"); +} + +function runGit(args) { + return execFileSync("git", args, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); +} + +function getArgValue(args, name) { + const index = args.indexOf(name); + return index >= 0 ? args[index + 1] : undefined; +} + +function pathMatches(filePath, patterns) { + return patterns.some((pattern) => { + if (typeof pattern === "string") return filePath === pattern || filePath.startsWith(`${pattern}/`); + return pattern.test(filePath); + }); +} + +const docPatterns = [ + "docs", + "mockups", + /^.*\.md$/, + /^.*\.mdx$/, + /^README(?:\..*)?$/i, + /^CHANGELOG(?:\..*)?$/i, + /^LICENSE(?:\..*)?$/i, +]; + +const workflowPatterns = [ + ".github/workflows", + ".github/actions", + ".github/pull_request_template.md", + "AGENTS.md", + "docs/codex-review-protocol.md", + "docs/process-hardening.md", + /^scripts\/(?:ci-change-scope|verify-pr-local|eval-rag-offline|check-github-action-pins|check-codex-autofix-workflow)\.mjs$/, +]; + +const codexAutofixPatterns = [ + ".github/workflows/codex-autofix-review-comments.yml", + "AGENTS.md", + "docs/codex-review-protocol.md", + "scripts/check-codex-autofix-workflow.mjs", +]; + +const uiPatterns = [ + "src/app", + "src/components", + "src/styles", + "public", + /^tests\/ui-.*\.spec\.ts$/, + /^tests\/playwright-.*\.ts$/, + /^playwright(?:\..*)?\.config\.ts$/, + /^scripts\/(run-playwright|playwright-base-url)\.mjs$/, +]; + +const dbPatterns = [ + "supabase", + "src/lib/supabase", + "src/app/api/answer", + "src/app/api/differentials", + "src/app/api/documents", + "src/app/api/eval-cases", + "src/app/api/health", + "src/app/api/images", + "src/app/api/ingestion", + "src/app/api/jobs", + "src/app/api/medications", + "src/app/api/registry", + "src/app/api/search", + "src/app/api/setup-status", + "src/app/api/upload", + "docs/database-drift-detection.md", + "docs/supabase-migration-reconciliation.md", + /^scripts\/(check-drift|generate-drift-manifest|check-m13-migration|check-retrieval-owner-migration|check-supabase-project|audit-tables|reindex|reindex-health|cleanup-abandoned-reindex-generations)\.ts$/, + /^tests\/(supabase|drift|private-rag|private-access|retrieval-owner).*\.test\.ts$/, +]; + +const ragEvalPatterns = [ + "scripts/fixtures", + "src/app/api/answer", + "src/app/api/search", + /^src\/lib\/(?:rag(?:-[^/]+)?|smart-rag-api|clinical-search|clinical-query-mode|retrieval(?:-[^/]+)?|answer(?:-[^/]+)?|citations|cross-document-synthesis|evidence(?:-[^/]+)?|ranking-config|source(?:-[^/]+)?|chunking|document-index-units|query-privacy|owner-scope|corpus-grounding|indexed-source-formatting)\.ts$/, + /^src\/components\/(?:.*\/)?(?:answer|source|citation)[^/]*\.tsx?$/i, + /^scripts\/(eval-|run-eval-safe|compare-retrieval-eval|retrieval-health|profile-retrieval|warm-retrieval-cache|tune-search-weights)/, + /^tests\/(rag|retrieval|answer|citations|evidence|eval|clinical-safety|source).*\.test\.ts$/, +]; + +const containerPatterns = [ + "Dockerfile", + "Dockerfile.worker", + ".dockerignore", + ".npmrc", + ".nvmrc", + "next.config.ts", + "package.json", + "package-lock.json", + "worker/python/requirements.txt", + /^scripts\/(check-node-engine|guard-next-build)\.(?:cjs|mjs)$/, +]; + +const sourcePatterns = ["src", "tests", "scripts", "worker", "playwright", "public", "supabase"]; + +const coveragePatterns = ["src", "tests"]; + +const buildPatterns = [ + "src", + "worker", + "public", + "next.config.ts", + "tsconfig.json", + "postcss.config.mjs", + "package.json", + "package-lock.json", + /^scripts\/(check-node-engine|guard-next-build|dev-free-port|ensure-local-server)\.(?:cjs|mjs)$/, +]; + +const staticConfigPatterns = [ + "eslint.config.mjs", + "playwright.config.ts", + "playwright.visual.config.ts", + "vitest.config.mts", +]; + +function classify(files) { + const normalized = [...new Set(files.map(normalizePath).filter(Boolean))].sort(); + const sourceChanged = normalized.some((file) => pathMatches(file, [...sourcePatterns, ...staticConfigPatterns])); + const coverageChanged = normalized.some((file) => pathMatches(file, coveragePatterns)); + const uiChanged = normalized.some((file) => pathMatches(file, uiPatterns)); + const dbChanged = normalized.some((file) => pathMatches(file, dbPatterns)); + const containerChanged = normalized.some((file) => pathMatches(file, containerPatterns)); + const ragEvalChanged = normalized.some((file) => pathMatches(file, ragEvalPatterns)); + const workflowChanged = normalized.some((file) => pathMatches(file, workflowPatterns)); + const codexAutofixChanged = normalized.some((file) => pathMatches(file, codexAutofixPatterns)); + const buildChanged = normalized.some((file) => pathMatches(file, buildPatterns)) || containerChanged; + const docsOnly = + normalized.length > 0 && + normalized.every((file) => pathMatches(file, docPatterns)) && + !sourceChanged && + !workflowChanged; + + return { + files: normalized, + docs_only: docsOnly, + source_changed: sourceChanged, + coverage_changed: coverageChanged, + ui_changed: uiChanged, + db_changed: dbChanged, + container_changed: containerChanged, + rag_eval_changed: ragEvalChanged, + workflow_changed: workflowChanged, + codex_autofix_changed: codexAutofixChanged, + build_changed: buildChanged, + }; +} + +function parseStatusPorcelain(raw) { + if (!raw) return []; + const fields = raw.split("\0").filter(Boolean); + const files = []; + + for (let index = 0; index < fields.length; index += 1) { + const entry = fields[index]; + const status = entry.slice(0, 2); + const pathPart = entry.slice(3); + if (status.includes("R") || status.includes("C")) { + const originalPath = fields[index + 1]; + files.push(pathPart); + if (originalPath) { + files.push(originalPath); + index += 1; + } + continue; + } + files.push(pathPart); + } + + return files; +} + +function changedFilesFromStatus() { + return parseStatusPorcelain(runGit(["status", "--porcelain=v1", "-z", "--untracked-files=all"])); +} + +function changedFilesFromRange(base, head) { + if (!base || !head || zeroSha.test(base) || zeroSha.test(head)) return null; + try { + const raw = runGit(["diff", "--name-only", `${base}...${head}`]); + return raw ? raw.split(/\r?\n/).filter(Boolean) : []; + } catch { + const raw = runGit(["diff", "--name-only", base, head]); + return raw ? raw.split(/\r?\n/).filter(Boolean) : []; + } +} + +function refExists(ref) { + try { + runGit(["rev-parse", "--verify", `${ref}^{commit}`]); + return true; + } catch { + return false; + } +} + +function isBaseBranchRef(ref) { + return /(?:^|\/)(?:main|master|develop|release\/.+)$/.test(ref); +} + +function resolveLocalBaseRef(args) { + const explicit = getArgValue(args, "--base-ref") ?? process.env.PR_BASE_REF ?? process.env.GITHUB_BASE_REF ?? ""; + if (explicit) { + if (!refExists(explicit)) throw new Error(`Local PR base ref does not exist: ${explicit}`); + return explicit; + } + + let upstream = ""; + try { + upstream = runGit(["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"]); + if (isBaseBranchRef(upstream)) return upstream; + } catch { + // A local-only feature branch can still resolve the remote default branch below. + } + + try { + const remoteHead = runGit(["symbolic-ref", "--quiet", "--short", "refs/remotes/origin/HEAD"]); + if (remoteHead && refExists(remoteHead)) return remoteHead; + } catch { + // Fall through to conventional base names. + } + + for (const candidate of ["origin/main", "origin/master", "origin/develop", "main", "master", "develop"]) { + if (refExists(candidate)) return candidate; + } + + return upstream || null; +} + +function changedFilesFromLocal(args) { + const statusFiles = changedFilesFromStatus(); + const baseRef = resolveLocalBaseRef(args); + if (!baseRef) return statusFiles; + + try { + const mergeBase = runGit(["merge-base", "HEAD", baseRef]); + const raw = runGit(["diff", "--name-only", `${mergeBase}...HEAD`]); + return [...(raw ? raw.split(/\r?\n/).filter(Boolean) : []), ...statusFiles]; + } catch { + return statusFiles; + } +} + +function resolveChangedFiles(args) { + const filesArg = getArgValue(args, "--files"); + if (filesArg) + return filesArg + .split(",") + .map((item) => item.trim()) + .filter(Boolean); + + const separator = args.indexOf("--"); + if (separator >= 0) return args.slice(separator + 1); + + const base = getArgValue(args, "--base") ?? process.env.BASE_SHA ?? ""; + const head = getArgValue(args, "--head") ?? process.env.HEAD_SHA ?? ""; + const ranged = changedFilesFromRange(base, head); + if (ranged) return ranged; + + if (process.env.GITHUB_ACTIONS === "true") { + return ["src/__ci_full_run__.ts", "supabase/__ci_full_run__.sql", "Dockerfile", ".github/workflows/ci.yml"]; + } + + return changedFilesFromLocal(args); +} + +function writeOutputs(result) { + for (const key of outputs) { + console.log(`${key}=${result[key]}`); + } + console.log(`changed_files=${result.files.join(",")}`); + + if (!process.env.GITHUB_OUTPUT) return; + const lines = [...outputs.map((key) => `${key}=${result[key]}`), `changed_files=${result.files.join(",")}`]; + appendFileSync(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`); +} + +function assertScope(name, files, expected) { + const result = classify(files); + for (const [key, value] of Object.entries(expected)) { + if (result[key] !== value) { + throw new Error(`${name}: expected ${key}=${value}, received ${result[key]} for ${files.join(", ")}`); + } + } +} + +function selfTest() { + assertScope("docs-only", ["docs/process-note.md"], { + docs_only: true, + source_changed: false, + build_changed: false, + }); + assertScope("tests-only", ["tests/rag-routing.test.ts"], { + source_changed: true, + coverage_changed: true, + build_changed: false, + }); + assertScope("ui", ["src/components/ClinicalDashboard.tsx", "tests/ui-smoke.spec.ts"], { + docs_only: false, + source_changed: true, + ui_changed: true, + build_changed: true, + }); + assertScope("db", ["supabase/migrations/20260710000000_example.sql"], { + db_changed: true, + source_changed: true, + build_changed: false, + }); + assertScope( + "rag", + [ + "src/app/api/answer/route.ts", + "src/lib/corpus-grounding.ts", + "src/components/clinical-dashboard/answer-content.tsx", + ], + { + rag_eval_changed: true, + source_changed: true, + }, + ); + assertScope("rag-fixture", ["src/lib/retrieval-selection.ts", "scripts/fixtures/rag-retrieval-golden.json"], { + rag_eval_changed: true, + source_changed: true, + }); + assertScope("database-access", ["src/app/api/documents/route.ts"], { + db_changed: true, + source_changed: true, + }); + assertScope("workflow", [".github/workflows/ci.yml", "docs/process-hardening.md"], { + workflow_changed: true, + docs_only: false, + build_changed: false, + }); + assertScope( + "codex-autofix", + [".github/workflows/codex-autofix-review-comments.yml", "AGENTS.md", "scripts/check-codex-autofix-workflow.mjs"], + { + workflow_changed: true, + codex_autofix_changed: true, + build_changed: false, + }, + ); + assertScope("package", ["package.json"], { + source_changed: false, + coverage_changed: false, + container_changed: true, + workflow_changed: false, + build_changed: true, + }); + assertScope("container", ["Dockerfile.worker", "worker/python/requirements.txt"], { + container_changed: true, + build_changed: true, + }); + assertScope("renamed-destination", parseStatusPorcelain("R src/lib/rag-new.ts\0docs/rag-old.md\0"), { + source_changed: true, + rag_eval_changed: true, + docs_only: false, + }); + console.log("CI change scope self-test passed."); +} + +const args = process.argv.slice(2); +if (args.includes("--self-test")) { + selfTest(); + process.exit(0); +} + +const result = classify(resolveChangedFiles(args)); +if (args.includes("--json")) { + console.log(JSON.stringify(result, null, 2)); +} else { + writeOutputs(result); +} diff --git a/scripts/eval-rag-offline.mjs b/scripts/eval-rag-offline.mjs new file mode 100644 index 000000000..ff8f35f6b --- /dev/null +++ b/scripts/eval-rag-offline.mjs @@ -0,0 +1,107 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const goldenPath = "scripts/fixtures/rag-retrieval-golden.json"; +const cases = JSON.parse(readFileSync(goldenPath, "utf8")); +const failures = []; + +function fail(message) { + failures.push(message); +} + +function hasText(value) { + return typeof value === "string" && value.trim().length > 0; +} + +function validateTerm(term, caseId, fieldName) { + if (Array.isArray(term)) { + if (term.length === 0 || !term.every(hasText)) { + fail(`${caseId}: ${fieldName} contains an empty alternative term group.`); + } + return; + } + if (!hasText(term)) fail(`${caseId}: ${fieldName} contains an empty term.`); +} + +function validateGoldenCases() { + if (!Array.isArray(cases) || cases.length < 10) { + fail(`${goldenPath} must contain the offline retrieval contract cases.`); + return; + } + + const ids = new Set(); + for (const item of cases) { + if (!hasText(item.id)) fail("Golden retrieval case is missing id."); + if (ids.has(item.id)) fail(`${item.id}: duplicate golden retrieval case id.`); + ids.add(item.id); + + if (!hasText(item.query)) fail(`${item.id}: query is required.`); + if (!hasText(item.expectedQueryClass)) fail(`${item.id}: expectedQueryClass is required.`); + if (!Number.isInteger(item.topK) || item.topK < 1 || item.topK > 50) { + fail(`${item.id}: topK must be an integer between 1 and 50.`); + } + if (!Array.isArray(item.expectedDocumentSubstrings)) { + fail(`${item.id}: expectedDocumentSubstrings must be an array.`); + } else { + for (const term of item.expectedDocumentSubstrings) validateTerm(term, item.id, "expectedDocumentSubstrings"); + } + if (!Array.isArray(item.expectedContentTerms) || item.expectedContentTerms.length === 0) { + fail(`${item.id}: expectedContentTerms must include at least one required term.`); + } else { + for (const term of item.expectedContentTerms) validateTerm(term, item.id, "expectedContentTerms"); + } + if (typeof item.expectTableEvidence !== "boolean") { + fail(`${item.id}: expectTableEvidence must be boolean.`); + } + } +} + +validateGoldenCases(); + +if (failures.length > 0) { + console.error("Offline RAG preflight failed:"); + for (const failure of failures) console.error(`- ${failure}`); + process.exit(1); +} + +console.log(`Offline RAG fixture schema passed (${cases.length} golden retrieval cases).`); + +const contractTests = [ + "tests/rag-offline-answer.test.ts", + "tests/rag-answer-fallback.test.ts", + "tests/retrieval-selection.test.ts", + "tests/citations.test.ts", + "tests/smart-rag-api.test.ts", +]; +const vitestPath = resolve("node_modules/vitest/vitest.mjs"); +const offlineEnv = { ...process.env }; +for (const key of [ + "RAG_PROVIDER_MODE", + "OPENAI_API_KEY", + "OPENAI_ORG_ID", + "OPENAI_PROJECT_ID", + "OPENAI_BASE_URL", + "NEXT_PUBLIC_SUPABASE_URL", + "NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY", + "NEXT_PUBLIC_SUPABASE_ANON_KEY", + "SUPABASE_URL", + "SUPABASE_ANON_KEY", + "SUPABASE_SECRET_KEY", + "SUPABASE_SERVICE_ROLE_KEY", +]) { + delete offlineEnv[key]; +} +const result = spawnSync(process.execPath, [vitestPath, "run", ...contractTests, "--reporter=dot"], { + env: offlineEnv, + stdio: "inherit", +}); + +if (result.error) { + console.error(`Offline RAG contract tests could not start: ${result.error.message}`); + process.exit(1); +} +if (result.status !== 0) process.exit(result.status ?? 1); + +console.log("Offline RAG production contract tests passed."); diff --git a/scripts/verify-pr-local.mjs b/scripts/verify-pr-local.mjs new file mode 100644 index 000000000..6c1fd0c4f --- /dev/null +++ b/scripts/verify-pr-local.mjs @@ -0,0 +1,40 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; + +const isWindows = process.platform === "win32"; + +function runNpmScript(script) { + console.log(`\n> npm run ${script}`); + const result = isWindows + ? spawnSync("cmd.exe", ["/d", "/s", "/c", `npm run ${script}`], { stdio: "inherit" }) + : spawnSync("npm", ["run", script], { stdio: "inherit" }); + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} + +function readScope() { + const result = spawnSync(process.execPath, ["scripts/ci-change-scope.mjs", "--json"], { + encoding: "utf8", + stdio: ["ignore", "pipe", "inherit"], + }); + if (result.status !== 0) process.exit(result.status ?? 1); + return JSON.parse(result.stdout); +} + +const scope = readScope(); +console.log(`Changed files: ${scope.files.length > 0 ? scope.files.join(", ") : "(none detected)"}`); + +for (const script of ["check:runtime", "format:check", "lint", "typecheck", "test"]) { + runNpmScript(script); +} + +if (scope.build_changed) { + runNpmScript("build"); +} else { + console.log("\nSkipping build: no build-affecting source, config, package, or container changes detected."); +} + +if (scope.rag_eval_changed) { + runNpmScript("eval:rag:offline"); +} diff --git a/tests/rag-offline-answer.test.ts b/tests/rag-offline-answer.test.ts index 59f6de995..be322688a 100644 --- a/tests/rag-offline-answer.test.ts +++ b/tests/rag-offline-answer.test.ts @@ -16,6 +16,20 @@ function source(overrides: Partial = {}): SearchResult { similarity: 0.95, hybrid_score: 0.95, text_rank: 1.2, + table_facts: [ + { + id: "clozapine-anc-threshold", + document_id: "clozapine-doc", + source_chunk_id: "clozapine-chunk-1", + source_image_id: null, + page_number: 11, + table_title: "Clozapine ANC thresholds", + row_label: "ANC below 1.5 x 10^9/L", + clinical_parameter: "ANC", + threshold_value: "below 1.5 x 10^9/L", + action: "Withhold clozapine and repeat FBC.", + }, + ], source_metadata: { source_title: "Clozapine source", publisher: "Local service", @@ -92,7 +106,7 @@ afterEach(() => { }); describe("source-only / offline answers", () => { - it("answers from sources deterministically without calling the model or embeddings", async () => { + it("returns the source/citation wire contract without calling the model or embeddings", async () => { const { answer, generateStructuredTextResult, embedTextWithTelemetry } = await answerOffline( "What ANC threshold should withhold clozapine?", [source()], @@ -103,7 +117,35 @@ describe("source-only / offline answers", () => { expect(answer.modelUsed).toBeNull(); expect(answer.routingMode).toBe("extractive"); expect(answer.routingReason).toContain("source_only"); - expect(answer.sources.length).toBeGreaterThan(0); + expect(answer).toMatchObject({ + answer: expect.any(String), + grounded: false, + confidence: "unsupported", + citations: [ + expect.objectContaining({ + chunk_id: "clozapine-chunk-1", + document_id: "clozapine-doc", + title: "Clozapine Prescribing Administration Monitoring", + file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf", + page_number: 11, + source_metadata: expect.objectContaining({ + jurisdiction: "Australia/WA", + document_status: "current", + clinical_validation_status: "approved", + }), + }), + ], + sources: [ + expect.objectContaining({ + id: "clozapine-chunk-1", + document_id: "clozapine-doc", + page_number: 11, + }), + ], + }); + expect(new Set(answer.sources.map((item) => item.id))).toEqual( + new Set(answer.citations.map((citation) => citation.chunk_id)), + ); // quality signalling for the UI disclosure expect(answer.answerQualityTier).toBe("source_only"); expect(answer.providerMode).toBe("offline"); @@ -119,6 +161,12 @@ describe("source-only / offline answers", () => { expect(generateStructuredTextResult).not.toHaveBeenCalled(); expect(answer.modelUsed).toBeNull(); expect(answer.routingMode).toBe("unsupported"); - expect(answer.grounded).toBe(false); + expect(answer).toMatchObject({ + answer: expect.any(String), + grounded: false, + confidence: "unsupported", + citations: [], + sources: [], + }); }); }); diff --git a/tests/ui-smoke.spec.ts b/tests/ui-smoke.spec.ts index 78331466e..0ed6c5f8e 100644 --- a/tests/ui-smoke.spec.ts +++ b/tests/ui-smoke.spec.ts @@ -31,6 +31,11 @@ async function gotoApp(page: Page, path: string) { await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => undefined); } +async function gotoCriticalApp(page: Page, path: string) { + await page.goto(path, { waitUntil: "domcontentloaded" }); + await expect(page.locator("#main-content, main").first()).toBeVisible({ timeout: 30_000 }); +} + async function expectSingleMedicationPage(page: Page) { // The medication route renders inside GlobalMockupSearchShell, whose Suspense // fallback and resolved client subtree both render `children`. During a @@ -791,13 +796,13 @@ test.describe("Clinical KB UI smoke coverage", () => { }); } - test("anonymous user can see enabled live search without a forced sign-in gate", async ({ page }) => { + test("anonymous user can see enabled live search without a forced sign-in gate @critical", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 900 }); await mockPrivateUnauthenticatedApi(page); await page.route(/\/api\/search(?:\?.*)?$/, async (route) => { await route.fulfill({ json: { results: [], telemetry: { retrieval_strategy: "text_fast_path" } } }); }); - await gotoApp(page, "/"); + await gotoCriticalApp(page, "/"); await waitForDemoDashboardReady(page); await expect(page.getByText("Create your Clinical Guide account")).toHaveCount(0); @@ -864,10 +869,10 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(page.getByRole("heading", { name: "Answer" })).toBeVisible(); }); - test("tablet shows icon rail without drawer trigger or expand control", async ({ page }) => { + test("tablet shows icon rail without drawer trigger or expand control @critical", async ({ page }) => { await page.setViewportSize({ width: 768, height: 1024 }); await mockDemoApi(page); - await gotoApp(page, "/?mode=answer"); + await gotoCriticalApp(page, "/?mode=answer"); await waitForDemoDashboardReady(page); await expect(page.getByRole("button", { name: "Open Clinical Guide menu" })).toHaveCount(0); @@ -953,7 +958,7 @@ test.describe("Clinical KB UI smoke coverage", () => { }) => { await page.setViewportSize({ width: 1280, height: 900 }); await mockDemoApi(page); - await gotoApp(page, "/"); + await gotoCriticalApp(page, "/"); await waitForDemoDashboardReady(page); const settings = accountSettingsDialog(page); @@ -1109,10 +1114,10 @@ test.describe("Clinical KB UI smoke coverage", () => { await expectNoPageHorizontalOverflow(page); }); - test("demo answer flow reaches a source-backed answer", async ({ browserName, page }) => { + test("demo answer flow reaches a source-backed answer @critical", async ({ browserName, page }) => { await page.setViewportSize({ width: 390, height: 820 }); await mockDemoApi(page); - await gotoApp(page, "/"); + await gotoCriticalApp(page, "/"); await waitForDemoDashboardReady(page); const question = "What clozapine monitoring items are shown in the table image?"; @@ -1954,7 +1959,7 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(appModeButton).toBeFocused(); }); - test("prescribing workflow uses in-app medication routes", async ({ page }) => { + test("prescribing workflow uses in-app medication routes @critical", async ({ page }) => { test.setTimeout(120_000); // Regression guard: navigating away from a mode home used to throw // "Cannot read properties of null (reading 'parentNode')" because the header @@ -1970,7 +1975,7 @@ test.describe("Clinical KB UI smoke coverage", () => { }); await page.setViewportSize({ width: 1280, height: 900 }); await mockDemoApi(page); - await gotoApp(page, "/?mode=prescribing&q=acamprosate%20renal%20dose&run=1"); + await gotoCriticalApp(page, "/?mode=prescribing&q=acamprosate%20renal%20dose&run=1"); const globalSearchInput = page.getByTestId("global-search-input"); await expect(page.getByRole("button", { name: "Mode Medication" })).toBeVisible({ timeout: 30_000 }); @@ -1983,7 +1988,7 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(page).toHaveURL(/\/medications\/acamprosate$/, { timeout: 30_000 }); await expectSingleMedicationPage(page); - await gotoApp(page, "/mockups/medication-prescribing"); + await gotoCriticalApp(page, "/mockups/medication-prescribing"); await expect(page).toHaveURL(/\/medications\/acamprosate$/); await expectSingleMedicationPage(page); expect(parentNodeErrors).toEqual([]); @@ -2018,7 +2023,7 @@ test.describe("Clinical KB UI smoke coverage", () => { test("document search mode lists matching documents and scope actions", async ({ page }) => { await page.setViewportSize({ width: 390, height: 820 }); await mockDemoApi(page); - await gotoApp(page, "/"); + await gotoCriticalApp(page, "/"); await waitForDemoDashboardReady(page); await switchToDocumentSearchMode(page); @@ -2103,10 +2108,10 @@ test.describe("Clinical KB UI smoke coverage", () => { await expectNoPageHorizontalOverflow(page); }); - test("search regressions avoid fetch errors and open viewer hits", async ({ page }) => { + test("search regressions avoid fetch errors and open viewer hits @critical", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 900 }); await mockDemoApi(page); - await gotoApp(page, "/"); + await gotoCriticalApp(page, "/"); await waitForDemoDashboardReady(page); await switchToDocumentSearchMode(page); @@ -2119,7 +2124,7 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(page.getByRole("heading", { name: /Search command centre|Find source evidence/ })).toBeVisible(); const demoDocId = "11111111-1111-4111-8111-111111111111"; - await gotoApp(page, `/documents/${demoDocId}?chunk=55555555-5555-4555-8555-555555555555`); + await gotoCriticalApp(page, `/documents/${demoDocId}?chunk=55555555-5555-4555-8555-555555555555`); await expect(page).toHaveURL(/chunk=55555555-5555-4555-8555-555555555555/); await expect(page.locator("#source-evidence").getByTestId("highlighted-source-passage")).toContainText( "Patient safety plan should include", From dbe20ec2143ff82df0c17ccef329f2357f8e95f2 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:58:17 +0800 Subject: [PATCH 2/2] fix: fail closed when detecting PR scope --- scripts/ci-change-scope.mjs | 85 +++++++++++++++++++++++++++++++------ 1 file changed, 72 insertions(+), 13 deletions(-) diff --git a/scripts/ci-change-scope.mjs b/scripts/ci-change-scope.mjs index 2f3fe30b5..881913e7e 100644 --- a/scripts/ci-change-scope.mjs +++ b/scripts/ci-change-scope.mjs @@ -4,6 +4,13 @@ import { appendFileSync } from "node:fs"; const zeroSha = /^0{40}$/; +const fullRunSentinelFiles = [ + "src/app/api/answer/__ci_full_run__.ts", + "supabase/__ci_full_run__.sql", + "Dockerfile", + ".github/workflows/codex-autofix-review-comments.yml", +]; + const outputs = [ "docs_only", "source_changed", @@ -28,6 +35,10 @@ function runGit(args) { return execFileSync("git", args, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } +function runGitRaw(args) { + return execFileSync("git", args, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }); +} + function getArgValue(args, name) { const index = args.indexOf(name); return index >= 0 ? args[index + 1] : undefined; @@ -68,6 +79,7 @@ const codexAutofixPatterns = [ ]; const uiPatterns = [ + "data", "src/app", "src/components", "src/styles", @@ -123,11 +135,12 @@ const containerPatterns = [ /^scripts\/(check-node-engine|guard-next-build)\.(?:cjs|mjs)$/, ]; -const sourcePatterns = ["src", "tests", "scripts", "worker", "playwright", "public", "supabase"]; +const sourcePatterns = ["data", "src", "tests", "scripts", "worker", "playwright", "public", "supabase"]; -const coveragePatterns = ["src", "tests"]; +const coveragePatterns = ["data", "src", "tests", "vitest.config.mts"]; const buildPatterns = [ + "data", "src", "worker", "public", @@ -203,17 +216,34 @@ function parseStatusPorcelain(raw) { } function changedFilesFromStatus() { - return parseStatusPorcelain(runGit(["status", "--porcelain=v1", "-z", "--untracked-files=all"])); + return parseStatusPorcelain(runGitRaw(["status", "--porcelain=v1", "-z", "--untracked-files=all"])); +} + +function parseNameStatus(raw) { + const fields = raw.split("\0").filter(Boolean); + const files = []; + + for (let index = 0; index < fields.length;) { + const status = fields[index++]; + const firstPath = fields[index++]; + if (!status || !firstPath) break; + + files.push(firstPath); + if (/^[RC]/.test(status)) { + const secondPath = fields[index++]; + if (secondPath) files.push(secondPath); + } + } + + return files; } function changedFilesFromRange(base, head) { if (!base || !head || zeroSha.test(base) || zeroSha.test(head)) return null; try { - const raw = runGit(["diff", "--name-only", `${base}...${head}`]); - return raw ? raw.split(/\r?\n/).filter(Boolean) : []; + return parseNameStatus(runGitRaw(["diff", "--name-status", "-z", "--find-renames", `${base}...${head}`])); } catch { - const raw = runGit(["diff", "--name-only", base, head]); - return raw ? raw.split(/\r?\n/).filter(Boolean) : []; + return parseNameStatus(runGitRaw(["diff", "--name-status", "-z", "--find-renames", base, head])); } } @@ -256,20 +286,19 @@ function resolveLocalBaseRef(args) { if (refExists(candidate)) return candidate; } - return upstream || null; + return null; } function changedFilesFromLocal(args) { const statusFiles = changedFilesFromStatus(); const baseRef = resolveLocalBaseRef(args); - if (!baseRef) return statusFiles; + if (!baseRef) return [...fullRunSentinelFiles, ...statusFiles]; try { const mergeBase = runGit(["merge-base", "HEAD", baseRef]); - const raw = runGit(["diff", "--name-only", `${mergeBase}...HEAD`]); - return [...(raw ? raw.split(/\r?\n/).filter(Boolean) : []), ...statusFiles]; + return [...(changedFilesFromRange(mergeBase, "HEAD") ?? []), ...statusFiles]; } catch { - return statusFiles; + return [...fullRunSentinelFiles, ...statusFiles]; } } @@ -290,7 +319,7 @@ function resolveChangedFiles(args) { if (ranged) return ranged; if (process.env.GITHUB_ACTIONS === "true") { - return ["src/__ci_full_run__.ts", "supabase/__ci_full_run__.sql", "Dockerfile", ".github/workflows/ci.yml"]; + return fullRunSentinelFiles; } return changedFilesFromLocal(args); @@ -317,6 +346,10 @@ function assertScope(name, files, expected) { } function selfTest() { + assertScope("unstaged-status", parseStatusPorcelain(" M scripts/ci-change-scope.mjs\0"), { + source_changed: true, + workflow_changed: true, + }); assertScope("docs-only", ["docs/process-note.md"], { docs_only: true, source_changed: false, @@ -327,6 +360,16 @@ function selfTest() { coverage_changed: true, build_changed: false, }); + assertScope("coverage-config", ["vitest.config.mts"], { + source_changed: true, + coverage_changed: true, + }); + assertScope("runtime-data", ["data/medications-snapshot.json"], { + source_changed: true, + coverage_changed: true, + ui_changed: true, + build_changed: true, + }); assertScope("ui", ["src/components/ClinicalDashboard.tsx", "tests/ui-smoke.spec.ts"], { docs_only: false, source_changed: true, @@ -388,6 +431,22 @@ function selfTest() { rag_eval_changed: true, docs_only: false, }); + assertScope("ranged-rename", parseNameStatus("R100\0src/lib/rag-old.ts\0docs/rag-old.md\0"), { + source_changed: true, + rag_eval_changed: true, + docs_only: false, + }); + assertScope("unknown-base-full-run", fullRunSentinelFiles, { + source_changed: true, + coverage_changed: true, + ui_changed: true, + db_changed: true, + container_changed: true, + rag_eval_changed: true, + workflow_changed: true, + codex_autofix_changed: true, + build_changed: true, + }); console.log("CI change scope self-test passed."); }