test: harden reliable test execution

.github/workflows/ci-triage.yml

@@ -4,12 +4,11 @@
 #   - main-side: the same job is also failing on the latest default-branch run
 #     (CI merges the PR branch with current main, so a main regression surfaces on
 #     every open PR — this has cost debugging time before).
-#   - possible known flake: a UI/Playwright job failed — check tests/flake-ledger.json.
 #   - needs investigation: everything else.
 #
 # SHIPPED INERT: this event-triggered workflow does nothing until the repo variable
 # CI_TRIAGE_ENABLED == "true". It never runs PR-authored code — it only reads job
-# metadata + the committed flake ledger via the trusted default-branch checkout, and
+# metadata via the trusted default-branch checkout, and
 # posts a comment with the built-in token.
 name: CI Triage
 
@@ -45,7 +44,6 @@ jobs:
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         with:
           script: |
-            const fs = require("fs");
             const run = context.payload.workflow_run;
 
             // Resolve the PR for this run (empty for fork PRs → skip quietly).
@@ -93,13 +91,11 @@ jobs:
               core.info(`main-side check skipped: ${e.message}`);
             }
 
-            const flakes = JSON.parse(fs.readFileSync("tests/flake-ledger.json", "utf8")).flakes || [];
-            const uiFlakeSpecs = flakes.map((f) => f.spec).filter((s) => /ui-/.test(s));
             const looksUi = (name) => /ui|playwright|browser|e2e/i.test(name);
 
             const lines = failed.map((name) => {
               if (mainFailingJobs.has(name)) return `- \`${name}\` — **main-side**: also failing on the latest \`main\` run, likely not your change.`;
-              if (looksUi(name)) return `- \`${name}\` — **possible known flake**: UI/Playwright job; check \`tests/flake-ledger.json\`${uiFlakeSpecs.length ? ` (${uiFlakeSpecs.join(", ")})` : ""} and re-run before bisecting.`;
+              if (looksUi(name)) return `- \`${name}\` — **needs investigation**: use the uploaded JUnit classification and trace; job type alone is not evidence of a known flake.`;
               return `- \`${name}\` — **needs investigation**.`;
             });
 

.github/workflows/ci.yml

@@ -108,9 +108,6 @@ jobs:
       - name: Typecheck
         run: npm run typecheck
 
-      - name: Unit tests
-        run: npm run test
-
   safety:
     name: Safety and config checks
     needs: changes
@@ -160,19 +157,17 @@ jobs:
         if: needs.changes.outputs.codex_autofix_changed == 'true'
         run: npm run check:codex-autofix-workflow
 
-      - name: Offline RAG preflight
-        # Runs for every non-docs change (this job's docs_only guard), NOT only
-        # files matching the rag_eval_changed allowlist. This offline grounding
-        # gate is cheap and clinical-safety-relevant, so a new retrieval file that
-        # falls outside the scope patterns must never silently skip it.
-        run: npm run eval:rag:offline
+      - name: Offline RAG fixture and manifest validation
+        run: npm run check:rag:fixtures
 
   coverage:
     name: Unit coverage
     needs: changes
     if: needs.changes.outputs.coverage_changed == 'true'
     runs-on: ubuntu-24.04
     timeout-minutes: 20
+    env:
+      FAST_CHECK_SEED: ${{ github.event_name == 'schedule' && github.run_id || '424242' }}
     steps:
       - name: Checkout
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -228,11 +223,11 @@ jobs:
         run: npm run check:deployment-readiness
 
   ui-critical:
-    name: Critical UI smoke
+    name: Production UI
     needs: changes
     if: needs.changes.outputs.ui_changed == 'true'
     runs-on: ubuntu-24.04
-    timeout-minutes: 15
+    timeout-minutes: 45
     steps:
       - name: Checkout
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -242,120 +237,53 @@ jobs:
       - name: Setup UI e2e environment
         uses: ./.github/actions/setup-ui-e2e
 
-      - name: Chromium critical UI smoke
-        run: npm run test:e2e:critical
+      - name: Chromium production journeys
+        run: npm run test:e2e:pr
 
-      - name: Upload UI diagnostics
+      - name: Classify exact failed test identities
         if: failure()
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
-        with:
-          name: critical-ui-diagnostics-${{ github.run_id }}
-          path: |
-            test-results/
-            playwright-report/
-          if-no-files-found: ignore
+        run: node scripts/classify-playwright-failures.mjs
 
-  # Production-journey regression suite. Blocking (PT-05): a red production
-  # journey must fail pr-required instead of normalising as an advisory red.
-  # Prototype /mockups specs run separately in ui-mockups below.
-  ui-regression:
-    name: UI regression
-    needs: changes
-    if: needs.changes.outputs.ui_changed == 'true'
-    runs-on: ubuntu-24.04
-    timeout-minutes: 45
-    steps:
-      - name: Checkout
-        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
-        with:
-          persist-credentials: false
-
-      - name: Setup UI e2e environment
-        uses: ./.github/actions/setup-ui-e2e
-
-      - name: Chromium stable regression suite
-        run: npm run test:e2e:regression
-
-      - name: Upload UI regression diagnostics
+      - name: Upload UI diagnostics
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: ui-regression-diagnostics-${{ github.run_id }}
+          name: production-ui-diagnostics-${{ github.run_id }}
           path: |
             test-results/
             playwright-report/
           if-no-files-found: ignore
 
-  # Keep the advisory quarantine lane available without paying the Node/browser
-  # setup cost while no specs are tagged. Stable regression excludes @quarantine,
-  # so removing this lane entirely would make a newly tagged test run nowhere.
-  ui-quarantine:
-    name: Advisory quarantined UI tests
+  # Quarantined production tests and mockup-only journeys share one advisory
+  # install/server without weakening the required production-ui job above.
+  ui-advisory:
+    name: Advisory UI
     needs: changes
     if: github.event_name == 'pull_request' && needs.changes.outputs.ui_changed == 'true'
     continue-on-error: true
     runs-on: ubuntu-24.04
-    timeout-minutes: 30
+    timeout-minutes: 45
     steps:
       - name: Checkout
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           persist-credentials: false
 
-      - name: Detect quarantined specs
-        id: quarantine
-        shell: bash
-        run: |
-          if git grep -q '@quarantine' -- ':(glob)tests/ui-*.spec.ts'; then
-            echo "present=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "present=false" >> "$GITHUB_OUTPUT"
-          fi
-
       - name: Setup UI e2e environment
-        if: steps.quarantine.outputs.present == 'true'
         uses: ./.github/actions/setup-ui-e2e
 
-      - name: Chromium quarantined tests (advisory)
-        if: steps.quarantine.outputs.present == 'true'
-        run: npm run test:e2e:quarantine
+      - name: Chromium quarantined and mockup journeys (advisory)
+        run: npm run test:e2e:advisory
 
-      - name: Upload quarantine diagnostics
-        if: failure() && steps.quarantine.outputs.present == 'true'
-        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
-        with:
-          name: quarantine-ui-diagnostics-${{ github.run_id }}
-          path: |
-            test-results/
-            playwright-report/
-          if-no-files-found: ignore
-
-  # Prototype /mockups specs live in their own advisory lane so a red mockup
-  # can never mask (or block on) a production-journey regression (PT-05).
-  ui-mockups:
-    name: Advisory mockup UI
-    needs: changes
-    if: github.event_name == 'pull_request' && needs.changes.outputs.ui_changed == 'true'
-    continue-on-error: true
-    runs-on: ubuntu-24.04
-    timeout-minutes: 30
-    steps:
-      - name: Checkout
-        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
-        with:
-          persist-credentials: false
-
-      - name: Setup UI e2e environment
-        uses: ./.github/actions/setup-ui-e2e
-
-      - name: Chromium mockup regression
-        run: npm run test:e2e:mockups
+      - name: Classify exact failed test identities
+        if: failure()
+        run: node scripts/classify-playwright-failures.mjs
 
-      - name: Upload mockup UI diagnostics
+      - name: Upload advisory UI diagnostics
         if: failure()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: mockup-ui-diagnostics-${{ github.run_id }}
+          name: advisory-ui-diagnostics-${{ github.run_id }}
           path: |
             test-results/
             playwright-report/
@@ -411,7 +339,7 @@ jobs:
 
   pr-required:
     name: PR required
-    needs: [changes, static-pr, safety, coverage, build, ui-critical, ui-regression, db-reset-verify]
+    needs: [changes, static-pr, safety, coverage, build, ui-critical, db-reset-verify]
     if: always()
     runs-on: ubuntu-24.04
     timeout-minutes: 5
@@ -429,7 +357,6 @@ jobs:
           COVERAGE_RESULT: ${{ needs.coverage.result }}
           BUILD_RESULT: ${{ needs.build.result }}
           UI_RESULT: ${{ needs.ui-critical.result }}
-          UI_REGRESSION_RESULT: ${{ needs.ui-regression.result }}
           DB_RESULT: ${{ needs.db-reset-verify.result }}
         run: |
           set -euo pipefail
@@ -474,11 +401,9 @@ jobs:
           fi
 
           if [ "$UI_CHANGED" = "true" ]; then
-            require_success "ui-critical" "$UI_RESULT"
-            require_success "ui-regression" "$UI_REGRESSION_RESULT"
+            require_success "production-ui" "$UI_RESULT"
           else
-            require_skipped_or_success "ui-critical" "$UI_RESULT"
-            require_skipped_or_success "ui-regression" "$UI_REGRESSION_RESULT"
+            require_skipped_or_success "production-ui" "$UI_RESULT"
           fi
 
           if [ "$DB_CHANGED" = "true" ]; then

.gitignore

@@ -27,6 +27,7 @@
 
 # next.js
 /.next/
+/.next-playwright/
 /out/
 
 # production

AGENTS.md

@@ -164,7 +164,8 @@ When a branch or PR review completes, record the reviewed branch/ref, HEAD SHA,
 
 # Process hardening phases
 
-- For non-trivial source/config/test changes, prefer `npm run verify:cheap` as the first broad gate and `npm run verify:pr-local` before PR handoff when the change is ready. The PR-local gate runs format plus `verify:cheap`, then conditionally adds the production build/client-bundle scan and code-backed offline RAG tests. Browser, dependency-audit, Docker/Supabase replay, and provider-backed checks remain separate gates. Use `npm run verify:pr-local -- --dry-run --files <comma-separated paths>` to inspect selection without running commands. The broader `--extended` plan is dry-run only unless explicit approval is reflected by `ALLOW_EXTENDED_PR_LOCAL=true`.
+- For non-trivial source/config/test changes, prefer `npm run verify:cheap` as the first broad gate and `npm run verify:pr-local` before PR handoff when the change is ready. The PR-local gate runs the full unit suite once, then conditionally adds the production build/client-bundle scan and RAG fixture/manifest validation. Browser, dependency-audit, Docker/Supabase replay, and provider-backed checks remain separate gates. Use `npm run verify:pr-local -- --dry-run --files <comma-separated paths>` to inspect selection without running commands. The broader `--extended` plan is dry-run only unless explicit approval is reflected by `ALLOW_EXTENDED_PR_LOCAL=true`.
+- Run one heavy Database command at a time across all worktrees. Do not install while a repository test, build, lint, typecheck, or server command is active. Avoid aggressive short-interval polling, and do not repeat an unchanged full gate after it passes.
 - For UI, frontend, browser, routing, styling, reduced-motion, or forced-colors changes, run `npm run ensure` before browser work and use `npm run verify:ui` as the Chromium UI gate.
 - For release or handoff confidence, use `npm run verify:release`; this includes the full Playwright project set.
 - For clinical ingestion, answer generation, source governance, privacy, production-readiness, or environment changes, run the smallest relevant domain check plus `npm run check:production-readiness`.
@@ -307,6 +308,7 @@ After completing `upload`, summarize the current branch and worktree state, whet
 - For non-trivial changes, start from concrete repo state: branch, `git status`, relevant package scripts, recent failures, and local logs such as `dev-server.log` when runtime behavior is involved. For architecture and module orientation, read `docs/codebase-index.md` (routes: `docs/site-map.md`).
 - For UI, browser, styling, routing, accessibility, or screenshot work, run `npm run ensure` before opening the app, then use browser QA and the smallest relevant UI proof before broader gates.
 - Prefer the smallest failing check first. For this repo, use focused Vitest or Playwright targets before widening to `npm run verify:cheap`, `npm run verify:ui`, or `npm run verify:release`.
+- Use `npm run test:focused -- --files <paths>` only for safe source-only iteration. It fails closed for deleted files and test/configuration infrastructure; follow its instruction to run `npm run test` in those cases.
 - When the user says `safely`, preserve unrelated staged, unstaged, and untracked work; stop only clearly repo-owned transient processes; and verify the result instead of doing broad cleanup.
 - After auth, Supabase, ingestion, answer generation, search/ranking, clinical output, or source-governance changes, run the smallest domain check plus `npm run check:production-readiness`. Run `npm run check:supabase-project` after Supabase env/config changes.
 - For handoff, archive-safety, or upload-style requests, inspect branch/upstream/status first, run the appropriate verification gate, and only commit or push when the request explicitly asks for that workflow.
@@ -423,4 +425,4 @@ Durable notes for Cloud Agents. Standard commands live in `README.md` and `packa
 - Live-mode caveat: `RAG_PROVIDER_MODE=auto` attempts OpenAI (fast → strong route); if generation fails the built-in quality gates it silently degrades to a deterministic "Source-only" answer that still cites real documents — this is expected, not a failure. The header sign-in UI exposes magic-link + OAuth only (no password field), but the `/api/answer` + retrieval flow works server-side without a browser session.
 - What still won't run in this VM even with secrets: `npm run worker` also needs the Python OCR stack (`worker/python/requirements.txt`) and heavy parsing deps; Supabase edge functions need Deno v2.x + deployment. `verify:release` additionally runs governance/eval gates. Treat missing-secret failures of `check:supabase-project`/`verify:release` in demo mode as expected, not regressions.
 - Dev server: `npm run dev` selects a stable per-project localhost port (e.g. `4461`), binds `0.0.0.0`, and prints the exact URL. Never assume port 3000/3001/3002. `npm run ensure` starts/verifies it in the background.
-- Verification without secrets: `npm run lint`, `npm run typecheck`, and `npm run test` (vitest) all pass offline. `npm run verify:cheap` also runs runtime, GitHub Actions pin, CI-scope, and sitemap checks. `npm run verify:pr-local` adds format, conditional build/client-bundle scanning, and code-backed offline RAG tests; browser, Docker/Supabase, audit, and provider checks remain separate.
+- Verification without secrets: `npm run lint`, `npm run typecheck`, and `npm run test` (vitest) all pass offline. `npm run verify:cheap` also runs runtime, GitHub Actions pin, CI-scope, and sitemap checks. `npm run verify:pr-local` adds format, conditional build/client-bundle scanning, and RAG fixture/manifest validation without repeating unit tests; browser, Docker/Supabase, audit, and provider checks remain separate. See `docs/testing.md` for lock, live-test, Playwright, and flake-ledger rules.

README.md

@@ -168,6 +168,7 @@ records vs archive). The most load-bearing entries:
 - `docs/codebase-index.md` — architecture and module map (start here)
 - `docs/site-map.md` — generated route map (`npm run sitemap:update`)
 - `docs/process-hardening.md` — verification gates, CI expectations, known limits
+- `docs/testing.md` — local test safety, focused/live commands, Playwright ownership, flake policy
 - `docs/clinical-governance.md` — deployment and source governance checklist
 - `docs/deployment-architecture.md` — app/worker/Supabase deployment topology
 - `docs/supabase-migration-reconciliation.md` — migration drift and repair policy
@@ -184,9 +185,9 @@ npm run verify:cheap    # check:runtime + check:github-actions + sitemap:check
                         # + brand:check + check:type-scale + check:icon-scale
                         # + lint + typecheck + test
 npm run verify:pr-local # closest local mirror of the PR gate: format + verify:cheap,
-                        # plus conditional build/client-bundle scan and offline RAG
-                        # tests when changed-file scope requires them
-npm run verify:ui       # check:runtime + test:e2e:chromium
+                        # plus conditional build/client-bundle scan and RAG
+                        # fixture validation; the full unit suite runs once
+npm run verify:ui       # check:runtime + required production Chromium journeys
 npm run verify:release  # check:runtime + lint + typecheck + test + build + test:e2e
                         # + check:production-readiness + governance:release
                         # + eval:quality:release (needs live Supabase + OpenAI keys)
@@ -197,12 +198,12 @@ inspect which checks a change would trigger without running them.
 
 CI is risk-scoped (`.github/workflows/ci.yml`): a `changes` job classifies
 changed paths, `static-pr` always runs runtime, action-pin, CI-scope, format,
-lint, typecheck, and unit checks, and `pr-required` is the single
+lint, and typecheck checks, and `pr-required` is the single
 always-reporting required aggregate (required PR checks are Gitleaks plus that
-aggregate). Coverage, build, safety/config checks, Chromium `ui-critical`
-smoke, and the repo-owned Supabase `db-reset-verify` migration replay run only
-when their file scopes apply; UI PRs also get a non-blocking advisory
-`ui-regression` job. The full Playwright browser matrix
+aggregate). One full unit run with coverage, build, safety/config checks, the
+production Chromium gate, and the repo-owned Supabase `db-reset-verify`
+migration replay run only when their file scopes apply; UI PRs also get one
+non-blocking advisory Chromium invocation. The full Playwright browser matrix
 (`release-browser-matrix`) runs on `main`, `release/*`, manual dispatch, and a
 weekly schedule. Docker image builds, live drift, and live eval canary checks
 are path-filtered, scheduled, or manual rather than required checks for every
@@ -221,8 +222,12 @@ npm run samples:check
 npm run lint
 npm run typecheck
 npm run test
+npm run test:focused -- --files src/lib/example.ts
+npm run test:live # requires ALLOW_PROVIDER_TESTS=true
 npm run test:coverage
 npm run test:e2e
+npm run test:e2e:pr
+npm run test:e2e:advisory
 npm run test:e2e:all
 npm run test:e2e:accessibility
 npm run test:e2e:chromium