From b518c1de9087c27d8a6ccd219dd15ae0c90d6a4a Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:24:15 +0800 Subject: [PATCH 1/2] test: harden reliable test execution --- .github/workflows/ci-triage.yml | 8 +- .github/workflows/ci.yml | 148 ++------- .gitignore | 1 + AGENTS.md | 6 +- README.md | 21 +- docs/branch-review-ledger.md | 2 + docs/codebase-index.md | 4 +- docs/process-hardening.md | 20 +- docs/testing.md | 38 +++ next.config.ts | 10 + package.json | 20 +- playwright.config.ts | 18 +- playwright.visual.config.ts | 2 +- scripts/check-format-changed.mjs | 27 ++ scripts/check-rag-fixtures.mjs | 56 ++++ scripts/child-process-result.mjs | 13 + scripts/ci-change-scope.mjs | 21 +- scripts/classify-playwright-failures.mjs | 53 +++ scripts/eval-rag-offline.mjs | 112 +------ scripts/flake-ledger.mjs | 128 +++++--- scripts/playwright-base-url.ts | 8 +- scripts/run-heavy.mjs | 38 +++ scripts/run-live-tests.mjs | 31 ++ scripts/run-playwright.mjs | 306 +++++++++--------- scripts/run-vitest.mjs | 51 +-- scripts/test-environment.mjs | 63 ++++ scripts/test-focused.mjs | 73 +++++ scripts/test-rag-offline.mjs | 17 + scripts/test-run-lock.mjs | 147 +++++++++ scripts/verify-pr-local.mjs | 38 ++- src/instrumentation.ts | 17 + src/proxy.ts | 16 +- tests/architecture-boundaries.test.ts | 14 +- tests/client-secret-surface.test.ts | 3 +- tests/demo-data.test.ts | 1 + tests/flake-ledger.json | 37 +-- tests/flake-ledger.test.ts | 103 ++++-- tests/instrumentation.test.ts | 24 ++ tests/property-accessible-table.test.ts | 1 + tests/property-chunking.test.ts | 1 + ...roperty-numeric-token-preservation.test.ts | 1 + tests/property-seed.ts | 13 + tests/proxy.test.ts | 28 +- tests/rag-answer-fallback.test.ts | 8 +- tests/test-runner-safety.test.ts | 161 +++++++++ tests/tsx-server-only-runner.test.ts | 6 +- tests/ui-accessibility.spec.ts | 2 +- tests/ui-formulation.spec.ts | 2 +- tests/ui-smoke.spec.ts | 244 +++++--------- tests/ui-stress.spec.ts | 35 +- tests/ui-tools-collapse.spec.ts | 2 +- tests/ui-tools-task-directory.spec.ts | 2 +- tests/ui-tools.spec.ts | 279 ++++++++-------- tests/ui-visual-artifacts.spec.ts | 3 - ...ts => universal-search-owner.live.test.ts} | 24 +- tests/universal-search.test.ts | 22 +- tests/verify-pr-local.test.ts | 4 +- vitest.config.mts | 33 +- 58 files changed, 1621 insertions(+), 945 deletions(-) create mode 100644 docs/testing.md create mode 100644 scripts/check-format-changed.mjs create mode 100644 scripts/check-rag-fixtures.mjs create mode 100644 scripts/child-process-result.mjs create mode 100644 scripts/classify-playwright-failures.mjs create mode 100644 scripts/run-heavy.mjs create mode 100644 scripts/run-live-tests.mjs create mode 100644 scripts/test-environment.mjs create mode 100644 scripts/test-focused.mjs create mode 100644 scripts/test-rag-offline.mjs create mode 100644 scripts/test-run-lock.mjs create mode 100644 tests/property-seed.ts create mode 100644 tests/test-runner-safety.test.ts rename tests/{universal-search-owner-live.test.ts => universal-search-owner.live.test.ts} (69%) diff --git a/.github/workflows/ci-triage.yml b/.github/workflows/ci-triage.yml index d465e0281..38d440adc 100644 --- a/.github/workflows/ci-triage.yml +++ b/.github/workflows/ci-triage.yml @@ -4,12 +4,11 @@ # - main-side: the same job is also failing on the latest default-branch run # (CI merges the PR branch with current main, so a main regression surfaces on # every open PR — this has cost debugging time before). -# - possible known flake: a UI/Playwright job failed — check tests/flake-ledger.json. # - needs investigation: everything else. # # SHIPPED INERT: this event-triggered workflow does nothing until the repo variable # CI_TRIAGE_ENABLED == "true". It never runs PR-authored code — it only reads job -# metadata + the committed flake ledger via the trusted default-branch checkout, and +# metadata via the trusted default-branch checkout, and # posts a comment with the built-in token. name: CI Triage @@ -45,7 +44,6 @@ jobs: uses: actions/github-script@v9 with: script: | - const fs = require("fs"); const run = context.payload.workflow_run; // Resolve the PR for this run (empty for fork PRs → skip quietly). @@ -93,13 +91,11 @@ jobs: core.info(`main-side check skipped: ${e.message}`); } - const flakes = JSON.parse(fs.readFileSync("tests/flake-ledger.json", "utf8")).flakes || []; - const uiFlakeSpecs = flakes.map((f) => f.spec).filter((s) => /ui-/.test(s)); const looksUi = (name) => /ui|playwright|browser|e2e/i.test(name); const lines = failed.map((name) => { if (mainFailingJobs.has(name)) return `- \`${name}\` — **main-side**: also failing on the latest \`main\` run, likely not your change.`; - if (looksUi(name)) return `- \`${name}\` — **possible known flake**: UI/Playwright job; check \`tests/flake-ledger.json\`${uiFlakeSpecs.length ? ` (${uiFlakeSpecs.join(", ")})` : ""} and re-run before bisecting.`; + if (looksUi(name)) return `- \`${name}\` — **needs investigation**: use the uploaded JUnit classification and trace; job type alone is not evidence of a known flake.`; return `- \`${name}\` — **needs investigation**.`; }); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ea554c7a..d326801e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,9 +97,6 @@ jobs: - name: Typecheck run: npm run typecheck - - name: Unit tests - run: npm run test - safety: name: Safety and config checks needs: changes @@ -140,9 +137,9 @@ jobs: if: needs.changes.outputs.codex_autofix_changed == 'true' run: npm run check:codex-autofix-workflow - - name: Offline RAG preflight + - name: Offline RAG fixture and manifest validation if: needs.changes.outputs.rag_eval_changed == 'true' - run: npm run eval:rag:offline + run: npm run check:rag:fixtures coverage: name: Unit coverage @@ -150,6 +147,8 @@ jobs: if: needs.changes.outputs.coverage_changed == 'true' runs-on: ubuntu-24.04 timeout-minutes: 20 + env: + FAST_CHECK_SEED: ${{ github.event_name == 'schedule' && github.run_id || '424242' }} steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -206,11 +205,11 @@ jobs: run: npm run check:deployment-readiness ui-critical: - name: Critical UI smoke + name: Production UI needs: changes if: needs.changes.outputs.ui_changed == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 45 steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -236,76 +235,35 @@ jobs: - name: Install Chromium browser run: npx playwright install --with-deps chromium - - name: Chromium critical UI smoke - run: npm run test:e2e:critical + - name: Chromium production journeys + run: npm run test:e2e:pr - - name: Upload UI diagnostics + - name: Classify exact failed test identities if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: critical-ui-diagnostics-${{ github.run_id }} - path: | - test-results/ - playwright-report/ - if-no-files-found: ignore - - # Production-journey regression suite. Blocking (PT-05): a red production - # journey must fail pr-required instead of normalising as an advisory red. - # Prototype /mockups specs run separately in ui-mockups below. - ui-regression: - name: UI regression - needs: changes - if: needs.changes.outputs.ui_changed == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 45 - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Setup Node.js - uses: actions/setup-node@v5 - with: - node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: package-lock.json + run: node scripts/classify-playwright-failures.mjs - - name: Install dependencies - run: npm ci - - - name: Restore Chromium browser cache - uses: actions/cache@v6 - with: - path: ~/.cache/ms-playwright - key: playwright-chromium-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - - - name: Install Chromium browser - run: npx playwright install --with-deps chromium - - - name: Chromium stable regression suite - run: npm run test:e2e:regression - - - name: Upload UI regression diagnostics + - name: Upload UI diagnostics if: failure() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: ui-regression-diagnostics-${{ github.run_id }} + name: production-ui-diagnostics-${{ github.run_id }} path: | test-results/ playwright-report/ if-no-files-found: ignore - ui-quarantine: - name: Advisory quarantined UI tests + # Quarantined production tests and mockup-only journeys share one advisory + # install/server without weakening the required production-ui job above. + ui-advisory: + name: Advisory UI needs: changes if: github.event_name == 'pull_request' && needs.changes.outputs.ui_changed == 'true' continue-on-error: true runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 45 steps: - name: Checkout - uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -320,7 +278,7 @@ jobs: run: npm ci - name: Restore Chromium browser cache - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ~/.cache/ms-playwright key: playwright-chromium-${{ runner.os }}-${{ hashFiles('package-lock.json') }} @@ -328,61 +286,18 @@ jobs: - name: Install Chromium browser run: npx playwright install --with-deps chromium - - name: Chromium quarantined tests (advisory) - run: npm run test:e2e:quarantine + - name: Chromium quarantined and mockup journeys (advisory) + run: npm run test:e2e:advisory - - name: Upload quarantine diagnostics + - name: Classify exact failed test identities if: failure() - uses: actions/upload-artifact@v7 - with: - name: quarantine-ui-diagnostics-${{ github.run_id }} - path: | - test-results/ - playwright-report/ - if-no-files-found: ignore - - # Prototype /mockups specs live in their own advisory lane so a red mockup - # can never mask (or block on) a production-journey regression (PT-05). - ui-mockups: - name: Advisory mockup UI - needs: changes - if: github.event_name == 'pull_request' && needs.changes.outputs.ui_changed == 'true' - continue-on-error: true - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - persist-credentials: false - - - name: Setup Node.js - uses: actions/setup-node@v5 - with: - node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: package-lock.json + run: node scripts/classify-playwright-failures.mjs - - name: Install dependencies - run: npm ci - - - name: Restore Chromium browser cache - uses: actions/cache@v6 - with: - path: ~/.cache/ms-playwright - key: playwright-chromium-${{ runner.os }}-${{ hashFiles('package-lock.json') }} - - - name: Install Chromium browser - run: npx playwright install --with-deps chromium - - - name: Chromium mockup regression - run: npm run test:e2e:mockups - - - name: Upload mockup UI diagnostics + - name: Upload advisory UI diagnostics if: failure() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: mockup-ui-diagnostics-${{ github.run_id }} + name: advisory-ui-diagnostics-${{ github.run_id }} path: | test-results/ playwright-report/ @@ -438,7 +353,7 @@ jobs: pr-required: name: PR required - needs: [changes, static-pr, safety, coverage, build, ui-critical, ui-regression, db-reset-verify] + needs: [changes, static-pr, safety, coverage, build, ui-critical, db-reset-verify] if: always() runs-on: ubuntu-24.04 timeout-minutes: 5 @@ -456,7 +371,6 @@ jobs: COVERAGE_RESULT: ${{ needs.coverage.result }} BUILD_RESULT: ${{ needs.build.result }} UI_RESULT: ${{ needs.ui-critical.result }} - UI_REGRESSION_RESULT: ${{ needs.ui-regression.result }} DB_RESULT: ${{ needs.db-reset-verify.result }} run: | set -euo pipefail @@ -501,11 +415,9 @@ jobs: fi if [ "$UI_CHANGED" = "true" ]; then - require_success "ui-critical" "$UI_RESULT" - require_success "ui-regression" "$UI_REGRESSION_RESULT" + require_success "production-ui" "$UI_RESULT" else - require_skipped_or_success "ui-critical" "$UI_RESULT" - require_skipped_or_success "ui-regression" "$UI_REGRESSION_RESULT" + require_skipped_or_success "production-ui" "$UI_RESULT" fi if [ "$DB_CHANGED" = "true" ]; then diff --git a/.gitignore b/.gitignore index 51d2059d4..66f2dcf3b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ # next.js /.next/ +/.next-playwright/ /out/ # production diff --git a/AGENTS.md b/AGENTS.md index 8b34383c1..d5d498c51 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -164,7 +164,8 @@ When a branch or PR review completes, record the reviewed branch/ref, HEAD SHA, # Process hardening phases -- For non-trivial source/config/test changes, prefer `npm run verify:cheap` as the first broad gate and `npm run verify:pr-local` before PR handoff when the change is ready. The PR-local gate runs format plus `verify:cheap`, then conditionally adds the production build/client-bundle scan and code-backed offline RAG tests. Browser, dependency-audit, Docker/Supabase replay, and provider-backed checks remain separate gates. Use `npm run verify:pr-local -- --dry-run --files ` to inspect selection without running commands. The broader `--extended` plan is dry-run only unless explicit approval is reflected by `ALLOW_EXTENDED_PR_LOCAL=true`. +- For non-trivial source/config/test changes, prefer `npm run verify:cheap` as the first broad gate and `npm run verify:pr-local` before PR handoff when the change is ready. The PR-local gate runs the full unit suite once, then conditionally adds the production build/client-bundle scan and RAG fixture/manifest validation. Browser, dependency-audit, Docker/Supabase replay, and provider-backed checks remain separate gates. Use `npm run verify:pr-local -- --dry-run --files ` to inspect selection without running commands. The broader `--extended` plan is dry-run only unless explicit approval is reflected by `ALLOW_EXTENDED_PR_LOCAL=true`. +- Run one heavy Database command at a time across all worktrees. Do not install while a repository test, build, lint, typecheck, or server command is active. Avoid aggressive short-interval polling, and do not repeat an unchanged full gate after it passes. - For UI, frontend, browser, routing, styling, reduced-motion, or forced-colors changes, run `npm run ensure` before browser work and use `npm run verify:ui` as the Chromium UI gate. - For release or handoff confidence, use `npm run verify:release`; this includes the full Playwright project set. - For clinical ingestion, answer generation, source governance, privacy, production-readiness, or environment changes, run the smallest relevant domain check plus `npm run check:production-readiness`. @@ -307,6 +308,7 @@ After completing `upload`, summarize the current branch and worktree state, whet - For non-trivial changes, start from concrete repo state: branch, `git status`, relevant package scripts, recent failures, and local logs such as `dev-server.log` when runtime behavior is involved. For architecture and module orientation, read `docs/codebase-index.md` (routes: `docs/site-map.md`). - For UI, browser, styling, routing, accessibility, or screenshot work, run `npm run ensure` before opening the app, then use browser QA and the smallest relevant UI proof before broader gates. - Prefer the smallest failing check first. For this repo, use focused Vitest or Playwright targets before widening to `npm run verify:cheap`, `npm run verify:ui`, or `npm run verify:release`. +- Use `npm run test:focused -- --files ` only for safe source-only iteration. It fails closed for deleted files and test/configuration infrastructure; follow its instruction to run `npm run test` in those cases. - When the user says `safely`, preserve unrelated staged, unstaged, and untracked work; stop only clearly repo-owned transient processes; and verify the result instead of doing broad cleanup. - After auth, Supabase, ingestion, answer generation, search/ranking, clinical output, or source-governance changes, run the smallest domain check plus `npm run check:production-readiness`. Run `npm run check:supabase-project` after Supabase env/config changes. - For handoff, archive-safety, or upload-style requests, inspect branch/upstream/status first, run the appropriate verification gate, and only commit or push when the request explicitly asks for that workflow. @@ -423,4 +425,4 @@ Durable notes for Cloud Agents. Standard commands live in `README.md` and `packa - Live-mode caveat: `RAG_PROVIDER_MODE=auto` attempts OpenAI (fast → strong route); if generation fails the built-in quality gates it silently degrades to a deterministic "Source-only" answer that still cites real documents — this is expected, not a failure. The header sign-in UI exposes magic-link + OAuth only (no password field), but the `/api/answer` + retrieval flow works server-side without a browser session. - What still won't run in this VM even with secrets: `npm run worker` also needs the Python OCR stack (`worker/python/requirements.txt`) and heavy parsing deps; Supabase edge functions need Deno v2.x + deployment. `verify:release` additionally runs governance/eval gates. Treat missing-secret failures of `check:supabase-project`/`verify:release` in demo mode as expected, not regressions. - Dev server: `npm run dev` selects a stable per-project localhost port (e.g. `4461`), binds `0.0.0.0`, and prints the exact URL. Never assume port 3000/3001/3002. `npm run ensure` starts/verifies it in the background. -- Verification without secrets: `npm run lint`, `npm run typecheck`, and `npm run test` (vitest) all pass offline. `npm run verify:cheap` also runs runtime, GitHub Actions pin, CI-scope, and sitemap checks. `npm run verify:pr-local` adds format, conditional build/client-bundle scanning, and code-backed offline RAG tests; browser, Docker/Supabase, audit, and provider checks remain separate. +- Verification without secrets: `npm run lint`, `npm run typecheck`, and `npm run test` (vitest) all pass offline. `npm run verify:cheap` also runs runtime, GitHub Actions pin, CI-scope, and sitemap checks. `npm run verify:pr-local` adds format, conditional build/client-bundle scanning, and RAG fixture/manifest validation without repeating unit tests; browser, Docker/Supabase, audit, and provider checks remain separate. See `docs/testing.md` for lock, live-test, Playwright, and flake-ledger rules. diff --git a/README.md b/README.md index 3b17a03c8..1bf8635a9 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ records vs archive). The most load-bearing entries: - `docs/codebase-index.md` — architecture and module map (start here) - `docs/site-map.md` — generated route map (`npm run sitemap:update`) - `docs/process-hardening.md` — verification gates, CI expectations, known limits +- `docs/testing.md` — local test safety, focused/live commands, Playwright ownership, flake policy - `docs/clinical-governance.md` — deployment and source governance checklist - `docs/deployment-architecture.md` — app/worker/Supabase deployment topology - `docs/supabase-migration-reconciliation.md` — migration drift and repair policy @@ -184,9 +185,9 @@ npm run verify:cheap # check:runtime + check:github-actions + sitemap:check # + brand:check + check:type-scale + check:icon-scale # + lint + typecheck + test npm run verify:pr-local # closest local mirror of the PR gate: format + verify:cheap, - # plus conditional build/client-bundle scan and offline RAG - # tests when changed-file scope requires them -npm run verify:ui # check:runtime + test:e2e:chromium + # plus conditional build/client-bundle scan and RAG + # fixture validation; the full unit suite runs once +npm run verify:ui # check:runtime + required production Chromium journeys npm run verify:release # check:runtime + lint + typecheck + test + build + test:e2e # + check:production-readiness + governance:release # + eval:quality:release (needs live Supabase + OpenAI keys) @@ -197,12 +198,12 @@ inspect which checks a change would trigger without running them. CI is risk-scoped (`.github/workflows/ci.yml`): a `changes` job classifies changed paths, `static-pr` always runs runtime, action-pin, CI-scope, format, -lint, typecheck, and unit checks, and `pr-required` is the single +lint, and typecheck checks, and `pr-required` is the single always-reporting required aggregate (required PR checks are Gitleaks plus that -aggregate). Coverage, build, safety/config checks, Chromium `ui-critical` -smoke, and the repo-owned Supabase `db-reset-verify` migration replay run only -when their file scopes apply; UI PRs also get a non-blocking advisory -`ui-regression` job. The full Playwright browser matrix +aggregate). One full unit run with coverage, build, safety/config checks, the +production Chromium gate, and the repo-owned Supabase `db-reset-verify` +migration replay run only when their file scopes apply; UI PRs also get one +non-blocking advisory Chromium invocation. The full Playwright browser matrix (`release-browser-matrix`) runs on `main`, `release/*`, manual dispatch, and a weekly schedule. Docker image builds, live drift, and live eval canary checks are path-filtered, scheduled, or manual rather than required checks for every @@ -221,8 +222,12 @@ npm run samples:check npm run lint npm run typecheck npm run test +npm run test:focused -- --files src/lib/example.ts +npm run test:live # requires ALLOW_PROVIDER_TESTS=true npm run test:coverage npm run test:e2e +npm run test:e2e:pr +npm run test:e2e:advisory npm run test:e2e:all npm run test:e2e:accessibility npm run test:e2e:chromium diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index 99a081e71..ff124a3c1 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -490,3 +490,5 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-14 | PR #655 / codex/release-blocker-remediation | 3ed3a7a2df37d7d15143ab7606e5748ac7ecca09 + reviewed follow-up diff | offline dose-route latency follow-up | The next isolated timeout was a short IM/PO agitation question receiving the same blanket ten-term AND expansion. Agitation dose/route retrieval now keeps only the dose and route signals present in the question; the exact case retrieves its expected source and five citations in 1.54 seconds without a model. All remaining RAG cases 23–44 passed individually, so no further deadline crash remains. | Focused clinical-search/retrieval Vitest 111/111; scoped ESLint; Prettier; `git diff --check`; live provider-free case 22 passed; live provider-free cases 23–44 passed individually. | | 2026-07-14 | PR #655 / codex/release-blocker-remediation | a3f3a89676015cd5f018c07e8c3ad9483f91cef6 + reviewed follow-up diff | offline adversarial-latency follow-up | The final blocking offline-quality failure was an adversarial secret-exfiltration query that correctly refused but first spent about 25 seconds in lexical retrieval. Adversarial manipulation now short-circuits at the search boundary before provider-client creation, cache access, classification, aliases, or Supabase work, and is never cached. | Focused Vitest 2/2; scoped ESLint; Prettier; full TypeScript; `git diff --check`; live provider-free adversarial case completed in 100 ms with 0 ms RPC time; `eval:quality:release:offline` passed with zero blocking failures and zero model, request-ID, token, cost, or generation-latency evidence. Flaky local browser/composite suites intentionally not repeated; hosted CI remains authoritative. | | 2026-07-14 | PR #666 / codex/release-blocker-remediation | 9b56eebe4b23ab783207445fb827c317c8d59be8 + reviewed follow-up diff | review-followup | One late P2 retrieval-contract gap was confirmed: the optimized agitation query retained IM/PO but could drop other already-supported amount, route, and frequency aliases. Medication evidence intent is now shared with retrieval selection, and focused agitation queries preserve requested numeric units, SC, SL, PRN, and frequency signals without restoring the broad ten-term expansion. | GitHub review-thread inspection; focused clinical-search/retrieval Vitest 112/112; scoped ESLint; Prettier; `git diff --check`. Hosted final-head TypeScript/build/CI and exact-head staging evidence remain required after push. | +| 2026-07-15 | HEAD (detached) 0c56f27a37af88a073d2bb695d2cf4c05067ff4f | 0c56f27a37af88a073d2bb695d2cf4c05067ff4f | test flakiness, runner safety, CI duplication, and local execution-churn audit | P1 test-integrity issues: `run-vitest.mjs` can report success when Vitest terminates by signal and force-kills concurrent same-worktree runs; the default Vitest glob can execute the live Supabase owner-search test when credentials are present, while that test silently passes on authentication failure. P2 cluster: Playwright server races/shared `.next` state and config-time persistent-server startup; CI reruns the full unit suite for coverage and PR-local RAG verification repeats 21 suites; known-flake tracking is substring-only, not tied to quarantine, and structurally requires a non-empty ledger; unseeded property tests; repeated module resets/source-graph parsing; long retrying browser matrices and swallowed `networkidle` timeouts; host-wide concurrent worktrees/install/test/server activity causes memory pressure, native-module lock failures, and invalidated runs. | Pure review, no source mutation except this ledger append. Offline `node scripts/flake-ledger.mjs --self-test` passed (confirming the non-empty invariant); ledger matching confirmed `narrow-viewport-parallel` applies to an unquarantined blocking title; `node scripts/verify-pr-local.mjs --dry-run --files src/lib/rag.ts` confirmed full test + build + offline RAG duplication. Full Vitest/Playwright not run because this worktree has no dependencies and concurrent repo work was active; provider-backed checks not run (confirmation boundary). | +| 2026-07-17 | codex/test-reliability-hardening | 0c56f27a37af88a073d2bb695d2cf4c05067ff4f + reviewed remediation diff | implementation and merge-readiness review of heavyweight locking, offline test isolation, focused/live test selection, Playwright ownership, flake classification, CI consolidation, and browser race fixes | No remaining high-confidence P0-P2 issue in the reviewed diff. Three scoped integration defects were fixed before handoff: advisory mockup tests were unreachable behind the production route boundary; explicit focused-test paths did not fail closed when deleted or missing; and JUnit spec identities were relative to the Playwright test directory while the ledger uses repository-relative paths. Provider scrubbing was expanded to all repository-specific Supabase/database/E2E credentials, and early Playwright server signal/launch failures now fail immediately. Residual risk is final integration with the latest `main` and the full production Chromium run. | Offline focused Vitest 36/36 then 22/22; full TypeScript passed; targeted Prettier passed; `git diff --check` passed. Full ESLint was externally interrupted without a result and remains required on the clean integrated tree. Provider-backed Supabase/OpenAI checks were not run. | diff --git a/docs/codebase-index.md b/docs/codebase-index.md index 26088386c..6389b7016 100644 --- a/docs/codebase-index.md +++ b/docs/codebase-index.md @@ -1,6 +1,6 @@ # Clinical KB — Codebase Index -Structured map for AI agents and onboarding. For live routes, see `docs/site-map.md` (`npm run sitemap:update` / `sitemap:check`). For agent rules and verification gates, see `AGENTS.md`. +Structured map for AI agents and onboarding. For live routes, see `docs/site-map.md` (`npm run sitemap:update` / `sitemap:check`). For agent rules and verification gates, see `AGENTS.md`; for test execution and flake policy, see `docs/testing.md`. **Stack:** Next.js 16, React 19, Supabase (pgvector, Storage, Auth), OpenAI, Python OCR worker. **Live Supabase:** `Clinical KB Database` — ref `sjrfecxgysukkwxsowpy` (never use stale `qjgitjyhxrwxsrydablr`). @@ -226,7 +226,7 @@ Golden retrieval fixture: `scripts/fixtures/rag-retrieval-golden.json` **Domain clusters in `tests/`:** RAG/answers, retrieval, ingestion/indexing, source governance, API routes, Supabase schema, shell/routing, UI formatting guards. -**Gates:** `verify:cheap` (lint + typecheck + unit), `verify:ui` (Chromium E2E), `verify:release` (full build + all browsers + production readiness). +**Gates:** `verify:cheap` (lint + typecheck + full offline unit suite), `verify:ui` (required production Chromium journeys), `verify:release` (full build + all browsers + production readiness). Use `test:focused` only for safe source-only iteration. --- diff --git a/docs/process-hardening.md b/docs/process-hardening.md index f9a4c19ed..d2f9a0985 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -12,10 +12,10 @@ artifact before release; see ## Phase 1 - Active now - `npm run verify:cheap` is the default broad local gate for source/config/test changes: `check:runtime`, `sitemap:check`, lint, typecheck, and unit tests. -- `npm run verify:pr-local` is the closest local mirror of the normal PR gate: runtime, format, lint, typecheck, unit tests, conditional build, and offline RAG preflight when changed-file scope requires it. Local scope resolves against the repository default base rather than a feature-branch upstream; set `PR_BASE_REF` explicitly for release-targeted PRs. -- `npm run verify:ui` is the default UI gate: `check:runtime` plus Chromium Playwright smoke, stress, and accessibility media checks (`test:e2e:chromium`). +- `npm run verify:pr-local` is the closest local mirror of the normal PR gate: runtime, format, lint, typecheck, one full unit run, conditional build, and RAG fixture/manifest validation when changed-file scope requires it. Local scope resolves against the repository default base rather than a feature-branch upstream; set `PR_BASE_REF` explicitly for release-targeted PRs. +- `npm run verify:ui` is the complete required production Chromium gate: `check:runtime` plus all non-quarantined production journeys (`test:e2e:pr`). - `npm run verify:release` is the release-confidence gate: `check:runtime`, lint, typecheck, unit tests, build, full Playwright browser matrix, `check:production-readiness`, `governance:release`, and `eval:quality:release` (the last step needs live Supabase and OpenAI keys). -- CI uses a risk-scoped PR gate: `changes` classifies paths, `static-pr` always runs runtime/action/scope/format/lint/typecheck/unit checks, and `pr-required` is the single always-reporting required aggregate. Coverage, build, critical UI smoke, migration replay, safety/config, Codex workflow validation, and the production-backed offline RAG preflight run only when their file scopes apply. UI PRs also run a non-blocking advisory Chromium regression job. The external `Supabase Preview` check may still replay migrations on branch databases when enabled. A gated `release-browser-matrix` job runs the full Playwright browser set on `main`, `release/*`, manual dispatch, and the weekly schedule. +- CI uses a risk-scoped PR gate: `changes` classifies paths, `static-pr` always runs runtime/action/scope/format/lint/typecheck checks, and `pr-required` is the single always-reporting required aggregate. One full unit run with coverage, build, one required production Chromium invocation, migration replay, safety/config, Codex workflow validation, and RAG fixture validation run only when their file scopes apply. UI PRs also run one non-blocking advisory Chromium invocation for quarantined and mockup journeys. The external `Supabase Preview` check may still replay migrations on branch databases when enabled. A gated `release-browser-matrix` job runs the full Playwright browser set on `main`, `release/*`, manual dispatch, and the weekly schedule. - `tests/ui-accessibility.spec.ts` covers reduced-motion and forced-colors dashboard usability so those modes are no longer only reviewed by inspection. - `tests/ui-tools.spec.ts` covers the Applications dashboard mode at mobile and desktop sizes, including the `/applications` compatibility redirect. - `AGENTS.md` now points future agents to these gates and to this document. @@ -138,11 +138,11 @@ passes `p_worker_id`. Ordered apply steps, R17 manual `CONCURRENTLY` index, and ## PR merge gate: risk-scoped CI + required aggregate (2026-07-10) -- CI now has one always-reporting required aggregate: `CI / PR required`. The aggregate depends on `changes`, `static-pr`, `safety`, `coverage`, `build`, `ui-critical`, `ui-regression`, and `db-reset-verify`, then enforces only the jobs whose scopes apply. -- `static-pr` is the deterministic baseline for every PR: runtime, action pin check, CI scope self-test, format, lint, typecheck, and unit tests. Coverage, build, safety/config, critical UI, production-backed offline RAG preflight, and migration replay are independent jobs so reruns stay focused. Coverage is limited to `src`/test changes; process-only changes do not trigger builds. +- CI now has one always-reporting required aggregate: `CI / PR required`. The aggregate depends on `changes`, `static-pr`, `safety`, `coverage`, `build`, `ui-critical`, and `db-reset-verify`, then enforces only the jobs whose scopes apply. +- `static-pr` is the deterministic baseline for every PR: runtime, action pin check, CI scope self-test, format, lint, and typecheck. Coverage is the one required full unit run. Build, safety/config, production UI, RAG fixture validation, and migration replay are independent jobs so reruns stay focused. Coverage includes source, tests, package/test-runner configuration, while process-only documentation does not trigger builds. - `db-reset-verify` is path-scoped to Supabase migrations/schema/config and database-access code. Do not also require an external Supabase Preview replay unless the repo owner intentionally wants duplicate migration replay. -- `ui-critical` is path-scoped to UI/routing/styling/browser-facing changes and runs only the `@critical` Chromium smoke subset. `ui-regression` (2026-07-13 audit, finding 8) runs the remaining stable Chromium cases via `test:e2e:regression` (`--grep-invert "@critical|@quarantine"`) and is **merge-blocking** through `pr-required` on UI changes. Genuinely flaky specs are tagged `@quarantine` and run in the advisory `ui-quarantine` job (`test:e2e:quarantine`, `--pass-with-no-tests`); fix and untag them rather than letting the quarantine grow. The full browser matrix remains main/release/manual/scheduled. -- 2026-07-13 flip-day triage: the whole 142-test regression set was green locally except one stale `/privacy` heading assertion (fixed — the page title is "Privacy & data handling" since the footer simplification) and three cold-dev-server timing timeouts that pass on a warmed server and have no CI failure history (`ui-tools` mode-home centering `:462`, differentials comparison wiring `:1367`, `ui-universal-search` grouped-result navigation `:102`). Watch those three; if one flakes in CI, tag it `@quarantine` rather than reverting the gate. +- `ui-critical` retains its job ID for branch-protection compatibility but runs one required production Chromium invocation covering all non-quarantined critical and regression journeys (`test:e2e:pr`). `ui-advisory` runs quarantined and mockup journeys together when UI scope applies. A JUnit failure is considered a known flake only when its exact spec/title matches the validated ledger. The full browser matrix remains main/release/manual/scheduled. +- The 2026-07-13 cold-server and historical ledger candidates are tracked through the reproduction policy in `docs/testing.md`: run each three times on the same SHA, fix fail/pass races, treat deterministic failures as regressions, and remove entries that do not reproduce. On `0c56f27a3`, the historical composer/tap/answer-fallback entries and three cold-route candidates did not reproduce in three runs; the narrow differential viewport reproduced once in three cold runs, was fixed with route-specific readiness before its single submit, then passed three of three. The ledger is intentionally empty. - Branch protection for `main` should require `CI / PR required` and `Secret Scan / Gitleaks`. Keep `SAST / Semgrep` required only if the repository owner accepts its external-rule/network dependency as part of the normal merge gate. Do not require path-filtered or scheduled/manual contexts such as `CI / Unit coverage`, `CI / Critical UI smoke`, `CI / Migration replay`, `Docker image build / app-image`, `Docker image build / worker-image`, `CI / release-browser-matrix`, `Eval Canary`, or `Live drift check`; they can be skipped on ordinary PRs and would leave branches stuck at "Expected - Waiting for status to be reported." ## CSS cascade layering (2026-07-02) @@ -348,9 +348,9 @@ the durable index for the tooling; `docs/operator-backlog.md` tracks the human-o (`INGESTION_AUTOPILOT_APPLY` unset → read-only); flip that repo var to `true` after a clean dry-run to allow real recovery. - **CI failure triage** (`.github/workflows/ci-triage.yml`): on PR CI failure, classifies each failed job - as main-side / possible-known-flake / needs-investigation. Inert until repo var `CI_TRIAGE_ENABLED=true` - (now set). Reads only job metadata + the trusted default-branch flake ledger (`tests/flake-ledger.json`, - `scripts/flake-ledger.mjs`); never runs PR code. + as main-side or needs-investigation. Inert until repo var `CI_TRIAGE_ENABLED=true` (now set). UI jobs use + their uploaded JUnit classification and trace; job names alone never produce a known-flake verdict. + The workflow reads only trusted default-branch job metadata and never runs PR code. - **Repo hygiene:** `check:env-parity` (env-var NAME reconciliation across `env.ts`, `check-ci-env.mjs`, and — opt-in, names-only — `gh secret list` / Railway) and `sweep:branch-ledger` (report-only branch inventory, cherry-pick-aware). diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 000000000..27c6214f4 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,38 @@ +# Testing and verification + +## Safe local execution + +Heavy commands (`lint`, `typecheck`, `build`, Vitest, Playwright, `verify:cheap`, and `verify:pr-local`) share one lock derived from Git's common directory. The lock therefore covers every worktree for this repository. Nested commands reuse their parent's token; an unrelated command fails immediately and prints the current owner. Only a lock whose recorded process is demonstrably dead is reclaimed. + +Run one heavy Database command at a time. Do not install packages while a repository test, build, lint, typecheck, or server command is active. Avoid short-interval polling, and do not repeat an unchanged broad gate after it has already passed. + +Ordinary Vitest and Playwright runs remove OpenAI, Supabase, database, and E2E credentials and force demo/offline mode. Provider tests use the `*.live.test.ts` suffix, are excluded from default discovery, and can only be started explicitly with `ALLOW_PROVIDER_TESTS=true npm run test:live`. + +## Commands + +| Command | Purpose | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `npm run test:focused -- --files ` | Local iteration using Vitest related-file selection. It fails closed for deleted files, test infrastructure, configuration, or an empty/unsafe mapping. | +| `npm run test` | Complete offline unit suite. | +| `npm run test:live` | Explicit provider suite; requires `ALLOW_PROVIDER_TESTS=true`. | +| `npm run test:e2e:pr` | Required production Chromium journeys, excluding mockups and quarantined tests. | +| `npm run test:e2e:advisory` | Quarantined and mockup journeys in one advisory invocation. | +| `npm run verify:cheap` | Broad offline local gate: runtime/config checks, lint, typecheck, and the full unit suite. | +| `npm run verify:pr-local` | PR-like local gate. Formatting is checked on the changed set, the full unit suite runs once, and RAG scope adds fixture/manifest validation. | +| `npm run verify:ui` | Complete required production Chromium gate. | + +Set `FAST_CHECK_SEED` to reproduce a property-test run. Local and ordinary CI runs default to `424242`; scheduled CI may derive a bounded seed from the run ID. + +## Playwright ownership + +The repository runner exclusively builds and serves each Playwright production app. It selects a safe port, verifies `/api/local-project-id`, uses an isolated `.next-playwright/` build directory, replaces provider configuration with inert loopback values, and removes its server and output on success, failure, or signal. Playwright configuration never starts a server. The production boot guard permits this demo profile only when the output is isolated, provider mode is offline, credentials are absent, and the Supabase URL is the inert `127.0.0.1:1` target. + +Blocking tests run with zero retries. CI publishes list, JUnit, and JSON reports. Failed-test classification parses JUnit test cases and uses exact spec/title matches; a job name is never enough to classify a failure as a known flake. + +## Flake policy + +`tests/flake-ledger.json` may be empty. Each entry must match the exact spec and title, and the test title must include `@quarantine` but not `@critical`. Entries require an owner, reproduction command, local tracking reference, first/last-seen dates, and an expiry no more than 30 days away. Reproduce a candidate three times on the same SHA before adding or retaining it: fix fail/pass races, treat repeatable failures as regressions, and remove entries that no longer reproduce. + +## CI topology + +PR CI keeps static checks separate from one required full unit run with coverage. UI scope uses one required production Chromium invocation for non-quarantined critical and regression journeys, plus one advisory invocation for quarantined and mockup journeys. Build, migration, security, and release behavior remain independently scoped and unchanged. diff --git a/next.config.ts b/next.config.ts index 2654b9e1d..fd659db45 100644 --- a/next.config.ts +++ b/next.config.ts @@ -4,6 +4,14 @@ import { fileURLToPath } from "node:url"; import { buildSecurityHeaders, resolveRuntimeFlags } from "./src/lib/security-headers"; const projectRoot = path.dirname(fileURLToPath(import.meta.url)); +const requestedDistDir = process.env.NEXT_DIST_DIR?.trim(); +if (requestedDistDir && !/^\.next-playwright\/[a-z0-9-]+\/dist$/i.test(requestedDistDir)) { + throw new Error("NEXT_DIST_DIR must be an owned .next-playwright//dist directory."); +} +const requestedTsConfigPath = process.env.NEXT_TSCONFIG_PATH?.trim(); +if (requestedTsConfigPath && !/^\.next-playwright\/[a-z0-9-]+\/tsconfig\.json$/i.test(requestedTsConfigPath)) { + throw new Error("NEXT_TSCONFIG_PATH must be an owned .next-playwright//tsconfig.json file."); +} // Static (non-CSP) headers for every route. The nonce'd CSP is emitted per // request from src/proxy.ts; both derive their runtime flags from the same helper. @@ -18,6 +26,8 @@ async function withOptionalBundleAnalyzer(config: NextConfig): Promise existsSync(path.join(projectRoot, file))); +if (changedFiles.length === 0) { + console.log("No changed files require a formatting check."); + process.exit(0); +} + +const prettierBin = path.join(projectRoot, "node_modules", "prettier", "bin", "prettier.cjs"); +const result = spawnSync(process.execPath, [prettierBin, "--check", "--ignore-unknown", "--", ...changedFiles], { + cwd: projectRoot, + stdio: "inherit", +}); +process.exit(childProcessExitCode(result)); diff --git a/scripts/check-rag-fixtures.mjs b/scripts/check-rag-fixtures.mjs new file mode 100644 index 000000000..0aaa93e41 --- /dev/null +++ b/scripts/check-rag-fixtures.mjs @@ -0,0 +1,56 @@ +#!/usr/bin/env node +import { readFileSync } from "node:fs"; +import { validateOfflineContractTests } from "./rag-offline-contract.mjs"; + +const goldenPath = "scripts/fixtures/rag-retrieval-golden.json"; +const cases = JSON.parse(readFileSync(goldenPath, "utf8")); +const failures = []; +const fail = (message) => failures.push(message); +const hasText = (value) => typeof value === "string" && value.trim().length > 0; + +function validateTerm(term, caseId, fieldName) { + if (Array.isArray(term)) { + if (term.length === 0 || !term.every(hasText)) + fail(`${caseId}: ${fieldName} contains an empty alternative term group.`); + return; + } + if (!hasText(term)) fail(`${caseId}: ${fieldName} contains an empty term.`); +} + +if (!Array.isArray(cases) || cases.length < 10) { + fail(`${goldenPath} must contain the offline retrieval contract cases.`); +} else { + const ids = new Set(); + for (const item of cases) { + if (!hasText(item.id)) fail("Golden retrieval case is missing id."); + if (ids.has(item.id)) fail(`${item.id}: duplicate golden retrieval case id.`); + ids.add(item.id); + if (!hasText(item.query)) fail(`${item.id}: query is required.`); + if (!hasText(item.expectedQueryClass)) fail(`${item.id}: expectedQueryClass is required.`); + if (!Number.isInteger(item.topK) || item.topK < 1 || item.topK > 50) fail(`${item.id}: topK must be 1-50.`); + if (!Array.isArray(item.expectedDocumentSubstrings)) { + fail(`${item.id}: expectedDocumentSubstrings must be an array.`); + } else { + for (const term of item.expectedDocumentSubstrings) validateTerm(term, item.id, "expectedDocumentSubstrings"); + } + if (!Array.isArray(item.expectedContentTerms) || item.expectedContentTerms.length === 0) { + fail(`${item.id}: expectedContentTerms must include at least one required term.`); + } else { + for (const term of item.expectedContentTerms) validateTerm(term, item.id, "expectedContentTerms"); + } + if (typeof item.expectTableEvidence !== "boolean") fail(`${item.id}: expectTableEvidence must be boolean.`); + } +} + +const contractTestsPath = "scripts/fixtures/rag-offline-contract-tests.json"; +const contractTests = JSON.parse(readFileSync(contractTestsPath, "utf8")); +failures.push(...validateOfflineContractTests(contractTests)); + +if (failures.length > 0) { + console.error("Offline RAG fixture and manifest validation failed:"); + for (const failure of failures) console.error(`- ${failure}`); + process.exit(1); +} +console.log( + `Offline RAG fixture and manifest validation passed (${cases.length} golden cases, ${contractTests.length} suites).`, +); diff --git a/scripts/child-process-result.mjs b/scripts/child-process-result.mjs new file mode 100644 index 000000000..fcf8887d0 --- /dev/null +++ b/scripts/child-process-result.mjs @@ -0,0 +1,13 @@ +export function childProcessExitCode(result) { + if (Number.isInteger(result?.status)) return result.status; + return 1; +} + +export function childProcessFailureSummary(result) { + const details = []; + if (typeof result?.status === "number") details.push(`status ${result.status}`); + else details.push("missing exit status"); + if (result?.signal) details.push(`signal ${result.signal}`); + if (result?.error) details.push(`launch error: ${result.error.message ?? String(result.error)}`); + return details.join(", "); +} diff --git a/scripts/ci-change-scope.mjs b/scripts/ci-change-scope.mjs index 4128368b0..50e4d3a74 100644 --- a/scripts/ci-change-scope.mjs +++ b/scripts/ci-change-scope.mjs @@ -118,6 +118,7 @@ const ragEvalPatterns = [ "src/app/api/search", /^src\/lib\/(?:rag(?:-[^/]+)?|smart-rag-api|clinical-search|clinical-query-mode|retrieval(?:-[^/]+)?|answer(?:-[^/]+)?|citations|cross-document-synthesis|evidence(?:-[^/]+)?|ranking-config|source(?:-[^/]+)?|chunking|document-index-units|query-privacy|owner-scope|corpus-grounding|indexed-source-formatting)\.ts$/, /^src\/components\/(?:.*\/)?(?:answer|source|citation)[^/]*\.tsx?$/i, + /^scripts\/(?:check-rag-fixtures|test-rag-offline)\.mjs$/, /^scripts\/(eval-|run-eval-safe|compare-retrieval-eval|retrieval-health|profile-retrieval|warm-retrieval-cache|tune-search-weights)/, /^tests\/(rag|retrieval|answer|citations|evidence|eval|clinical-safety|source).*\.test\.ts$/, ]; @@ -139,7 +140,14 @@ const containerPatterns = [ const sourcePatterns = ["data", "src", "tests", "scripts", "worker", "playwright", "public", "supabase"]; -const coveragePatterns = ["data", "src", "tests", "vitest.config.mts"]; +const coveragePatterns = [ + "data", + "src", + "tests", + "package.json", + "vitest.config.mts", + /^scripts\/(?:child-process-result|run-heavy|run-live-tests|run-vitest|test-environment|test-focused|test-run-lock)\.mjs$/, +]; const buildPatterns = [ "bundle-budget.json", @@ -374,6 +382,11 @@ function selfTest() { source_changed: true, coverage_changed: true, }); + assertScope("test-runner", ["scripts/run-vitest.mjs", "scripts/run-playwright.mjs"], { + source_changed: true, + coverage_changed: true, + ui_changed: true, + }); assertScope("runtime-data", ["data/medications-snapshot.json"], { source_changed: true, coverage_changed: true, @@ -407,6 +420,10 @@ function selfTest() { rag_eval_changed: true, source_changed: true, }); + assertScope("rag-fixture-checker", ["scripts/check-rag-fixtures.mjs"], { + rag_eval_changed: true, + source_changed: true, + }); assertScope("database-access", ["src/app/api/documents/route.ts"], { db_changed: true, source_changed: true, @@ -427,7 +444,7 @@ function selfTest() { ); assertScope("package", ["package.json"], { source_changed: false, - coverage_changed: false, + coverage_changed: true, container_changed: true, workflow_changed: false, build_changed: true, diff --git a/scripts/classify-playwright-failures.mjs b/scripts/classify-playwright-failures.mjs new file mode 100644 index 000000000..740e699cd --- /dev/null +++ b/scripts/classify-playwright-failures.mjs @@ -0,0 +1,53 @@ +#!/usr/bin/env node +import { existsSync, readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +import { loadFlakeLedger, matchFlake } from "./flake-ledger.mjs"; + +const decode = (value) => + value + .replaceAll(""", '"') + .replaceAll("'", "'") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll("&", "&"); + +const attribute = (attributes, name) => { + const match = attributes.match(new RegExp(`\\b${name}="([^"]*)"`)); + return match ? decode(match[1]) : ""; +}; + +const repositorySpec = (classname) => { + const normalized = classname.trim().replaceAll("\\", "/").replace(/^\.\//, ""); + return normalized.startsWith("tests/") ? normalized : `tests/${normalized}`; +}; + +export function failedTestcasesFromJunit(xml) { + return [...xml.matchAll(/]*)>([\s\S]*?)<\/testcase>/g)] + .filter((match) => /<(?:failure|error)\b/.test(match[2])) + .map((match) => ({ spec: repositorySpec(attribute(match[1], "classname")), title: attribute(match[1], "name") })) + .filter((testcase) => testcase.spec && testcase.title); +} + +function main() { + const reportPath = process.argv[2] ?? "test-results/playwright-junit.xml"; + if (!existsSync(reportPath)) { + console.error(`Playwright JUnit report not found: ${reportPath}`); + process.exitCode = 1; + return; + } + + const failures = failedTestcasesFromJunit(readFileSync(reportPath, "utf8")); + if (failures.length === 0) { + console.log("No failed Playwright testcases were found in the JUnit report."); + return; + } + + const flakes = loadFlakeLedger(); + console.log("Playwright failure classification:"); + for (const { spec, title } of failures) { + const known = matchFlake(spec, title, flakes); + console.log(`- ${spec} :: ${title}: ${known ? `known quarantine (${known.id})` : "needs investigation"}`); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main(); diff --git a/scripts/eval-rag-offline.mjs b/scripts/eval-rag-offline.mjs index 8606ab77a..36b66f96c 100644 --- a/scripts/eval-rag-offline.mjs +++ b/scripts/eval-rag-offline.mjs @@ -1,109 +1,13 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; -import { readFileSync } from "node:fs"; -import { resolve } from "node:path"; -import { validateOfflineContractTests } from "./rag-offline-contract.mjs"; +import { childProcessExitCode } from "./child-process-result.mjs"; -const goldenPath = "scripts/fixtures/rag-retrieval-golden.json"; -const cases = JSON.parse(readFileSync(goldenPath, "utf8")); -const failures = []; - -function fail(message) { - failures.push(message); -} - -function hasText(value) { - return typeof value === "string" && value.trim().length > 0; -} - -function validateTerm(term, caseId, fieldName) { - if (Array.isArray(term)) { - if (term.length === 0 || !term.every(hasText)) { - fail(`${caseId}: ${fieldName} contains an empty alternative term group.`); - } - return; - } - if (!hasText(term)) fail(`${caseId}: ${fieldName} contains an empty term.`); -} - -function validateGoldenCases() { - if (!Array.isArray(cases) || cases.length < 10) { - fail(`${goldenPath} must contain the offline retrieval contract cases.`); - return; - } - - const ids = new Set(); - for (const item of cases) { - if (!hasText(item.id)) fail("Golden retrieval case is missing id."); - if (ids.has(item.id)) fail(`${item.id}: duplicate golden retrieval case id.`); - ids.add(item.id); - - if (!hasText(item.query)) fail(`${item.id}: query is required.`); - if (!hasText(item.expectedQueryClass)) fail(`${item.id}: expectedQueryClass is required.`); - if (!Number.isInteger(item.topK) || item.topK < 1 || item.topK > 50) { - fail(`${item.id}: topK must be an integer between 1 and 50.`); - } - if (!Array.isArray(item.expectedDocumentSubstrings)) { - fail(`${item.id}: expectedDocumentSubstrings must be an array.`); - } else { - for (const term of item.expectedDocumentSubstrings) validateTerm(term, item.id, "expectedDocumentSubstrings"); - } - if (!Array.isArray(item.expectedContentTerms) || item.expectedContentTerms.length === 0) { - fail(`${item.id}: expectedContentTerms must include at least one required term.`); - } else { - for (const term of item.expectedContentTerms) validateTerm(term, item.id, "expectedContentTerms"); - } - if (typeof item.expectTableEvidence !== "boolean") { - fail(`${item.id}: expectTableEvidence must be boolean.`); - } - } -} - -validateGoldenCases(); - -if (failures.length > 0) { - console.error("Offline RAG preflight failed:"); - for (const failure of failures) console.error(`- ${failure}`); - process.exit(1); -} - -console.log(`Offline RAG fixture schema passed (${cases.length} golden retrieval cases).`); - -const contractTestsPath = "scripts/fixtures/rag-offline-contract-tests.json"; -const contractTests = JSON.parse(readFileSync(contractTestsPath, "utf8")); -const contractTestFailures = validateOfflineContractTests(contractTests); -if (contractTestFailures.length > 0) { - console.error(`${contractTestsPath} failed the offline safety contract:`); - for (const failure of contractTestFailures) console.error(`- ${failure}`); - process.exit(1); -} -const vitestPath = resolve("node_modules/vitest/vitest.mjs"); -const offlineEnv = { ...process.env }; -for (const key of [ - "RAG_PROVIDER_MODE", - "OPENAI_API_KEY", - "OPENAI_ORG_ID", - "OPENAI_PROJECT_ID", - "OPENAI_BASE_URL", - "NEXT_PUBLIC_SUPABASE_URL", - "NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY", - "NEXT_PUBLIC_SUPABASE_ANON_KEY", - "SUPABASE_URL", - "SUPABASE_ANON_KEY", - "SUPABASE_SECRET_KEY", - "SUPABASE_SERVICE_ROLE_KEY", +for (const [script, args] of [ + ["scripts/check-rag-fixtures.mjs", []], + ["scripts/test-rag-offline.mjs", process.argv.slice(2)], ]) { - delete offlineEnv[key]; -} -const result = spawnSync(process.execPath, [vitestPath, "run", ...contractTests, "--reporter=dot"], { - env: offlineEnv, - stdio: "inherit", -}); - -if (result.error) { - console.error(`Offline RAG contract tests could not start: ${result.error.message}`); - process.exit(1); + const result = spawnSync(process.execPath, [script, ...args], { stdio: "inherit" }); + const exitCode = childProcessExitCode(result); + if (exitCode !== 0) process.exit(exitCode); } -if (result.status !== 0) process.exit(result.status ?? 1); - -console.log("Offline RAG production contract tests passed."); +console.log("Offline RAG fixture and production-contract checks passed."); diff --git a/scripts/flake-ledger.mjs b/scripts/flake-ledger.mjs index 3f615e6bc..e3906916a 100644 --- a/scripts/flake-ledger.mjs +++ b/scripts/flake-ledger.mjs @@ -1,77 +1,111 @@ #!/usr/bin/env node -/** - * flake-ledger — loader + matcher for the known-flaky Playwright specs recorded in - * tests/flake-ledger.json. - * - * Purpose: stop re-diagnosing the same flakes from memory every time CI goes red. - * The CI failure-triage workflow uses isKnownFlake() to attribute a failed test to - * a known flake; a serial re-run of only-flaky failures can be layered on top. - * - * CLI: - * node scripts/flake-ledger.mjs --list print the ledger - * node scripts/flake-ledger.mjs --self-test validate shape + matcher - * node scripts/flake-ledger.mjs --match "" → prints matching id or "none" - */ import { readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -const LEDGER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "tests", "flake-ledger.json"); +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const ledgerPath = path.join(projectRoot, "tests", "flake-ledger.json"); +const requiredFields = [ + "id", + "title", + "spec", + "reason", + "owner", + "reproduction", + "firstSeen", + "lastSeen", + "expires", + "tracking", +]; +const dayMs = 24 * 60 * 60 * 1000; -export function loadFlakeLedger(ledgerPath = LEDGER_PATH) { - const raw = JSON.parse(readFileSync(ledgerPath, "utf8")); - const flakes = Array.isArray(raw.flakes) ? raw.flakes : []; +function dateValue(value, field, id) { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) throw new Error(`${id}: ${field} must be YYYY-MM-DD`); + const parsed = Date.parse(`${value}T23:59:59Z`); + if (!Number.isFinite(parsed)) throw new Error(`${id}: ${field} is not a valid date`); + return parsed; +} + +export function validateFlakeLedgerEntries(flakes, { root = projectRoot, now = Date.now() } = {}) { + const ids = new Set(); + const today = Date.parse(`${new Date(now).toISOString().slice(0, 10)}T00:00:00Z`); + const latestAllowedExpiry = today + 31 * dayMs - 1; for (const flake of flakes) { - if (!flake.id || !flake.match || !flake.spec || !flake.reason) { - throw new Error(`flake-ledger entry missing required field (id/match/spec/reason): ${JSON.stringify(flake)}`); - } + const missing = requiredFields.filter((field) => typeof flake[field] !== "string" || !flake[field].trim()); + if (missing.length) throw new Error(`flake-ledger ${flake.id ?? "entry"} missing: ${missing.join(", ")}`); + if (ids.has(flake.id)) throw new Error(`duplicate flake-ledger id: ${flake.id}`); + ids.add(flake.id); + + const firstSeen = dateValue(flake.firstSeen, "firstSeen", flake.id); + const lastSeen = dateValue(flake.lastSeen, "lastSeen", flake.id); + const expires = dateValue(flake.expires, "expires", flake.id); + if (firstSeen > lastSeen) throw new Error(`${flake.id}: firstSeen cannot be after lastSeen`); + if (lastSeen > expires) throw new Error(`${flake.id}: lastSeen cannot be after expires`); + if (expires < now) throw new Error(`${flake.id}: ledger entry expired ${flake.expires}`); + if (expires > latestAllowedExpiry) throw new Error(`${flake.id}: expiry must be within 30 days`); + if (!flake.title.includes("@quarantine")) throw new Error(`${flake.id}: exact title must include @quarantine`); + if (flake.title.includes("@critical")) + throw new Error(`${flake.id}: a test cannot be both @quarantine and @critical`); + + const specPath = path.join(root, flake.spec); + const spec = readFileSync(specPath, "utf8"); + if (!spec.includes(flake.title)) throw new Error(`${flake.id}: exact title is not present in ${flake.spec}`); } return flakes; } -/** Return the matching flake entry for a test title, or null. Case-insensitive substring. */ -export function matchFlake(testTitle, flakes = loadFlakeLedger()) { - if (!testTitle) return null; - const haystack = String(testTitle).toLowerCase(); - return flakes.find((flake) => haystack.includes(String(flake.match).toLowerCase())) ?? null; +export function loadFlakeLedger(sourcePath = ledgerPath) { + const raw = JSON.parse(readFileSync(sourcePath, "utf8")); + return validateFlakeLedgerEntries(Array.isArray(raw.flakes) ? raw.flakes : []); +} + +function normalizeSpec(spec) { + return String(spec ?? "") + .trim() + .replaceAll("\\", "/") + .replace(/^\.\//, "") + .toLowerCase(); } -export function isKnownFlake(testTitle, flakes = loadFlakeLedger()) { - return matchFlake(testTitle, flakes) !== null; +export function matchFlake(spec, testTitle, flakes = loadFlakeLedger()) { + const normalizedSpec = normalizeSpec(spec); + const title = String(testTitle ?? "") + .trim() + .toLowerCase(); + if (!normalizedSpec || !title) return null; + return ( + flakes.find( + (flake) => normalizeSpec(flake.spec) === normalizedSpec && flake.title.trim().toLowerCase() === title, + ) ?? null + ); } function selfTest() { const flakes = loadFlakeLedger(); - const assert = (cond, label) => { - if (!cond) { - console.error(`✖ self-test failed: ${label}`); - process.exitCode = 1; - throw new Error(label); - } - }; - assert(flakes.length > 0, "ledger is non-empty"); - const ids = new Set(flakes.map((f) => f.id)); - assert(ids.size === flakes.length, "flake ids are unique"); - assert(isKnownFlake("composer hero renders on hydrate", flakes), "matches a known flake by title substring"); - assert(!isKnownFlake("a totally unrelated passing test", flakes), "does not match an unrelated title"); - assert(matchFlake("", flakes) === null, "empty title matches nothing"); - if (process.exitCode !== 1) console.error("[flake-ledger] self-test passed"); + if (new Set(flakes.map((flake) => flake.id)).size !== flakes.length) throw new Error("flake ids are not unique"); + const sample = [{ spec: "tests/example.spec.ts", title: "exact quarantined title @quarantine", id: "sample" }]; + if (matchFlake("tests/example.spec.ts", "EXACT QUARANTINED TITLE @QUARANTINE", sample)?.id !== "sample") + throw new Error("exact title did not match"); + if (matchFlake("tests/example.spec.ts", "prefix exact quarantined title @quarantine", sample)) + throw new Error("partial title unexpectedly matched"); + if (matchFlake("tests/other.spec.ts", "exact quarantined title @quarantine", sample)) + throw new Error("cross-spec title unexpectedly matched"); + console.error(`[flake-ledger] self-test passed (${flakes.length} active entries)`); } function main() { if (process.argv.includes("--self-test")) return selfTest(); if (process.argv.includes("--list")) { - for (const flake of loadFlakeLedger()) console.log(`${flake.id}\t${flake.spec}\t"${flake.match}"`); + for (const flake of loadFlakeLedger()) console.log(`${flake.id}\t${flake.spec}\t${flake.title}`); return; } const matchIndex = process.argv.indexOf("--match"); if (matchIndex >= 0) { - const hit = matchFlake(process.argv[matchIndex + 1], loadFlakeLedger()); - console.log(hit ? hit.id : "none"); + console.log(matchFlake(process.argv[matchIndex + 1], process.argv[matchIndex + 2])?.id ?? "none"); return; } - console.error('usage: flake-ledger.mjs [--list | --self-test | --match ""]'); + console.error('usage: flake-ledger.mjs [--list | --self-test | --match "<exact spec>" "<exact title>"]'); + process.exitCode = 1; } -const invokedDirectly = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; -if (invokedDirectly) main(); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main(); diff --git a/scripts/playwright-base-url.ts b/scripts/playwright-base-url.ts index 391733dc2..3c9b4ae16 100644 --- a/scripts/playwright-base-url.ts +++ b/scripts/playwright-base-url.ts @@ -81,7 +81,7 @@ function findExistingLocalProjectUrl() { ); } -export function getPlaywrightBaseUrl() { +export function getPlaywrightBaseUrl({ allowEnsure = true }: { allowEnsure?: boolean } = {}) { const configuredBaseUrl = process.env.PLAYWRIGHT_BASE_URL; if (configuredBaseUrl) { if (!localUrlPattern.test(configuredBaseUrl)) { @@ -94,6 +94,12 @@ export function getPlaywrightBaseUrl() { const existingUrl = findExistingLocalProjectUrl(); if (existingUrl) return existingUrl; + if (!allowEnsure) { + throw new Error( + "Playwright requires a runner-owned local server. Use the repository npm scripts instead of invoking the Playwright CLI directly.", + ); + } + const result = spawnSync(process.execPath, [ensureScript, "--print-url"], { cwd: projectRoot, encoding: "utf8", diff --git a/scripts/run-heavy.mjs b/scripts/run-heavy.mjs new file mode 100644 index 000000000..5e7b038d4 --- /dev/null +++ b/scripts/run-heavy.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; +import { acquireHeavyRunLock } from "./test-run-lock.mjs"; + +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const args = process.argv.slice(2); +if (args[0] !== "--npm-script" || !args[1]) { + console.error("Usage: node scripts/run-heavy.mjs --npm-script <script> [forwarded arguments]"); + process.exit(2); +} + +const script = args[1]; +const forwarded = args.slice(2); +const lock = acquireHeavyRunLock({ projectRoot, command: `npm run ${script}` }); +let exitCode = 1; +try { + const npmExecPath = process.env.npm_execpath; + const result = npmExecPath + ? spawnSync(process.execPath, [npmExecPath, "run", script, ...(forwarded.length ? ["--", ...forwarded] : [])], { + cwd: projectRoot, + env: lock.environment, + stdio: "inherit", + }) + : spawnSync( + process.platform === "win32" ? "cmd.exe" : "npm", + process.platform === "win32" + ? ["/d", "/s", "/c", `npm run ${script}`] + : ["run", script, ...(forwarded.length ? ["--", ...forwarded] : [])], + { cwd: projectRoot, env: lock.environment, stdio: "inherit" }, + ); + exitCode = childProcessExitCode(result); +} finally { + lock.release(); +} +process.exit(exitCode); diff --git a/scripts/run-live-tests.mjs b/scripts/run-live-tests.mjs new file mode 100644 index 000000000..42432f84c --- /dev/null +++ b/scripts/run-live-tests.mjs @@ -0,0 +1,31 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; +import { requireProviderTestPermission } from "./test-environment.mjs"; +import { acquireHeavyRunLock } from "./test-run-lock.mjs"; + +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const vitestBin = path.join(projectRoot, "node_modules", "vitest", "vitest.mjs"); + +try { + requireProviderTestPermission(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} + +const lock = acquireHeavyRunLock({ projectRoot, command: "vitest live provider tests" }); +let exitCode = 1; +try { + const result = spawnSync(process.execPath, [vitestBin, "run", ...process.argv.slice(2)], { + cwd: projectRoot, + env: { ...lock.environment, NODE_ENV: "test" }, + stdio: "inherit", + }); + exitCode = childProcessExitCode(result); +} finally { + lock.release(); +} +process.exit(exitCode); diff --git a/scripts/run-playwright.mjs b/scripts/run-playwright.mjs index 7171dbca1..a971fc227 100644 --- a/scripts/run-playwright.mjs +++ b/scripts/run-playwright.mjs @@ -1,9 +1,13 @@ #!/usr/bin/env node import { spawn, spawnSync } from "node:child_process"; +import { mkdirSync, rmdirSync, rmSync, writeFileSync } from "node:fs"; import http from "node:http"; import net from "node:net"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { childProcessExitCode, childProcessFailureSummary } from "./child-process-result.mjs"; +import { offlineTestEnvironment } from "./test-environment.mjs"; +import { acquireHeavyRunLock } from "./test-run-lock.mjs"; import { appName, isReservedDevPort, @@ -21,9 +25,34 @@ const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), " const playwrightBin = path.join(projectRoot, "node_modules", "playwright", "cli.js"); const nextBin = path.join(projectRoot, "node_modules", "next", "dist", "bin", "next"); const identityPath = "/api/local-project-id"; -const startupTimeoutMs = 120_000; +const startupTimeoutMs = 180_000; const missingErrorComponentsNeedle = "missing required error components"; -const routeSmokePaths = ["/", "/applications"]; +const routeSmokePaths = [ + "/", + "/applications", + "/?mode=tools", + "/documents/search?mode=documents", + "/forms/transport-crisis-form", +]; +const playwrightArgs = process.argv.slice(2); +const mockupProjectRequested = playwrightArgs.some( + (argument, index) => + argument === "--project=chromium-mockups" || + (argument === "--project" && playwrightArgs[index + 1] === "chromium-mockups"), +); +const runId = `${process.pid}-${Date.now()}`; +const relativeRunRoot = `.next-playwright/${runId}`; +const absoluteRunRoot = path.join(projectRoot, relativeRunRoot); +const relativeDistDir = `${relativeRunRoot}/dist`; +const relativeTsConfigPath = `${relativeRunRoot}/tsconfig.json`; + +let lock; +try { + lock = acquireHeavyRunLock({ projectRoot, command: `playwright ${playwrightArgs.join(" ")}` }); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); @@ -32,11 +61,7 @@ function sleep(ms) { function canListenOnHost(port, host) { return new Promise((resolve) => { const server = net.createServer(); - server.once("error", (error) => { - // An unsupported address family (e.g. no IPv6 in the container) cannot - // hold the port, so it must not disqualify it as busy. - resolve(error.code === "EAFNOSUPPORT" || error.code === "EADDRNOTAVAIL"); - }); + server.once("error", (error) => resolve(error.code === "EAFNOSUPPORT" || error.code === "EADDRNOTAVAIL")); server.once("listening", () => server.close(() => resolve(true))); server.listen(port, host); }); @@ -59,9 +84,7 @@ function canConnectToHost(port, host) { } async function canListen(port) { - for (const host of ["127.0.0.1", "localhost", "::1"]) { - if (await canConnectToHost(port, host)) return false; - } + for (const host of ["127.0.0.1", "localhost", "::1"]) if (await canConnectToHost(port, host)) return false; for (const host of ["127.0.0.1", "localhost", "::1", "0.0.0.0", "::"]) { if (!(await canListenOnHost(port, host))) return false; } @@ -70,25 +93,20 @@ async function canListen(port) { async function findFreePort(startPort) { for (let port = startPort; port <= projectPortEnd; port += 1) { - if (isReservedDevPort(port)) continue; - if (await canListen(port)) return port; + if (!isReservedDevPort(port) && (await canListen(port))) return port; } throw new Error(`No free Playwright server port found from ${startPort} to ${projectPortEnd}.`); } -function requestJson(url) { +function request(url, { json = false, timeoutMs = 30_000 } = {}) { return new Promise((resolve) => { - const request = http.get(url, { timeout: 5000 }, (response) => { + const pending = http.get(url, { timeout: timeoutMs }, (response) => { let body = ""; response.setEncoding("utf8"); - response.on("data", (chunk) => { - body += chunk; - }); + response.on("data", (chunk) => (body += chunk)); response.on("end", () => { - if (response.statusCode !== 200) { - resolve(null); - return; - } + if (!response.statusCode || response.statusCode < 200 || response.statusCode >= 400) return resolve(null); + if (!json) return resolve(body); try { resolve(JSON.parse(body)); } catch { @@ -96,60 +114,14 @@ function requestJson(url) { } }); }); - request.on("timeout", () => { - request.destroy(); - resolve(null); - }); - request.on("error", () => resolve(null)); - }); -} - -function requestText(url, timeoutMs = 30_000) { - return new Promise((resolve) => { - const request = http.get(url, { timeout: timeoutMs }, (response) => { - let body = ""; - response.setEncoding("utf8"); - response.on("data", (chunk) => { - body += chunk; - }); - response.on("end", () => { - if (!response.statusCode || response.statusCode < 200 || response.statusCode >= 400) { - resolve(null); - return; - } - resolve(body); - }); - }); - request.on("timeout", () => { - request.destroy(); + pending.on("timeout", () => { + pending.destroy(); resolve(null); }); - request.on("error", () => resolve(null)); + pending.on("error", () => resolve(null)); }); } -async function hasHealthyRouteComponents(baseUrl) { - for (const smokePath of routeSmokePaths) { - const body = await requestText(`${baseUrl}${smokePath}`); - if (!body || body.includes(missingErrorComponentsNeedle)) { - return false; - } - } - return true; -} - -async function waitForServer(baseUrl) { - const startedAt = Date.now(); - while (Date.now() - startedAt < startupTimeoutMs) { - const payload = await requestJson(`${baseUrl}${identityPath}`); - if (isVerifiedProjectPayload(payload) && (await hasHealthyRouteComponents(baseUrl))) { - return; - } - await sleep(500); - } - throw new Error(`Timed out waiting for Clinical KB at ${baseUrl}.`); -} - function isVerifiedProjectPayload(payload) { return ( payload?.appName === appName && @@ -158,22 +130,36 @@ function isVerifiedProjectPayload(payload) { ); } -async function findExistingProjectServer() { - const baseUrl = `http://localhost:${stableProjectPort(projectRoot)}`; - const payload = await requestJson(`${baseUrl}${identityPath}`); - return isVerifiedProjectPayload(payload) ? baseUrl : null; -} - -function runPlaywright(baseUrl) { - return spawnSync(process.execPath, [playwrightBin, "test", ...process.argv.slice(2)], { - cwd: projectRoot, - env: { ...process.env, PLAYWRIGHT_BASE_URL: baseUrl }, - stdio: "inherit", - }); +async function waitForServer(baseUrl, server) { + const startedAt = Date.now(); + while (Date.now() - startedAt < startupTimeoutMs) { + if (serverLaunchError) { + throw new Error(`Playwright-owned Next server failed to launch: ${serverLaunchError.message}`); + } + if (server.exitCode !== null || server.signalCode) { + throw new Error( + `Playwright-owned Next server exited before readiness (${server.exitCode !== null ? `code ${server.exitCode}` : `signal ${server.signalCode}`}).`, + ); + } + const payload = await request(`${baseUrl}${identityPath}`, { json: true, timeoutMs: 5000 }); + if (isVerifiedProjectPayload(payload)) { + let healthy = true; + for (const smokePath of routeSmokePaths) { + const body = await request(`${baseUrl}${smokePath}`); + if (!body || body.includes(missingErrorComponentsNeedle)) { + healthy = false; + break; + } + } + if (healthy) return; + } + await sleep(500); + } + throw new Error(`Timed out waiting for the Playwright-owned Clinical KB server at ${baseUrl}.`); } -function stopProcessTree(child) { - if (!child.pid || child.exitCode !== null) return; +function stopOwnedProcessTree(child) { + if (!child?.pid || child.exitCode !== null) return; if (process.platform === "win32") { spawnSync("taskkill", ["/PID", String(child.pid), "/T", "/F"], { stdio: "ignore" }); return; @@ -185,87 +171,101 @@ function stopProcessTree(child) { } } -function stopExistingProjectDevServers() { - if (process.platform !== "win32") return; - - const command = ` -$repo = $env:PLAYWRIGHT_PROJECT_ROOT -$patterns = @( - "*$repo\\node_modules\\next\\dist\\bin\\next dev*", - "*$repo\\node_modules\\next\\dist\\server\\lib\\start-server.js*", - "*$repo\\.next\\dev\\build\\*" -) -$targets = Get-CimInstance Win32_Process | Where-Object { - if (-not $_.CommandLine) { return $false } - foreach ($pattern in $patterns) { - if ($_.CommandLine -like $pattern) { return $true } - } - return $false -} -foreach ($target in $targets) { - Stop-Process -Id $target.ProcessId -Force -ErrorAction SilentlyContinue -} -`; - - spawnSync("powershell.exe", ["-NoProfile", "-Command", command], { - cwd: projectRoot, - env: { ...process.env, PLAYWRIGHT_PROJECT_ROOT: projectRoot }, - stdio: "ignore", - }); -} - -const existingBaseUrl = await findExistingProjectServer(); -if (existingBaseUrl) { - if (await hasHealthyRouteComponents(existingBaseUrl)) { - console.log(`Using existing Clinical KB server at ${existingBaseUrl}`); - const result = runPlaywright(existingBaseUrl); - process.exit(result.status ?? (result.signal ? 1 : 0)); +let server; +let serverLaunchError; +let cleaned = false; +function cleanup() { + if (cleaned) return; + cleaned = true; + try { + stopOwnedProcessTree(server); + rmSync(absoluteRunRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + try { + rmdirSync(path.dirname(absoluteRunRoot)); + } catch (error) { + if (error?.code !== "ENOENT" && error?.code !== "ENOTEMPTY") throw error; + } + } catch (error) { + console.error(`Playwright cleanup warning: ${error instanceof Error ? error.message : String(error)}`); + } finally { + lock.release(); } - - console.log(`Existing Clinical KB server at ${existingBaseUrl} failed route-component smoke; restarting it.`); - stopExistingProjectDevServers(); - await sleep(1000); } -stopExistingProjectDevServers(); -await sleep(1000); - -const port = await findFreePort(stableProjectPort(projectRoot)); -const baseUrl = `http://localhost:${port}`; -console.log(`Starting Playwright-owned Clinical KB server at ${baseUrl}`); - -const server = spawn(process.execPath, [nextBin, "dev", "--hostname", "0.0.0.0", "--port", String(port)], { - cwd: projectRoot, - detached: process.platform !== "win32", - env: { ...process.env, PORT: String(port), PLAYWRIGHT_BASE_URL: baseUrl }, - stdio: ["ignore", "inherit", "inherit"], - windowsHide: true, -}); - -let stopping = false; -const stop = () => { - if (stopping) return; - stopping = true; - stopProcessTree(server); -}; - process.once("SIGINT", () => { - stop(); + cleanup(); process.exit(130); }); process.once("SIGTERM", () => { - stop(); + cleanup(); process.exit(143); }); -process.once("exit", stop); +process.once("exit", cleanup); try { - await waitForServer(baseUrl); - const result = runPlaywright(baseUrl); - stop(); - process.exit(result.status ?? (result.signal ? 1 : 0)); + const port = await findFreePort(stableProjectPort(projectRoot)); + const baseUrl = `http://localhost:${port}`; + mkdirSync(absoluteRunRoot, { recursive: true }); + writeFileSync( + path.join(absoluteRunRoot, "tsconfig.json"), + `${JSON.stringify( + { + extends: "../../tsconfig.json", + compilerOptions: { + baseUrl: "../..", + paths: { "@/*": ["src/*"] }, + }, + }, + null, + 2, + )}\n`, + "utf8", + ); + const offlineEnv = offlineTestEnvironment(lock.environment, { + PORT: String(port), + PLAYWRIGHT_BASE_URL: baseUrl, + NEXT_DIST_DIR: relativeDistDir, + NEXT_TSCONFIG_PATH: relativeTsConfigPath, + NODE_ENV: "production", + PLAYWRIGHT_OFFLINE_MODE: "true", + NEXT_PUBLIC_MOCKUPS_ENABLED: mockupProjectRequested ? "true" : "false", + }); + console.log(`Building isolated production Playwright app (${relativeRunRoot})`); + + const buildResult = spawnSync(process.execPath, ["--max-old-space-size=8192", nextBin, "build", "--webpack"], { + cwd: projectRoot, + env: offlineEnv, + stdio: "inherit", + }); + const buildExitCode = childProcessExitCode(buildResult); + if (buildExitCode !== 0) { + throw new Error(`Playwright production build failed (${childProcessFailureSummary(buildResult)}).`); + } + + console.log(`Starting isolated production Playwright server at ${baseUrl} (${relativeRunRoot})`); + + server = spawn(process.execPath, [nextBin, "start", "--hostname", "0.0.0.0", "--port", String(port)], { + cwd: projectRoot, + detached: process.platform !== "win32", + env: offlineEnv, + stdio: ["ignore", "inherit", "inherit"], + windowsHide: true, + }); + server.once("error", (error) => { + serverLaunchError = error; + }); + + await waitForServer(baseUrl, server); + const result = spawnSync(process.execPath, [playwrightBin, "test", ...playwrightArgs], { + cwd: projectRoot, + env: offlineEnv, + stdio: "inherit", + }); + const exitCode = childProcessExitCode(result); + cleanup(); + process.exit(exitCode); } catch (error) { - stop(); + cleanup(); console.error(error instanceof Error ? error.message : String(error)); process.exit(1); } diff --git a/scripts/run-vitest.mjs b/scripts/run-vitest.mjs index 61a24f75d..1ff55d271 100644 --- a/scripts/run-vitest.mjs +++ b/scripts/run-vitest.mjs @@ -2,46 +2,23 @@ import { spawnSync } from "node:child_process"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; +import { offlineTestEnvironment } from "./test-environment.mjs"; +import { acquireHeavyRunLock } from "./test-run-lock.mjs"; const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const vitestBin = path.join(projectRoot, "node_modules", "vitest", "vitest.mjs"); -const vitestNeedle = vitestBin.toLowerCase(); -const powershell = [ - "Get-CimInstance Win32_Process", - "| Where-Object {", - " $_.CommandLine -and", - ` $_.CommandLine.ToLower().Contains('${vitestNeedle}') -and`, - " $_.CommandLine -notlike '*run-vitest.mjs*' -and", - " $_.CommandLine -notlike '*Get-CimInstance*'", - "}", - "| ForEach-Object { $_.ProcessId }", -].join(" "); - -let runningPids = []; +const args = process.argv.slice(2); +const lock = acquireHeavyRunLock({ projectRoot, command: `vitest ${args.join(" ")}` }); +let exitCode = 1; try { - const probe = spawnSync("powershell.exe", ["-NoProfile", "-Command", powershell], { - encoding: "utf8", - stdio: ["ignore", "pipe", "ignore"], + const result = spawnSync(process.execPath, [vitestBin, ...args], { + cwd: projectRoot, + env: offlineTestEnvironment(lock.environment, { NODE_ENV: "test" }), + stdio: "inherit", }); - const output = probe.status === 0 ? probe.stdout.trim() : ""; - runningPids = output - .split(/\r?\n/) - .map((line) => Number.parseInt(line.trim(), 10)) - .filter(Number.isFinite); -} catch { - runningPids = []; -} - -for (const pid of runningPids) { - try { - spawnSync("taskkill", ["/F", "/T", "/PID", String(pid)], { stdio: "ignore" }); - } catch { - // best effort cleanup - } + exitCode = childProcessExitCode(result); +} finally { + lock.release(); } - -const args = process.argv.slice(2); -const result = spawnSync(process.execPath, [vitestBin, ...args], { - stdio: "inherit", -}); -process.exit(result.status ?? 0); +process.exit(exitCode); diff --git a/scripts/test-environment.mjs b/scripts/test-environment.mjs new file mode 100644 index 000000000..c304fe297 --- /dev/null +++ b/scripts/test-environment.mjs @@ -0,0 +1,63 @@ +const providerEnvironmentKeys = Object.freeze([ + "OPENAI_API_KEY", + "OPENAI_ORG_ID", + "OPENAI_PROJECT_ID", + "OPENAI_BASE_URL", + "NEXT_PUBLIC_SUPABASE_URL", + "NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY", + "NEXT_PUBLIC_SUPABASE_ANON_KEY", + "SUPABASE_URL", + "SUPABASE_ANON_KEY", + "SUPABASE_PUBLISHABLE_KEY", + "SUPABASE_SECRET_KEY", + "SUPABASE_SERVICE_ROLE_KEY", + "SUPABASE_ACCESS_TOKEN", + "SUPABASE_DB_URL", + "SUPABASE_PROJECT_REF", + "SUPABASE_PROJECT_NAME", + "SUPABASE_STAGING_PROJECT_REF", + "SUPABASE_STAGING_PROJECT_NAME", + "DATABASE_URL", + "POSTGRES_PASSWORD", + "E2E_AUTH_ENABLED", + "E2E_USER_EMAIL", + "E2E_USER_PASSWORD", + "ALLOW_PROVIDER_TESTS", +]); + +const offlineUrlValues = Object.freeze({ + NEXT_PUBLIC_SUPABASE_URL: "http://127.0.0.1:1", + SUPABASE_URL: "http://127.0.0.1:1", + SUPABASE_DB_URL: "postgresql://offline:offline@127.0.0.1:1/offline", + DATABASE_URL: "postgresql://offline:offline@127.0.0.1:1/offline", +}); + +/** + * @param {Record<string, string | undefined>} source + * @param {Record<string, string | undefined>} overrides + */ +export function offlineTestEnvironment(source = process.env, overrides = {}) { + const environment = { ...source }; + // Explicit values both scrub inherited secrets and prevent Next/Vite from + // repopulating the same names from a repository-local env file. URL-shaped + // settings use inert loopback values so the runtime env schema still parses. + for (const key of providerEnvironmentKeys) environment[key] = offlineUrlValues[key] ?? ""; + + return { + ...environment, + RAG_PROVIDER_MODE: "offline", + NEXT_PUBLIC_DEMO_MODE: "true", + ...overrides, + }; +} + +/** @param {Record<string, string | undefined>} environment */ +export function requireProviderTestPermission(environment = process.env) { + if (environment.ALLOW_PROVIDER_TESTS !== "true") { + throw new Error( + "Live provider tests are disabled. Set ALLOW_PROVIDER_TESTS=true only after explicit provider-test approval.", + ); + } +} + +export { offlineUrlValues, providerEnvironmentKeys }; diff --git a/scripts/test-focused.mjs b/scripts/test-focused.mjs new file mode 100644 index 000000000..d1065b9a7 --- /dev/null +++ b/scripts/test-focused.mjs @@ -0,0 +1,73 @@ +#!/usr/bin/env node +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; + +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const unsafeSelectionPattern = + /^(?:tests\/|scripts\/|\.github\/|package(?:-lock)?\.json$|tsconfig|vitest\.config|next\.config|eslint)/i; + +function git(args) { + return execFileSync("git", args, { cwd: projectRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }); +} + +function lines(value) { + return value + .split(/\r?\n/) + .map((item) => item.trim()) + .filter(Boolean); +} + +function defaultSelection() { + let base = "HEAD"; + try { + base = git(["merge-base", "HEAD", "origin/main"]).trim(); + } catch { + // A local-only checkout can still select working-tree changes from HEAD. + } + const changed = lines(git(["diff", "--name-only", base])); + const untracked = lines(git(["ls-files", "--others", "--exclude-standard"])); + const deleted = new Set(lines(git(["diff", "--name-only", "--diff-filter=D", base]))); + return { files: [...new Set([...changed, ...untracked])], deleted }; +} + +function failClosed(reason) { + console.error(`Focused test selection is unsafe: ${reason}`); + console.error("Run the full unit suite with: npm run test"); + process.exit(2); +} + +const args = process.argv.slice(2); +const filesIndex = args.indexOf("--files"); +const explicitFiles = filesIndex >= 0 ? lines((args[filesIndex + 1] ?? "").replaceAll(",", "\n")) : null; +if (filesIndex >= 0 && explicitFiles?.length === 0) failClosed("--files did not contain a path"); + +const selection = explicitFiles ? { files: explicitFiles, deleted: new Set() } : defaultSelection(); +if (selection.files.length === 0) failClosed("no changed files were found"); +const missing = selection.files.filter((file) => !existsSync(path.resolve(projectRoot, file))); +if (missing.length > 0) failClosed(`deleted or missing paths require the full suite (${missing.join(", ")})`); +if (selection.deleted.size > 0) + failClosed(`deleted paths require the full suite (${[...selection.deleted].join(", ")})`); +const unsafe = selection.files.filter((file) => unsafeSelectionPattern.test(file.replaceAll("\\", "/"))); +if (unsafe.length > 0) failClosed(`test or configuration paths changed (${unsafe.join(", ")})`); + +const selectionArgs = new Set(); +if (filesIndex >= 0) { + selectionArgs.add(filesIndex); + selectionArgs.add(filesIndex + 1); +} +const forwarded = args.filter((argument, index) => !selectionArgs.has(index) && argument !== "--dry-run"); +console.log(`Focused test inputs: ${selection.files.join(", ")}`); +if (args.includes("--dry-run")) { + console.log("Command: vitest related --run <changed files>"); + process.exit(0); +} + +const runnerPath = path.join(projectRoot, "scripts", "run-vitest.mjs"); +const result = spawnSync(process.execPath, [runnerPath, "related", "--run", ...selection.files, ...forwarded], { + cwd: projectRoot, + stdio: "inherit", +}); +process.exit(childProcessExitCode(result)); diff --git a/scripts/test-rag-offline.mjs b/scripts/test-rag-offline.mjs new file mode 100644 index 000000000..738fbcd83 --- /dev/null +++ b/scripts/test-rag-offline.mjs @@ -0,0 +1,17 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; + +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const contractTests = JSON.parse( + readFileSync(path.join(projectRoot, "scripts/fixtures/rag-offline-contract-tests.json"), "utf8"), +); +const result = spawnSync( + process.execPath, + [path.join(projectRoot, "scripts/run-vitest.mjs"), "run", ...contractTests, ...process.argv.slice(2)], + { cwd: projectRoot, stdio: "inherit" }, +); +process.exit(childProcessExitCode(result)); diff --git a/scripts/test-run-lock.mjs b/scripts/test-run-lock.mjs new file mode 100644 index 000000000..c59a4e64e --- /dev/null +++ b/scripts/test-run-lock.mjs @@ -0,0 +1,147 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +const tokenEnvironmentKey = "CLINICAL_KB_HEAVY_LOCK_TOKEN"; +const pathEnvironmentKey = "CLINICAL_KB_HEAVY_LOCK_PATH"; +const incompleteLockGraceMs = 30_000; + +function processIsAlive(pid) { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code === "EPERM"; + } +} + +function normalizeIdentity(value) { + const resolved = path.resolve(value); + return process.platform === "win32" ? resolved.toLowerCase() : resolved; +} + +function resolveRepositoryIdentity(projectRoot) { + const result = spawnSync("git", ["rev-parse", "--path-format=absolute", "--git-common-dir"], { + cwd: projectRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + if (result.status !== 0 || !result.stdout.trim()) { + throw new Error("Could not resolve the shared Git directory for the Database heavyweight-run lock."); + } + return normalizeIdentity(result.stdout.trim()); +} + +function lockPathFor(repositoryIdentity, baseDirectory = os.tmpdir()) { + const repositoryId = createHash("sha256").update(normalizeIdentity(repositoryIdentity)).digest("hex").slice(0, 20); + return path.join(baseDirectory, "clinical-kb-heavy-locks", `${repositoryId}.lock`); +} + +function readOwner(lockPath) { + try { + return JSON.parse(readFileSync(path.join(lockPath, "owner.json"), "utf8")); + } catch { + return null; + } +} + +function lockIsOldEnoughToRecover(lockPath, now = Date.now()) { + try { + return now - statSync(lockPath).mtimeMs >= incompleteLockGraceMs; + } catch { + return false; + } +} + +/** + * @param {{ + * projectRoot: string; + * command?: string; + * environment?: Record<string, string | undefined>; + * baseDirectory?: string; + * repositoryIdentity?: string; + * processId?: number; + * }} options + */ +export function acquireHeavyRunLock({ + projectRoot, + command = process.argv.join(" "), + environment = process.env, + baseDirectory, + repositoryIdentity = resolveRepositoryIdentity(projectRoot), + processId = process.pid, +}) { + if (!projectRoot) throw new Error("projectRoot is required for the Database heavyweight-run lock."); + const lockPath = lockPathFor(repositoryIdentity, baseDirectory); + const inheritedToken = environment[tokenEnvironmentKey]; + const inheritedPath = environment[pathEnvironmentKey]; + + if (inheritedToken || inheritedPath) { + if (!inheritedToken || normalizeIdentity(inheritedPath) !== normalizeIdentity(lockPath)) { + throw new Error("The inherited Database heavyweight-run lock does not match this repository."); + } + const owner = readOwner(lockPath); + if (owner?.token !== inheritedToken || !processIsAlive(owner.pid)) { + throw new Error("The inherited Database heavyweight-run lock is no longer owned by a live parent process."); + } + return { path: lockPath, owner, environment: { ...environment }, reentrant: true, release() {} }; + } + + mkdirSync(path.dirname(lockPath), { recursive: true }); + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + mkdirSync(lockPath); + const token = randomUUID(); + const owner = { + pid: processId, + token, + command, + worktree: path.resolve(projectRoot), + repositoryIdentity: normalizeIdentity(repositoryIdentity), + startedAt: new Date().toISOString(), + }; + writeFileSync(path.join(lockPath, "owner.json"), `${JSON.stringify(owner, null, 2)}\n`, "utf8"); + let released = false; + return { + path: lockPath, + owner, + reentrant: false, + environment: { + ...environment, + [tokenEnvironmentKey]: token, + [pathEnvironmentKey]: lockPath, + }, + release() { + if (released) return; + released = true; + if (readOwner(lockPath)?.token === token) rmSync(lockPath, { recursive: true, force: true }); + }, + }; + } catch (error) { + if (error?.code !== "EEXIST") throw error; + const owner = readOwner(lockPath); + if (owner && processIsAlive(owner.pid)) { + throw new Error( + `Another Database heavyweight command is active (PID ${owner.pid}, worktree ${owner.worktree ?? "unknown"}, started ${owner.startedAt ?? "unknown"}): ${owner.command ?? "unknown command"}`, + ); + } + if (!owner && !lockIsOldEnoughToRecover(lockPath)) { + throw new Error(`A Database heavyweight lock is being initialized at ${lockPath}; retry after it settles.`); + } + rmSync(lockPath, { recursive: true, force: true }); + } + } + throw new Error(`Could not acquire the Database heavyweight-run lock at ${lockPath}.`); +} + +export const testRunLockInternals = { + lockPathFor, + processIsAlive, + readOwner, + resolveRepositoryIdentity, + tokenEnvironmentKey, + pathEnvironmentKey, +}; diff --git a/scripts/verify-pr-local.mjs b/scripts/verify-pr-local.mjs index 61d95f9a1..c93796ae8 100644 --- a/scripts/verify-pr-local.mjs +++ b/scripts/verify-pr-local.mjs @@ -1,8 +1,13 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; +import { acquireHeavyRunLock } from "./test-run-lock.mjs"; const isWindows = process.platform === "win32"; -const baseScripts = ["check:runtime", "format:check", "lint", "typecheck", "test"]; +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const baseScripts = ["check:runtime", "format:changed", "lint", "typecheck", "test"]; function parseArgs(args) { const options = { dryRun: false, extended: false, files: undefined }; @@ -43,14 +48,12 @@ function parseArgs(args) { return options; } -function runNpmScript(script) { +function runNpmScript(script, environment) { console.log(`\n> npm run ${script}`); const result = isWindows - ? spawnSync("cmd.exe", ["/d", "/s", "/c", `npm run ${script}`], { stdio: "inherit" }) - : spawnSync("npm", ["run", script], { stdio: "inherit" }); - if (result.status !== 0) { - process.exit(result.status ?? 1); - } + ? spawnSync("cmd.exe", ["/d", "/s", "/c", `npm run ${script}`], { env: environment, stdio: "inherit" }) + : spawnSync("npm", ["run", script], { env: environment, stdio: "inherit" }); + return childProcessExitCode(result); } function readScope(files) { @@ -60,14 +63,15 @@ function readScope(files) { encoding: "utf8", stdio: ["ignore", "pipe", "inherit"], }); - if (result.status !== 0) process.exit(result.status ?? 1); + if (childProcessExitCode(result) !== 0) process.exit(childProcessExitCode(result)); return JSON.parse(result.stdout); } function selectedScripts(scope, extended) { const scripts = [...baseScripts]; if (scope.build_changed) scripts.push("build"); - if (scope.rag_eval_changed) scripts.push("eval:rag:offline"); + // Full unit testing already includes every offline RAG contract suite. + if (scope.rag_eval_changed) scripts.push("check:rag:fixtures"); if (extended && scope.ui_changed) scripts.push("verify:ui"); return scripts; } @@ -81,13 +85,25 @@ if (options.dryRun) { console.log("\nPR-local verification plan (dry run):"); for (const script of scripts) console.log(`- npm run ${script}`); if (!scope.build_changed) console.log("- build skipped: no build-affecting changes detected"); - if (!scope.rag_eval_changed) console.log("- offline RAG evaluation skipped: no RAG-affecting changes detected"); + if (!scope.rag_eval_changed) + console.log("- offline RAG fixture validation skipped: no RAG-affecting changes detected"); if (options.extended && !scope.ui_changed) console.log("- Chromium UI gate skipped: no UI-affecting changes detected"); process.exit(0); } -for (const script of scripts) runNpmScript(script); +const lock = acquireHeavyRunLock({ projectRoot, command: "npm run verify:pr-local" }); +let exitCode = 0; +try { + for (const script of scripts) { + exitCode = runNpmScript(script, lock.environment); + if (exitCode !== 0) break; + } +} finally { + lock.release(); +} + +if (exitCode !== 0) process.exit(exitCode); if (!scope.build_changed) console.log("\nSkipping build: no build-affecting source, config, package, or container changes detected."); diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 2c1686055..b83237985 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -10,6 +10,23 @@ export async function register() { if (process.env.NEXT_RUNTIME !== "nodejs") return; if (process.env.NODE_ENV !== "production") return; + // Playwright validates a real production build, but its runner must remain a + // provider-free demo. Permit that otherwise-invalid combination only for the + // runner's isolated output and inert loopback environment. A partial or + // externally-addressed configuration still falls through to the production + // guard below and fails closed. + if (process.env.PLAYWRIGHT_OFFLINE_MODE === "true") { + const isolatedOutput = /^\.next-playwright\/[a-z0-9-]+\/dist$/i.test(process.env.NEXT_DIST_DIR ?? ""); + const providerFree = + process.env.NEXT_PUBLIC_DEMO_MODE === "true" && + process.env.RAG_PROVIDER_MODE === "offline" && + process.env.NEXT_PUBLIC_SUPABASE_URL === "http://127.0.0.1:1" && + !process.env.SUPABASE_SERVICE_ROLE_KEY && + !process.env.OPENAI_API_KEY; + if (isolatedOutput && providerFree) return; + throw new Error("Refusing to start: invalid isolated Playwright offline environment."); + } + // Defense in depth: no-auth must never be active in a production build (even // though isLocalNoAuthMode() already hard-guards on NODE_ENV). if (process.env.LOCAL_NO_AUTH === "true" || process.env.NEXT_PUBLIC_LOCAL_NO_AUTH === "true") { diff --git a/src/proxy.ts b/src/proxy.ts index e387fcd5f..da0b65a26 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -77,7 +77,7 @@ export async function proxy(request: NextRequest) { return withCsp(NextResponse.redirect(url)); } - if (pathname.startsWith("/mockups") && process.env.NODE_ENV === "production") { + if (shouldBlockProductionMockups(pathname)) { return withCsp(new NextResponse(null, { status: 404 })); } @@ -109,6 +109,20 @@ export async function proxy(request: NextRequest) { return withCsp(response); } +export function shouldBlockProductionMockups( + pathname: string, + environment: Record<string, string | undefined> = process.env, +) { + if (!pathname.startsWith("/mockups") || environment.NODE_ENV !== "production") return false; + + // Mockups remain unavailable in every normal production process. The one + // exception is the repository-owned, isolated Playwright server when its + // advisory project explicitly opts into mockup coverage. Instrumentation + // separately refuses PLAYWRIGHT_OFFLINE_MODE outside the inert loopback and + // isolated .next-playwright profile. + return !(environment.PLAYWRIGHT_OFFLINE_MODE === "true" && environment.NEXT_PUBLIC_MOCKUPS_ENABLED === "true"); +} + export const config = { // Run on everything except static assets and image files. API routes stay in // the matcher so cookie-authenticated requests can return rotated cookies and diff --git a/tests/architecture-boundaries.test.ts b/tests/architecture-boundaries.test.ts index 323aec37c..a2d75037f 100644 --- a/tests/architecture-boundaries.test.ts +++ b/tests/architecture-boundaries.test.ts @@ -116,7 +116,9 @@ function resolveModule(fromFile: string, specifier: string, fileSet: Set<string> return candidates.map((candidate) => path.resolve(candidate)).find((candidate) => fileSet.has(candidate)) ?? null; } -function runtimeGraph() { +let runtimeGraphCache: ReturnType<typeof buildRuntimeGraph> | undefined; + +function buildRuntimeGraph() { const files = sourceFiles(); const fileSet = new Set(files); const graph = new Map<string, string[]>(); @@ -134,6 +136,11 @@ function runtimeGraph() { return { files, fileSet, graph, parsed }; } +function runtimeGraph() { + runtimeGraphCache ??= buildRuntimeGraph(); + return runtimeGraphCache; +} + function runtimeCycles(graph: Map<string, string[]>) { let nextIndex = 0; const indexes = new Map<string, number>(); @@ -229,8 +236,9 @@ describe("architecture boundaries", () => { }); it("does not make runtime source modules depend on operational scripts", () => { - const scriptImports = sourceFiles().flatMap((file) => { - const modules = moduleSpecifiers(file); + const { files, parsed } = runtimeGraph(); + const scriptImports = files.flatMap((file) => { + const modules = parsed.get(file)!; return [...modules.staticImports, ...modules.dynamicImports] .filter((specifier) => path.resolve(path.dirname(file), specifier).startsWith(path.join(projectRoot, "scripts")), diff --git a/tests/client-secret-surface.test.ts b/tests/client-secret-surface.test.ts index fff4b038e..98d223ad2 100644 --- a/tests/client-secret-surface.test.ts +++ b/tests/client-secret-surface.test.ts @@ -144,7 +144,8 @@ describe("client environment isolation", () => { scripts: Record<string, string>; }; expect(packageJson.scripts["check:client-bundle-secrets"]).toContain("check-client-bundle-secrets.mjs"); - expect(packageJson.scripts.build).toContain("check-client-bundle-secrets.mjs"); + expect(packageJson.scripts.build).toContain("run-heavy.mjs"); + expect(packageJson.scripts["build:internal"]).toContain("check-client-bundle-secrets.mjs"); const fixtureRoot = mkdtempSync(join(tmpdir(), "clinical-kb-client-bundle-")); try { diff --git a/tests/demo-data.test.ts b/tests/demo-data.test.ts index 70d570bfc..c181c8c17 100644 --- a/tests/demo-data.test.ts +++ b/tests/demo-data.test.ts @@ -110,6 +110,7 @@ describe("demoSearch production safety guard", () => { it("refuses to serve synthetic demo data to a live production request", async () => { vi.stubEnv("NODE_ENV", "production"); + vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "false"); const { demoSearch: guarded } = await import("../src/lib/demo-data"); expect(() => guarded("lithium toxicity")).toThrow(/synthetic demo data/i); }); diff --git a/tests/flake-ledger.json b/tests/flake-ledger.json index 0db897684..8f2f9f59c 100644 --- a/tests/flake-ledger.json +++ b/tests/flake-ledger.json @@ -1,37 +1,4 @@ { - "$comment": "Known-flaky Playwright specs. Durable knowledge so a failure gets attributed to a known flake instead of re-diagnosed from memory each time. Consumed by scripts/flake-ledger.mjs (loader + isKnownFlake) and the CI failure-triage workflow. `match` is a case-insensitive substring of the test title; keep entries narrow. Removing an entry means 'we believe this is fixed' — do that deliberately.", - "flakes": [ - { - "id": "raf-portal-hydration", - "match": "composer hero", - "spec": "tests/ui-smoke.spec.ts", - "reason": "Hero-composer hydration depends on a rAF portal that starves under a headless render; fixed with a MutationObserver microtask (#502/#504) but can still flake on a cold render.", - "since": "2026-07-11", - "refs": ["#502", "#504"] - }, - { - "id": "tap-target-subpixel", - "match": "tap target", - "spec": "tests/ui-smoke.spec.ts", - "reason": "min-h-11 tap-target rect rounds sub-pixel below 44px on some DPRs; mitigated by min-h-12 but historically flaky.", - "since": "2026-07-11", - "refs": [] - }, - { - "id": "narrow-viewport-parallel", - "match": "narrow viewport", - "spec": "tests/ui-tools.spec.ts", - "reason": "Differentials narrow-viewport assertions race only under parallel workers; the suite runs workers:1 but sharded CI can still interleave.", - "since": "2026-07-11", - "refs": [] - }, - { - "id": "rag-answer-fallback-timeout", - "match": "answer fallback", - "spec": "tests/ui-tools.spec.ts", - "reason": "The RAG answer-fallback path can exceed the 60s test timeout when the provider route is cold; a pre-existing advisory flake.", - "since": "2026-07-11", - "refs": [] - } - ] + "$comment": "Only reproduced @quarantine Playwright tests belong here. Entries use an exact title and expire within 30 days. The ledger is intentionally empty after stale, non-reproducible entries were removed.", + "flakes": [] } diff --git a/tests/flake-ledger.test.ts b/tests/flake-ledger.test.ts index fd8dae544..bdde8fb7b 100644 --- a/tests/flake-ledger.test.ts +++ b/tests/flake-ledger.test.ts @@ -1,31 +1,92 @@ -import { describe, expect, it } from "vitest"; -import { isKnownFlake, loadFlakeLedger, matchFlake } from "../scripts/flake-ledger.mjs"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { failedTestcasesFromJunit } from "../scripts/classify-playwright-failures.mjs"; +import { loadFlakeLedger, matchFlake, validateFlakeLedgerEntries } from "../scripts/flake-ledger.mjs"; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +function isoDate(offsetDays: number) { + return new Date(Date.now() + offsetDays * 24 * 60 * 60 * 1000).toISOString().slice(0, 10); +} + +function validEntry(overrides: Record<string, string> = {}) { + return { + id: "sample-flake", + title: "exact flaky journey @quarantine", + spec: "tests/sample.spec.ts", + reason: "Reproduced fail/pass variation on the same SHA.", + owner: "test-infrastructure", + reproduction: "npm run test:e2e:advisory -- --grep exact --repeat-each=3", + firstSeen: isoDate(-2), + lastSeen: isoDate(-1), + expires: isoDate(20), + tracking: "docs/process-hardening.md#known-flakes", + ...overrides, + }; +} + +function temporarySpec(source: string) { + const root = mkdtempSync(path.join(os.tmpdir(), "clinical-kb-flake-ledger-test-")); + temporaryDirectories.push(root); + mkdirSync(path.join(root, "tests")); + writeFileSync(path.join(root, "tests", "sample.spec.ts"), source); + return root; +} describe("flake ledger", () => { - it("loads and validates the committed ledger", () => { - const flakes = loadFlakeLedger(); - expect(flakes.length).toBeGreaterThan(0); - for (const flake of flakes) { - expect(flake.id).toBeTruthy(); - expect(flake.match).toBeTruthy(); - expect(flake.spec).toMatch(/^tests\//); - expect(flake.reason).toBeTruthy(); - } + it("allows the committed ledger to be empty", () => { + expect(loadFlakeLedger()).toEqual([]); + }); + + it("matches only the exact case-insensitive spec and title identity", () => { + const flakes = [validEntry()]; + expect(matchFlake("tests/sample.spec.ts", "EXACT FLAKY JOURNEY @QUARANTINE", flakes)?.id).toBe("sample-flake"); + expect(matchFlake("tests/sample.spec.ts", "prefix exact flaky journey @quarantine", flakes)).toBeNull(); + expect(matchFlake("tests/other.spec.ts", "exact flaky journey @quarantine", flakes)).toBeNull(); + expect(matchFlake("", "")).toBeNull(); + }); + + it("requires exact quarantined titles and complete ownership metadata", () => { + expect(() => validateFlakeLedgerEntries([validEntry({ title: "" })])).toThrow(/missing: title/); + expect(() => validateFlakeLedgerEntries([validEntry({ title: "not tagged" })])).toThrow(/include @quarantine/); }); - it("has unique ids", () => { - const ids = loadFlakeLedger().map((f: { id: string }) => f.id); - expect(new Set(ids).size).toBe(ids.length); + it("rejects critical overlap and expiry beyond 30 days", () => { + expect(() => validateFlakeLedgerEntries([validEntry({ title: "unsafe @quarantine @critical" })])).toThrow( + /cannot be both @quarantine and @critical/, + ); + expect(() => validateFlakeLedgerEntries([validEntry({ expires: isoDate(31) })])).toThrow(/within 30 days/); + const root = temporarySpec('test("exact flaky journey @quarantine", () => {});'); + expect(validateFlakeLedgerEntries([validEntry({ expires: isoDate(30) })], { root })).toHaveLength(1); }); - it("matches a known flake by case-insensitive title substring", () => { - expect(isKnownFlake("Composer Hero mounts after hydration")).toBe(true); - const hit = matchFlake("the tap target is at least 44px"); - expect(hit?.id).toBe("tap-target-subpixel"); + it("requires the exact title to exist in the referenced spec", () => { + const entry = validEntry(); + const root = temporarySpec(`test(${JSON.stringify(entry.title)}, () => {});`); + expect(validateFlakeLedgerEntries([entry], { root })).toHaveLength(1); + expect(() => validateFlakeLedgerEntries([validEntry({ title: "missing @quarantine" })], { root })).toThrow( + /exact title is not present/, + ); }); - it("does not match an unrelated title or an empty string", () => { - expect(isKnownFlake("renders the medication results grid")).toBe(false); - expect(matchFlake("")).toBeNull(); + it("extracts only exact failed testcase identities from JUnit", () => { + const failures = failedTestcasesFromJunit(` + <testsuite> + <testcase name="exact flaky journey @quarantine" classname="sample.spec.ts"> + <failure message="failed" /> + </testcase> + <testcase name="passing journey" classname="tests/sample.spec.ts"></testcase> + </testsuite> + `); + expect(failures).toEqual([{ spec: "tests/sample.spec.ts", title: "exact flaky journey @quarantine" }]); + const flakes = [validEntry()]; + expect(matchFlake(failures[0]?.spec, failures[0]?.title, flakes)?.id).toBe("sample-flake"); + expect(matchFlake("tests/other.spec.ts", failures[0]?.title, flakes)).toBeNull(); }); }); diff --git a/tests/instrumentation.test.ts b/tests/instrumentation.test.ts index eeebfe99d..af61b7c99 100644 --- a/tests/instrumentation.test.ts +++ b/tests/instrumentation.test.ts @@ -22,6 +22,8 @@ const ENV_KEYS = [ "RAG_QUERY_HASH_SECRET", "NEXT_PUBLIC_LOCAL_NO_AUTH", "LOCAL_NO_AUTH", + "PLAYWRIGHT_OFFLINE_MODE", + "NEXT_DIST_DIR", ] as const; async function loadInstrumentation(overrides: Partial<Record<(typeof ENV_KEYS)[number], string | undefined>>) { @@ -64,6 +66,28 @@ describe("instrumentation boot guard", () => { await expect(register()).rejects.toThrow(/demo mode is enabled/); }); + it("allows only the isolated provider-free Playwright production profile", async () => { + const valid = await loadRegister({ + ...PRODUCTION_NODE, + PLAYWRIGHT_OFFLINE_MODE: "true", + NEXT_DIST_DIR: ".next-playwright/123-456/dist", + NEXT_PUBLIC_DEMO_MODE: "true", + NEXT_PUBLIC_SUPABASE_URL: "http://127.0.0.1:1", + RAG_PROVIDER_MODE: "offline", + }); + await expect(valid()).resolves.toBeUndefined(); + + const external = await loadRegister({ + ...PRODUCTION_NODE, + PLAYWRIGHT_OFFLINE_MODE: "true", + NEXT_DIST_DIR: ".next-playwright/123-456/dist", + NEXT_PUBLIC_DEMO_MODE: "true", + NEXT_PUBLIC_SUPABASE_URL: MATCHING_URL, + RAG_PROVIDER_MODE: "offline", + }); + await expect(external()).rejects.toThrow(/invalid isolated Playwright offline environment/); + }); + it("refuses to start a production server with local no-auth enabled", async () => { const register = await loadRegister({ ...PRODUCTION_NODE, LOCAL_NO_AUTH: "true" }); await expect(register()).rejects.toThrow(/no-auth mode is enabled/); diff --git a/tests/property-accessible-table.test.ts b/tests/property-accessible-table.test.ts index a6ff21482..12d48f3cc 100644 --- a/tests/property-accessible-table.test.ts +++ b/tests/property-accessible-table.test.ts @@ -1,4 +1,5 @@ import fc from "fast-check"; +import "./property-seed"; import { describe, expect, it } from "vitest"; import { normalizeAccessibleTable } from "../src/lib/accessible-table-normalization"; import { formatAnswerForClipboard, formatWardNote } from "../src/lib/ward-output"; diff --git a/tests/property-chunking.test.ts b/tests/property-chunking.test.ts index a0547e4d6..6422ea7a1 100644 --- a/tests/property-chunking.test.ts +++ b/tests/property-chunking.test.ts @@ -1,4 +1,5 @@ import fc from "fast-check"; +import "./property-seed"; import { describe, expect, it } from "vitest"; import { chunkTextWithOverlap } from "../src/lib/chunking"; diff --git a/tests/property-numeric-token-preservation.test.ts b/tests/property-numeric-token-preservation.test.ts index af645922b..48071c9f4 100644 --- a/tests/property-numeric-token-preservation.test.ts +++ b/tests/property-numeric-token-preservation.test.ts @@ -1,4 +1,5 @@ import fc from "fast-check"; +import "./property-seed"; import { describe, expect, it } from "vitest"; import { clinicalProseUsefulness, diff --git a/tests/property-seed.ts b/tests/property-seed.ts new file mode 100644 index 000000000..b76b82425 --- /dev/null +++ b/tests/property-seed.ts @@ -0,0 +1,13 @@ +import fc from "fast-check"; + +const defaultPropertySeed = 424_242; +const parsedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? `${defaultPropertySeed}`, 10); +if (!Number.isSafeInteger(parsedSeed)) { + throw new Error(`FAST_CHECK_SEED must be a safe integer; received ${process.env.FAST_CHECK_SEED}.`); +} +const propertySeed = ((parsedSeed % 2_147_483_647) + 2_147_483_647) % 2_147_483_647 || defaultPropertySeed; + +fc.configureGlobal({ seed: propertySeed, verbose: true }); +console.info(`[fast-check] replay seed: ${propertySeed}`); + +export { propertySeed }; diff --git a/tests/proxy.test.ts b/tests/proxy.test.ts index fead5901c..a3b47cc61 100644 --- a/tests/proxy.test.ts +++ b/tests/proxy.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { NextRequest } from "next/server"; -import { proxy } from "../src/proxy"; +import { proxy, shouldBlockProductionMockups } from "../src/proxy"; // The proxy owns the per-request nonce CSP (see src/proxy.ts). CI's verify:ui // only exercises the *dev* CSP path (Turbopack keeps 'unsafe-inline'); these @@ -75,3 +75,29 @@ describe("proxy content-security-policy", () => { expect(res.headers.get("x-middleware-request-x-nonce")).toBe(cspNonce); }); }); + +describe("production mockup boundary", () => { + it("blocks ordinary production traffic and permits only the explicit isolated Playwright advisory profile", () => { + expect(shouldBlockProductionMockups("/mockups/tools-workflow-board", { NODE_ENV: "production" })).toBe(true); + expect( + shouldBlockProductionMockups("/mockups/tools-workflow-board", { + NODE_ENV: "production", + PLAYWRIGHT_OFFLINE_MODE: "true", + }), + ).toBe(true); + expect( + shouldBlockProductionMockups("/mockups/tools-workflow-board", { + NODE_ENV: "production", + NEXT_PUBLIC_MOCKUPS_ENABLED: "true", + }), + ).toBe(true); + expect( + shouldBlockProductionMockups("/mockups/tools-workflow-board", { + NODE_ENV: "production", + PLAYWRIGHT_OFFLINE_MODE: "true", + NEXT_PUBLIC_MOCKUPS_ENABLED: "true", + }), + ).toBe(false); + expect(shouldBlockProductionMockups("/applications", { NODE_ENV: "production" })).toBe(false); + }); +}); diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 18af9825b..33defe169 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { RagAnswer, SearchResult } from "../src/lib/types"; function retrievalRpcBaseName(name: string) { @@ -139,6 +139,12 @@ async function answerFromTextSources( }); } +beforeEach(() => { + // The default runner removes real provider credentials and selects offline mode. + // This suite supplies a fully mocked provider and must exercise those fake paths. + vi.stubEnv("RAG_PROVIDER_MODE", "auto"); +}); + afterEach(() => { vi.restoreAllMocks(); vi.resetModules(); diff --git a/tests/test-runner-safety.test.ts b/tests/test-runner-safety.test.ts new file mode 100644 index 000000000..29bf7bd1c --- /dev/null +++ b/tests/test-runner-safety.test.ts @@ -0,0 +1,161 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { childProcessExitCode, childProcessFailureSummary } from "../scripts/child-process-result.mjs"; +import { + offlineTestEnvironment, + offlineUrlValues, + providerEnvironmentKeys, + requireProviderTestPermission, +} from "../scripts/test-environment.mjs"; +import { acquireHeavyRunLock } from "../scripts/test-run-lock.mjs"; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +function temporaryDirectory(prefix: string) { + const directory = mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryDirectories.push(directory); + return directory; +} + +describe("child process results", () => { + it("never treats a missing status, signal, or launch error as success", () => { + expect(childProcessExitCode({ status: null, signal: "SIGTERM" })).toBe(1); + expect(childProcessExitCode({ status: null, signal: null })).toBe(1); + expect(childProcessExitCode({ status: null, error: new Error("launch failed") })).toBe(1); + expect(childProcessExitCode({ status: 0 })).toBe(0); + expect(childProcessExitCode({ status: 7 })).toBe(7); + expect(childProcessFailureSummary({ status: null, signal: "SIGTERM" })).toBe("missing exit status, signal SIGTERM"); + }); + + it("maps a real missing executable launch to failure", () => { + const result = spawnSync(`clinical-kb-missing-command-${Date.now()}`, []); + expect(result.error).toBeTruthy(); + expect(childProcessExitCode(result)).toBe(1); + }); +}); + +describe("repository-wide heavyweight lock", () => { + it("blocks another worktree but permits a nested child with the owner token", () => { + const baseDirectory = temporaryDirectory("clinical-kb-lock-"); + const repositoryIdentity = path.join(baseDirectory, "shared.git"); + const first = acquireHeavyRunLock({ + projectRoot: path.join(baseDirectory, "worktree-a"), + repositoryIdentity, + baseDirectory, + environment: {}, + command: "first", + }); + + expect(() => + acquireHeavyRunLock({ + projectRoot: path.join(baseDirectory, "worktree-b"), + repositoryIdentity, + baseDirectory, + environment: {}, + command: "second", + }), + ).toThrow(/Another Database heavyweight command is active/); + + const nested = acquireHeavyRunLock({ + projectRoot: path.join(baseDirectory, "worktree-b"), + repositoryIdentity, + baseDirectory, + environment: first.environment, + command: "nested", + }); + expect(nested.reentrant).toBe(true); + nested.release(); + first.release(); + }); + + it("recovers a dead owner without allowing the old token to release the replacement", () => { + const baseDirectory = temporaryDirectory("clinical-kb-stale-lock-"); + const repositoryIdentity = path.join(baseDirectory, "shared.git"); + const stale = acquireHeavyRunLock({ + projectRoot: path.join(baseDirectory, "worktree-a"), + repositoryIdentity, + baseDirectory, + environment: {}, + processId: 2_147_483_647, + command: "dead", + }); + const replacement = acquireHeavyRunLock({ + projectRoot: path.join(baseDirectory, "worktree-b"), + repositoryIdentity, + baseDirectory, + environment: {}, + command: "replacement", + }); + + stale.release(); + expect(readFileSync(path.join(replacement.path, "owner.json"), "utf8")).toContain(replacement.owner.token); + replacement.release(); + }); +}); + +describe("provider-safe test environment", () => { + it("removes provider credentials and explicit live-test permission", () => { + const source = Object.fromEntries(providerEnvironmentKeys.map((key) => [key, `secret-${key}`])); + const environment: Record<string, string | undefined> = offlineTestEnvironment({ + ...source, + SAFE_VALUE: "kept", + }); + + expect(environment).toMatchObject({ + SAFE_VALUE: "kept", + RAG_PROVIDER_MODE: "offline", + NEXT_PUBLIC_DEMO_MODE: "true", + }); + for (const key of providerEnvironmentKeys) { + expect(environment[key]).toBe(offlineUrlValues[key as keyof typeof offlineUrlValues] ?? ""); + } + }); + + it("requires explicit permission before live tests can run", () => { + expect(() => requireProviderTestPermission({})).toThrow(/ALLOW_PROVIDER_TESTS=true/); + expect(() => requireProviderTestPermission({ ALLOW_PROVIDER_TESTS: "true" })).not.toThrow(); + }); + + it("keeps live tests out of default Vitest discovery", () => { + const config = readFileSync(new URL("../vitest.config.mts", import.meta.url), "utf8"); + expect(config).toContain('exclude: liveProviderTests ? [] : ["tests/**/*.live.test.ts"]'); + }); + + it("refuses the live-test command before collection when permission is absent", () => { + const result = spawnSync(process.execPath, ["scripts/run-live-tests.mjs"], { + cwd: path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."), + env: { ...process.env, ALLOW_PROVIDER_TESTS: "" }, + encoding: "utf8", + }); + expect(childProcessExitCode(result)).toBe(1); + expect(`${result.stdout}${result.stderr}`).toContain("Live provider tests are disabled"); + }); + + it("fails focused selection closed for a deleted or missing explicit source path", () => { + const result = spawnSync(process.execPath, ["scripts/test-focused.mjs", "--files", "src/missing-source.ts"], { + cwd: path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."), + encoding: "utf8", + }); + expect(childProcessExitCode(result)).toBe(2); + expect(`${result.stdout}${result.stderr}`).toContain("deleted or missing paths require the full suite"); + }); + + it("builds and starts an isolated production server for Playwright", () => { + const runner = readFileSync(new URL("../scripts/run-playwright.mjs", import.meta.url), "utf8"); + expect(runner).toContain('["--max-old-space-size=8192", nextBin, "build", "--webpack"]'); + expect(runner).toContain('[nextBin, "start", "--hostname"'); + expect(runner).not.toContain('[nextBin, "dev", "--hostname"'); + expect(runner).toContain('NODE_ENV: "production"'); + expect(runner).toContain('PLAYWRIGHT_OFFLINE_MODE: "true"'); + expect(runner).toContain('NEXT_PUBLIC_MOCKUPS_ENABLED: mockupProjectRequested ? "true" : "false"'); + expect(runner).not.toContain("supabase.co"); + }); +}); diff --git a/tests/tsx-server-only-runner.test.ts b/tests/tsx-server-only-runner.test.ts index 8f8543c5d..74cfe16b5 100644 --- a/tests/tsx-server-only-runner.test.ts +++ b/tests/tsx-server-only-runner.test.ts @@ -47,11 +47,11 @@ describe("standalone TSX server-only compatibility", () => { ); }); - it("bounds Vitest workers and scopes stale-process cleanup to this checkout", () => { + it("bounds Vitest workers and uses the shared non-destructive run lock", () => { const runner = readFileSync(new URL("../scripts/run-vitest.mjs", import.meta.url), "utf8"); const config = readFileSync(new URL("../vitest.config.mts", import.meta.url), "utf8"); - expect(runner).toContain("const vitestNeedle = vitestBin.toLowerCase()"); - expect(runner).not.toContain("repoNeedle"); + expect(runner).toContain("acquireHeavyRunLock"); + expect(runner).not.toContain("taskkill"); expect(config).toContain("maxWorkers: 2"); expect(config).toContain("testTimeout: 30_000"); }); diff --git a/tests/ui-accessibility.spec.ts b/tests/ui-accessibility.spec.ts index ec02f3635..074aa4307 100644 --- a/tests/ui-accessibility.spec.ts +++ b/tests/ui-accessibility.spec.ts @@ -35,7 +35,7 @@ async function mockMinimalDashboardApi(page: Page) { async function gotoApp(page: Page) { await page.goto("/", { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => undefined); + await expect(page.locator("body")).toBeVisible(); } async function expectNoPageHorizontalOverflow(page: Page) { diff --git a/tests/ui-formulation.spec.ts b/tests/ui-formulation.spec.ts index ca14a5205..460f1f0cb 100644 --- a/tests/ui-formulation.spec.ts +++ b/tests/ui-formulation.spec.ts @@ -20,7 +20,7 @@ async function blockExternalRequests(page: Page) { async function gotoApp(page: Page, path: string) { await page.goto(path, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => undefined); + await expect(page.locator("body")).toBeVisible(); } async function expectNoHorizontalOverflow(page: Page) { diff --git a/tests/ui-smoke.spec.ts b/tests/ui-smoke.spec.ts index 923ce8ed0..fd3d7b030 100644 --- a/tests/ui-smoke.spec.ts +++ b/tests/ui-smoke.spec.ts @@ -50,10 +50,10 @@ async function installClipboardMock(page: Page) { async function gotoApp(page: Page, path: string) { await page.goto(path, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => undefined); + await expect(page.locator("body")).toBeVisible(); } -async function waitForReactEventHandler(locator: Locator, eventName: "onScroll") { +async function waitForReactEventHandler(locator: Locator, eventName: "onChange" | "onClick" | "onScroll" | "onSubmit") { await expect .poll( async () => @@ -102,6 +102,20 @@ function visibleAnswerSubmitButton(page: Page) { return page.locator('[aria-label="Generate source-backed answer"]:visible').first(); } +async function submitDocumentSearch(page: Page) { + const submit = page.getByRole("button", { name: "Find matching documents" }); + await expect(submit).toBeEnabled(); + await waitForReactEventHandler(submit.locator("xpath=ancestor::form[1]"), "onSubmit"); + const response = page.waitForResponse( + (candidate) => new URL(candidate.url()).pathname === "/api/search" && candidate.ok(), + { timeout: 30_000 }, + ); + await Promise.all([response, submit.click()]); + await expect(page.getByRole("heading", { name: "Finding matching documents" })).toHaveCount(0, { + timeout: 30_000, + }); +} + function visibleAnswerFollowUpSuggestions(page: Page) { return page .locator( @@ -135,10 +149,9 @@ async function switchToDocumentSearchMode(page: Page) { const legacyDocumentsMode = page.getByRole("button", { name: "Switch to document search mode" }); if (await isVisibleWithoutThrow(legacyDocumentsMode)) { await expect(legacyDocumentsMode).toBeEnabled(); - await expect(async () => { - await legacyDocumentsMode.click(); - await expect(legacyDocumentsMode).toHaveAttribute("aria-pressed", "true", { timeout: uiAssertionTimeoutMs }); - }).toPass({ timeout: 8_000 }); + await waitForReactEventHandler(legacyDocumentsMode, "onClick"); + await legacyDocumentsMode.click(); + await expect(legacyDocumentsMode).toHaveAttribute("aria-pressed", "true", { timeout: uiAssertionTimeoutMs }); return; } @@ -149,18 +162,15 @@ async function switchToDocumentSearchMode(page: Page) { ); } await expect(appModeMenu).toBeEnabled(); - - await expect(async () => { - if ((await appModeMenu.getAttribute("aria-expanded")) !== "true") { - await appModeMenu.click({ force: true }); - } - const appModeGroup = page.getByRole("menu", { name: "Choose app mode" }); - await expect(appModeGroup).toBeVisible({ timeout: uiAssertionTimeoutMs }); - const documentsMode = appModeGroup.getByRole("menuitemradio", { name: /^Documents\b/ }); - await expect(documentsMode).toBeVisible({ timeout: uiAssertionTimeoutMs }); - await documentsMode.click({ force: true }); - await expect(appModeMenu).toHaveAccessibleName("Mode Documents", { timeout: uiAssertionTimeoutMs }); - }).toPass({ timeout: 8_000 }); + await waitForReactEventHandler(appModeMenu, "onClick"); + await appModeMenu.click({ force: true }); + const appModeGroup = page.getByRole("menu", { name: "Choose app mode" }); + await expect(appModeGroup).toBeVisible({ timeout: uiAssertionTimeoutMs }); + const documentsMode = appModeGroup.getByRole("menuitemradio", { name: /^Documents\b/ }); + await expect(documentsMode).toBeVisible({ timeout: uiAssertionTimeoutMs }); + await waitForReactEventHandler(documentsMode, "onClick"); + await documentsMode.click({ force: true }); + await expect(appModeMenu).toHaveAccessibleName("Mode Documents", { timeout: uiAssertionTimeoutMs }); } const readySetupChecks = [ @@ -652,23 +662,20 @@ async function openScopeControl(page: Page) { const composer = page.locator('[aria-label^="Search indexed guidelines by question or keyword"]:visible').first(); - await expect(async () => { - await composer.click(); - const scopeOption = page.getByRole("option", { name: /Scope sources/i }); - if (await scopeOption.isVisible({ timeout: 2_000 }).catch(() => false)) { - await scopeOption.click(); - } else { - const actionMenu = page.getByRole("button", { name: "Open answer options" }); - await expect(actionMenu).toBeVisible(); - await actionMenu.click(); - const actionsMenu = page.getByTestId("daily-actions-menu"); - await expect(actionsMenu).toBeVisible({ timeout: uiAssertionTimeoutMs }); - await actionsMenu.getByRole("menuitem", { name: /^Scope\b/ }).click(); - } - await expect(page.getByTestId("scope-command-popover")).toBeVisible({ - timeout: uiAssertionTimeoutMs, - }); - }).toPass({ timeout: 15_000 }); + await composer.click(); + const scopeOption = page.getByRole("option", { name: /Scope sources/i }); + if (await scopeOption.isVisible({ timeout: 2_000 }).catch(() => false)) { + await scopeOption.click(); + } else { + const actionMenu = page.getByRole("button", { name: "Open answer options" }); + await expect(actionMenu).toBeVisible(); + await waitForReactEventHandler(actionMenu, "onClick"); + await actionMenu.click(); + const actionsMenu = page.getByTestId("daily-actions-menu"); + await expect(actionsMenu).toBeVisible({ timeout: uiAssertionTimeoutMs }); + await actionsMenu.getByRole("menuitem", { name: /^Scope\b/ }).click(); + } + await expect(page.getByTestId("scope-command-popover")).toBeVisible({ timeout: uiAssertionTimeoutMs }); } async function expectMinTouchTarget(locator: Locator, minSize = 44) { @@ -707,18 +714,17 @@ async function openMobileTableFullscreen(page: Page, clinicalTable: Locator) { await scrollMobileTableExpandClearOfFooter(page, clinicalTable); const expandButton = clinicalTable.getByTestId("table-expand-button"); const tableDialog = page.getByTestId("table-fullscreen-dialog"); - await expect(async () => { - if (await tableDialog.isVisible().catch(() => false)) return; - await expect(expandButton).toBeVisible(); - await expandButton.click(); - await expect(tableDialog).toBeVisible({ timeout: 2_000 }); - }).toPass({ timeout: 15_000 }); + await expect(expandButton).toBeVisible(); + await waitForReactEventHandler(expandButton, "onClick"); + await expandButton.click(); + await expect(tableDialog).toBeVisible({ timeout: 15_000 }); return tableDialog; } async function openMobileClinicalGuideMenu(page: Page) { const trigger = page.getByRole("button", { name: "Open Clinical Guide menu" }); await expect(trigger).toBeVisible(); + await waitForReactEventHandler(trigger, "onClick"); await trigger.click(); const menu = page.getByRole("dialog", { name: "Clinical Guide" }); @@ -770,11 +776,9 @@ async function openGuide(page: Page) { const trigger = (await expandedGuide.isVisible().catch(() => false)) ? expandedGuide : railGuide; await expect(trigger).toBeVisible(); await expect(trigger).toBeEnabled(); - await expect(async () => { - if (await dialog.isVisible().catch(() => false)) return; - await trigger.click(); - await expect(dialog).toBeVisible({ timeout: uiAssertionTimeoutMs }); - }).toPass({ timeout: 10_000 }); + await waitForReactEventHandler(trigger, "onClick"); + await trigger.click(); + await expect(dialog).toBeVisible({ timeout: uiAssertionTimeoutMs }); } else { const menu = await openMobileClinicalGuideMenu(page); await menu.getByRole("button", { name: "Guide & help" }).click(); @@ -825,59 +829,15 @@ async function expectAccountSetupSurface(setup: Locator) { await expect(setup).toContainText("Do not enter patient-identifying information."); } -async function openUploadDrawer(page: Page) { +async function expectAdminOnlyUploadNotice(page: Page) { const uploadButton = page.getByRole("button", { name: /Upload document/i }); - const uploadDrawer = page.getByRole("dialog", { name: "Upload and indexing" }); await expect(uploadButton).toBeVisible(); - - await expect(async () => { - if (await uploadDrawer.isVisible().catch(() => false)) return; - await uploadButton.click(); - await expect(uploadDrawer).toBeVisible({ timeout: uiAssertionTimeoutMs }); - }).toPass({ timeout: 8_000 }); - - return uploadDrawer; -} - -async function openUploadSurface(page: Page) { - const uploadButton = page.getByRole("button", { name: /Upload document/i }); - const uploadDialog = page.getByRole("dialog", { name: "Upload and indexing" }); - const inlineUploadPanel = page.getByRole("tabpanel", { name: "Upload", exact: true }); - const visibleUploadTab = page - .locator('[role="tab"]:visible') - .filter({ hasText: /^Upload\b/ }) - .first(); - await expect(uploadButton).toBeVisible(); - await expect(async () => { - if ( - (await uploadDialog.isVisible().catch(() => false)) || - (await inlineUploadPanel.isVisible().catch(() => false)) || - (await visibleUploadTab.isVisible().catch(() => false)) - ) { - return; - } - await uploadButton.click(); - await expect - .poll( - async () => - (await uploadDialog.isVisible().catch(() => false)) || - (await inlineUploadPanel.isVisible().catch(() => false)) || - (await visibleUploadTab.isVisible().catch(() => false)), - ) - .toBe(true); - }).toPass({ timeout: 8_000 }); - - if (await uploadDialog.isVisible().catch(() => false)) { - const uploadTab = uploadDialog.getByRole("tab", { name: /^Upload\b/ }); - if (await uploadTab.isVisible().catch(() => false)) await uploadTab.click(); - const dialogUploadPanel = uploadDialog.getByRole("tabpanel", { name: "Upload", exact: true }); - await expect(dialogUploadPanel).toBeVisible(); - return dialogUploadPanel; - } - - if (!(await inlineUploadPanel.isVisible().catch(() => false))) await visibleUploadTab.click(); - await expect(inlineUploadPanel).toBeVisible(); - return inlineUploadPanel; + await waitForReactEventHandler(uploadButton, "onClick"); + await uploadButton.click(); + await expect(page.getByRole("alert").filter({ hasText: "Upload and indexing tools are admin-only." })).toContainText( + "Use the source library to open indexed documents.", + ); + await expect(page.getByRole("dialog", { name: "Upload and indexing" })).toHaveCount(0); } async function dismissOverlayByHeaderClick(page: Page) { @@ -892,11 +852,9 @@ async function openDailyActions(page: Page, triggerName: string | RegExp = /^Ope await expect(trigger).toBeVisible(); await expect(trigger).toBeEnabled(); - await expect(async () => { - if (await menu.isVisible().catch(() => false)) return; - await trigger.click(); - await expect(menu).toBeVisible({ timeout: uiAssertionTimeoutMs }); - }).toPass({ timeout: 20_000 }); + await waitForReactEventHandler(trigger, "onClick"); + await trigger.click(); + await expect(menu).toBeVisible({ timeout: uiAssertionTimeoutMs }); return menu; } @@ -1075,8 +1033,6 @@ test.describe("Clinical KB UI smoke coverage", () => { await gotoApp(page, route.path); if (route.path.includes("mode=answer")) { await waitForDemoDashboardReady(page); - } else { - await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => undefined); } const activeLink = page.getByRole("link", { name: route.label, exact: true }); @@ -1205,7 +1161,7 @@ test.describe("Clinical KB UI smoke coverage", () => { await expectNoPageHorizontalOverflow(page); }); - test("private mode unauthenticated dashboard gates real-mode search", async ({ page }) => { + test("offline browser gate remains in demo mode when private endpoints are mocked", async ({ page }) => { await page.setViewportSize({ width: 390, height: 820 }); const answerRequests: string[] = []; const unsafeLocalProjectPayload = { @@ -1235,7 +1191,7 @@ test.describe("Clinical KB UI smoke coverage", () => { const questionInput = visibleQuestionInput(page); await questionInput.fill("lithium monitoring"); - await expect(page.getByRole("button", { name: "Generate source-backed answer" })).toBeDisabled(); + await expect(page.getByRole("button", { name: "Generate source-backed answer" })).toBeEnabled(); await expect(page.getByTestId("answer-grounding-chip")).toHaveCount(0); expect(answerRequests).toEqual([]); await expect(page.getByRole("heading", { level: 1, name: "Clinical Guide" })).toBeVisible(); @@ -1657,14 +1613,6 @@ test.describe("Clinical KB UI smoke coverage", () => { await composerPrivacyLink.focus(); await expect(composerPrivacyLink).toBeFocused(); - const uploadSurface = await openUploadSurface(page); - await expect(uploadSurface.getByRole("note")).toBeVisible(); - await expect(uploadSurface.getByText("Do not enter patient-identifiable information.")).toBeVisible(); - const uploadPrivacyLink = uploadSurface.getByRole("link", { name: "Privacy and data processing" }); - await expect(uploadPrivacyLink).toBeVisible(); - await uploadPrivacyLink.focus(); - await expect(uploadPrivacyLink).toBeFocused(); - await page.goto("/privacy", { waitUntil: "domcontentloaded" }); await expect(page.getByRole("main")).toBeVisible(); await expect(page.getByRole("heading", { level: 1, name: "Privacy & data handling" })).toBeVisible(); @@ -1989,16 +1937,9 @@ test.describe("Clinical KB UI smoke coverage", () => { const medicationLink = strip.getByRole("link", { name: "Clozapine", exact: true }); await expect(medicationLink).toHaveAttribute("href", "/medications/clozapine"); - // Re-click on each retry: a single click can be swallowed while the answer - // surface is still hydrating (same guard as the launcher mode switches), and - // app-router URLs only commit after the destination responds — a cold dev - // compile of /medications/[slug] can take ~30s when this spec runs without - // the @critical warm-up journeys. - await expect(async () => { - if (/\/medications\/clozapine/.test(page.url())) return; - await medicationLink.click(); - await expect(page).toHaveURL(/\/medications\/clozapine/, { timeout: 5_000 }); - }).toPass({ timeout: 45_000 }); + await waitForReactEventHandler(medicationLink, "onClick"); + await medicationLink.click(); + await expect(page).toHaveURL(/\/medications\/clozapine/, { timeout: 45_000 }); await expectNoPageHorizontalOverflow(page); }); @@ -2666,9 +2607,14 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(page.getByText("Source library workspace")).toHaveCount(0); await expect(page.getByText("Document display")).toHaveCount(0); + // The mode switch above is covered independently. Submit from the canonical + // route so a dev-only cross-segment remount cannot abort the mocked POST. + await gotoApp(page, "/documents/search?mode=documents"); const questionInput = visibleQuestionInput(page); + await expect(questionInput).toBeVisible(); + await waitForReactEventHandler(questionInput, "onChange"); await questionInput.fill("lithium monitoring"); - await page.getByRole("button", { name: "Find matching documents" }).click(); + await submitDocumentSearch(page); await expect(page).toHaveURL(/\/documents\/search\?.*q=lithium\+monitoring/); const documentResults = page.getByRole("article").filter({ hasText: "Synthetic Lithium Monitoring Protocol" }); @@ -2715,11 +2661,13 @@ test.describe("Clinical KB UI smoke coverage", () => { await gotoApp(page, "/"); await waitForDemoDashboardReady(page); - await switchToDocumentSearchMode(page); + await gotoApp(page, "/documents/search?mode=documents"); const questionInput = visibleQuestionInput(page); + await expect(questionInput).toBeVisible(); + await waitForReactEventHandler(questionInput, "onChange"); await questionInput.fill("what is the best coffee machine for my kitchen"); - await page.getByRole("button", { name: "Find matching documents" }).click(); + await submitDocumentSearch(page); await expect(page).toHaveURL(/\/documents\/search\?/); await expect(page.locator("body")).not.toContainText(/failed to fetch|Search failed/i); await expect(page.getByRole("heading", { name: "No matching documents" }).first()).toBeVisible(); @@ -3226,58 +3174,26 @@ test.describe("Clinical KB UI smoke coverage", () => { expect(JSON.stringify(payload)).not.toMatch(/sk-|service_role|eyJ/i); }); - test("upload drawer exposes setup checklist and explicit upload labels", async ({ page, request }) => { + test("production upload action remains admin-only for unauthenticated users", async ({ page, request }) => { await page.setViewportSize({ width: 414, height: 820 }); await mockPrivateUnauthenticatedApi(page); - // Upload availability depends on the checkout's env config: env-less servers run in - // read-only demo mode while .env.local local-auth servers accept uploads. The browser - // mocks above do not decide enablement, so read the real server flag and branch the - // enablement assertions on it to keep this test green in both configurations. const setupStatusResponse = await request.get("/api/setup-status"); expect(setupStatusResponse.ok()).toBe(true); - const serverDemoMode = (await setupStatusResponse.json()).demoMode === true; + expect((await setupStatusResponse.json()).demoMode).toBe(true); await gotoApp(page, "/"); await expect(visibleQuestionInput(page)).toBeVisible(); - const uploadDrawer = await openUploadDrawer(page); - - await uploadDrawer.getByRole("tab", { name: /Setup/ }).click(); - await expect(uploadDrawer.getByText("First-run setup checklist")).toBeVisible(); - await expect(uploadDrawer.getByText(".env.local configured")).toBeVisible(); - await expect(uploadDrawer.getByText("Clinical KB Database target")).toBeVisible(); - await expect(uploadDrawer.getByText("supabase/schema.sql applied")).toBeVisible(); - await expect(uploadDrawer.getByText("Search RPC and vector indexes")).toBeVisible(); - await expect(uploadDrawer.getByText("OpenAI API key available")).toBeVisible(); - await expect(uploadDrawer.getByText("npm run worker running")).toBeVisible(); - const uploadTab = uploadDrawer.getByRole("tab", { name: /Upload/ }); - await uploadTab.click(); - await expect(uploadDrawer.getByText("Clinical upload")).toBeVisible(); - await expect(uploadDrawer.getByText("Guideline PDF files")).toBeVisible(); - if (serverDemoMode) { - await expect(uploadTab).toContainText("Locked"); - await expect(uploadDrawer.getByRole("button", { name: "Guideline PDF files" })).toBeDisabled(); - } else { - await expect(uploadTab).toContainText("Ready"); - await expect(uploadDrawer.getByRole("button", { name: "Guideline PDF files" })).toBeEnabled(); - } - await expect(uploadDrawer.getByRole("button", { name: "Upload guidelines" })).toBeVisible(); + await expectAdminOnlyUploadNotice(page); await expectNoPageHorizontalOverflow(page); }); - test("upload drawer disables uploads in demo mode", async ({ page }) => { + test("demo upload action cannot bypass the production admin gate", async ({ page }) => { await page.setViewportSize({ width: 414, height: 820 }); await mockDemoApi(page); await gotoApp(page, "/"); - const uploadDrawer = await openUploadDrawer(page); - - await uploadDrawer.getByRole("tab", { name: /Jobs/ }).click(); - await expect(uploadDrawer.getByText("Indexing progress")).toBeVisible(); - await uploadDrawer.getByRole("tab", { name: /Upload/ }).click(); - await expect(uploadDrawer.getByText("Read-only status")).toBeVisible(); - await expect(uploadDrawer.getByRole("button", { name: "Guideline PDF files" })).toBeDisabled(); - await expect(uploadDrawer.getByRole("button", { name: "Upload guidelines" })).toBeVisible(); + await expectAdminOnlyUploadNotice(page); await expectNoPageHorizontalOverflow(page); }); diff --git a/tests/ui-stress.spec.ts b/tests/ui-stress.spec.ts index 280d9260e..1b86c7982 100644 --- a/tests/ui-stress.spec.ts +++ b/tests/ui-stress.spec.ts @@ -275,7 +275,7 @@ test.describe("Clinical KB long-content stress coverage", () => { await mockStressData(page); await page.setViewportSize({ width: viewport.width, height: viewport.height }); await page.goto("/?mode=documents", { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => undefined); + await expect(page.locator("body")).toBeVisible(); if (viewport.name === "mobile") { const dailyActions = await openDailyActions(page); @@ -283,33 +283,18 @@ test.describe("Clinical KB long-content stress coverage", () => { // the tap lands on Upload rather than an adjacent row mid-animation. await dailyActions.getByRole("menuitem", { name: /Upload(?: PDF)?/ }).click(); await expect(dailyActions).toBeHidden(); - const uploadSurface = page.getByRole("dialog", { name: "Upload and indexing" }); - await expect(uploadSurface).toBeVisible(); - await expect(uploadSurface.getByRole("button", { name: "Show indexed document files" })).toContainText( - "24 indexed", - { timeout: 20_000 }, - ); - const closeUploadSheet = page.getByRole("button", { name: "Close Upload and indexing" }); - if (await closeUploadSheet.isVisible().catch(() => false)) { - await closeUploadSheet.click(); - } + await expect( + page.getByRole("alert").filter({ hasText: "Upload and indexing tools are admin-only." }), + ).toContainText("Use the source library to open indexed documents."); + await expect(page.getByRole("dialog", { name: "Upload and indexing" })).toHaveCount(0); } await expectNoPageHorizontalOverflow(page); - const legacyAnswerModeToggle = page.getByRole("button", { name: "Switch to answer mode" }); - if (await legacyAnswerModeToggle.isVisible().catch(() => false)) { - await legacyAnswerModeToggle.click(); - } else { - const appModeMenu = page.getByRole("button", { name: /^Mode / }); - await expect(appModeMenu).toBeVisible(); - await appModeMenu.click({ force: true }); - const answerMode = page - .getByRole("menu", { name: "Choose app mode" }) - .getByRole("menuitemradio", { name: /^Answer\b/ }); - await expect(answerMode).toBeVisible(); - await answerMode.click({ force: true }); - await expect(page.getByRole("button", { name: "Mode Answer" })).toBeVisible(); - } + // Mode-menu behavior is covered by the launcher suites. This stress test + // owns only dense answer rendering, so enter that route directly rather + // than making its result depend on an unrelated menu transition. + await page.goto("/?mode=answer", { waitUntil: "domcontentloaded" }); + await expect(page.getByRole("button", { name: "Mode Answer" })).toBeVisible(); await page .locator('[aria-label^="Search indexed guidelines by question or keyword"]:visible') diff --git a/tests/ui-tools-collapse.spec.ts b/tests/ui-tools-collapse.spec.ts index d5d31b011..216cb69bf 100644 --- a/tests/ui-tools-collapse.spec.ts +++ b/tests/ui-tools-collapse.spec.ts @@ -6,7 +6,7 @@ import { expect, test, type Page } from "playwright/test"; async function goto(page: Page, path: string) { await page.goto(path, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => undefined); + await expect(page.locator("body")).toBeVisible(); } // The shell's expanded sidebar (now the desktop default) contributes its own diff --git a/tests/ui-tools-task-directory.spec.ts b/tests/ui-tools-task-directory.spec.ts index fe01d6731..44f65a366 100644 --- a/tests/ui-tools-task-directory.spec.ts +++ b/tests/ui-tools-task-directory.spec.ts @@ -7,7 +7,7 @@ const PATH = "/mockups/tools-task-directory"; async function goto(page: Page, path: string) { await page.goto(path, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => undefined); + await expect(page.locator("body")).toBeVisible(); } async function expectNoHorizontalOverflow(page: Page) { diff --git a/tests/ui-tools.spec.ts b/tests/ui-tools.spec.ts index d3f4e0d72..b1fbbfc49 100644 --- a/tests/ui-tools.spec.ts +++ b/tests/ui-tools.spec.ts @@ -49,6 +49,32 @@ async function blockExternalRequests(page: Page) { }); } +function waitForDifferentialCatalogQuery(page: Page, query: string) { + const expectedQuery = query.trim().toLowerCase(); + return Promise.all( + ["diagnosis", "presentation"].map((kind) => + page.waitForResponse( + (response) => { + const url = new URL(response.url()); + return ( + url.pathname === "/api/differentials" && + url.searchParams.get("kind") === kind && + url.searchParams.get("q")?.trim().toLowerCase() === expectedQuery && + response.ok() + ); + }, + { timeout: 30_000 }, + ), + ), + ); +} + +async function submitDifferentialSearch(page: Page, query: string) { + const submit = page.locator('button[aria-label="Search differential presentations"]:visible'); + await expect(submit).toBeEnabled(); + await Promise.all([waitForDifferentialCatalogQuery(page, query), submit.click()]); +} + async function mockAnswerDashboardApi(page: Page) { await blockExternalRequests(page); await page.route(/\/api\/local-project-id$/, async (route) => { @@ -181,10 +207,22 @@ function launcherLaunchLink(page: Page, title: string) { async function gotoLauncher(page: Page, path = "/?mode=tools") { await page.goto(path, { waitUntil: "domcontentloaded" }); - // Most launcher routes keep background fetches alive, so a long networkidle wait - // burns the full timeout on every navigation without adding signal. A short grace - // period is enough; the per-route assertions below wait on real UI readiness. - await page.waitForLoadState("networkidle", { timeout: 2_000 }).catch(() => undefined); + await expect(page.locator("body")).toBeVisible(); +} + +async function waitForReactEventHandler(locator: Locator, eventName: "onClick" | "onSubmit" = "onClick") { + await expect + .poll( + async () => + locator.evaluate((element, reactEventName) => { + const propsKey = Object.keys(element).find((key) => key.startsWith("__reactProps$")); + if (!propsKey) return false; + const props = (element as unknown as Record<string, Record<string, unknown>>)[propsKey]; + return typeof props?.[reactEventName] === "function"; + }, eventName), + { timeout: 15_000 }, + ) + .toBe(true); } async function expectNoPageHorizontalOverflow(page: Page) { @@ -263,11 +301,9 @@ async function openAppModeMenu(page: Page, currentMode: string) { const menu = page.getByRole("menu", { name: "Choose app mode" }); await expect(trigger).toBeVisible(); - await expect(async () => { - if (await menu.isVisible().catch(() => false)) return; - await trigger.click(); - await expect(menu).toBeVisible({ timeout: 5_000 }); - }).toPass({ timeout: 20_000 }); + await waitForReactEventHandler(trigger); + await trigger.click(); + await expect(menu).toBeVisible({ timeout: 20_000 }); return menu; } @@ -382,15 +418,11 @@ test.describe("Clinical KB tools launcher", () => { await page.addInitScript(() => window.localStorage.setItem("clinical-kb-sidebar-collapsed", "1")); await gotoLauncher(page, "/?mode=answer"); - // Re-open + re-click on each retry: a single click can be swallowed while the - // launcher is still hydrating, leaving the route on /?mode=answer. Mirrors the - // Forms switch below, which already guards against the same flake. - await expect(async () => { - if (/\/services$/.test(page.url())) return; - const menu = await openAppModeMenu(page, "Answer"); - await menu.getByRole("menuitemradio", { name: /^Services\b/ }).click(); - await expect(page).toHaveURL(/\/services$/, { timeout: 3_000 }); - }).toPass({ timeout: 20_000 }); + const answerMenu = await openAppModeMenu(page, "Answer"); + const servicesMode = answerMenu.getByRole("menuitemradio", { name: /^Services\b/ }); + await waitForReactEventHandler(servicesMode); + await servicesMode.click(); + await expect(page).toHaveURL(/\/services$/, { timeout: 20_000 }); await expect(page).toHaveURL(/\/services$/); await expect(page.getByRole("button", { name: "Mode Services" })).toBeVisible(); @@ -416,12 +448,10 @@ test.describe("Clinical KB tools launcher", () => { await expect(servicesMenu.getByRole("menuitemradio", { name: /^Differentials\b/ })).toBeVisible(); await expect(servicesMenu.getByRole("menuitemradio", { name: /^Medication\b/ })).toBeVisible(); await expect(servicesMenu.getByRole("menuitemradio", { name: /^Tools\b/ })).toBeVisible(); - await expect(async () => { - if (/\/forms$/.test(page.url())) return; - const menu = await openAppModeMenu(page, "Services"); - await menu.getByRole("menuitemradio", { name: /^Forms\b/ }).click(); - await expect(page).toHaveURL(/\/forms$/, { timeout: 2_000 }); - }).toPass({ timeout: 20_000 }); + const formsMode = servicesMenu.getByRole("menuitemradio", { name: /^Forms\b/ }); + await waitForReactEventHandler(formsMode); + await formsMode.click(); + await expect(page).toHaveURL(/\/forms$/, { timeout: 20_000 }); await expect(page.getByRole("button", { name: "Mode Forms" })).toBeVisible(); await expect(page.getByTestId("forms-home")).toBeVisible(); await expect(page.getByTestId("form-search-results")).toHaveCount(0); @@ -516,20 +546,19 @@ test.describe("Clinical KB tools launcher", () => { // one dashboard-shell home and one standalone-shell home; the full 5-route // design spec stays in the advisory "mode home routes center the shared // search on mobile" test above. - test("mode home search composer is always present at phone and desktop widths @critical", async ({ page }) => { - test.setTimeout(120_000); - await mockAnswerDashboardApi(page); - - for (const viewport of [ - { name: "phone", width: 390, height: 820 }, - { name: "desktop", width: 1280, height: 900 }, + for (const viewport of [ + { name: "phone", width: 390, height: 820 }, + { name: "desktop", width: 1280, height: 900 }, + ] as const) { + for (const home of [ + { path: "/?mode=answer", testId: "answer-empty-state" }, + { path: "/services", testId: "services-home" }, ] as const) { - await page.setViewportSize({ width: viewport.width, height: viewport.height }); - - for (const home of [ - { path: "/?mode=answer", testId: "answer-empty-state" }, - { path: "/services", testId: "services-home" }, - ] as const) { + test(`mode home search composer is present at ${viewport.name} width on ${home.path} @critical`, async ({ + page, + }) => { + await mockAnswerDashboardApi(page); + await page.setViewportSize({ width: viewport.width, height: viewport.height }); await gotoLauncher(page, home.path); await expect(page.getByTestId(home.testId)).toBeVisible(); // The composer must never vanish: exactly one visible search input. @@ -537,9 +566,9 @@ test.describe("Clinical KB tools launcher", () => { // Hero-centred design: the input lives inside the mode-home hero at // every width, phones included. await expect(page.getByTestId(home.testId).getByTestId("global-search-input")).toBeVisible(); - } + }); } - }); + } test("answer composer keeps the PHI warning visible before submission @critical", async ({ page }) => { await mockAnswerDashboardApi(page); @@ -565,25 +594,18 @@ test.describe("Clinical KB tools launcher", () => { } }); - test("all mode home heroes share identical sizing on mobile", async ({ page }) => { - test.setTimeout(150_000); - await mockAnswerDashboardApi(page); - await page.setViewportSize({ width: 390, height: 820 }); - - // Every mode home renders the shared compact ModeHomeHero, so the icon box - // and type scale must be identical across modes. Baseline: Answer. - let baseline: { iconWidth: number; iconHeight: number; headingFontSize: number; subtitleFontSize: number } | null = - null; - - for (const home of [ - { path: "/?mode=answer", testId: "answer-empty-state", heroTestId: "answer-empty-state" }, - { path: "/?mode=documents", testId: "document-search-empty-state", heroTestId: "document-search-empty-state" }, - { path: "/?mode=prescribing", testId: "medication-home", heroTestId: "medication-home" }, - { path: "/?mode=tools", testId: "tools-home", heroTestId: "tools-home" }, - { path: "/services", testId: "services-home", heroTestId: "services-home-template" }, - { path: "/forms", testId: "forms-home", heroTestId: "forms-home-template" }, - { path: "/differentials", testId: "differentials-home", heroTestId: "differentials-home-template" }, - ] as const) { + for (const home of [ + { path: "/?mode=answer", testId: "answer-empty-state", heroTestId: "answer-empty-state" }, + { path: "/?mode=documents", testId: "document-search-empty-state", heroTestId: "document-search-empty-state" }, + { path: "/?mode=prescribing", testId: "medication-home", heroTestId: "medication-home" }, + { path: "/?mode=tools", testId: "tools-home", heroTestId: "tools-home" }, + { path: "/services", testId: "services-home", heroTestId: "services-home-template" }, + { path: "/forms", testId: "forms-home", heroTestId: "forms-home-template" }, + { path: "/differentials", testId: "differentials-home", heroTestId: "differentials-home-template" }, + ] as const) { + test(`mode home hero uses the shared mobile sizing on ${home.path}`, async ({ page }) => { + await mockAnswerDashboardApi(page); + await page.setViewportSize({ width: 390, height: 820 }); await gotoLauncher(page, home.path); const homeRegion = page.getByTestId(home.testId); await expect(homeRegion).toBeVisible(); @@ -608,20 +630,14 @@ test.describe("Clinical KB tools launcher", () => { headingFontSize, subtitleFontSize, }; - if (!baseline) { - baseline = metrics; - // Compact hero mobile scale: 2.75rem icon, 1.45rem heading, 0.875rem subtitle. - expect(metrics.iconWidth).toBe(44); - expect(metrics.iconHeight).toBe(44); - expect(metrics.headingFontSize).toBeCloseTo(23.2, 1); - expect(metrics.subtitleFontSize).toBeCloseTo(14, 1); - } else { - expect(metrics, `${home.path} hero metrics`).toEqual(baseline); - } + expect(metrics.iconWidth).toBe(44); + expect(metrics.iconHeight).toBe(44); + expect(metrics.headingFontSize).toBeCloseTo(23.2, 1); + expect(metrics.subtitleFontSize).toBeCloseTo(14, 1); await expectNoPageHorizontalOverflow(page); - } - }); + }); + } test("phone bottom-dock search opens the bounded command results sheet", async ({ page }) => { await page.setViewportSize({ width: 390, height: 820 }); @@ -629,7 +645,6 @@ test.describe("Clinical KB tools launcher", () => { await expect(page.getByRole("button", { name: "Mode Services" })).toBeVisible(); const input = visibleGlobalSearchInput(page).first(); await expect(input).toBeVisible(); - await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => undefined); // Phone searches use the same accessible listbox as larger viewports, bounded // above the bottom dock so results stay reachable without horizontal overflow. @@ -681,30 +696,27 @@ test.describe("Clinical KB tools launcher", () => { await expectNoPageHorizontalOverflow(page); }); - test("mode home routes center the shared search from tablet up", async ({ page }) => { - test.setTimeout(150_000); - await mockAnswerDashboardApi(page); - - for (const viewport of [ - { name: "tablet", width: 768, height: 1024 }, - { name: "desktop", width: 1280, height: 900 }, + for (const viewport of [ + { name: "tablet", width: 768, height: 1024 }, + { name: "desktop", width: 1280, height: 900 }, + ] as const) { + for (const home of [ + { path: "/?mode=answer", testId: "answer-empty-state", heading: "How can I help?", headingLevel: 2 }, + { path: "/?mode=documents", testId: "document-search-empty-state", heading: "Documents", headingLevel: 2 }, + { + path: "/?mode=prescribing", + testId: "medication-home", + heading: "Medication prescribing", + headingLevel: 2, + }, + { path: "/services", testId: "services-home", heading: "Find a service", headingLevel: 1 }, + { path: "/forms", testId: "forms-home", heading: "Forms", headingLevel: 1 }, + { path: "/differentials", testId: "differentials-home", heading: "Differentials", headingLevel: 1 }, + { path: "/tools", testId: "tools-home", heading: "Tools", headingLevel: 1 }, ] as const) { - await page.setViewportSize({ width: viewport.width, height: viewport.height }); - - for (const home of [ - { path: "/?mode=answer", testId: "answer-empty-state", heading: "How can I help?", headingLevel: 2 }, - { path: "/?mode=documents", testId: "document-search-empty-state", heading: "Documents", headingLevel: 2 }, - { - path: "/?mode=prescribing", - testId: "medication-home", - heading: "Medication prescribing", - headingLevel: 2, - }, - { path: "/services", testId: "services-home", heading: "Find a service", headingLevel: 1 }, - { path: "/forms", testId: "forms-home", heading: "Forms", headingLevel: 1 }, - { path: "/differentials", testId: "differentials-home", heading: "Differentials", headingLevel: 1 }, - { path: "/tools", testId: "tools-home", heading: "Tools", headingLevel: 1 }, - ] as const) { + test(`mode home search is centered at ${viewport.name} width on ${home.path}`, async ({ page }) => { + await mockAnswerDashboardApi(page); + await page.setViewportSize({ width: viewport.width, height: viewport.height }); await gotoLauncher(page, home.path); await expect(page.getByTestId(home.testId)).toBeVisible(); await expect(visibleGlobalSearchInput(page)).toHaveCount(1); @@ -735,31 +747,28 @@ test.describe("Clinical KB tools launcher", () => { expect(metrics?.formRight ?? 0).toBeLessThanOrEqual((metrics?.homeRight ?? viewport.width) + 1); expect(Math.abs((metrics?.formCenterX ?? 0) - (metrics?.homeCenterX ?? 0))).toBeLessThanOrEqual(24); await expectNoPageHorizontalOverflow(page); - } + }); } - }); - - test("search result and detail routes keep top search from tablet up", async ({ page }) => { - test.setTimeout(240_000); + } - for (const viewport of [ - { name: "mobile", width: 390, height: 820 }, - { name: "tablet", width: 768, height: 1024 }, - { name: "desktop", width: 1280, height: 900 }, + for (const viewport of [ + { name: "mobile", width: 390, height: 820 }, + { name: "tablet", width: 768, height: 1024 }, + { name: "desktop", width: 1280, height: 900 }, + ] as const) { + for (const route of [ + { path: "/services?q=13YARN&focus=1&run=1", modeButton: "Mode Services", compactBottomSearch: true }, + { path: "/services/13yarn", modeButton: "Mode Services", compactBottomSearch: false }, + { path: "/forms?q=transport&focus=1&run=1", modeButton: "Mode Forms", compactBottomSearch: true }, + { path: "/favourites?q=lithium&focus=1&run=1", modeButton: "Mode Favourites", compactBottomSearch: true }, + { + path: "/differentials?q=acute+confusion&focus=1&run=1", + modeButton: "Mode Differentials", + compactBottomSearch: true, + }, ] as const) { - await page.setViewportSize({ width: viewport.width, height: viewport.height }); - - for (const route of [ - { path: "/services?q=13YARN&focus=1&run=1", modeButton: "Mode Services", compactBottomSearch: true }, - { path: "/services/13yarn", modeButton: "Mode Services", compactBottomSearch: false }, - { path: "/forms?q=transport&focus=1&run=1", modeButton: "Mode Forms", compactBottomSearch: true }, - { path: "/favourites?q=lithium&focus=1&run=1", modeButton: "Mode Favourites", compactBottomSearch: true }, - { - path: "/differentials?q=acute+confusion&focus=1&run=1", - modeButton: "Mode Differentials", - compactBottomSearch: true, - }, - ] as const) { + test(`search route keeps the correct composer at ${viewport.name} width on ${route.path}`, async ({ page }) => { + await page.setViewportSize({ width: viewport.width, height: viewport.height }); await gotoLauncher(page, route.path); await expect(page.getByRole("button", { name: route.modeButton })).toBeVisible({ timeout: 20_000 }); await expect(visibleGlobalSearchInput(page), `${route.path} at ${viewport.name}`).toHaveCount(1, { @@ -785,9 +794,9 @@ test.describe("Clinical KB tools launcher", () => { } await expectNoPageHorizontalOverflow(page); - } + }); } - }); + } test("mode home deep links preserve focus=1 on initial load", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 900 }); @@ -877,7 +886,7 @@ test.describe("Clinical KB tools launcher", () => { test("form detail pages keep the shared forms search wired to form results", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 900 }); await gotoLauncher(page, "/forms/transport-crisis-form"); - await expect(page.getByTestId("form-detail-page")).toBeVisible(); + await expect(page.getByTestId("form-detail-page")).toBeVisible({ timeout: 30_000 }); // Structural coverage — runs on every browser, WebKit included: the form // detail page renders inside the shared shell with the Forms-mode composer @@ -888,25 +897,14 @@ test.describe("Clinical KB tools launcher", () => { const formsSearchInput = page.locator('input[placeholder="Search forms..."]:visible').first(); await expect(formsSearchInput).toBeVisible(); - // The shell now seeds its composer state from the URL and only re-syncs on a - // real navigation, so a programmatic fill on this no-query detail route is no - // longer wiped by a mount-time frame — the race that used to break CI WebKit - // (and could flake Firefox). Drive the fill-and-submit as one retried unit - // regardless, so any residual cross-browser navigation-timing jitter cannot - // flake the route assertion; the assertions below still verify the result. + await expect(page.getByText("Loading your forms registry...")).toBeHidden({ timeout: 30_000 }); const formsSearchButton = page.getByRole("button", { name: "Search forms" }); - await expect(async () => { - // A previous attempt's click may have navigated late — after the inner URL - // wait timed out and triggered a retry. If we have already routed to the - // results page the detail-page input is gone, so re-filling would throw; - // treat the completed navigation as success instead. - if (/\/forms\?/.test(page.url())) return; - await formsSearchInput.fill("transport forms"); - await expect(formsSearchButton).toBeEnabled({ timeout: 1_000 }); - await formsSearchButton.click(); - await expect(page).toHaveURL(/\/forms\?/, { timeout: 2_000 }); - }).toPass({ timeout: 20_000 }); - await expect(page.getByTestId("form-search-results")).toBeVisible(); + await formsSearchInput.fill("transport forms"); + await expect(formsSearchButton).toBeEnabled(); + await waitForReactEventHandler(formsSearchButton.locator("xpath=ancestor::form[1]"), "onSubmit"); + await formsSearchButton.click(); + await expect(page).toHaveURL(/\/forms\?.*\bq=transport(?:\+|%20)forms\b/, { timeout: 20_000 }); + await expect(page.getByTestId("form-search-results")).toBeVisible({ timeout: 20_000 }); await expect(page.getByTestId("form-search-result-transport-crisis-form")).toContainText("Transport order"); await expect( page.getByTestId("form-search-result-transport-crisis-form").getByLabel("Open Transport order"), @@ -1134,8 +1132,7 @@ test.describe("Clinical KB tools launcher", () => { await gotoLauncher(page, "/differentials"); await expect(page.getByRole("button", { name: "Mode Differentials" })).toBeVisible(); await page.locator('input[placeholder="Ask or search a presentation"]:visible').first().fill("acute confusion"); - await expect(page.locator('button[aria-label="Search differential presentations"]:visible')).toBeEnabled(); - await page.locator('button[aria-label="Search differential presentations"]:visible').click(); + await submitDifferentialSearch(page, "acute confusion"); await expect.poll(() => searchRequests.length).toBeGreaterThan(0); expect(searchRequests.at(-1)).toMatchObject({ @@ -1202,7 +1199,7 @@ test.describe("Clinical KB tools launcher", () => { await gotoLauncher(page, "/differentials"); await page.locator('input[placeholder="Ask or search a presentation"]:visible').first().fill("acute confusion"); - await page.locator('button[aria-label="Search differential presentations"]:visible').click(); + await submitDifferentialSearch(page, "acute confusion"); await expect(page.getByTestId("differentials-search-results")).toBeVisible(); const tabs = page.getByTestId("differential-result-type-tabs"); @@ -1298,7 +1295,7 @@ test.describe("Clinical KB tools launcher", () => { await gotoLauncher(page, "/differentials"); await page.locator('input[placeholder="Ask or search a presentation"]:visible').first().fill("acute confusion"); - await page.locator('button[aria-label="Search differential presentations"]:visible').click(); + await submitDifferentialSearch(page, "acute confusion"); await expect(page.getByTestId("differentials-search-results")).toBeVisible(); const tabs = page.getByTestId("differential-result-type-tabs"); diff --git a/tests/ui-visual-artifacts.spec.ts b/tests/ui-visual-artifacts.spec.ts index 7eb0b1cc7..454126b14 100644 --- a/tests/ui-visual-artifacts.spec.ts +++ b/tests/ui-visual-artifacts.spec.ts @@ -2,8 +2,6 @@ import { expect, test, type Page, type TestInfo } from "playwright/test"; const documentPath = "/documents/11111111-1111-4111-8111-111111111111?page=1&chunk=44444444-4444-4444-8444-444444444442"; -const uiLoadTimeoutMs = 15_000; - async function attachViewportScreenshot( page: Page, testInfo: TestInfo, @@ -13,7 +11,6 @@ async function attachViewportScreenshot( ) { await page.setViewportSize(viewport); await page.goto(path, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: uiLoadTimeoutMs }).catch(() => undefined); await expect(page.locator("body")).toBeVisible(); await testInfo.attach(name, { diff --git a/tests/universal-search-owner-live.test.ts b/tests/universal-search-owner.live.test.ts similarity index 69% rename from tests/universal-search-owner-live.test.ts rename to tests/universal-search-owner.live.test.ts index 156a53452..a87dc4c94 100644 --- a/tests/universal-search-owner-live.test.ts +++ b/tests/universal-search-owner.live.test.ts @@ -1,14 +1,6 @@ import { describe, expect, it } from "vitest"; import { isUsableBrowserSupabaseKey } from "../src/lib/supabase/client"; -// Live owner-auth coverage for universal search. The header sign-in UI is magic-link/OAuth -// only (no password field), so browser-login Playwright coverage is not feasible; instead -// this signs in with the E2E password user via supabase-js and exercises the real route -// handler with a genuine bearer token — the same path the typeahead hook uses. -// -// Skips (never fails) when the live env is absent: demo/offline environments and keys-free -// CI run the mocked coverage in tests/universal-search.test.ts instead. - const liveEnvReady = Boolean( process.env.E2E_USER_EMAIL && process.env.E2E_USER_PASSWORD && @@ -19,7 +11,11 @@ const liveEnvReady = Boolean( process.env.NEXT_PUBLIC_DEMO_MODE !== "true", ); -describe.skipIf(!liveEnvReady)("GET /api/search/universal (live owner auth)", () => { +if (!liveEnvReady) { + throw new Error("Live owner-search tests require complete, non-placeholder E2E and Supabase credentials."); +} + +describe("GET /api/search/universal (live owner auth)", () => { it("serves owner-scoped registry groups through a real session token", { timeout: 45_000 }, async () => { const { createClient } = await import("@supabase/supabase-js"); const supabase = createClient( @@ -31,11 +27,7 @@ describe.skipIf(!liveEnvReady)("GET /api/search/universal (live owner auth)", () email: process.env.E2E_USER_EMAIL!, password: process.env.E2E_USER_PASSWORD!, }); - if (error) { - // Env advertises live credentials but Supabase rejected them (stale secret, wrong project). - console.warn(`Skipping live owner-auth test: ${error.message}`); - return; - } + if (error) throw new Error(`Live owner-auth sign-in failed: ${error.message}`); const token = data.session?.access_token; expect(token).toBeTruthy(); @@ -52,7 +44,6 @@ describe.skipIf(!liveEnvReady)("GET /api/search/universal (live owner auth)", () publicAccess?: boolean; groups: Array<{ kind: string; error?: boolean; items: Array<{ href: string }> }>; }; - // Owner path: neither the demo nor the public-fixture ladder rung served this. expect(payload.demoMode).toBeUndefined(); expect(payload.publicAccess).toBeUndefined(); @@ -64,6 +55,7 @@ describe.skipIf(!liveEnvReady)("GET /api/search/universal (live owner auth)", () const documents = payload.groups.find((group) => group.kind === "documents"); expect(documents?.error).toBeUndefined(); - await supabase.auth.signOut(); + const signOut = await supabase.auth.signOut(); + if (signOut.error) throw new Error(`Live owner-auth sign-out failed: ${signOut.error.message}`); }); }); diff --git a/tests/universal-search.test.ts b/tests/universal-search.test.ts index 22c02caac..f149da675 100644 --- a/tests/universal-search.test.ts +++ b/tests/universal-search.test.ts @@ -1,8 +1,18 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +let resetModulesAfterTest = false; + +function isolateNextModuleImport() { + vi.resetModules(); + resetModulesAfterTest = true; +} + afterEach(() => { vi.unstubAllEnvs(); - vi.resetModules(); + if (resetModulesAfterTest) { + vi.resetModules(); + resetModulesAfterTest = false; + } }); async function loadUniversalSearch() { @@ -81,6 +91,7 @@ describe("runUniversalSearch (demo/fixtures path)", () => { }); it("isolates a failing domain instead of blanking the response", async () => { + isolateNextModuleImport(); vi.doMock("@/lib/tools-catalog", async (importOriginal) => { const actual = await importOriginal<typeof import("../src/lib/tools-catalog")>(); return { @@ -101,6 +112,7 @@ describe("runUniversalSearch (demo/fixtures path)", () => { }); it("isolates a failing presentations adapter without touching the differentials group", async () => { + isolateNextModuleImport(); vi.doMock("@/lib/differentials", async (importOriginal) => { const actual = await importOriginal<typeof import("../src/lib/differentials")>(); return { @@ -134,6 +146,7 @@ describe("runUniversalSearch (demo/fixtures path)", () => { }); it("keeps registry hrefs when document search uses related-document mapping", async () => { + isolateNextModuleImport(); vi.doMock("@/lib/rag", async (importOriginal) => { const actual = await importOriginal<typeof import("../src/lib/rag")>(); return { @@ -196,6 +209,7 @@ describe("runUniversalSearch (demo/fixtures path)", () => { describe("GET /api/search/universal (demo mode)", () => { it("serves fixture-backed groups with demoMode flagged", async () => { + isolateNextModuleImport(); vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "true"); const { GET } = await import("../src/app/api/search/universal/route"); const response = await GET(new Request("http://localhost/api/search/universal?q=acamprosate&limit=3")); @@ -211,6 +225,7 @@ describe("GET /api/search/universal (demo mode)", () => { }); it("rejects queries under the minimum length", async () => { + isolateNextModuleImport(); vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "true"); const { GET } = await import("../src/app/api/search/universal/route"); const response = await GET(new Request("http://localhost/api/search/universal?q=a")); @@ -218,6 +233,7 @@ describe("GET /api/search/universal (demo mode)", () => { }); it("ignores unknown domains in the CSV filter", async () => { + isolateNextModuleImport(); vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "true"); const { GET } = await import("../src/app/api/search/universal/route"); const response = await GET(new Request("http://localhost/api/search/universal?q=monitoring&domains=tools,bogus")); @@ -227,6 +243,7 @@ describe("GET /api/search/universal (demo mode)", () => { }); it("serves the presentations domain through the CSV filter", async () => { + isolateNextModuleImport(); vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "true"); const { GET } = await import("../src/app/api/search/universal/route"); const response = await GET( @@ -239,6 +256,7 @@ describe("GET /api/search/universal (demo mode)", () => { }); it("accepts a mode context and rejects unknown modes", async () => { + isolateNextModuleImport(); vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "true"); const { GET } = await import("../src/app/api/search/universal/route"); const response = await GET( @@ -321,6 +339,7 @@ describe("runUniversalSearch (query intelligence & ranking)", () => { it("passes the ORIGINAL (uncorrected) query to the documents domain", async () => { const captured: string[] = []; + isolateNextModuleImport(); vi.doMock("@/lib/demo-data", async (importOriginal) => { const actual = await importOriginal<typeof import("../src/lib/demo-data")>(); return { @@ -453,6 +472,7 @@ describe("GET /api/search/universal (live public/owner path)", () => { client: ReturnType<typeof createSupabaseMock>, runUniversalSearch: ReturnType<typeof createRunMock>, ) { + isolateNextModuleImport(); // env:{} keeps isDemoMode false (forcing the live path) while leaving the Supabase public // keys unset, so the cookie-session probe resolves anonymous without a network call. vi.doMock("@/lib/env", () => ({ diff --git a/tests/verify-pr-local.test.ts b/tests/verify-pr-local.test.ts index eef8010ef..092a7d990 100644 --- a/tests/verify-pr-local.test.ts +++ b/tests/verify-pr-local.test.ts @@ -17,6 +17,7 @@ describe("verify-pr-local CLI", () => { expect(output).toContain("Changed files: docs/frontend-architecture.md"); expect(output).toContain("PR-local verification plan (dry run)"); expect(output).toContain("- npm run check:runtime"); + expect(output).toContain("- npm run format:changed"); expect(output).toContain("- build skipped"); expect(output).not.toContain("\n> npm run "); }); @@ -25,7 +26,8 @@ describe("verify-pr-local CLI", () => { const output = dryRun("src/app/api/answer/stream/route.ts", "--extended"); expect(output).toContain("- npm run build"); - expect(output).toContain("- npm run eval:rag:offline"); + expect(output).toContain("- npm run check:rag:fixtures"); + expect(output).not.toContain("- npm run eval:rag:offline"); expect(output).toContain("- npm run verify:ui"); }); diff --git a/vitest.config.mts b/vitest.config.mts index 76e6689a7..74e13c719 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -1,3 +1,5 @@ +const liveProviderTests = process.env.ALLOW_PROVIDER_TESTS === "true"; + const config = { test: { // Route and RAG tests cold-import large Next.js module graphs inside the test @@ -36,21 +38,26 @@ const config = { // Node environment, unchanged glob — existing tests behave exactly as before. name: "node", environment: "node", - include: ["tests/**/*.test.ts"], - }, - }, - { - extends: true, - test: { - // Interactive component tier: @testing-library/react under jsdom. Kept on a - // distinct `*.dom.test.tsx` glob so it can never collect the node suite's - // `*.test.ts` files (and vice versa). - name: "jsdom", - environment: "jsdom", - include: ["tests/**/*.dom.test.tsx"], - setupFiles: ["tests/setup/jsdom.setup.ts"], + include: liveProviderTests ? ["tests/**/*.live.test.ts"] : ["tests/**/*.test.ts"], + exclude: liveProviderTests ? [] : ["tests/**/*.live.test.ts"], }, }, + ...(!liveProviderTests + ? [ + { + extends: true, + test: { + // Interactive component tier: @testing-library/react under jsdom. Kept on a + // distinct `*.dom.test.tsx` glob so it can never collect the node suite's + // `*.test.ts` files (and vice versa). + name: "jsdom", + environment: "jsdom", + include: ["tests/**/*.dom.test.tsx"], + setupFiles: ["tests/setup/jsdom.setup.ts"], + }, + }, + ] + : []), ], }, resolve: { From 6202835ab1cf3703af311b9afdf372f74c63e040 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 17 Jul 2026 05:50:11 +0000 Subject: [PATCH 2/2] merge: resolve main conflicts for test reliability branch --- tests/tsx-server-only-runner.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/tsx-server-only-runner.test.ts b/tests/tsx-server-only-runner.test.ts index 74cfe16b5..f60532388 100644 --- a/tests/tsx-server-only-runner.test.ts +++ b/tests/tsx-server-only-runner.test.ts @@ -52,7 +52,9 @@ describe("standalone TSX server-only compatibility", () => { const config = readFileSync(new URL("../vitest.config.mts", import.meta.url), "utf8"); expect(runner).toContain("acquireHeavyRunLock"); expect(runner).not.toContain("taskkill"); - expect(config).toContain("maxWorkers: 2"); + expect(config).toMatch( + /maxWorkers:\s*process\.env\.VITEST_MAX_WORKERS\s*\?\s*Number\(process\.env\.VITEST_MAX_WORKERS\)\s*:\s*\d+/, + ); expect(config).toContain("testTimeout: 30_000"); }); });