diff --git a/.github/workflows/ci-triage.yml b/.github/workflows/ci-triage.yml index 2dde0837a..f9b3ab35b 100644 --- a/.github/workflows/ci-triage.yml +++ b/.github/workflows/ci-triage.yml @@ -4,12 +4,11 @@ # - main-side: the same job is also failing on the latest default-branch run # (CI merges the PR branch with current main, so a main regression surfaces on # every open PR — this has cost debugging time before). -# - possible known flake: a UI/Playwright job failed — check tests/flake-ledger.json. # - needs investigation: everything else. # # SHIPPED INERT: this event-triggered workflow does nothing until the repo variable # CI_TRIAGE_ENABLED == "true". It never runs PR-authored code — it only reads job -# metadata + the committed flake ledger via the trusted default-branch checkout, and +# metadata via the trusted default-branch checkout, and # posts a comment with the built-in token. name: CI Triage @@ -45,7 +44,6 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const fs = require("fs"); const run = context.payload.workflow_run; // Resolve the PR for this run (empty for fork PRs → skip quietly). @@ -93,13 +91,11 @@ jobs: core.info(`main-side check skipped: ${e.message}`); } - const flakes = JSON.parse(fs.readFileSync("tests/flake-ledger.json", "utf8")).flakes || []; - const uiFlakeSpecs = flakes.map((f) => f.spec).filter((s) => /ui-/.test(s)); const looksUi = (name) => /ui|playwright|browser|e2e/i.test(name); const lines = failed.map((name) => { if (mainFailingJobs.has(name)) return `- \`${name}\` — **main-side**: also failing on the latest \`main\` run, likely not your change.`; - if (looksUi(name)) return `- \`${name}\` — **possible known flake**: UI/Playwright job; check \`tests/flake-ledger.json\`${uiFlakeSpecs.length ? ` (${uiFlakeSpecs.join(", ")})` : ""} and re-run before bisecting.`; + if (looksUi(name)) return `- \`${name}\` — **needs investigation**: use the uploaded JUnit classification and trace; job type alone is not evidence of a known flake.`; return `- \`${name}\` — **needs investigation**.`; }); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9e3cff36..f2e21311a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,9 +108,6 @@ jobs: - name: Typecheck run: npm run typecheck - - name: Unit tests - run: npm run test - safety: name: Safety and config checks needs: changes @@ -160,12 +157,8 @@ jobs: if: needs.changes.outputs.codex_autofix_changed == 'true' run: npm run check:codex-autofix-workflow - - name: Offline RAG preflight - # Runs for every non-docs change (this job's docs_only guard), NOT only - # files matching the rag_eval_changed allowlist. This offline grounding - # gate is cheap and clinical-safety-relevant, so a new retrieval file that - # falls outside the scope patterns must never silently skip it. - run: npm run eval:rag:offline + - name: Offline RAG fixture and manifest validation + run: npm run check:rag:fixtures coverage: name: Unit coverage @@ -173,6 +166,8 @@ jobs: if: needs.changes.outputs.coverage_changed == 'true' runs-on: ubuntu-24.04 timeout-minutes: 20 + env: + FAST_CHECK_SEED: ${{ github.event_name == 'schedule' && github.run_id || '424242' }} steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -228,11 +223,11 @@ jobs: run: npm run check:deployment-readiness ui-critical: - name: Critical UI smoke + name: Production UI needs: changes if: needs.changes.outputs.ui_changed == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 45 steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -242,120 +237,53 @@ jobs: - name: Setup UI e2e environment uses: ./.github/actions/setup-ui-e2e - - name: Chromium critical UI smoke - run: npm run test:e2e:critical + - name: Chromium production journeys + run: npm run test:e2e:pr - - name: Upload UI diagnostics + - name: Classify exact failed test identities if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: critical-ui-diagnostics-${{ github.run_id }} - path: | - test-results/ - playwright-report/ - if-no-files-found: ignore + run: node scripts/classify-playwright-failures.mjs - # Production-journey regression suite. Blocking (PT-05): a red production - # journey must fail pr-required instead of normalising as an advisory red. - # Prototype /mockups specs run separately in ui-mockups below. - ui-regression: - name: UI regression - needs: changes - if: needs.changes.outputs.ui_changed == 'true' - runs-on: ubuntu-24.04 - timeout-minutes: 45 - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Setup UI e2e environment - uses: ./.github/actions/setup-ui-e2e - - - name: Chromium stable regression suite - run: npm run test:e2e:regression - - - name: Upload UI regression diagnostics + - name: Upload UI diagnostics if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: ui-regression-diagnostics-${{ github.run_id }} + name: production-ui-diagnostics-${{ github.run_id }} path: | test-results/ playwright-report/ if-no-files-found: ignore - # Keep the advisory quarantine lane available without paying the Node/browser - # setup cost while no specs are tagged. Stable regression excludes @quarantine, - # so removing this lane entirely would make a newly tagged test run nowhere. - ui-quarantine: - name: Advisory quarantined UI tests + # Quarantined production tests and mockup-only journeys share one advisory + # install/server without weakening the required production-ui job above. + ui-advisory: + name: Advisory UI needs: changes if: github.event_name == 'pull_request' && needs.changes.outputs.ui_changed == 'true' continue-on-error: true runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 45 steps: - name: Checkout uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - name: Detect quarantined specs - id: quarantine - shell: bash - run: | - if git grep -q '@quarantine' -- ':(glob)tests/ui-*.spec.ts'; then - echo "present=true" >> "$GITHUB_OUTPUT" - else - echo "present=false" >> "$GITHUB_OUTPUT" - fi - - name: Setup UI e2e environment - if: steps.quarantine.outputs.present == 'true' uses: ./.github/actions/setup-ui-e2e - - name: Chromium quarantined tests (advisory) - if: steps.quarantine.outputs.present == 'true' - run: npm run test:e2e:quarantine + - name: Chromium quarantined and mockup journeys (advisory) + run: npm run test:e2e:advisory - - name: Upload quarantine diagnostics - if: failure() && steps.quarantine.outputs.present == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: quarantine-ui-diagnostics-${{ github.run_id }} - path: | - test-results/ - playwright-report/ - if-no-files-found: ignore - - # Prototype /mockups specs live in their own advisory lane so a red mockup - # can never mask (or block on) a production-journey regression (PT-05). - ui-mockups: - name: Advisory mockup UI - needs: changes - if: github.event_name == 'pull_request' && needs.changes.outputs.ui_changed == 'true' - continue-on-error: true - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - name: Checkout - uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3 - with: - persist-credentials: false - - - name: Setup UI e2e environment - uses: ./.github/actions/setup-ui-e2e - - - name: Chromium mockup regression - run: npm run test:e2e:mockups + - name: Classify exact failed test identities + if: failure() + run: node scripts/classify-playwright-failures.mjs - - name: Upload mockup UI diagnostics + - name: Upload advisory UI diagnostics if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: mockup-ui-diagnostics-${{ github.run_id }} + name: advisory-ui-diagnostics-${{ github.run_id }} path: | test-results/ playwright-report/ @@ -411,7 +339,7 @@ jobs: pr-required: name: PR required - needs: [changes, static-pr, safety, coverage, build, ui-critical, ui-regression, db-reset-verify] + needs: [changes, static-pr, safety, coverage, build, ui-critical, db-reset-verify] if: always() runs-on: ubuntu-24.04 timeout-minutes: 5 @@ -429,7 +357,6 @@ jobs: COVERAGE_RESULT: ${{ needs.coverage.result }} BUILD_RESULT: ${{ needs.build.result }} UI_RESULT: ${{ needs.ui-critical.result }} - UI_REGRESSION_RESULT: ${{ needs.ui-regression.result }} DB_RESULT: ${{ needs.db-reset-verify.result }} run: | set -euo pipefail @@ -474,11 +401,9 @@ jobs: fi if [ "$UI_CHANGED" = "true" ]; then - require_success "ui-critical" "$UI_RESULT" - require_success "ui-regression" "$UI_REGRESSION_RESULT" + require_success "production-ui" "$UI_RESULT" else - require_skipped_or_success "ui-critical" "$UI_RESULT" - require_skipped_or_success "ui-regression" "$UI_REGRESSION_RESULT" + require_skipped_or_success "production-ui" "$UI_RESULT" fi if [ "$DB_CHANGED" = "true" ]; then diff --git a/.gitignore b/.gitignore index 51d2059d4..66f2dcf3b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ # next.js /.next/ +/.next-playwright/ /out/ # production diff --git a/AGENTS.md b/AGENTS.md index 8b34383c1..d5d498c51 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -164,7 +164,8 @@ When a branch or PR review completes, record the reviewed branch/ref, HEAD SHA, # Process hardening phases -- For non-trivial source/config/test changes, prefer `npm run verify:cheap` as the first broad gate and `npm run verify:pr-local` before PR handoff when the change is ready. The PR-local gate runs format plus `verify:cheap`, then conditionally adds the production build/client-bundle scan and code-backed offline RAG tests. Browser, dependency-audit, Docker/Supabase replay, and provider-backed checks remain separate gates. Use `npm run verify:pr-local -- --dry-run --files ` to inspect selection without running commands. The broader `--extended` plan is dry-run only unless explicit approval is reflected by `ALLOW_EXTENDED_PR_LOCAL=true`. +- For non-trivial source/config/test changes, prefer `npm run verify:cheap` as the first broad gate and `npm run verify:pr-local` before PR handoff when the change is ready. The PR-local gate runs the full unit suite once, then conditionally adds the production build/client-bundle scan and RAG fixture/manifest validation. Browser, dependency-audit, Docker/Supabase replay, and provider-backed checks remain separate gates. Use `npm run verify:pr-local -- --dry-run --files ` to inspect selection without running commands. The broader `--extended` plan is dry-run only unless explicit approval is reflected by `ALLOW_EXTENDED_PR_LOCAL=true`. +- Run one heavy Database command at a time across all worktrees. Do not install while a repository test, build, lint, typecheck, or server command is active. Avoid aggressive short-interval polling, and do not repeat an unchanged full gate after it passes. - For UI, frontend, browser, routing, styling, reduced-motion, or forced-colors changes, run `npm run ensure` before browser work and use `npm run verify:ui` as the Chromium UI gate. - For release or handoff confidence, use `npm run verify:release`; this includes the full Playwright project set. - For clinical ingestion, answer generation, source governance, privacy, production-readiness, or environment changes, run the smallest relevant domain check plus `npm run check:production-readiness`. @@ -307,6 +308,7 @@ After completing `upload`, summarize the current branch and worktree state, whet - For non-trivial changes, start from concrete repo state: branch, `git status`, relevant package scripts, recent failures, and local logs such as `dev-server.log` when runtime behavior is involved. For architecture and module orientation, read `docs/codebase-index.md` (routes: `docs/site-map.md`). - For UI, browser, styling, routing, accessibility, or screenshot work, run `npm run ensure` before opening the app, then use browser QA and the smallest relevant UI proof before broader gates. - Prefer the smallest failing check first. For this repo, use focused Vitest or Playwright targets before widening to `npm run verify:cheap`, `npm run verify:ui`, or `npm run verify:release`. +- Use `npm run test:focused -- --files ` only for safe source-only iteration. It fails closed for deleted files and test/configuration infrastructure; follow its instruction to run `npm run test` in those cases. - When the user says `safely`, preserve unrelated staged, unstaged, and untracked work; stop only clearly repo-owned transient processes; and verify the result instead of doing broad cleanup. - After auth, Supabase, ingestion, answer generation, search/ranking, clinical output, or source-governance changes, run the smallest domain check plus `npm run check:production-readiness`. Run `npm run check:supabase-project` after Supabase env/config changes. - For handoff, archive-safety, or upload-style requests, inspect branch/upstream/status first, run the appropriate verification gate, and only commit or push when the request explicitly asks for that workflow. @@ -423,4 +425,4 @@ Durable notes for Cloud Agents. Standard commands live in `README.md` and `packa - Live-mode caveat: `RAG_PROVIDER_MODE=auto` attempts OpenAI (fast → strong route); if generation fails the built-in quality gates it silently degrades to a deterministic "Source-only" answer that still cites real documents — this is expected, not a failure. The header sign-in UI exposes magic-link + OAuth only (no password field), but the `/api/answer` + retrieval flow works server-side without a browser session. - What still won't run in this VM even with secrets: `npm run worker` also needs the Python OCR stack (`worker/python/requirements.txt`) and heavy parsing deps; Supabase edge functions need Deno v2.x + deployment. `verify:release` additionally runs governance/eval gates. Treat missing-secret failures of `check:supabase-project`/`verify:release` in demo mode as expected, not regressions. - Dev server: `npm run dev` selects a stable per-project localhost port (e.g. `4461`), binds `0.0.0.0`, and prints the exact URL. Never assume port 3000/3001/3002. `npm run ensure` starts/verifies it in the background. -- Verification without secrets: `npm run lint`, `npm run typecheck`, and `npm run test` (vitest) all pass offline. `npm run verify:cheap` also runs runtime, GitHub Actions pin, CI-scope, and sitemap checks. `npm run verify:pr-local` adds format, conditional build/client-bundle scanning, and code-backed offline RAG tests; browser, Docker/Supabase, audit, and provider checks remain separate. +- Verification without secrets: `npm run lint`, `npm run typecheck`, and `npm run test` (vitest) all pass offline. `npm run verify:cheap` also runs runtime, GitHub Actions pin, CI-scope, and sitemap checks. `npm run verify:pr-local` adds format, conditional build/client-bundle scanning, and RAG fixture/manifest validation without repeating unit tests; browser, Docker/Supabase, audit, and provider checks remain separate. See `docs/testing.md` for lock, live-test, Playwright, and flake-ledger rules. diff --git a/README.md b/README.md index 3b17a03c8..1bf8635a9 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ records vs archive). The most load-bearing entries: - `docs/codebase-index.md` — architecture and module map (start here) - `docs/site-map.md` — generated route map (`npm run sitemap:update`) - `docs/process-hardening.md` — verification gates, CI expectations, known limits +- `docs/testing.md` — local test safety, focused/live commands, Playwright ownership, flake policy - `docs/clinical-governance.md` — deployment and source governance checklist - `docs/deployment-architecture.md` — app/worker/Supabase deployment topology - `docs/supabase-migration-reconciliation.md` — migration drift and repair policy @@ -184,9 +185,9 @@ npm run verify:cheap # check:runtime + check:github-actions + sitemap:check # + brand:check + check:type-scale + check:icon-scale # + lint + typecheck + test npm run verify:pr-local # closest local mirror of the PR gate: format + verify:cheap, - # plus conditional build/client-bundle scan and offline RAG - # tests when changed-file scope requires them -npm run verify:ui # check:runtime + test:e2e:chromium + # plus conditional build/client-bundle scan and RAG + # fixture validation; the full unit suite runs once +npm run verify:ui # check:runtime + required production Chromium journeys npm run verify:release # check:runtime + lint + typecheck + test + build + test:e2e # + check:production-readiness + governance:release # + eval:quality:release (needs live Supabase + OpenAI keys) @@ -197,12 +198,12 @@ inspect which checks a change would trigger without running them. CI is risk-scoped (`.github/workflows/ci.yml`): a `changes` job classifies changed paths, `static-pr` always runs runtime, action-pin, CI-scope, format, -lint, typecheck, and unit checks, and `pr-required` is the single +lint, and typecheck checks, and `pr-required` is the single always-reporting required aggregate (required PR checks are Gitleaks plus that -aggregate). Coverage, build, safety/config checks, Chromium `ui-critical` -smoke, and the repo-owned Supabase `db-reset-verify` migration replay run only -when their file scopes apply; UI PRs also get a non-blocking advisory -`ui-regression` job. The full Playwright browser matrix +aggregate). One full unit run with coverage, build, safety/config checks, the +production Chromium gate, and the repo-owned Supabase `db-reset-verify` +migration replay run only when their file scopes apply; UI PRs also get one +non-blocking advisory Chromium invocation. The full Playwright browser matrix (`release-browser-matrix`) runs on `main`, `release/*`, manual dispatch, and a weekly schedule. Docker image builds, live drift, and live eval canary checks are path-filtered, scheduled, or manual rather than required checks for every @@ -221,8 +222,12 @@ npm run samples:check npm run lint npm run typecheck npm run test +npm run test:focused -- --files src/lib/example.ts +npm run test:live # requires ALLOW_PROVIDER_TESTS=true npm run test:coverage npm run test:e2e +npm run test:e2e:pr +npm run test:e2e:advisory npm run test:e2e:all npm run test:e2e:accessibility npm run test:e2e:chromium diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index 4b668c49e..9fb17c89c 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -560,6 +560,8 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-15 | codex/outstanding-work-cleanup | 0c56f27a37af88a073d2bb695d2cf4c05067ff4f + working-tree diff | repo-wide outstanding-work reconciliation and cleanup | No high-confidence P0/P1 remained in the locally executable scope. Fixed the stale architecture index for DSM/specifier routes, pruned redundant Knip configuration, removed the unused `SignedImage` default export, and reconciled maintained docs so completed/superseded plans no longer present as active work. Retained explicitly blocked work: provider-gated operator actions, the live-eval shadow reindex harness, deep-memory section-ownership design, and RAG follow-ups that require live evidence or product/security decisions. Full Knip findings were triaged rather than mass-deleted; unresolved-import scan is clean. | Offline `npm run verify:cheap` passed with 2,417 tests/1 skipped; docs links 806, script refs 264, codebase index 35/35; Knip unresolved scan clean; typecheck clean; focused Vitest 15/15; env-name parity clean; `git diff --check`. Provider-backed Supabase/OpenAI/GitHub/Railway, dependency audit, browser, build, drift, and release gates not run. | | 2026-07-15 | codex/design-polish-pass | 0c56f27a37af88a073d2bb695d2cf4c05067ff4f + reviewed working diff | full live design, responsive, UX, accessibility, design-system, routing, performance, lint, testing, documentation, and release-readiness review | No P0/P1 reproduced. Fixed the P2 duplicate phone scroll surface in the shared standalone shell and added a regression test; fixed mode-menu Tab dismissal; explicitly hid decorative dynamic icons; restricted press scaling to motion-safe environments; moved sheet backdrops and forced-colour glass/backdrop behavior onto design tokens. External target fidelity remains blocked because no independent Figma/mockup source was supplied. | Live 30-route desktop + 21-route phone sweep; targeted 320/390/639/768/1440/1920 proofs; `npm run verify:cheap` (2417 passed/1 skipped); `npm run verify:ui` (175/175); focused keyboard 1/1; accessibility media/axe 5/5; scoped Prettier, ESLint, TypeScript, type-scale, and icon-scale passed. Provider-backed checks not run. | | 2026-07-15 | codex/documents-closed-default | 49f63791bced2b1764a11ab723aea94b45b026b6 | documents viewer disclosure defaults and related defect hunt | Fixed the inconsistent default-open document viewer sections by making indexed text, high-yield summary, tables/diagrams, and indexing details a native mutually exclusive closed disclosure group. The section navigation opens its requested disclosure and deep-linked evidence still reveals its target. The hunt also removed the explicitly open nested table-review queue, preserved printable summary content through the browser print lifecycle, and added cold-server readiness guards to the affected viewer tests. No other high-confidence default-open defect remains in the live Documents scope. | `npm run verify:cheap`; TypeScript; focused ESLint/Prettier; clean-worktree mocked Chromium coverage for deep-linked evidence, structured summary, closed/mutually-exclusive disclosures, navigation opening, and print state restore; `git diff --check`. Turbopack could not run through the local external `node_modules` junction, so clean browser verification used Next's supported Webpack dev mode. No Supabase/OpenAI/live-provider checks run. | +| 2026-07-15 | HEAD (detached) 0c56f27a37af88a073d2bb695d2cf4c05067ff4f | 0c56f27a37af88a073d2bb695d2cf4c05067ff4f | test flakiness, runner safety, CI duplication, and local execution-churn audit | P1 test-integrity issues: `run-vitest.mjs` can report success when Vitest terminates by signal and force-kills concurrent same-worktree runs; the default Vitest glob can execute the live Supabase owner-search test when credentials are present, while that test silently passes on authentication failure. P2 cluster: Playwright server races/shared `.next` state and config-time persistent-server startup; CI reruns the full unit suite for coverage and PR-local RAG verification repeats 21 suites; known-flake tracking is substring-only, not tied to quarantine, and structurally requires a non-empty ledger; unseeded property tests; repeated module resets/source-graph parsing; long retrying browser matrices and swallowed `networkidle` timeouts; host-wide concurrent worktrees/install/test/server activity causes memory pressure, native-module lock failures, and invalidated runs. | Pure review, no source mutation except this ledger append. Offline `node scripts/flake-ledger.mjs --self-test` passed (confirming the non-empty invariant); ledger matching confirmed `narrow-viewport-parallel` applies to an unquarantined blocking title; `node scripts/verify-pr-local.mjs --dry-run --files src/lib/rag.ts` confirmed full test + build + offline RAG duplication. Full Vitest/Playwright not run because this worktree has no dependencies and concurrent repo work was active; provider-backed checks not run (confirmation boundary). | +| 2026-07-17 | codex/test-reliability-hardening | 0c56f27a37af88a073d2bb695d2cf4c05067ff4f + reviewed remediation diff | implementation and merge-readiness review of heavyweight locking, offline test isolation, focused/live test selection, Playwright ownership, flake classification, CI consolidation, and browser race fixes | No remaining high-confidence P0-P2 issue in the reviewed diff. Three scoped integration defects were fixed before handoff: advisory mockup tests were unreachable behind the production route boundary; explicit focused-test paths did not fail closed when deleted or missing; and JUnit spec identities were relative to the Playwright test directory while the ledger uses repository-relative paths. Provider scrubbing was expanded to all repository-specific Supabase/database/E2E credentials, and early Playwright server signal/launch failures now fail immediately. Residual risk is final integration with the latest `main` and the full production Chromium run. | Offline focused Vitest 36/36 then 22/22; full TypeScript passed; targeted Prettier passed; `git diff --check` passed. Full ESLint was externally interrupted without a result and remains required on the clean integrated tree. Provider-backed Supabase/OpenAI checks were not run. | | 2026-07-17 | codex/scroll-geometry-stability-20260717 | f88a41ae5fb62ae39d5d1d33fd5e21490e1ea60e | phone scroll geometry and boundary stability review | Confirmed two coupled P2 interaction defects: hiding the fixed bottom composer removed its reserved space and changed the scroll viewport, while collapsing the in-flow header at the bottom produced a browser-driven scrollTop clamp that was misread as an upward gesture. The dock now keeps stable clearance, hide/reveal uses directional hysteresis, and geometry clamps are rebased without revealing chrome. No other high-confidence defect remains in the changed scope. | Focused Vitest 8/8; TypeScript; scoped ESLint; Prettier; `git diff --check`. `npm run verify:cheap` reached the full Vitest phase but was stopped after making no progress under concurrent local Vitest workloads. Turbopack rejected the external node_modules junction and Webpack did not reach the identity endpoint, so exact-head hosted UI and required CI remain the merge gates. No Supabase/OpenAI/live-provider checks run. | | 2026-07-17 | codex/header-footer-scroll-timing-20260717 | 8298bfdcb40c207dbac1128e83c07b4aba782e32 | header and bottom-composer scroll timing, motion, responsive behavior, and merge readiness | Added deliberate hide/reveal travel thresholds with direction-reset handling, aligned header and composer easing/durations, and preserved reduced-motion and breakpoint behavior. No remaining high-confidence P0-P2 defect was found in the scoped diff or focused live behavior. | Focused Vitest 7/7; targeted Chromium UI 5/5; scoped ESLint; Prettier; full TypeScript; full lint; `git diff --check`. `verify:cheap` reached the aggregate Vitest phase, where two repository graph scans exceeded their 30-second test timeout under local disk contention; isolated assertions passed until the same timeout. No Supabase/OpenAI/live-provider checks run. | | 2026-07-15 | HEAD / main snapshot (detached review worktree) | 0c56f27a37af88a073d2bb695d2cf4c05067ff4f | comprehensive repository review | Changes requested: two P1 defects (high-risk clinical claim support can accept a different trigger condition on lexical overlap; document-mode URL auto-run loops on navigation and leaves search loading indefinitely), one P2 supply-chain guard gap (the action-pin checker accepts mutable major tags and ignores SHA-pinned major versions), and one P3 orientation-doc gap (DSM and legacy specifier routes are absent from the codebase index). No P0 found. | Node 24/npm 11 and `npm ls --depth=0`; format, runtime, lint, TypeScript, sitemap/brand/icon/type-scale, docs links/scripts, CI/action/Codex guards; coverage 259 files passed/1 skipped and 2,417 tests passed/1 skipped; offline RAG 36 fixtures plus 282 tests; production build/client-secret scan; deployment boot smoke; critical Chromium 8/10 with two reproducible document-search failures; accessibility 5/5; viewport/focus checks through 1920x1080. Provider-backed governance, quality, drift, tenancy, hosted CI, and the cross-browser matrix were blocked or skipped by policy/targeted failures. | diff --git a/docs/codebase-index.md b/docs/codebase-index.md index 22b5a64bb..dfebc4880 100644 --- a/docs/codebase-index.md +++ b/docs/codebase-index.md @@ -1,6 +1,6 @@ # Clinical KB — Codebase Index -Structured map for AI agents and onboarding. For live routes, see `docs/site-map.md` (`npm run sitemap:update` / `sitemap:check`). For agent rules and verification gates, see `AGENTS.md`. +Structured map for AI agents and onboarding. For live routes, see `docs/site-map.md` (`npm run sitemap:update` / `sitemap:check`). For agent rules and verification gates, see `AGENTS.md`; for test execution and flake policy, see `docs/testing.md`. **Stack:** Next.js 16, React 19, Supabase (pgvector, Storage, Auth), OpenAI, Python OCR worker. **Live Supabase:** `Clinical KB Database` — ref `sjrfecxgysukkwxsowpy` (never use stale `qjgitjyhxrwxsrydablr`). @@ -231,7 +231,7 @@ Golden retrieval fixture: `scripts/fixtures/rag-retrieval-golden.json` **Domain clusters in `tests/`:** RAG/answers, retrieval, ingestion/indexing, source governance, API routes, Supabase schema, shell/routing, UI formatting guards. -**Gates:** `verify:cheap` (lint + typecheck + unit), `verify:ui` (Chromium E2E), `verify:release` (full build + all browsers + production readiness). +**Gates:** `verify:cheap` (lint + typecheck + full offline unit suite), `verify:ui` (required production Chromium journeys), `verify:release` (full build + all browsers + production readiness). Use `test:focused` only for safe source-only iteration. --- diff --git a/docs/process-hardening.md b/docs/process-hardening.md index f9a4c19ed..5ef94eb9f 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -12,10 +12,10 @@ artifact before release; see ## Phase 1 - Active now - `npm run verify:cheap` is the default broad local gate for source/config/test changes: `check:runtime`, `sitemap:check`, lint, typecheck, and unit tests. -- `npm run verify:pr-local` is the closest local mirror of the normal PR gate: runtime, format, lint, typecheck, unit tests, conditional build, and offline RAG preflight when changed-file scope requires it. Local scope resolves against the repository default base rather than a feature-branch upstream; set `PR_BASE_REF` explicitly for release-targeted PRs. -- `npm run verify:ui` is the default UI gate: `check:runtime` plus Chromium Playwright smoke, stress, and accessibility media checks (`test:e2e:chromium`). +- `npm run verify:pr-local` is the closest local mirror of the normal PR gate: runtime, format, lint, typecheck, one full unit run, conditional build, and RAG fixture/manifest validation when changed-file scope requires it. Local scope resolves against the repository default base rather than a feature-branch upstream; set `PR_BASE_REF` explicitly for release-targeted PRs. +- `npm run verify:ui` is the complete required production Chromium gate: `check:runtime` plus all non-quarantined production journeys (`test:e2e:pr`). - `npm run verify:release` is the release-confidence gate: `check:runtime`, lint, typecheck, unit tests, build, full Playwright browser matrix, `check:production-readiness`, `governance:release`, and `eval:quality:release` (the last step needs live Supabase and OpenAI keys). -- CI uses a risk-scoped PR gate: `changes` classifies paths, `static-pr` always runs runtime/action/scope/format/lint/typecheck/unit checks, and `pr-required` is the single always-reporting required aggregate. Coverage, build, critical UI smoke, migration replay, safety/config, Codex workflow validation, and the production-backed offline RAG preflight run only when their file scopes apply. UI PRs also run a non-blocking advisory Chromium regression job. The external `Supabase Preview` check may still replay migrations on branch databases when enabled. A gated `release-browser-matrix` job runs the full Playwright browser set on `main`, `release/*`, manual dispatch, and the weekly schedule. +- CI uses a risk-scoped PR gate: `changes` classifies paths, `static-pr` always runs runtime/action/scope/format/lint/typecheck checks, and `pr-required` is the single always-reporting required aggregate. One full unit run with coverage, build, one required production Chromium invocation, migration replay, safety/config, Codex workflow validation, and RAG fixture validation run only when their file scopes apply. UI PRs also run one non-blocking advisory Chromium invocation for quarantined and mockup journeys. The external `Supabase Preview` check may still replay migrations on branch databases when enabled. A gated `release-browser-matrix` job runs the full Playwright browser set on `main`, `release/*`, manual dispatch, and the weekly schedule. - `tests/ui-accessibility.spec.ts` covers reduced-motion and forced-colors dashboard usability so those modes are no longer only reviewed by inspection. - `tests/ui-tools.spec.ts` covers the Applications dashboard mode at mobile and desktop sizes, including the `/applications` compatibility redirect. - `AGENTS.md` now points future agents to these gates and to this document. @@ -138,11 +138,11 @@ passes `p_worker_id`. Ordered apply steps, R17 manual `CONCURRENTLY` index, and ## PR merge gate: risk-scoped CI + required aggregate (2026-07-10) -- CI now has one always-reporting required aggregate: `CI / PR required`. The aggregate depends on `changes`, `static-pr`, `safety`, `coverage`, `build`, `ui-critical`, `ui-regression`, and `db-reset-verify`, then enforces only the jobs whose scopes apply. -- `static-pr` is the deterministic baseline for every PR: runtime, action pin check, CI scope self-test, format, lint, typecheck, and unit tests. Coverage, build, safety/config, critical UI, production-backed offline RAG preflight, and migration replay are independent jobs so reruns stay focused. Coverage is limited to `src`/test changes; process-only changes do not trigger builds. +- CI now has one always-reporting required aggregate: `CI / PR required`. The aggregate depends on `changes`, `static-pr`, `safety`, `coverage`, `build`, `ui-critical`, and `db-reset-verify`, then enforces only the jobs whose scopes apply. +- `static-pr` is the deterministic baseline for every PR: runtime, action pin check, CI scope self-test, format, lint, and typecheck. Coverage is the one required full unit run. Build, safety/config (the `safety` job includes RAG fixture validation), production UI, and migration replay are independent jobs so reruns stay focused. Coverage includes source, tests, package/test-runner configuration, while process-only documentation does not trigger builds. - `db-reset-verify` is path-scoped to Supabase migrations/schema/config and database-access code. Do not also require an external Supabase Preview replay unless the repo owner intentionally wants duplicate migration replay. -- `ui-critical` is path-scoped to UI/routing/styling/browser-facing changes and runs only the `@critical` Chromium smoke subset. `ui-regression` (2026-07-13 audit, finding 8) runs the remaining stable Chromium cases via `test:e2e:regression` (`--grep-invert "@critical|@quarantine"`) and is **merge-blocking** through `pr-required` on UI changes. Genuinely flaky specs are tagged `@quarantine` and run in the advisory `ui-quarantine` job (`test:e2e:quarantine`, `--pass-with-no-tests`); fix and untag them rather than letting the quarantine grow. The full browser matrix remains main/release/manual/scheduled. -- 2026-07-13 flip-day triage: the whole 142-test regression set was green locally except one stale `/privacy` heading assertion (fixed — the page title is "Privacy & data handling" since the footer simplification) and three cold-dev-server timing timeouts that pass on a warmed server and have no CI failure history (`ui-tools` mode-home centering `:462`, differentials comparison wiring `:1367`, `ui-universal-search` grouped-result navigation `:102`). Watch those three; if one flakes in CI, tag it `@quarantine` rather than reverting the gate. +- `ui-critical` retains its job ID for branch-protection compatibility but runs one required production Chromium invocation covering all non-quarantined critical and regression journeys (`test:e2e:pr`). `ui-advisory` runs quarantined and mockup journeys together when UI scope applies. A JUnit failure is considered a known flake only when its exact spec/title matches the validated ledger. The full browser matrix remains main/release/manual/scheduled. +- The 2026-07-13 cold-server and historical ledger candidates are tracked through the reproduction policy in `docs/testing.md`: run each three times on the same SHA, fix fail/pass races, treat deterministic failures as regressions, and remove entries that do not reproduce. On `0c56f27a3`, the historical composer/tap/answer-fallback entries and three cold-route candidates did not reproduce in three runs; the narrow differential viewport reproduced once in three cold runs, was fixed with route-specific readiness before its single submit, then passed three of three. The ledger is intentionally empty. - Branch protection for `main` should require `CI / PR required` and `Secret Scan / Gitleaks`. Keep `SAST / Semgrep` required only if the repository owner accepts its external-rule/network dependency as part of the normal merge gate. Do not require path-filtered or scheduled/manual contexts such as `CI / Unit coverage`, `CI / Critical UI smoke`, `CI / Migration replay`, `Docker image build / app-image`, `Docker image build / worker-image`, `CI / release-browser-matrix`, `Eval Canary`, or `Live drift check`; they can be skipped on ordinary PRs and would leave branches stuck at "Expected - Waiting for status to be reported." ## CSS cascade layering (2026-07-02) @@ -348,9 +348,9 @@ the durable index for the tooling; `docs/operator-backlog.md` tracks the human-o (`INGESTION_AUTOPILOT_APPLY` unset → read-only); flip that repo var to `true` after a clean dry-run to allow real recovery. - **CI failure triage** (`.github/workflows/ci-triage.yml`): on PR CI failure, classifies each failed job - as main-side / possible-known-flake / needs-investigation. Inert until repo var `CI_TRIAGE_ENABLED=true` - (now set). Reads only job metadata + the trusted default-branch flake ledger (`tests/flake-ledger.json`, - `scripts/flake-ledger.mjs`); never runs PR code. + as main-side or needs-investigation. Inert until repo var `CI_TRIAGE_ENABLED=true` (now set). UI jobs use + their uploaded JUnit classification and trace; job names alone never produce a known-flake verdict. + The workflow reads only trusted default-branch job metadata and never runs PR code. - **Repo hygiene:** `check:env-parity` (env-var NAME reconciliation across `env.ts`, `check-ci-env.mjs`, and — opt-in, names-only — `gh secret list` / Railway) and `sweep:branch-ledger` (report-only branch inventory, cherry-pick-aware). diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 000000000..1230c07cc --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,38 @@ +# Testing and verification + +## Safe local execution + +Heavy commands (`lint`, `typecheck`, `build`, Vitest, Playwright, `verify:cheap`, and `verify:pr-local`) share one lock derived from Git's common directory. The lock therefore covers every worktree for this repository. Nested commands reuse their parent's token; an unrelated command fails immediately and prints the current owner. A lock with an owner is reclaimed only when its recorded process is demonstrably dead; an ownerless initialization lock is reclaimed only after its initialization grace period. + +Run one heavy Database command at a time. Do not install packages while a repository test, build, lint, typecheck, or server command is active. Avoid short-interval polling, and do not repeat an unchanged broad gate after it has already passed. + +Ordinary Vitest and Playwright runs remove OpenAI, Supabase, database, and E2E credentials and force demo/offline mode. Provider tests use the `*.live.test.ts` suffix, are excluded from default discovery, and can only be started explicitly with `ALLOW_PROVIDER_TESTS=true npm run test:live`. + +## Commands + +| Command | Purpose | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `npm run test:focused -- --files ` | Local iteration using Vitest related-file selection. It fails closed for deleted files, test infrastructure, configuration, or an empty/unsafe mapping. | +| `npm run test` | Complete offline unit suite. | +| `npm run test:live` | Explicit provider suite; requires `ALLOW_PROVIDER_TESTS=true`. | +| `npm run test:e2e:pr` | Required production Chromium journeys, excluding mockups and quarantined tests. | +| `npm run test:e2e:advisory` | Quarantined and mockup journeys in one advisory invocation. | +| `npm run verify:cheap` | Broad offline local gate: runtime/config checks, lint, typecheck, and the full unit suite. | +| `npm run verify:pr-local` | PR-like local gate. Formatting is checked on the changed set, the full unit suite runs once, and RAG scope adds fixture/manifest validation. | +| `npm run verify:ui` | Complete required production Chromium gate. | + +Set `FAST_CHECK_SEED` to reproduce a property-test run. Local and ordinary CI runs default to `424242`; scheduled CI may derive a bounded seed from the run ID. + +## Playwright ownership + +The repository runner exclusively builds and serves each Playwright production app. It selects a safe port, verifies `/api/local-project-id`, uses an isolated `.next-playwright/` build directory, replaces provider configuration with inert loopback values, and removes its server and output on success, failure, or signal. Playwright configuration never starts a server. The production boot guard permits this demo profile only when the output is isolated, provider mode is offline, credentials are absent, and the Supabase URL is the inert `127.0.0.1:1` target. + +Blocking tests run with zero retries. CI publishes list, JUnit, and JSON reports. Failed-test classification parses JUnit test cases and uses exact spec/title matches; a job name is never enough to classify a failure as a known flake. + +## Flake policy + +`tests/flake-ledger.json` may be empty. Each entry must match the exact spec and title, and the test title must include `@quarantine` but not `@critical`. Entries require an owner, reproduction command, local tracking reference, first/last-seen dates, and an expiry no more than 30 days away. Reproduce a candidate three times on the same SHA before adding or retaining it: fix fail/pass races, treat repeatable failures as regressions, and remove entries that no longer reproduce. + +## CI topology + +PR CI keeps static checks separate from one required full unit run with coverage. UI scope uses one required production Chromium invocation for non-quarantined critical and regression journeys, plus one advisory invocation for quarantined and mockup journeys. Build, migration, security, and release behavior remain independently scoped and unchanged. diff --git a/next.config.ts b/next.config.ts index a2bf26000..7ad9f31bd 100644 --- a/next.config.ts +++ b/next.config.ts @@ -4,6 +4,14 @@ import { fileURLToPath } from "node:url"; import { buildSecurityHeaders, resolveRuntimeFlags } from "./src/lib/security-headers"; const projectRoot = path.dirname(fileURLToPath(import.meta.url)); +const requestedDistDir = process.env.NEXT_DIST_DIR?.trim(); +if (requestedDistDir && !/^\.next-playwright\/[a-z0-9-]+\/dist$/i.test(requestedDistDir)) { + throw new Error("NEXT_DIST_DIR must be an owned .next-playwright//dist directory."); +} +const requestedTsConfigPath = process.env.NEXT_TSCONFIG_PATH?.trim(); +if (requestedTsConfigPath && !/^\.next-playwright\/[a-z0-9-]+\/tsconfig\.json$/i.test(requestedTsConfigPath)) { + throw new Error("NEXT_TSCONFIG_PATH must be an owned .next-playwright//tsconfig.json file."); +} // Static (non-CSP) headers for every route. The nonce'd CSP is emitted per // request from src/proxy.ts; both derive their runtime flags from the same helper. @@ -18,6 +26,8 @@ async function withOptionalBundleAnalyzer(config: NextConfig): Promise existsSync(path.join(projectRoot, file))); +if (changedFiles.length === 0) { + console.log("No changed files require a formatting check."); + process.exit(0); +} + +const prettierBin = path.join(projectRoot, "node_modules", "prettier", "bin", "prettier.cjs"); +const result = spawnSync(process.execPath, [prettierBin, "--check", "--ignore-unknown", "--", ...changedFiles], { + cwd: projectRoot, + stdio: "inherit", +}); +process.exit(childProcessExitCode(result)); diff --git a/scripts/check-rag-fixtures.mjs b/scripts/check-rag-fixtures.mjs new file mode 100644 index 000000000..0aaa93e41 --- /dev/null +++ b/scripts/check-rag-fixtures.mjs @@ -0,0 +1,56 @@ +#!/usr/bin/env node +import { readFileSync } from "node:fs"; +import { validateOfflineContractTests } from "./rag-offline-contract.mjs"; + +const goldenPath = "scripts/fixtures/rag-retrieval-golden.json"; +const cases = JSON.parse(readFileSync(goldenPath, "utf8")); +const failures = []; +const fail = (message) => failures.push(message); +const hasText = (value) => typeof value === "string" && value.trim().length > 0; + +function validateTerm(term, caseId, fieldName) { + if (Array.isArray(term)) { + if (term.length === 0 || !term.every(hasText)) + fail(`${caseId}: ${fieldName} contains an empty alternative term group.`); + return; + } + if (!hasText(term)) fail(`${caseId}: ${fieldName} contains an empty term.`); +} + +if (!Array.isArray(cases) || cases.length < 10) { + fail(`${goldenPath} must contain the offline retrieval contract cases.`); +} else { + const ids = new Set(); + for (const item of cases) { + if (!hasText(item.id)) fail("Golden retrieval case is missing id."); + if (ids.has(item.id)) fail(`${item.id}: duplicate golden retrieval case id.`); + ids.add(item.id); + if (!hasText(item.query)) fail(`${item.id}: query is required.`); + if (!hasText(item.expectedQueryClass)) fail(`${item.id}: expectedQueryClass is required.`); + if (!Number.isInteger(item.topK) || item.topK < 1 || item.topK > 50) fail(`${item.id}: topK must be 1-50.`); + if (!Array.isArray(item.expectedDocumentSubstrings)) { + fail(`${item.id}: expectedDocumentSubstrings must be an array.`); + } else { + for (const term of item.expectedDocumentSubstrings) validateTerm(term, item.id, "expectedDocumentSubstrings"); + } + if (!Array.isArray(item.expectedContentTerms) || item.expectedContentTerms.length === 0) { + fail(`${item.id}: expectedContentTerms must include at least one required term.`); + } else { + for (const term of item.expectedContentTerms) validateTerm(term, item.id, "expectedContentTerms"); + } + if (typeof item.expectTableEvidence !== "boolean") fail(`${item.id}: expectTableEvidence must be boolean.`); + } +} + +const contractTestsPath = "scripts/fixtures/rag-offline-contract-tests.json"; +const contractTests = JSON.parse(readFileSync(contractTestsPath, "utf8")); +failures.push(...validateOfflineContractTests(contractTests)); + +if (failures.length > 0) { + console.error("Offline RAG fixture and manifest validation failed:"); + for (const failure of failures) console.error(`- ${failure}`); + process.exit(1); +} +console.log( + `Offline RAG fixture and manifest validation passed (${cases.length} golden cases, ${contractTests.length} suites).`, +); diff --git a/scripts/child-process-result.mjs b/scripts/child-process-result.mjs new file mode 100644 index 000000000..fcf8887d0 --- /dev/null +++ b/scripts/child-process-result.mjs @@ -0,0 +1,13 @@ +export function childProcessExitCode(result) { + if (Number.isInteger(result?.status)) return result.status; + return 1; +} + +export function childProcessFailureSummary(result) { + const details = []; + if (typeof result?.status === "number") details.push(`status ${result.status}`); + else details.push("missing exit status"); + if (result?.signal) details.push(`signal ${result.signal}`); + if (result?.error) details.push(`launch error: ${result.error.message ?? String(result.error)}`); + return details.join(", "); +} diff --git a/scripts/ci-change-scope.mjs b/scripts/ci-change-scope.mjs index 446f9cd96..b5738a34b 100644 --- a/scripts/ci-change-scope.mjs +++ b/scripts/ci-change-scope.mjs @@ -126,6 +126,7 @@ const ragEvalPatterns = [ "src/app/api/search", /^src\/lib\/(?:rag(?:-[^/]+)?|smart-rag-api|clinical-search|clinical-query-mode|retrieval(?:-[^/]+)?|answer(?:-[^/]+)?|citations|cross-document-synthesis|evidence(?:-[^/]+)?|ranking-config|source(?:-[^/]+)?|chunking|document-index-units|query-privacy|owner-scope|corpus-grounding|indexed-source-formatting)\.ts$/, /^src\/components\/(?:.*\/)?(?:answer|source|citation)[^/]*\.tsx?$/i, + /^scripts\/(?:check-rag-fixtures|test-rag-offline)\.mjs$/, /^scripts\/(eval-|run-eval-safe|compare-retrieval-eval|retrieval-health|profile-retrieval|warm-retrieval-cache|tune-search-weights)/, /^tests\/(rag|retrieval|answer|citations|evidence|eval|clinical-safety|source).*\.test\.ts$/, ]; @@ -147,8 +148,6 @@ const containerPatterns = [ const sourcePatterns = ["data", "src", "tests", "scripts", "worker", "playwright", "public", "supabase"]; -const coveragePatterns = ["data", "src", "tests", "vitest.config.mts"]; - const buildPatterns = [ "bundle-budget.json", "data", @@ -180,7 +179,10 @@ const lockfilePatterns = ["package.json", "package-lock.json", ".npmrc"]; function classify(files) { const normalized = [...new Set(files.map(normalizePath).filter(Boolean))].sort(); const sourceChanged = normalized.some((file) => pathMatches(file, [...sourcePatterns, ...staticConfigPatterns])); - const coverageChanged = normalized.some((file) => pathMatches(file, coveragePatterns)); + // Preserve the pre-consolidation unit gate for every non-documentation + // change. Narrower signals still scope build/UI/database work, but must not + // leave runtime, worker, or configuration changes without unit coverage. + const coverageChanged = normalized.some((file) => !pathMatches(file, docPatterns)); const uiChanged = normalized.some((file) => pathMatches(file, uiPatterns)); const dbChanged = normalized.some((file) => pathMatches(file, dbPatterns)); const containerChanged = normalized.some((file) => pathMatches(file, containerPatterns)); @@ -391,6 +393,20 @@ function selfTest() { source_changed: true, coverage_changed: true, }); + assertScope("build-config-keeps-coverage", ["next.config.ts", "postcss.config.mjs", "tsconfig.json"], { + coverage_changed: true, + build_changed: true, + }); + assertScope("worker-keeps-coverage", ["worker/index.ts"], { + source_changed: true, + coverage_changed: true, + build_changed: true, + }); + assertScope("test-runner", ["scripts/run-vitest.mjs", "scripts/run-playwright.mjs"], { + source_changed: true, + coverage_changed: true, + ui_changed: true, + }); assertScope("runtime-data", ["data/medications-snapshot.json"], { source_changed: true, coverage_changed: true, @@ -424,9 +440,13 @@ function selfTest() { rag_eval_changed: true, source_changed: true, }); + assertScope("rag-fixture-checker", ["scripts/check-rag-fixtures.mjs"], { + rag_eval_changed: true, + source_changed: true, + }); // A RAG-relevant lib file outside ragEvalPatterns must still be caught as a // source change (so static-pr and the non-docs safety job / verify:pr-local, - // which run the offline grounding gate, always execute). Guards the "silent + // which run fixture validation, always execute). Guards the "silent // scope narrowing" gap. assertScope("rag-lib-outside-allowlist", ["src/lib/hybrid-reranker.ts"], { source_changed: true, @@ -453,7 +473,7 @@ function selfTest() { ); assertScope("package", ["package.json"], { source_changed: false, - coverage_changed: false, + coverage_changed: true, container_changed: true, workflow_changed: false, build_changed: true, diff --git a/scripts/classify-playwright-failures.mjs b/scripts/classify-playwright-failures.mjs new file mode 100644 index 000000000..7f7335442 --- /dev/null +++ b/scripts/classify-playwright-failures.mjs @@ -0,0 +1,56 @@ +#!/usr/bin/env node +import { existsSync, readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +import { loadFlakeLedger, matchFlake } from "./flake-ledger.mjs"; + +const decode = (value) => + value + .replaceAll(""", '"') + .replaceAll("'", "'") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll("&", "&"); + +const attribute = (attributes, name) => { + const match = attributes.match(new RegExp(`\\b${name}="([^"]*)"`)); + return match ? decode(match[1]) : ""; +}; + +const repositorySpec = (classname) => { + const normalized = classname.trim().replaceAll("\\", "/").replace(/^\.\//, ""); + return normalized.startsWith("tests/") ? normalized : `tests/${normalized}`; +}; + +export function failedTestcasesFromJunit(xml) { + return [...xml.matchAll(/]*)>([\s\S]*?)<\/testcase>/g)] + .filter((match) => { + const body = match[2].replace(//g, "").replace(//g, ""); + return /<(?:failure|error)\b/.test(body); + }) + .map((match) => ({ spec: repositorySpec(attribute(match[1], "classname")), title: attribute(match[1], "name") })) + .filter((testcase) => testcase.spec && testcase.title); +} + +function main() { + const reportPath = process.argv[2] ?? "test-results/playwright-junit.xml"; + if (!existsSync(reportPath)) { + console.error(`Playwright JUnit report not found: ${reportPath}`); + process.exitCode = 1; + return; + } + + const failures = failedTestcasesFromJunit(readFileSync(reportPath, "utf8")); + if (failures.length === 0) { + console.log("No failed Playwright testcases were found in the JUnit report."); + return; + } + + const flakes = loadFlakeLedger(); + console.log("Playwright failure classification:"); + for (const { spec, title } of failures) { + const known = matchFlake(spec, title, flakes); + console.log(`- ${spec} :: ${title}: ${known ? `known quarantine (${known.id})` : "needs investigation"}`); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main(); diff --git a/scripts/eval-rag-offline.mjs b/scripts/eval-rag-offline.mjs index 8606ab77a..bd5a119fe 100644 --- a/scripts/eval-rag-offline.mjs +++ b/scripts/eval-rag-offline.mjs @@ -1,109 +1,17 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; -import { readFileSync } from "node:fs"; -import { resolve } from "node:path"; -import { validateOfflineContractTests } from "./rag-offline-contract.mjs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; -const goldenPath = "scripts/fixtures/rag-retrieval-golden.json"; -const cases = JSON.parse(readFileSync(goldenPath, "utf8")); -const failures = []; +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -function fail(message) { - failures.push(message); -} - -function hasText(value) { - return typeof value === "string" && value.trim().length > 0; -} - -function validateTerm(term, caseId, fieldName) { - if (Array.isArray(term)) { - if (term.length === 0 || !term.every(hasText)) { - fail(`${caseId}: ${fieldName} contains an empty alternative term group.`); - } - return; - } - if (!hasText(term)) fail(`${caseId}: ${fieldName} contains an empty term.`); -} - -function validateGoldenCases() { - if (!Array.isArray(cases) || cases.length < 10) { - fail(`${goldenPath} must contain the offline retrieval contract cases.`); - return; - } - - const ids = new Set(); - for (const item of cases) { - if (!hasText(item.id)) fail("Golden retrieval case is missing id."); - if (ids.has(item.id)) fail(`${item.id}: duplicate golden retrieval case id.`); - ids.add(item.id); - - if (!hasText(item.query)) fail(`${item.id}: query is required.`); - if (!hasText(item.expectedQueryClass)) fail(`${item.id}: expectedQueryClass is required.`); - if (!Number.isInteger(item.topK) || item.topK < 1 || item.topK > 50) { - fail(`${item.id}: topK must be an integer between 1 and 50.`); - } - if (!Array.isArray(item.expectedDocumentSubstrings)) { - fail(`${item.id}: expectedDocumentSubstrings must be an array.`); - } else { - for (const term of item.expectedDocumentSubstrings) validateTerm(term, item.id, "expectedDocumentSubstrings"); - } - if (!Array.isArray(item.expectedContentTerms) || item.expectedContentTerms.length === 0) { - fail(`${item.id}: expectedContentTerms must include at least one required term.`); - } else { - for (const term of item.expectedContentTerms) validateTerm(term, item.id, "expectedContentTerms"); - } - if (typeof item.expectTableEvidence !== "boolean") { - fail(`${item.id}: expectTableEvidence must be boolean.`); - } - } -} - -validateGoldenCases(); - -if (failures.length > 0) { - console.error("Offline RAG preflight failed:"); - for (const failure of failures) console.error(`- ${failure}`); - process.exit(1); -} - -console.log(`Offline RAG fixture schema passed (${cases.length} golden retrieval cases).`); - -const contractTestsPath = "scripts/fixtures/rag-offline-contract-tests.json"; -const contractTests = JSON.parse(readFileSync(contractTestsPath, "utf8")); -const contractTestFailures = validateOfflineContractTests(contractTests); -if (contractTestFailures.length > 0) { - console.error(`${contractTestsPath} failed the offline safety contract:`); - for (const failure of contractTestFailures) console.error(`- ${failure}`); - process.exit(1); -} -const vitestPath = resolve("node_modules/vitest/vitest.mjs"); -const offlineEnv = { ...process.env }; -for (const key of [ - "RAG_PROVIDER_MODE", - "OPENAI_API_KEY", - "OPENAI_ORG_ID", - "OPENAI_PROJECT_ID", - "OPENAI_BASE_URL", - "NEXT_PUBLIC_SUPABASE_URL", - "NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY", - "NEXT_PUBLIC_SUPABASE_ANON_KEY", - "SUPABASE_URL", - "SUPABASE_ANON_KEY", - "SUPABASE_SECRET_KEY", - "SUPABASE_SERVICE_ROLE_KEY", +for (const [script, args] of [ + ["scripts/check-rag-fixtures.mjs", []], + ["scripts/test-rag-offline.mjs", process.argv.slice(2)], ]) { - delete offlineEnv[key]; -} -const result = spawnSync(process.execPath, [vitestPath, "run", ...contractTests, "--reporter=dot"], { - env: offlineEnv, - stdio: "inherit", -}); - -if (result.error) { - console.error(`Offline RAG contract tests could not start: ${result.error.message}`); - process.exit(1); + const result = spawnSync(process.execPath, [script, ...args], { cwd: projectRoot, stdio: "inherit" }); + const exitCode = childProcessExitCode(result); + if (exitCode !== 0) process.exit(exitCode); } -if (result.status !== 0) process.exit(result.status ?? 1); - -console.log("Offline RAG production contract tests passed."); +console.log("Offline RAG fixture and production-contract checks passed."); diff --git a/scripts/flake-ledger.mjs b/scripts/flake-ledger.mjs index 3f615e6bc..a6de7793e 100644 --- a/scripts/flake-ledger.mjs +++ b/scripts/flake-ledger.mjs @@ -1,77 +1,113 @@ #!/usr/bin/env node -/** - * flake-ledger — loader + matcher for the known-flaky Playwright specs recorded in - * tests/flake-ledger.json. - * - * Purpose: stop re-diagnosing the same flakes from memory every time CI goes red. - * The CI failure-triage workflow uses isKnownFlake() to attribute a failed test to - * a known flake; a serial re-run of only-flaky failures can be layered on top. - * - * CLI: - * node scripts/flake-ledger.mjs --list print the ledger - * node scripts/flake-ledger.mjs --self-test validate shape + matcher - * node scripts/flake-ledger.mjs --match "" → prints matching id or "none" - */ import { readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -const LEDGER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "tests", "flake-ledger.json"); +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const ledgerPath = path.join(projectRoot, "tests", "flake-ledger.json"); +const requiredFields = [ + "id", + "title", + "spec", + "reason", + "owner", + "reproduction", + "firstSeen", + "lastSeen", + "expires", + "tracking", +]; +const dayMs = 24 * 60 * 60 * 1000; -export function loadFlakeLedger(ledgerPath = LEDGER_PATH) { - const raw = JSON.parse(readFileSync(ledgerPath, "utf8")); - const flakes = Array.isArray(raw.flakes) ? raw.flakes : []; +function dateValue(value, field, id) { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) throw new Error(`${id}: ${field} must be YYYY-MM-DD`); + const parsed = Date.parse(`${value}T23:59:59Z`); + if (!Number.isFinite(parsed) || new Date(parsed).toISOString().slice(0, 10) !== value) { + throw new Error(`${id}: ${field} is not a valid date`); + } + return parsed; +} + +export function validateFlakeLedgerEntries(flakes, { root = projectRoot, now = Date.now() } = {}) { + const ids = new Set(); + const today = Date.parse(`${new Date(now).toISOString().slice(0, 10)}T00:00:00Z`); + const latestAllowedExpiry = today + 31 * dayMs - 1; for (const flake of flakes) { - if (!flake.id || !flake.match || !flake.spec || !flake.reason) { - throw new Error(`flake-ledger entry missing required field (id/match/spec/reason): ${JSON.stringify(flake)}`); - } + const missing = requiredFields.filter((field) => typeof flake[field] !== "string" || !flake[field].trim()); + if (missing.length) throw new Error(`flake-ledger ${flake.id ?? "entry"} missing: ${missing.join(", ")}`); + if (ids.has(flake.id)) throw new Error(`duplicate flake-ledger id: ${flake.id}`); + ids.add(flake.id); + + const firstSeen = dateValue(flake.firstSeen, "firstSeen", flake.id); + const lastSeen = dateValue(flake.lastSeen, "lastSeen", flake.id); + const expires = dateValue(flake.expires, "expires", flake.id); + if (firstSeen > lastSeen) throw new Error(`${flake.id}: firstSeen cannot be after lastSeen`); + if (lastSeen > expires) throw new Error(`${flake.id}: lastSeen cannot be after expires`); + if (expires < now) throw new Error(`${flake.id}: ledger entry expired ${flake.expires}`); + if (expires > latestAllowedExpiry) throw new Error(`${flake.id}: expiry must be within 30 days`); + if (!flake.title.includes("@quarantine")) throw new Error(`${flake.id}: exact title must include @quarantine`); + if (flake.title.includes("@critical")) + throw new Error(`${flake.id}: a test cannot be both @quarantine and @critical`); + + const specPath = path.join(root, flake.spec); + const spec = readFileSync(specPath, "utf8"); + if (!spec.includes(flake.title)) throw new Error(`${flake.id}: exact title is not present in ${flake.spec}`); } return flakes; } -/** Return the matching flake entry for a test title, or null. Case-insensitive substring. */ -export function matchFlake(testTitle, flakes = loadFlakeLedger()) { - if (!testTitle) return null; - const haystack = String(testTitle).toLowerCase(); - return flakes.find((flake) => haystack.includes(String(flake.match).toLowerCase())) ?? null; +export function loadFlakeLedger(sourcePath = ledgerPath) { + const raw = JSON.parse(readFileSync(sourcePath, "utf8")); + return validateFlakeLedgerEntries(Array.isArray(raw.flakes) ? raw.flakes : []); +} + +function normalizeSpec(spec) { + return String(spec ?? "") + .trim() + .replaceAll("\\", "/") + .replace(/^\.\//, "") + .toLowerCase(); } -export function isKnownFlake(testTitle, flakes = loadFlakeLedger()) { - return matchFlake(testTitle, flakes) !== null; +export function matchFlake(spec, testTitle, flakes = loadFlakeLedger()) { + const normalizedSpec = normalizeSpec(spec); + const title = String(testTitle ?? "") + .trim() + .toLowerCase(); + if (!normalizedSpec || !title) return null; + return ( + flakes.find( + (flake) => normalizeSpec(flake.spec) === normalizedSpec && flake.title.trim().toLowerCase() === title, + ) ?? null + ); } function selfTest() { const flakes = loadFlakeLedger(); - const assert = (cond, label) => { - if (!cond) { - console.error(`✖ self-test failed: ${label}`); - process.exitCode = 1; - throw new Error(label); - } - }; - assert(flakes.length > 0, "ledger is non-empty"); - const ids = new Set(flakes.map((f) => f.id)); - assert(ids.size === flakes.length, "flake ids are unique"); - assert(isKnownFlake("composer hero renders on hydrate", flakes), "matches a known flake by title substring"); - assert(!isKnownFlake("a totally unrelated passing test", flakes), "does not match an unrelated title"); - assert(matchFlake("", flakes) === null, "empty title matches nothing"); - if (process.exitCode !== 1) console.error("[flake-ledger] self-test passed"); + if (new Set(flakes.map((flake) => flake.id)).size !== flakes.length) throw new Error("flake ids are not unique"); + const sample = [{ spec: "tests/example.spec.ts", title: "exact quarantined title @quarantine", id: "sample" }]; + if (matchFlake("tests/example.spec.ts", "EXACT QUARANTINED TITLE @QUARANTINE", sample)?.id !== "sample") + throw new Error("exact title did not match"); + if (matchFlake("tests/example.spec.ts", "prefix exact quarantined title @quarantine", sample)) + throw new Error("partial title unexpectedly matched"); + if (matchFlake("tests/other.spec.ts", "exact quarantined title @quarantine", sample)) + throw new Error("cross-spec title unexpectedly matched"); + console.error(`[flake-ledger] self-test passed (${flakes.length} active entries)`); } function main() { if (process.argv.includes("--self-test")) return selfTest(); if (process.argv.includes("--list")) { - for (const flake of loadFlakeLedger()) console.log(`${flake.id}\t${flake.spec}\t"${flake.match}"`); + for (const flake of loadFlakeLedger()) console.log(`${flake.id}\t${flake.spec}\t${flake.title}`); return; } const matchIndex = process.argv.indexOf("--match"); if (matchIndex >= 0) { - const hit = matchFlake(process.argv[matchIndex + 1], loadFlakeLedger()); - console.log(hit ? hit.id : "none"); + console.log(matchFlake(process.argv[matchIndex + 1], process.argv[matchIndex + 2])?.id ?? "none"); return; } - console.error('usage: flake-ledger.mjs [--list | --self-test | --match ""]'); + console.error('usage: flake-ledger.mjs [--list | --self-test | --match "<exact spec>" "<exact title>"]'); + process.exitCode = 1; } -const invokedDirectly = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; -if (invokedDirectly) main(); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main(); diff --git a/scripts/playwright-base-url.ts b/scripts/playwright-base-url.ts index 391733dc2..88722b86d 100644 --- a/scripts/playwright-base-url.ts +++ b/scripts/playwright-base-url.ts @@ -81,7 +81,7 @@ function findExistingLocalProjectUrl() { ); } -export function getPlaywrightBaseUrl() { +export function getPlaywrightBaseUrl({ allowEnsure = true }: { allowEnsure?: boolean } = {}) { const configuredBaseUrl = process.env.PLAYWRIGHT_BASE_URL; if (configuredBaseUrl) { if (!localUrlPattern.test(configuredBaseUrl)) { @@ -91,6 +91,12 @@ export function getPlaywrightBaseUrl() { return configuredBaseUrl; } + if (!allowEnsure) { + throw new Error( + "Playwright requires a runner-owned local server. Use the repository npm scripts instead of invoking the Playwright CLI directly.", + ); + } + const existingUrl = findExistingLocalProjectUrl(); if (existingUrl) return existingUrl; diff --git a/scripts/run-heavy.mjs b/scripts/run-heavy.mjs new file mode 100644 index 000000000..5e7b038d4 --- /dev/null +++ b/scripts/run-heavy.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; +import { acquireHeavyRunLock } from "./test-run-lock.mjs"; + +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const args = process.argv.slice(2); +if (args[0] !== "--npm-script" || !args[1]) { + console.error("Usage: node scripts/run-heavy.mjs --npm-script <script> [forwarded arguments]"); + process.exit(2); +} + +const script = args[1]; +const forwarded = args.slice(2); +const lock = acquireHeavyRunLock({ projectRoot, command: `npm run ${script}` }); +let exitCode = 1; +try { + const npmExecPath = process.env.npm_execpath; + const result = npmExecPath + ? spawnSync(process.execPath, [npmExecPath, "run", script, ...(forwarded.length ? ["--", ...forwarded] : [])], { + cwd: projectRoot, + env: lock.environment, + stdio: "inherit", + }) + : spawnSync( + process.platform === "win32" ? "cmd.exe" : "npm", + process.platform === "win32" + ? ["/d", "/s", "/c", `npm run ${script}`] + : ["run", script, ...(forwarded.length ? ["--", ...forwarded] : [])], + { cwd: projectRoot, env: lock.environment, stdio: "inherit" }, + ); + exitCode = childProcessExitCode(result); +} finally { + lock.release(); +} +process.exit(exitCode); diff --git a/scripts/run-live-tests.mjs b/scripts/run-live-tests.mjs new file mode 100644 index 000000000..42432f84c --- /dev/null +++ b/scripts/run-live-tests.mjs @@ -0,0 +1,31 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; +import { requireProviderTestPermission } from "./test-environment.mjs"; +import { acquireHeavyRunLock } from "./test-run-lock.mjs"; + +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const vitestBin = path.join(projectRoot, "node_modules", "vitest", "vitest.mjs"); + +try { + requireProviderTestPermission(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} + +const lock = acquireHeavyRunLock({ projectRoot, command: "vitest live provider tests" }); +let exitCode = 1; +try { + const result = spawnSync(process.execPath, [vitestBin, "run", ...process.argv.slice(2)], { + cwd: projectRoot, + env: { ...lock.environment, NODE_ENV: "test" }, + stdio: "inherit", + }); + exitCode = childProcessExitCode(result); +} finally { + lock.release(); +} +process.exit(exitCode); diff --git a/scripts/run-playwright.mjs b/scripts/run-playwright.mjs index 7482f47eb..83565373a 100644 --- a/scripts/run-playwright.mjs +++ b/scripts/run-playwright.mjs @@ -1,14 +1,18 @@ #!/usr/bin/env node import { spawn, spawnSync } from "node:child_process"; +import { mkdirSync, rmdirSync, rmSync, writeFileSync } from "node:fs"; import http from "node:http"; import net from "node:net"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { childProcessExitCode, childProcessFailureSummary } from "./child-process-result.mjs"; +import { offlineTestEnvironment } from "./test-environment.mjs"; +import { acquireHeavyRunLock } from "./test-run-lock.mjs"; import { appName, + circularProjectPortRange, isReservedDevPort, localProjectId, - projectPortEnd, stableProjectPort, } from "../src/lib/local-server-utils.mjs"; @@ -21,23 +25,39 @@ const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), " const playwrightBin = path.join(projectRoot, "node_modules", "playwright", "cli.js"); const nextBin = path.join(projectRoot, "node_modules", "next", "dist", "bin", "next"); const identityPath = "/api/local-project-id"; -const startupTimeoutMs = 120_000; +const startupTimeoutMs = 180_000; const missingErrorComponentsNeedle = "missing required error components"; -const routeSmokePaths = ["/", "/applications"]; -// `next dev` compiles each route lazily on its first request, so the first test -// to hit a route races the compile and can exceed the test timeout (the documented -// cold-start flakes: ui-tools mode-home, differentials, universal-search). We warm -// every distinct page route the suite touches BEFORE running Playwright, so every -// navigation lands on an already-compiled route. `/?mode=…` variants share the `/` -// route, so warming `/` covers them. -// -// Warming is STRICTLY SEQUENTIAL: `next dev` uses Turbopack, whose persistent -// (RocksDB) cache cannot service concurrent compilations — firing the warms in -// parallel corrupts it ("Only a single write operation is allowed at a time" → -// TurbopackInternalError / missing build-manifest.json). One route at a time -// compiles cleanly. Best-effort with a generous per-route compile timeout. -const warmupPaths = ["/", "/applications", "/tools", "/services", "/favourites", "/medications"]; -const warmupTimeoutMs = 90_000; +const routeSmokePaths = [ + "/", + "/applications", + "/?mode=tools", + "/documents/search?mode=documents", + "/forms/transport-crisis-form", +]; +const playwrightArgs = process.argv.slice(2); +const explicitProjectRequested = playwrightArgs.some( + (argument) => argument === "--project" || argument.startsWith("--project="), +); +const mockupProjectRequested = + !explicitProjectRequested || + playwrightArgs.some( + (argument, index) => + argument === "--project=chromium-mockups" || + (argument === "--project" && playwrightArgs[index + 1] === "chromium-mockups"), + ); +const runId = `${process.pid}-${Date.now()}`; +const relativeRunRoot = `.next-playwright/${runId}`; +const absoluteRunRoot = path.join(projectRoot, relativeRunRoot); +const relativeDistDir = `${relativeRunRoot}/dist`; +const relativeTsConfigPath = `${relativeRunRoot}/tsconfig.json`; + +let lock; +try { + lock = acquireHeavyRunLock({ projectRoot, command: `playwright ${playwrightArgs.join(" ")}` }); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); @@ -46,11 +66,7 @@ function sleep(ms) { function canListenOnHost(port, host) { return new Promise((resolve) => { const server = net.createServer(); - server.once("error", (error) => { - // An unsupported address family (e.g. no IPv6 in the container) cannot - // hold the port, so it must not disqualify it as busy. - resolve(error.code === "EAFNOSUPPORT" || error.code === "EADDRNOTAVAIL"); - }); + server.once("error", (error) => resolve(error.code === "EAFNOSUPPORT" || error.code === "EADDRNOTAVAIL")); server.once("listening", () => server.close(() => resolve(true))); server.listen(port, host); }); @@ -73,9 +89,7 @@ function canConnectToHost(port, host) { } async function canListen(port) { - for (const host of ["127.0.0.1", "localhost", "::1"]) { - if (await canConnectToHost(port, host)) return false; - } + for (const host of ["127.0.0.1", "localhost", "::1"]) if (await canConnectToHost(port, host)) return false; for (const host of ["127.0.0.1", "localhost", "::1", "0.0.0.0", "::"]) { if (!(await canListenOnHost(port, host))) return false; } @@ -83,26 +97,21 @@ async function canListen(port) { } async function findFreePort(startPort) { - for (let port = startPort; port <= projectPortEnd; port += 1) { - if (isReservedDevPort(port)) continue; - if (await canListen(port)) return port; + for (const port of circularProjectPortRange(startPort)) { + if (!isReservedDevPort(port) && (await canListen(port))) return port; } - throw new Error(`No free Playwright server port found from ${startPort} to ${projectPortEnd}.`); + throw new Error("No free Playwright server port found in the configured project range."); } -function requestJson(url) { +function request(url, { json = false, timeoutMs = 30_000 } = {}) { return new Promise((resolve) => { - const request = http.get(url, { timeout: 5000 }, (response) => { + const pending = http.get(url, { timeout: timeoutMs }, (response) => { let body = ""; response.setEncoding("utf8"); - response.on("data", (chunk) => { - body += chunk; - }); + response.on("data", (chunk) => (body += chunk)); response.on("end", () => { - if (response.statusCode !== 200) { - resolve(null); - return; - } + if (!response.statusCode || response.statusCode < 200 || response.statusCode >= 400) return resolve(null); + if (!json) return resolve(body); try { resolve(JSON.parse(body)); } catch { @@ -110,70 +119,14 @@ function requestJson(url) { } }); }); - request.on("timeout", () => { - request.destroy(); - resolve(null); - }); - request.on("error", () => resolve(null)); - }); -} - -function requestText(url, timeoutMs = 30_000) { - return new Promise((resolve) => { - const request = http.get(url, { timeout: timeoutMs }, (response) => { - let body = ""; - response.setEncoding("utf8"); - response.on("data", (chunk) => { - body += chunk; - }); - response.on("end", () => { - if (!response.statusCode || response.statusCode < 200 || response.statusCode >= 400) { - resolve(null); - return; - } - resolve(body); - }); - }); - request.on("timeout", () => { - request.destroy(); + pending.on("timeout", () => { + pending.destroy(); resolve(null); }); - request.on("error", () => resolve(null)); + pending.on("error", () => resolve(null)); }); } -async function hasHealthyRouteComponents(baseUrl) { - for (const smokePath of routeSmokePaths) { - const body = await requestText(`${baseUrl}${smokePath}`); - if (!body || body.includes(missingErrorComponentsNeedle)) { - return false; - } - } - return true; -} - -// Pre-compile the page routes the suite navigates to so the first test to hit each -// one does not race `next dev`'s lazy on-demand compile. Sequential (see warmupPaths -// note) + best-effort: a route that is slow or 404s here is not fatal — the tests -// still assert real readiness. -async function warmRoutes(baseUrl) { - for (const routePath of warmupPaths) { - await requestText(`${baseUrl}${routePath}`, warmupTimeoutMs); - } -} - -async function waitForServer(baseUrl) { - const startedAt = Date.now(); - while (Date.now() - startedAt < startupTimeoutMs) { - const payload = await requestJson(`${baseUrl}${identityPath}`); - if (isVerifiedProjectPayload(payload) && (await hasHealthyRouteComponents(baseUrl))) { - return; - } - await sleep(500); - } - throw new Error(`Timed out waiting for Clinical KB at ${baseUrl}.`); -} - function isVerifiedProjectPayload(payload) { return ( payload?.appName === appName && @@ -182,22 +135,36 @@ function isVerifiedProjectPayload(payload) { ); } -async function findExistingProjectServer() { - const baseUrl = `http://localhost:${stableProjectPort(projectRoot)}`; - const payload = await requestJson(`${baseUrl}${identityPath}`); - return isVerifiedProjectPayload(payload) ? baseUrl : null; -} - -function runPlaywright(baseUrl) { - return spawnSync(process.execPath, [playwrightBin, "test", ...process.argv.slice(2)], { - cwd: projectRoot, - env: { ...process.env, PLAYWRIGHT_BASE_URL: baseUrl }, - stdio: "inherit", - }); +async function waitForServer(baseUrl, server) { + const startedAt = Date.now(); + while (Date.now() - startedAt < startupTimeoutMs) { + if (serverLaunchError) { + throw new Error(`Playwright-owned Next server failed to launch: ${serverLaunchError.message}`); + } + if (server.exitCode !== null || server.signalCode) { + throw new Error( + `Playwright-owned Next server exited before readiness (${server.exitCode !== null ? `code ${server.exitCode}` : `signal ${server.signalCode}`}).`, + ); + } + const payload = await request(`${baseUrl}${identityPath}`, { json: true, timeoutMs: 5000 }); + if (isVerifiedProjectPayload(payload)) { + let healthy = true; + for (const smokePath of routeSmokePaths) { + const body = await request(`${baseUrl}${smokePath}`); + if (!body || body.includes(missingErrorComponentsNeedle)) { + healthy = false; + break; + } + } + if (healthy) return; + } + await sleep(500); + } + throw new Error(`Timed out waiting for the Playwright-owned Clinical KB server at ${baseUrl}.`); } -function stopProcessTree(child) { - if (!child.pid || child.exitCode !== null) return; +function stopOwnedProcessTree(child) { + if (!child?.pid || child.exitCode !== null) return; if (process.platform === "win32") { spawnSync("taskkill", ["/PID", String(child.pid), "/T", "/F"], { stdio: "ignore" }); return; @@ -209,89 +176,101 @@ function stopProcessTree(child) { } } -function stopExistingProjectDevServers() { - if (process.platform !== "win32") return; - - const command = ` -$repo = $env:PLAYWRIGHT_PROJECT_ROOT -$patterns = @( - "*$repo\\node_modules\\next\\dist\\bin\\next dev*", - "*$repo\\node_modules\\next\\dist\\server\\lib\\start-server.js*", - "*$repo\\.next\\dev\\build\\*" -) -$targets = Get-CimInstance Win32_Process | Where-Object { - if (-not $_.CommandLine) { return $false } - foreach ($pattern in $patterns) { - if ($_.CommandLine -like $pattern) { return $true } - } - return $false -} -foreach ($target in $targets) { - Stop-Process -Id $target.ProcessId -Force -ErrorAction SilentlyContinue -} -`; - - spawnSync("powershell.exe", ["-NoProfile", "-Command", command], { - cwd: projectRoot, - env: { ...process.env, PLAYWRIGHT_PROJECT_ROOT: projectRoot }, - stdio: "ignore", - }); -} - -const existingBaseUrl = await findExistingProjectServer(); -if (existingBaseUrl) { - if (await hasHealthyRouteComponents(existingBaseUrl)) { - console.log(`Using existing Clinical KB server at ${existingBaseUrl}`); - await warmRoutes(existingBaseUrl); - const result = runPlaywright(existingBaseUrl); - process.exit(result.status ?? (result.signal ? 1 : 0)); +let server; +let serverLaunchError; +let cleaned = false; +function cleanup() { + if (cleaned) return; + cleaned = true; + try { + stopOwnedProcessTree(server); + rmSync(absoluteRunRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + try { + rmdirSync(path.dirname(absoluteRunRoot)); + } catch (error) { + if (error?.code !== "ENOENT" && error?.code !== "ENOTEMPTY") throw error; + } + } catch (error) { + console.error(`Playwright cleanup warning: ${error instanceof Error ? error.message : String(error)}`); + } finally { + lock.release(); } - - console.log(`Existing Clinical KB server at ${existingBaseUrl} failed route-component smoke; restarting it.`); - stopExistingProjectDevServers(); - await sleep(1000); } -stopExistingProjectDevServers(); -await sleep(1000); - -const port = await findFreePort(stableProjectPort(projectRoot)); -const baseUrl = `http://localhost:${port}`; -console.log(`Starting Playwright-owned Clinical KB server at ${baseUrl}`); - -const server = spawn(process.execPath, [nextBin, "dev", "--hostname", "0.0.0.0", "--port", String(port)], { - cwd: projectRoot, - detached: process.platform !== "win32", - env: { ...process.env, PORT: String(port), PLAYWRIGHT_BASE_URL: baseUrl }, - stdio: ["ignore", "inherit", "inherit"], - windowsHide: true, -}); - -let stopping = false; -const stop = () => { - if (stopping) return; - stopping = true; - stopProcessTree(server); -}; - process.once("SIGINT", () => { - stop(); + cleanup(); process.exit(130); }); process.once("SIGTERM", () => { - stop(); + cleanup(); process.exit(143); }); -process.once("exit", stop); +process.once("exit", cleanup); try { - await waitForServer(baseUrl); - await warmRoutes(baseUrl); - const result = runPlaywright(baseUrl); - stop(); - process.exit(result.status ?? (result.signal ? 1 : 0)); + const port = await findFreePort(stableProjectPort(projectRoot)); + const baseUrl = `http://localhost:${port}`; + mkdirSync(absoluteRunRoot, { recursive: true }); + writeFileSync( + path.join(absoluteRunRoot, "tsconfig.json"), + `${JSON.stringify( + { + extends: "../../tsconfig.json", + compilerOptions: { + baseUrl: "../..", + paths: { "@/*": ["src/*"] }, + }, + }, + null, + 2, + )}\n`, + "utf8", + ); + const offlineEnv = offlineTestEnvironment(lock.environment, { + PORT: String(port), + PLAYWRIGHT_BASE_URL: baseUrl, + NEXT_DIST_DIR: relativeDistDir, + NEXT_TSCONFIG_PATH: relativeTsConfigPath, + NODE_ENV: "production", + PLAYWRIGHT_OFFLINE_MODE: "true", + NEXT_PUBLIC_MOCKUPS_ENABLED: mockupProjectRequested ? "true" : "false", + }); + console.log(`Building isolated production Playwright app (${relativeRunRoot})`); + + const buildResult = spawnSync(process.execPath, ["--max-old-space-size=8192", nextBin, "build", "--webpack"], { + cwd: projectRoot, + env: offlineEnv, + stdio: "inherit", + }); + const buildExitCode = childProcessExitCode(buildResult); + if (buildExitCode !== 0) { + throw new Error(`Playwright production build failed (${childProcessFailureSummary(buildResult)}).`); + } + + console.log(`Starting isolated production Playwright server at ${baseUrl} (${relativeRunRoot})`); + + server = spawn(process.execPath, [nextBin, "start", "--hostname", "0.0.0.0", "--port", String(port)], { + cwd: projectRoot, + detached: process.platform !== "win32", + env: offlineEnv, + stdio: ["ignore", "inherit", "inherit"], + windowsHide: true, + }); + server.once("error", (error) => { + serverLaunchError = error; + }); + + await waitForServer(baseUrl, server); + const result = spawnSync(process.execPath, [playwrightBin, "test", ...playwrightArgs], { + cwd: projectRoot, + env: offlineEnv, + stdio: "inherit", + }); + const exitCode = childProcessExitCode(result); + cleanup(); + process.exit(exitCode); } catch (error) { - stop(); + cleanup(); console.error(error instanceof Error ? error.message : String(error)); process.exit(1); } diff --git a/scripts/run-vitest.mjs b/scripts/run-vitest.mjs index 61a24f75d..1ff55d271 100644 --- a/scripts/run-vitest.mjs +++ b/scripts/run-vitest.mjs @@ -2,46 +2,23 @@ import { spawnSync } from "node:child_process"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; +import { offlineTestEnvironment } from "./test-environment.mjs"; +import { acquireHeavyRunLock } from "./test-run-lock.mjs"; const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const vitestBin = path.join(projectRoot, "node_modules", "vitest", "vitest.mjs"); -const vitestNeedle = vitestBin.toLowerCase(); -const powershell = [ - "Get-CimInstance Win32_Process", - "| Where-Object {", - " $_.CommandLine -and", - ` $_.CommandLine.ToLower().Contains('${vitestNeedle}') -and`, - " $_.CommandLine -notlike '*run-vitest.mjs*' -and", - " $_.CommandLine -notlike '*Get-CimInstance*'", - "}", - "| ForEach-Object { $_.ProcessId }", -].join(" "); - -let runningPids = []; +const args = process.argv.slice(2); +const lock = acquireHeavyRunLock({ projectRoot, command: `vitest ${args.join(" ")}` }); +let exitCode = 1; try { - const probe = spawnSync("powershell.exe", ["-NoProfile", "-Command", powershell], { - encoding: "utf8", - stdio: ["ignore", "pipe", "ignore"], + const result = spawnSync(process.execPath, [vitestBin, ...args], { + cwd: projectRoot, + env: offlineTestEnvironment(lock.environment, { NODE_ENV: "test" }), + stdio: "inherit", }); - const output = probe.status === 0 ? probe.stdout.trim() : ""; - runningPids = output - .split(/\r?\n/) - .map((line) => Number.parseInt(line.trim(), 10)) - .filter(Number.isFinite); -} catch { - runningPids = []; -} - -for (const pid of runningPids) { - try { - spawnSync("taskkill", ["/F", "/T", "/PID", String(pid)], { stdio: "ignore" }); - } catch { - // best effort cleanup - } + exitCode = childProcessExitCode(result); +} finally { + lock.release(); } - -const args = process.argv.slice(2); -const result = spawnSync(process.execPath, [vitestBin, ...args], { - stdio: "inherit", -}); -process.exit(result.status ?? 0); +process.exit(exitCode); diff --git a/scripts/test-environment.mjs b/scripts/test-environment.mjs new file mode 100644 index 000000000..c304fe297 --- /dev/null +++ b/scripts/test-environment.mjs @@ -0,0 +1,63 @@ +const providerEnvironmentKeys = Object.freeze([ + "OPENAI_API_KEY", + "OPENAI_ORG_ID", + "OPENAI_PROJECT_ID", + "OPENAI_BASE_URL", + "NEXT_PUBLIC_SUPABASE_URL", + "NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY", + "NEXT_PUBLIC_SUPABASE_ANON_KEY", + "SUPABASE_URL", + "SUPABASE_ANON_KEY", + "SUPABASE_PUBLISHABLE_KEY", + "SUPABASE_SECRET_KEY", + "SUPABASE_SERVICE_ROLE_KEY", + "SUPABASE_ACCESS_TOKEN", + "SUPABASE_DB_URL", + "SUPABASE_PROJECT_REF", + "SUPABASE_PROJECT_NAME", + "SUPABASE_STAGING_PROJECT_REF", + "SUPABASE_STAGING_PROJECT_NAME", + "DATABASE_URL", + "POSTGRES_PASSWORD", + "E2E_AUTH_ENABLED", + "E2E_USER_EMAIL", + "E2E_USER_PASSWORD", + "ALLOW_PROVIDER_TESTS", +]); + +const offlineUrlValues = Object.freeze({ + NEXT_PUBLIC_SUPABASE_URL: "http://127.0.0.1:1", + SUPABASE_URL: "http://127.0.0.1:1", + SUPABASE_DB_URL: "postgresql://offline:offline@127.0.0.1:1/offline", + DATABASE_URL: "postgresql://offline:offline@127.0.0.1:1/offline", +}); + +/** + * @param {Record<string, string | undefined>} source + * @param {Record<string, string | undefined>} overrides + */ +export function offlineTestEnvironment(source = process.env, overrides = {}) { + const environment = { ...source }; + // Explicit values both scrub inherited secrets and prevent Next/Vite from + // repopulating the same names from a repository-local env file. URL-shaped + // settings use inert loopback values so the runtime env schema still parses. + for (const key of providerEnvironmentKeys) environment[key] = offlineUrlValues[key] ?? ""; + + return { + ...environment, + RAG_PROVIDER_MODE: "offline", + NEXT_PUBLIC_DEMO_MODE: "true", + ...overrides, + }; +} + +/** @param {Record<string, string | undefined>} environment */ +export function requireProviderTestPermission(environment = process.env) { + if (environment.ALLOW_PROVIDER_TESTS !== "true") { + throw new Error( + "Live provider tests are disabled. Set ALLOW_PROVIDER_TESTS=true only after explicit provider-test approval.", + ); + } +} + +export { offlineUrlValues, providerEnvironmentKeys }; diff --git a/scripts/test-focused.mjs b/scripts/test-focused.mjs new file mode 100644 index 000000000..d1065b9a7 --- /dev/null +++ b/scripts/test-focused.mjs @@ -0,0 +1,73 @@ +#!/usr/bin/env node +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; + +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const unsafeSelectionPattern = + /^(?:tests\/|scripts\/|\.github\/|package(?:-lock)?\.json$|tsconfig|vitest\.config|next\.config|eslint)/i; + +function git(args) { + return execFileSync("git", args, { cwd: projectRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }); +} + +function lines(value) { + return value + .split(/\r?\n/) + .map((item) => item.trim()) + .filter(Boolean); +} + +function defaultSelection() { + let base = "HEAD"; + try { + base = git(["merge-base", "HEAD", "origin/main"]).trim(); + } catch { + // A local-only checkout can still select working-tree changes from HEAD. + } + const changed = lines(git(["diff", "--name-only", base])); + const untracked = lines(git(["ls-files", "--others", "--exclude-standard"])); + const deleted = new Set(lines(git(["diff", "--name-only", "--diff-filter=D", base]))); + return { files: [...new Set([...changed, ...untracked])], deleted }; +} + +function failClosed(reason) { + console.error(`Focused test selection is unsafe: ${reason}`); + console.error("Run the full unit suite with: npm run test"); + process.exit(2); +} + +const args = process.argv.slice(2); +const filesIndex = args.indexOf("--files"); +const explicitFiles = filesIndex >= 0 ? lines((args[filesIndex + 1] ?? "").replaceAll(",", "\n")) : null; +if (filesIndex >= 0 && explicitFiles?.length === 0) failClosed("--files did not contain a path"); + +const selection = explicitFiles ? { files: explicitFiles, deleted: new Set() } : defaultSelection(); +if (selection.files.length === 0) failClosed("no changed files were found"); +const missing = selection.files.filter((file) => !existsSync(path.resolve(projectRoot, file))); +if (missing.length > 0) failClosed(`deleted or missing paths require the full suite (${missing.join(", ")})`); +if (selection.deleted.size > 0) + failClosed(`deleted paths require the full suite (${[...selection.deleted].join(", ")})`); +const unsafe = selection.files.filter((file) => unsafeSelectionPattern.test(file.replaceAll("\\", "/"))); +if (unsafe.length > 0) failClosed(`test or configuration paths changed (${unsafe.join(", ")})`); + +const selectionArgs = new Set(); +if (filesIndex >= 0) { + selectionArgs.add(filesIndex); + selectionArgs.add(filesIndex + 1); +} +const forwarded = args.filter((argument, index) => !selectionArgs.has(index) && argument !== "--dry-run"); +console.log(`Focused test inputs: ${selection.files.join(", ")}`); +if (args.includes("--dry-run")) { + console.log("Command: vitest related --run <changed files>"); + process.exit(0); +} + +const runnerPath = path.join(projectRoot, "scripts", "run-vitest.mjs"); +const result = spawnSync(process.execPath, [runnerPath, "related", "--run", ...selection.files, ...forwarded], { + cwd: projectRoot, + stdio: "inherit", +}); +process.exit(childProcessExitCode(result)); diff --git a/scripts/test-rag-offline.mjs b/scripts/test-rag-offline.mjs new file mode 100644 index 000000000..738fbcd83 --- /dev/null +++ b/scripts/test-rag-offline.mjs @@ -0,0 +1,17 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; + +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const contractTests = JSON.parse( + readFileSync(path.join(projectRoot, "scripts/fixtures/rag-offline-contract-tests.json"), "utf8"), +); +const result = spawnSync( + process.execPath, + [path.join(projectRoot, "scripts/run-vitest.mjs"), "run", ...contractTests, ...process.argv.slice(2)], + { cwd: projectRoot, stdio: "inherit" }, +); +process.exit(childProcessExitCode(result)); diff --git a/scripts/test-run-lock.mjs b/scripts/test-run-lock.mjs new file mode 100644 index 000000000..d2f635680 --- /dev/null +++ b/scripts/test-run-lock.mjs @@ -0,0 +1,156 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +const tokenEnvironmentKey = "CLINICAL_KB_HEAVY_LOCK_TOKEN"; +const pathEnvironmentKey = "CLINICAL_KB_HEAVY_LOCK_PATH"; +const incompleteLockGraceMs = 30_000; + +function processIsAlive(pid) { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code === "EPERM"; + } +} + +function normalizeIdentity(value) { + const resolved = path.resolve(value); + return process.platform === "win32" ? resolved.toLowerCase() : resolved; +} + +function resolveRepositoryIdentity(projectRoot) { + const result = spawnSync("git", ["rev-parse", "--path-format=absolute", "--git-common-dir"], { + cwd: projectRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + const gitCommonDirectory = typeof result.stdout === "string" ? result.stdout.trim() : ""; + if (result.status === 0 && gitCommonDirectory) return normalizeIdentity(gitCommonDirectory); + + // Docker build contexts intentionally omit .git. They cannot share worktrees + // with the host, so a validated workspace-local identity keeps nested build + // commands safe without weakening the common-directory lock in Git checkouts. + try { + const manifest = JSON.parse(readFileSync(path.join(projectRoot, "package.json"), "utf8")); + if (manifest.name === "prompt-for-codex-medical-knowledge-base") return normalizeIdentity(projectRoot); + } catch { + // Preserve the explicit error below for an invalid working directory. + } + throw new Error("Could not resolve the shared Git directory for the Database heavyweight-run lock."); +} + +function lockPathFor(repositoryIdentity, baseDirectory = os.tmpdir()) { + const repositoryId = createHash("sha256").update(normalizeIdentity(repositoryIdentity)).digest("hex").slice(0, 20); + return path.join(baseDirectory, "clinical-kb-heavy-locks", `${repositoryId}.lock`); +} + +function readOwner(lockPath) { + try { + return JSON.parse(readFileSync(path.join(lockPath, "owner.json"), "utf8")); + } catch { + return null; + } +} + +function lockIsOldEnoughToRecover(lockPath, now = Date.now()) { + try { + return now - statSync(lockPath).mtimeMs >= incompleteLockGraceMs; + } catch { + return false; + } +} + +/** + * @param {{ + * projectRoot: string; + * command?: string; + * environment?: Record<string, string | undefined>; + * baseDirectory?: string; + * repositoryIdentity?: string; + * processId?: number; + * }} options + */ +export function acquireHeavyRunLock({ + projectRoot, + command = process.argv.join(" "), + environment = process.env, + baseDirectory, + repositoryIdentity = resolveRepositoryIdentity(projectRoot), + processId = process.pid, +}) { + if (!projectRoot) throw new Error("projectRoot is required for the Database heavyweight-run lock."); + const lockPath = lockPathFor(repositoryIdentity, baseDirectory); + const inheritedToken = environment[tokenEnvironmentKey]; + const inheritedPath = environment[pathEnvironmentKey]; + + if (inheritedToken || inheritedPath) { + if (!inheritedToken || normalizeIdentity(inheritedPath) !== normalizeIdentity(lockPath)) { + throw new Error("The inherited Database heavyweight-run lock does not match this repository."); + } + const owner = readOwner(lockPath); + if (owner?.token !== inheritedToken || !processIsAlive(owner.pid)) { + throw new Error("The inherited Database heavyweight-run lock is no longer owned by a live parent process."); + } + return { path: lockPath, owner, environment: { ...environment }, reentrant: true, release() {} }; + } + + mkdirSync(path.dirname(lockPath), { recursive: true }); + for (let attempt = 0; attempt < 2; attempt += 1) { + try { + mkdirSync(lockPath); + const token = randomUUID(); + const owner = { + pid: processId, + token, + command, + worktree: path.resolve(projectRoot), + repositoryIdentity: normalizeIdentity(repositoryIdentity), + startedAt: new Date().toISOString(), + }; + writeFileSync(path.join(lockPath, "owner.json"), `${JSON.stringify(owner, null, 2)}\n`, "utf8"); + let released = false; + return { + path: lockPath, + owner, + reentrant: false, + environment: { + ...environment, + [tokenEnvironmentKey]: token, + [pathEnvironmentKey]: lockPath, + }, + release() { + if (released) return; + released = true; + if (readOwner(lockPath)?.token === token) rmSync(lockPath, { recursive: true, force: true }); + }, + }; + } catch (error) { + if (error?.code !== "EEXIST") throw error; + const owner = readOwner(lockPath); + if (owner && processIsAlive(owner.pid)) { + throw new Error( + `Another Database heavyweight command is active (PID ${owner.pid}, worktree ${owner.worktree ?? "unknown"}, started ${owner.startedAt ?? "unknown"}): ${owner.command ?? "unknown command"}`, + ); + } + if (!owner && !lockIsOldEnoughToRecover(lockPath)) { + throw new Error(`A Database heavyweight lock is being initialized at ${lockPath}; retry after it settles.`); + } + rmSync(lockPath, { recursive: true, force: true }); + } + } + throw new Error(`Could not acquire the Database heavyweight-run lock at ${lockPath}.`); +} + +export const testRunLockInternals = { + lockPathFor, + processIsAlive, + readOwner, + resolveRepositoryIdentity, + tokenEnvironmentKey, + pathEnvironmentKey, +}; diff --git a/scripts/verify-pr-local.mjs b/scripts/verify-pr-local.mjs index 29cf24c3b..5455e9740 100644 --- a/scripts/verify-pr-local.mjs +++ b/scripts/verify-pr-local.mjs @@ -1,8 +1,13 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { childProcessExitCode } from "./child-process-result.mjs"; +import { acquireHeavyRunLock } from "./test-run-lock.mjs"; const isWindows = process.platform === "win32"; -const baseScripts = ["check:runtime", "format:check", "lint", "typecheck", "test"]; +const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const baseScripts = ["check:runtime", "format:changed", "lint", "typecheck", "test"]; function parseArgs(args) { const options = { dryRun: false, extended: false, files: undefined }; @@ -43,14 +48,12 @@ function parseArgs(args) { return options; } -function runNpmScript(script) { +function runNpmScript(script, environment) { console.log(`\n> npm run ${script}`); const result = isWindows - ? spawnSync("cmd.exe", ["/d", "/s", "/c", `npm run ${script}`], { stdio: "inherit" }) - : spawnSync("npm", ["run", script], { stdio: "inherit" }); - if (result.status !== 0) { - process.exit(result.status ?? 1); - } + ? spawnSync("cmd.exe", ["/d", "/s", "/c", `npm run ${script}`], { env: environment, stdio: "inherit" }) + : spawnSync("npm", ["run", script], { env: environment, stdio: "inherit" }); + return childProcessExitCode(result); } function readScope(files) { @@ -60,14 +63,15 @@ function readScope(files) { encoding: "utf8", stdio: ["ignore", "pipe", "inherit"], }); - if (result.status !== 0) process.exit(result.status ?? 1); + if (childProcessExitCode(result) !== 0) process.exit(childProcessExitCode(result)); return JSON.parse(result.stdout); } function selectedScripts(scope, extended) { const scripts = [...baseScripts]; if (scope.build_changed) scripts.push("build"); - if (!scope.docs_only) scripts.push("eval:rag:offline"); + // Full unit testing already includes every offline RAG contract suite. + if (!scope.docs_only) scripts.push("check:rag:fixtures"); if (extended && scope.ui_changed) scripts.push("verify:ui"); return scripts; } @@ -81,13 +85,24 @@ if (options.dryRun) { console.log("\nPR-local verification plan (dry run):"); for (const script of scripts) console.log(`- npm run ${script}`); if (!scope.build_changed) console.log("- build skipped: no build-affecting changes detected"); - if (scope.docs_only) console.log("- offline RAG evaluation skipped: docs-only change"); + if (scope.docs_only) console.log("- offline RAG fixture validation skipped: docs-only change"); if (options.extended && !scope.ui_changed) console.log("- Chromium UI gate skipped: no UI-affecting changes detected"); process.exit(0); } -for (const script of scripts) runNpmScript(script); +const lock = acquireHeavyRunLock({ projectRoot, command: "npm run verify:pr-local" }); +let exitCode = 0; +try { + for (const script of scripts) { + exitCode = runNpmScript(script, lock.environment); + if (exitCode !== 0) break; + } +} finally { + lock.release(); +} + +if (exitCode !== 0) process.exit(exitCode); if (!scope.build_changed) console.log("\nSkipping build: no build-affecting source, config, package, or container changes detected."); diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 2c1686055..b83237985 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -10,6 +10,23 @@ export async function register() { if (process.env.NEXT_RUNTIME !== "nodejs") return; if (process.env.NODE_ENV !== "production") return; + // Playwright validates a real production build, but its runner must remain a + // provider-free demo. Permit that otherwise-invalid combination only for the + // runner's isolated output and inert loopback environment. A partial or + // externally-addressed configuration still falls through to the production + // guard below and fails closed. + if (process.env.PLAYWRIGHT_OFFLINE_MODE === "true") { + const isolatedOutput = /^\.next-playwright\/[a-z0-9-]+\/dist$/i.test(process.env.NEXT_DIST_DIR ?? ""); + const providerFree = + process.env.NEXT_PUBLIC_DEMO_MODE === "true" && + process.env.RAG_PROVIDER_MODE === "offline" && + process.env.NEXT_PUBLIC_SUPABASE_URL === "http://127.0.0.1:1" && + !process.env.SUPABASE_SERVICE_ROLE_KEY && + !process.env.OPENAI_API_KEY; + if (isolatedOutput && providerFree) return; + throw new Error("Refusing to start: invalid isolated Playwright offline environment."); + } + // Defense in depth: no-auth must never be active in a production build (even // though isLocalNoAuthMode() already hard-guards on NODE_ENV). if (process.env.LOCAL_NO_AUTH === "true" || process.env.NEXT_PUBLIC_LOCAL_NO_AUTH === "true") { diff --git a/src/lib/local-server-utils.mjs b/src/lib/local-server-utils.mjs index 30eae3ada..d7c694cfa 100644 --- a/src/lib/local-server-utils.mjs +++ b/src/lib/local-server-utils.mjs @@ -33,6 +33,17 @@ export function stableProjectPort(projectRoot, platform = process.platform) { return port; } +export function circularProjectPortRange(startPort) { + if (!Number.isInteger(startPort) || startPort < projectPortStart || startPort > projectPortEnd) { + throw new Error(`Project port must be between ${projectPortStart} and ${projectPortEnd}: ${startPort}`); + } + const count = projectPortEnd - projectPortStart + 1; + return Array.from( + { length: count }, + (_, index) => projectPortStart + ((startPort - projectPortStart + index) % count), + ); +} + export function localProjectId(projectRoot, platform = process.platform) { return `clinical-kb:${projectHash(projectRoot, platform).toString("hex").slice(0, 12)}`; } diff --git a/src/proxy.ts b/src/proxy.ts index da089ed92..2367aeb9e 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -89,7 +89,7 @@ export async function proxy(request: NextRequest) { return withCsp(NextResponse.redirect(url)); } - if (pathname.startsWith("/mockups") && process.env.NODE_ENV === "production") { + if (shouldBlockProductionMockups(pathname)) { return withCsp(new NextResponse(null, { status: 404 })); } @@ -121,6 +121,20 @@ export async function proxy(request: NextRequest) { return withCsp(response); } +export function shouldBlockProductionMockups( + pathname: string, + environment: Record<string, string | undefined> = process.env, +) { + if (!pathname.startsWith("/mockups") || environment.NODE_ENV !== "production") return false; + + // Mockups remain unavailable in every normal production process. The one + // exception is the repository-owned, isolated Playwright server when its + // advisory project explicitly opts into mockup coverage. Instrumentation + // separately refuses PLAYWRIGHT_OFFLINE_MODE outside the inert loopback and + // isolated .next-playwright profile. + return !(environment.PLAYWRIGHT_OFFLINE_MODE === "true" && environment.NEXT_PUBLIC_MOCKUPS_ENABLED === "true"); +} + export const config = { // Run on everything except static assets and image files. API routes stay in // the matcher so cookie-authenticated requests can return rotated cookies and diff --git a/tests/architecture-boundaries.test.ts b/tests/architecture-boundaries.test.ts index 323aec37c..9a0c6c401 100644 --- a/tests/architecture-boundaries.test.ts +++ b/tests/architecture-boundaries.test.ts @@ -116,7 +116,9 @@ function resolveModule(fromFile: string, specifier: string, fileSet: Set<string> return candidates.map((candidate) => path.resolve(candidate)).find((candidate) => fileSet.has(candidate)) ?? null; } -function runtimeGraph() { +let runtimeGraphCache: ReturnType<typeof buildRuntimeGraph> | undefined; + +function buildRuntimeGraph() { const files = sourceFiles(); const fileSet = new Set(files); const graph = new Map<string, string[]>(); @@ -134,6 +136,11 @@ function runtimeGraph() { return { files, fileSet, graph, parsed }; } +function runtimeGraph() { + runtimeGraphCache ??= buildRuntimeGraph(); + return runtimeGraphCache; +} + function runtimeCycles(graph: Map<string, string[]>) { let nextIndex = 0; const indexes = new Map<string, number>(); @@ -195,7 +202,7 @@ describe("architecture boundaries", () => { expect(runtimeCycles(new Map([[file, [file]]]))).toEqual([[file]]); }); - it("has no runtime import cycles", () => { + it("has no runtime import cycles", { timeout: 60_000 }, () => { const { graph } = runtimeGraph(); const cycles = runtimeCycles(graph).map((cycle) => cycle.map(relative).sort()); expect(cycles).toEqual([]); @@ -229,8 +236,9 @@ describe("architecture boundaries", () => { }); it("does not make runtime source modules depend on operational scripts", () => { - const scriptImports = sourceFiles().flatMap((file) => { - const modules = moduleSpecifiers(file); + const { files, parsed } = runtimeGraph(); + const scriptImports = files.flatMap((file) => { + const modules = parsed.get(file)!; return [...modules.staticImports, ...modules.dynamicImports] .filter((specifier) => path.resolve(path.dirname(file), specifier).startsWith(path.join(projectRoot, "scripts")), diff --git a/tests/ci-cache-safety.test.ts b/tests/ci-cache-safety.test.ts index 712de2a50..aad6f69e0 100644 --- a/tests/ci-cache-safety.test.ts +++ b/tests/ci-cache-safety.test.ts @@ -15,11 +15,12 @@ describe("CI cache safety", () => { expect(cacheKey).toContain(".npmrc"); }); - it("keeps quarantined UI specs in an advisory lane without unconditional browser setup", () => { - expect(workflow).toContain("ui-quarantine:"); - expect(workflow).toContain("git grep -q '@quarantine'"); - expect(workflow).toContain("run: npm run test:e2e:quarantine"); - expect(workflow).toContain("if: steps.quarantine.outputs.present == 'true'"); + it("keeps quarantined and mockup UI specs in one advisory lane", () => { + expect(workflow).toContain("ui-advisory:"); + expect(workflow).toContain("uses: ./.github/actions/setup-ui-e2e"); + expect(workflow).toContain("run: npm run test:e2e:advisory"); + expect(workflow).not.toContain("ui-quarantine:"); + expect(workflow).not.toContain("ui-mockups:"); }); it("installs Playwright system dependencies when browser caches hit", () => { diff --git a/tests/client-secret-surface.test.ts b/tests/client-secret-surface.test.ts index fff4b038e..98d223ad2 100644 --- a/tests/client-secret-surface.test.ts +++ b/tests/client-secret-surface.test.ts @@ -144,7 +144,8 @@ describe("client environment isolation", () => { scripts: Record<string, string>; }; expect(packageJson.scripts["check:client-bundle-secrets"]).toContain("check-client-bundle-secrets.mjs"); - expect(packageJson.scripts.build).toContain("check-client-bundle-secrets.mjs"); + expect(packageJson.scripts.build).toContain("run-heavy.mjs"); + expect(packageJson.scripts["build:internal"]).toContain("check-client-bundle-secrets.mjs"); const fixtureRoot = mkdtempSync(join(tmpdir(), "clinical-kb-client-bundle-")); try { diff --git a/tests/demo-data.test.ts b/tests/demo-data.test.ts index 70d570bfc..c181c8c17 100644 --- a/tests/demo-data.test.ts +++ b/tests/demo-data.test.ts @@ -110,6 +110,7 @@ describe("demoSearch production safety guard", () => { it("refuses to serve synthetic demo data to a live production request", async () => { vi.stubEnv("NODE_ENV", "production"); + vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "false"); const { demoSearch: guarded } = await import("../src/lib/demo-data"); expect(() => guarded("lithium toxicity")).toThrow(/synthetic demo data/i); }); diff --git a/tests/flake-ledger.json b/tests/flake-ledger.json index 0db897684..8f2f9f59c 100644 --- a/tests/flake-ledger.json +++ b/tests/flake-ledger.json @@ -1,37 +1,4 @@ { - "$comment": "Known-flaky Playwright specs. Durable knowledge so a failure gets attributed to a known flake instead of re-diagnosed from memory each time. Consumed by scripts/flake-ledger.mjs (loader + isKnownFlake) and the CI failure-triage workflow. `match` is a case-insensitive substring of the test title; keep entries narrow. Removing an entry means 'we believe this is fixed' — do that deliberately.", - "flakes": [ - { - "id": "raf-portal-hydration", - "match": "composer hero", - "spec": "tests/ui-smoke.spec.ts", - "reason": "Hero-composer hydration depends on a rAF portal that starves under a headless render; fixed with a MutationObserver microtask (#502/#504) but can still flake on a cold render.", - "since": "2026-07-11", - "refs": ["#502", "#504"] - }, - { - "id": "tap-target-subpixel", - "match": "tap target", - "spec": "tests/ui-smoke.spec.ts", - "reason": "min-h-11 tap-target rect rounds sub-pixel below 44px on some DPRs; mitigated by min-h-12 but historically flaky.", - "since": "2026-07-11", - "refs": [] - }, - { - "id": "narrow-viewport-parallel", - "match": "narrow viewport", - "spec": "tests/ui-tools.spec.ts", - "reason": "Differentials narrow-viewport assertions race only under parallel workers; the suite runs workers:1 but sharded CI can still interleave.", - "since": "2026-07-11", - "refs": [] - }, - { - "id": "rag-answer-fallback-timeout", - "match": "answer fallback", - "spec": "tests/ui-tools.spec.ts", - "reason": "The RAG answer-fallback path can exceed the 60s test timeout when the provider route is cold; a pre-existing advisory flake.", - "since": "2026-07-11", - "refs": [] - } - ] + "$comment": "Only reproduced @quarantine Playwright tests belong here. Entries use an exact title and expire within 30 days. The ledger is intentionally empty after stale, non-reproducible entries were removed.", + "flakes": [] } diff --git a/tests/flake-ledger.test.ts b/tests/flake-ledger.test.ts index fd8dae544..f5033bbb7 100644 --- a/tests/flake-ledger.test.ts +++ b/tests/flake-ledger.test.ts @@ -1,31 +1,103 @@ -import { describe, expect, it } from "vitest"; -import { isKnownFlake, loadFlakeLedger, matchFlake } from "../scripts/flake-ledger.mjs"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { failedTestcasesFromJunit } from "../scripts/classify-playwright-failures.mjs"; +import { loadFlakeLedger, matchFlake, validateFlakeLedgerEntries } from "../scripts/flake-ledger.mjs"; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +function isoDate(offsetDays: number, now = Date.now()) { + return new Date(now + offsetDays * 24 * 60 * 60 * 1000).toISOString().slice(0, 10); +} + +function validEntry(overrides: Record<string, string> = {}) { + return { + id: "sample-flake", + title: "exact flaky journey @quarantine", + spec: "tests/sample.spec.ts", + reason: "Reproduced fail/pass variation on the same SHA.", + owner: "test-infrastructure", + reproduction: "npm run test:e2e:advisory -- --grep exact --repeat-each=3", + firstSeen: isoDate(-2), + lastSeen: isoDate(-1), + expires: isoDate(20), + tracking: "docs/process-hardening.md#known-flakes", + ...overrides, + }; +} + +function temporarySpec(source: string) { + const root = mkdtempSync(path.join(os.tmpdir(), "clinical-kb-flake-ledger-test-")); + temporaryDirectories.push(root); + mkdirSync(path.join(root, "tests")); + writeFileSync(path.join(root, "tests", "sample.spec.ts"), source); + return root; +} describe("flake ledger", () => { - it("loads and validates the committed ledger", () => { - const flakes = loadFlakeLedger(); - expect(flakes.length).toBeGreaterThan(0); - for (const flake of flakes) { - expect(flake.id).toBeTruthy(); - expect(flake.match).toBeTruthy(); - expect(flake.spec).toMatch(/^tests\//); - expect(flake.reason).toBeTruthy(); - } + it("allows the committed ledger to be empty", () => { + expect(loadFlakeLedger()).toEqual([]); + }); + + it("matches only the exact case-insensitive spec and title identity", () => { + const flakes = [validEntry()]; + expect(matchFlake("tests/sample.spec.ts", "EXACT FLAKY JOURNEY @QUARANTINE", flakes)?.id).toBe("sample-flake"); + expect(matchFlake("tests/sample.spec.ts", "prefix exact flaky journey @quarantine", flakes)).toBeNull(); + expect(matchFlake("tests/other.spec.ts", "exact flaky journey @quarantine", flakes)).toBeNull(); + expect(matchFlake("", "")).toBeNull(); + }); + + it("requires exact quarantined titles and complete ownership metadata", () => { + expect(() => validateFlakeLedgerEntries([validEntry({ title: "" })])).toThrow(/missing: title/); + expect(() => validateFlakeLedgerEntries([validEntry({ title: "not tagged" })])).toThrow(/include @quarantine/); + }); + + it("rejects critical overlap and expiry beyond 30 days", () => { + const now = Date.now(); + expect(() => validateFlakeLedgerEntries([validEntry({ title: "unsafe @quarantine @critical" })])).toThrow( + /cannot be both @quarantine and @critical/, + ); + expect(() => validateFlakeLedgerEntries([validEntry({ expires: isoDate(31, now) })], { now })).toThrow( + /within 30 days/, + ); + const root = temporarySpec('test("exact flaky journey @quarantine", () => {});'); + expect(validateFlakeLedgerEntries([validEntry({ expires: isoDate(30, now) })], { root, now })).toHaveLength(1); }); - it("has unique ids", () => { - const ids = loadFlakeLedger().map((f: { id: string }) => f.id); - expect(new Set(ids).size).toBe(ids.length); + it("rejects calendar-invalid ledger dates", () => { + expect(() => validateFlakeLedgerEntries([validEntry({ firstSeen: "2026-02-30" })])).toThrow(/not a valid date/); }); - it("matches a known flake by case-insensitive title substring", () => { - expect(isKnownFlake("Composer Hero mounts after hydration")).toBe(true); - const hit = matchFlake("the tap target is at least 44px"); - expect(hit?.id).toBe("tap-target-subpixel"); + it("requires the exact title to exist in the referenced spec", () => { + const entry = validEntry(); + const root = temporarySpec(`test(${JSON.stringify(entry.title)}, () => {});`); + expect(validateFlakeLedgerEntries([entry], { root })).toHaveLength(1); + expect(() => validateFlakeLedgerEntries([validEntry({ title: "missing @quarantine" })], { root })).toThrow( + /exact title is not present/, + ); }); - it("does not match an unrelated title or an empty string", () => { - expect(isKnownFlake("renders the medication results grid")).toBe(false); - expect(matchFlake("")).toBeNull(); + it("extracts only exact failed testcase identities from JUnit", () => { + const failures = failedTestcasesFromJunit(` + <testsuite> + <testcase name="exact flaky journey @quarantine" classname="sample.spec.ts"> + <failure message="failed" /> + </testcase> + <testcase name="passing journey" classname="tests/sample.spec.ts"></testcase> + <testcase name="diagnostic journey" classname="tests/sample.spec.ts"> + <system-out><![CDATA[diagnostic text containing <failure> but no failure element]]></system-out> + <!-- <error>diagnostic comment</error> --> + </testcase> + </testsuite> + `); + expect(failures).toEqual([{ spec: "tests/sample.spec.ts", title: "exact flaky journey @quarantine" }]); + const flakes = [validEntry()]; + expect(matchFlake(failures[0]?.spec, failures[0]?.title, flakes)?.id).toBe("sample-flake"); + expect(matchFlake("tests/other.spec.ts", failures[0]?.title, flakes)).toBeNull(); }); }); diff --git a/tests/instrumentation.test.ts b/tests/instrumentation.test.ts index eeebfe99d..af61b7c99 100644 --- a/tests/instrumentation.test.ts +++ b/tests/instrumentation.test.ts @@ -22,6 +22,8 @@ const ENV_KEYS = [ "RAG_QUERY_HASH_SECRET", "NEXT_PUBLIC_LOCAL_NO_AUTH", "LOCAL_NO_AUTH", + "PLAYWRIGHT_OFFLINE_MODE", + "NEXT_DIST_DIR", ] as const; async function loadInstrumentation(overrides: Partial<Record<(typeof ENV_KEYS)[number], string | undefined>>) { @@ -64,6 +66,28 @@ describe("instrumentation boot guard", () => { await expect(register()).rejects.toThrow(/demo mode is enabled/); }); + it("allows only the isolated provider-free Playwright production profile", async () => { + const valid = await loadRegister({ + ...PRODUCTION_NODE, + PLAYWRIGHT_OFFLINE_MODE: "true", + NEXT_DIST_DIR: ".next-playwright/123-456/dist", + NEXT_PUBLIC_DEMO_MODE: "true", + NEXT_PUBLIC_SUPABASE_URL: "http://127.0.0.1:1", + RAG_PROVIDER_MODE: "offline", + }); + await expect(valid()).resolves.toBeUndefined(); + + const external = await loadRegister({ + ...PRODUCTION_NODE, + PLAYWRIGHT_OFFLINE_MODE: "true", + NEXT_DIST_DIR: ".next-playwright/123-456/dist", + NEXT_PUBLIC_DEMO_MODE: "true", + NEXT_PUBLIC_SUPABASE_URL: MATCHING_URL, + RAG_PROVIDER_MODE: "offline", + }); + await expect(external()).rejects.toThrow(/invalid isolated Playwright offline environment/); + }); + it("refuses to start a production server with local no-auth enabled", async () => { const register = await loadRegister({ ...PRODUCTION_NODE, LOCAL_NO_AUTH: "true" }); await expect(register()).rejects.toThrow(/no-auth mode is enabled/); diff --git a/tests/local-server-utils.test.ts b/tests/local-server-utils.test.ts index efca5b614..6337b917d 100644 --- a/tests/local-server-utils.test.ts +++ b/tests/local-server-utils.test.ts @@ -1,6 +1,13 @@ import { describe, expect, it } from "vitest"; -import { localProjectId, normalizeProjectRoot, stableProjectPort } from "../src/lib/local-server-utils.mjs"; +import { + circularProjectPortRange, + localProjectId, + normalizeProjectRoot, + projectPortEnd, + projectPortStart, + stableProjectPort, +} from "../src/lib/local-server-utils.mjs"; describe("local server project identity", () => { it("normalizes Windows roots case-insensitively", () => { @@ -32,4 +39,11 @@ describe("local server project identity", () => { expect(port).toBeLessThanOrEqual(4599); } }); + + it("scans the full port range circularly from the preferred port", () => { + const ports = circularProjectPortRange(projectPortEnd); + expect(ports.slice(0, 3)).toEqual([projectPortEnd, projectPortStart, projectPortStart + 1]); + expect(new Set(ports).size).toBe(projectPortEnd - projectPortStart + 1); + expect(ports.at(-1)).toBe(projectPortEnd - 1); + }); }); diff --git a/tests/property-accessible-table.test.ts b/tests/property-accessible-table.test.ts index a6ff21482..12d48f3cc 100644 --- a/tests/property-accessible-table.test.ts +++ b/tests/property-accessible-table.test.ts @@ -1,4 +1,5 @@ import fc from "fast-check"; +import "./property-seed"; import { describe, expect, it } from "vitest"; import { normalizeAccessibleTable } from "../src/lib/accessible-table-normalization"; import { formatAnswerForClipboard, formatWardNote } from "../src/lib/ward-output"; diff --git a/tests/property-chunking.test.ts b/tests/property-chunking.test.ts index a0547e4d6..6422ea7a1 100644 --- a/tests/property-chunking.test.ts +++ b/tests/property-chunking.test.ts @@ -1,4 +1,5 @@ import fc from "fast-check"; +import "./property-seed"; import { describe, expect, it } from "vitest"; import { chunkTextWithOverlap } from "../src/lib/chunking"; diff --git a/tests/property-numeric-token-preservation.test.ts b/tests/property-numeric-token-preservation.test.ts index af645922b..48071c9f4 100644 --- a/tests/property-numeric-token-preservation.test.ts +++ b/tests/property-numeric-token-preservation.test.ts @@ -1,4 +1,5 @@ import fc from "fast-check"; +import "./property-seed"; import { describe, expect, it } from "vitest"; import { clinicalProseUsefulness, diff --git a/tests/property-seed-validation.test.ts b/tests/property-seed-validation.test.ts new file mode 100644 index 000000000..6e8c2b265 --- /dev/null +++ b/tests/property-seed-validation.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { parsePropertySeed } from "./property-seed"; + +describe("FAST_CHECK_SEED validation", () => { + it.each(["123junk", "1.9", "1e3", "", "9007199254740992"])("rejects malformed seed %j", (seed) => { + expect(() => parsePropertySeed(seed)).toThrow(/must be a safe integer/); + }); + + it("accepts signed safe integers", () => { + expect(parsePropertySeed("+123")).toBe(123); + expect(parsePropertySeed("-123")).toBe(-123); + }); +}); diff --git a/tests/property-seed.ts b/tests/property-seed.ts new file mode 100644 index 000000000..a2b3dea9f --- /dev/null +++ b/tests/property-seed.ts @@ -0,0 +1,17 @@ +import fc from "fast-check"; + +const defaultPropertySeed = 424_242; +export function parsePropertySeed(rawSeed = `${defaultPropertySeed}`) { + const parsedSeed = Number(rawSeed); + if (!/^[+-]?\d+$/.test(rawSeed) || !Number.isSafeInteger(parsedSeed)) { + throw new Error(`FAST_CHECK_SEED must be a safe integer; received ${rawSeed}.`); + } + return parsedSeed; +} +const parsedSeed = parsePropertySeed(process.env.FAST_CHECK_SEED); +const propertySeed = ((parsedSeed % 2_147_483_647) + 2_147_483_647) % 2_147_483_647 || defaultPropertySeed; + +fc.configureGlobal({ seed: propertySeed, verbose: true }); +console.info(`[fast-check] replay seed: ${propertySeed}`); + +export { propertySeed }; diff --git a/tests/proxy.test.ts b/tests/proxy.test.ts index fead5901c..a3b47cc61 100644 --- a/tests/proxy.test.ts +++ b/tests/proxy.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { NextRequest } from "next/server"; -import { proxy } from "../src/proxy"; +import { proxy, shouldBlockProductionMockups } from "../src/proxy"; // The proxy owns the per-request nonce CSP (see src/proxy.ts). CI's verify:ui // only exercises the *dev* CSP path (Turbopack keeps 'unsafe-inline'); these @@ -75,3 +75,29 @@ describe("proxy content-security-policy", () => { expect(res.headers.get("x-middleware-request-x-nonce")).toBe(cspNonce); }); }); + +describe("production mockup boundary", () => { + it("blocks ordinary production traffic and permits only the explicit isolated Playwright advisory profile", () => { + expect(shouldBlockProductionMockups("/mockups/tools-workflow-board", { NODE_ENV: "production" })).toBe(true); + expect( + shouldBlockProductionMockups("/mockups/tools-workflow-board", { + NODE_ENV: "production", + PLAYWRIGHT_OFFLINE_MODE: "true", + }), + ).toBe(true); + expect( + shouldBlockProductionMockups("/mockups/tools-workflow-board", { + NODE_ENV: "production", + NEXT_PUBLIC_MOCKUPS_ENABLED: "true", + }), + ).toBe(true); + expect( + shouldBlockProductionMockups("/mockups/tools-workflow-board", { + NODE_ENV: "production", + PLAYWRIGHT_OFFLINE_MODE: "true", + NEXT_PUBLIC_MOCKUPS_ENABLED: "true", + }), + ).toBe(false); + expect(shouldBlockProductionMockups("/applications", { NODE_ENV: "production" })).toBe(false); + }); +}); diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 18af9825b..33defe169 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { RagAnswer, SearchResult } from "../src/lib/types"; function retrievalRpcBaseName(name: string) { @@ -139,6 +139,12 @@ async function answerFromTextSources( }); } +beforeEach(() => { + // The default runner removes real provider credentials and selects offline mode. + // This suite supplies a fully mocked provider and must exercise those fake paths. + vi.stubEnv("RAG_PROVIDER_MODE", "auto"); +}); + afterEach(() => { vi.restoreAllMocks(); vi.resetModules(); diff --git a/tests/test-runner-safety.test.ts b/tests/test-runner-safety.test.ts new file mode 100644 index 000000000..1e39508d0 --- /dev/null +++ b/tests/test-runner-safety.test.ts @@ -0,0 +1,187 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { childProcessExitCode, childProcessFailureSummary } from "../scripts/child-process-result.mjs"; +import { + offlineTestEnvironment, + offlineUrlValues, + providerEnvironmentKeys, + requireProviderTestPermission, +} from "../scripts/test-environment.mjs"; +import { acquireHeavyRunLock } from "../scripts/test-run-lock.mjs"; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +function temporaryDirectory(prefix: string) { + const directory = mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryDirectories.push(directory); + return directory; +} + +describe("child process results", () => { + it("never treats a missing status, signal, or launch error as success", () => { + expect(childProcessExitCode({ status: null, signal: "SIGTERM" })).toBe(1); + expect(childProcessExitCode({ status: null, signal: null })).toBe(1); + expect(childProcessExitCode({ status: null, error: new Error("launch failed") })).toBe(1); + expect(childProcessExitCode({ status: 0 })).toBe(0); + expect(childProcessExitCode({ status: 7 })).toBe(7); + expect(childProcessFailureSummary({ status: null, signal: "SIGTERM" })).toBe("missing exit status, signal SIGTERM"); + }); + + it("maps a real missing executable launch to failure", () => { + const result = spawnSync(`clinical-kb-missing-command-${Date.now()}`, []); + expect(result.error).toBeTruthy(); + expect(childProcessExitCode(result)).toBe(1); + }); +}); + +describe("repository-wide heavyweight lock", () => { + it("uses a workspace-local identity only when a packaged build context has no Git metadata", () => { + const projectRoot = temporaryDirectory("clinical-kb-no-git-"); + const baseDirectory = temporaryDirectory("clinical-kb-no-git-lock-"); + writeFileSync( + path.join(projectRoot, "package.json"), + JSON.stringify({ name: "prompt-for-codex-medical-knowledge-base" }), + ); + + const lock = acquireHeavyRunLock({ projectRoot, baseDirectory, environment: {}, command: "docker build" }); + const expectedIdentity = path.resolve(projectRoot); + expect(lock.owner.repositoryIdentity).toBe( + process.platform === "win32" ? expectedIdentity.toLowerCase() : expectedIdentity, + ); + lock.release(); + }); + + it("blocks another worktree but permits a nested child with the owner token", () => { + const baseDirectory = temporaryDirectory("clinical-kb-lock-"); + const repositoryIdentity = path.join(baseDirectory, "shared.git"); + const first = acquireHeavyRunLock({ + projectRoot: path.join(baseDirectory, "worktree-a"), + repositoryIdentity, + baseDirectory, + environment: {}, + command: "first", + }); + + expect(() => + acquireHeavyRunLock({ + projectRoot: path.join(baseDirectory, "worktree-b"), + repositoryIdentity, + baseDirectory, + environment: {}, + command: "second", + }), + ).toThrow(/Another Database heavyweight command is active/); + + const nested = acquireHeavyRunLock({ + projectRoot: path.join(baseDirectory, "worktree-b"), + repositoryIdentity, + baseDirectory, + environment: first.environment, + command: "nested", + }); + expect(nested.reentrant).toBe(true); + nested.release(); + first.release(); + }); + + it("recovers a dead owner without allowing the old token to release the replacement", () => { + const baseDirectory = temporaryDirectory("clinical-kb-stale-lock-"); + const repositoryIdentity = path.join(baseDirectory, "shared.git"); + const stale = acquireHeavyRunLock({ + projectRoot: path.join(baseDirectory, "worktree-a"), + repositoryIdentity, + baseDirectory, + environment: {}, + processId: 2_147_483_647, + command: "dead", + }); + const replacement = acquireHeavyRunLock({ + projectRoot: path.join(baseDirectory, "worktree-b"), + repositoryIdentity, + baseDirectory, + environment: {}, + command: "replacement", + }); + + stale.release(); + expect(readFileSync(path.join(replacement.path, "owner.json"), "utf8")).toContain(replacement.owner.token); + replacement.release(); + }); +}); + +describe("provider-safe test environment", () => { + it("removes provider credentials and explicit live-test permission", () => { + const source = Object.fromEntries(providerEnvironmentKeys.map((key) => [key, `secret-${key}`])); + const environment: Record<string, string | undefined> = offlineTestEnvironment({ + ...source, + SAFE_VALUE: "kept", + }); + + expect(environment).toMatchObject({ + SAFE_VALUE: "kept", + RAG_PROVIDER_MODE: "offline", + NEXT_PUBLIC_DEMO_MODE: "true", + }); + for (const key of providerEnvironmentKeys) { + expect(environment[key]).toBe(offlineUrlValues[key as keyof typeof offlineUrlValues] ?? ""); + } + }); + + it("requires explicit permission before live tests can run", () => { + expect(() => requireProviderTestPermission({})).toThrow(/ALLOW_PROVIDER_TESTS=true/); + expect(() => requireProviderTestPermission({ ALLOW_PROVIDER_TESTS: "true" })).not.toThrow(); + }); + + it("keeps live tests out of default Vitest discovery", () => { + const config = readFileSync(new URL("../vitest.config.mts", import.meta.url), "utf8"); + expect(config).toContain('exclude: liveProviderTests ? [] : ["tests/**/*.live.test.ts"]'); + }); + + it("refuses the live-test command before collection when permission is absent", () => { + const result = spawnSync(process.execPath, ["scripts/run-live-tests.mjs"], { + cwd: path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."), + env: { ...process.env, ALLOW_PROVIDER_TESTS: "" }, + encoding: "utf8", + }); + expect(childProcessExitCode(result)).toBe(1); + expect(`${result.stdout}${result.stderr}`).toContain("Live provider tests are disabled"); + }); + + it("fails focused selection closed for a deleted or missing explicit source path", () => { + const result = spawnSync(process.execPath, ["scripts/test-focused.mjs", "--files", "src/missing-source.ts"], { + cwd: path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."), + encoding: "utf8", + }); + expect(childProcessExitCode(result)).toBe(2); + expect(`${result.stdout}${result.stderr}`).toContain("deleted or missing paths require the full suite"); + }); + + it("builds and starts an isolated production server for Playwright", () => { + const runner = readFileSync(new URL("../scripts/run-playwright.mjs", import.meta.url), "utf8"); + const baseUrl = readFileSync(new URL("../scripts/playwright-base-url.ts", import.meta.url), "utf8"); + const ragRunner = readFileSync(new URL("../scripts/eval-rag-offline.mjs", import.meta.url), "utf8"); + const packageJson = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as { + scripts: Record<string, string>; + }; + expect(runner).toContain('["--max-old-space-size=8192", nextBin, "build", "--webpack"]'); + expect(runner).toContain('[nextBin, "start", "--hostname"'); + expect(runner).not.toContain('[nextBin, "dev", "--hostname"'); + expect(runner).toContain('NODE_ENV: "production"'); + expect(runner).toContain('PLAYWRIGHT_OFFLINE_MODE: "true"'); + expect(runner).toContain('NEXT_PUBLIC_MOCKUPS_ENABLED: mockupProjectRequested ? "true" : "false"'); + expect(runner).toContain("!explicitProjectRequested ||"); + expect(runner).not.toContain("supabase.co"); + expect(packageJson.scripts["test:e2e:pr"]).toContain('--grep-invert "@quarantine|@mockup"'); + expect(packageJson.scripts["test:e2e:regression"]).toContain('--grep-invert "@critical|@quarantine|@mockup"'); + expect(baseUrl.indexOf("if (!allowEnsure)")).toBeLessThan(baseUrl.indexOf("findExistingLocalProjectUrl();")); + expect(ragRunner).toContain("cwd: projectRoot"); + }); +}); diff --git a/tests/tsx-server-only-runner.test.ts b/tests/tsx-server-only-runner.test.ts index 0b88c2daf..4d1345268 100644 --- a/tests/tsx-server-only-runner.test.ts +++ b/tests/tsx-server-only-runner.test.ts @@ -47,11 +47,11 @@ describe("standalone TSX server-only compatibility", () => { ); }); - it("bounds Vitest workers and scopes stale-process cleanup to this checkout", () => { + it("bounds Vitest workers and uses the shared non-destructive run lock", () => { const runner = readFileSync(new URL("../scripts/run-vitest.mjs", import.meta.url), "utf8"); const config = readFileSync(new URL("../vitest.config.mts", import.meta.url), "utf8"); - expect(runner).toContain("const vitestNeedle = vitestBin.toLowerCase()"); - expect(runner).not.toContain("repoNeedle"); + expect(runner).toContain("acquireHeavyRunLock"); + expect(runner).not.toContain("taskkill"); // Workers stay bounded to a finite default (tunable via VITEST_MAX_WORKERS) so a // parallel run can never spawn unlimited workers and thrash the host. expect(config).toMatch( diff --git a/tests/ui-accessibility.spec.ts b/tests/ui-accessibility.spec.ts index 24c5f42d3..dd87de012 100644 --- a/tests/ui-accessibility.spec.ts +++ b/tests/ui-accessibility.spec.ts @@ -35,13 +35,7 @@ async function mockMinimalDashboardApi(page: Page) { async function gotoApp(page: Page) { await page.goto("/", { waitUntil: "domcontentloaded" }); - // Deterministic app-shell mount wait (networkidle burned the full timeout on - // routes with persistent background fetches; per-test assertions gate readiness). - await page - .locator("#main-content") - .first() - .waitFor({ state: "visible", timeout: 15_000 }) - .catch(() => undefined); + await expect(page.locator("#main-content").first()).toBeVisible({ timeout: 15_000 }); } async function expectNoPageHorizontalOverflow(page: Page) { diff --git a/tests/ui-formulation.spec.ts b/tests/ui-formulation.spec.ts index 09f1616fd..ff3d8e051 100644 --- a/tests/ui-formulation.spec.ts +++ b/tests/ui-formulation.spec.ts @@ -20,13 +20,7 @@ async function blockExternalRequests(page: Page) { async function gotoApp(page: Page, path: string) { await page.goto(path, { waitUntil: "domcontentloaded" }); - // Deterministic app-shell mount wait (networkidle burned the full timeout on - // routes with persistent background fetches; per-test assertions gate readiness). - await page - .locator("#main-content") - .first() - .waitFor({ state: "visible", timeout: 15_000 }) - .catch(() => undefined); + await expect(page.locator("#main-content").first()).toBeVisible({ timeout: 15_000 }); } async function expectNoHorizontalOverflow(page: Page) { diff --git a/tests/ui-smoke.spec.ts b/tests/ui-smoke.spec.ts index b4c3948a8..716ac1b16 100644 --- a/tests/ui-smoke.spec.ts +++ b/tests/ui-smoke.spec.ts @@ -50,18 +50,10 @@ async function installClipboardMock(page: Page) { async function gotoApp(page: Page, path: string) { await page.goto(path, { waitUntil: "domcontentloaded" }); - // Wait for the app shell to mount (deterministic) rather than for the network to - // idle — these routes keep background fetches alive, so networkidle burned the - // full timeout on every navigation. Best-effort: the per-test assertions below - // still gate real readiness if the shell selector never resolves. - await page - .locator("#main-content") - .first() - .waitFor({ state: "visible", timeout: 15_000 }) - .catch(() => undefined); + await expect(page.locator("#main-content").first()).toBeVisible({ timeout: 15_000 }); } -async function waitForReactEventHandler(locator: Locator, eventName: "onScroll") { +async function waitForReactEventHandler(locator: Locator, eventName: "onChange" | "onClick" | "onScroll" | "onSubmit") { await expect .poll( async () => @@ -110,6 +102,20 @@ function visibleAnswerSubmitButton(page: Page) { return page.locator('[aria-label="Generate source-backed answer"]:visible').first(); } +async function submitDocumentSearch(page: Page) { + const submit = page.getByRole("button", { name: "Find matching documents" }); + await expect(submit).toBeEnabled(); + await waitForReactEventHandler(submit.locator("xpath=ancestor::form[1]"), "onSubmit"); + const response = page.waitForResponse( + (candidate) => new URL(candidate.url()).pathname === "/api/search" && candidate.ok(), + { timeout: 30_000 }, + ); + await Promise.all([response, submit.click()]); + await expect(page.getByRole("heading", { name: "Finding matching documents" })).toHaveCount(0, { + timeout: 30_000, + }); +} + function visibleAnswerFollowUpSuggestions(page: Page) { return page .locator( @@ -143,10 +149,9 @@ async function switchToDocumentSearchMode(page: Page) { const legacyDocumentsMode = page.getByRole("button", { name: "Switch to document search mode" }); if (await isVisibleWithoutThrow(legacyDocumentsMode)) { await expect(legacyDocumentsMode).toBeEnabled(); - await expect(async () => { - await legacyDocumentsMode.click(); - await expect(legacyDocumentsMode).toHaveAttribute("aria-pressed", "true", { timeout: uiAssertionTimeoutMs }); - }).toPass({ timeout: 8_000 }); + await waitForReactEventHandler(legacyDocumentsMode, "onClick"); + await legacyDocumentsMode.click(); + await expect(legacyDocumentsMode).toHaveAttribute("aria-pressed", "true", { timeout: uiAssertionTimeoutMs }); return; } @@ -157,18 +162,15 @@ async function switchToDocumentSearchMode(page: Page) { ); } await expect(appModeMenu).toBeEnabled(); - - await expect(async () => { - if ((await appModeMenu.getAttribute("aria-expanded")) !== "true") { - await appModeMenu.click({ force: true }); - } - const appModeGroup = page.getByRole("menu", { name: "Choose app mode" }); - await expect(appModeGroup).toBeVisible({ timeout: uiAssertionTimeoutMs }); - const documentsMode = appModeGroup.getByRole("menuitemradio", { name: /^Documents\b/ }); - await expect(documentsMode).toBeVisible({ timeout: uiAssertionTimeoutMs }); - await documentsMode.click({ force: true }); - await expect(appModeMenu).toHaveAccessibleName("Mode Documents", { timeout: uiAssertionTimeoutMs }); - }).toPass({ timeout: 8_000 }); + await waitForReactEventHandler(appModeMenu, "onClick"); + await appModeMenu.click({ force: true }); + const appModeGroup = page.getByRole("menu", { name: "Choose app mode" }); + await expect(appModeGroup).toBeVisible({ timeout: uiAssertionTimeoutMs }); + const documentsMode = appModeGroup.getByRole("menuitemradio", { name: /^Documents\b/ }); + await expect(documentsMode).toBeVisible({ timeout: uiAssertionTimeoutMs }); + await waitForReactEventHandler(documentsMode, "onClick"); + await documentsMode.click({ force: true }); + await expect(appModeMenu).toHaveAccessibleName("Mode Documents", { timeout: uiAssertionTimeoutMs }); } const readySetupChecks = [ @@ -660,23 +662,20 @@ async function openScopeControl(page: Page) { const composer = page.locator('[aria-label^="Search indexed guidelines by question or keyword"]:visible').first(); - await expect(async () => { - await composer.click(); - const scopeOption = page.getByRole("option", { name: /Scope sources/i }); - if (await scopeOption.isVisible({ timeout: 2_000 }).catch(() => false)) { - await scopeOption.click(); - } else { - const actionMenu = page.getByRole("button", { name: "Open answer options" }); - await expect(actionMenu).toBeVisible(); - await actionMenu.click(); - const actionsMenu = page.getByTestId("daily-actions-menu"); - await expect(actionsMenu).toBeVisible({ timeout: uiAssertionTimeoutMs }); - await actionsMenu.getByRole("menuitem", { name: /^Scope\b/ }).click(); - } - await expect(page.getByTestId("scope-command-popover")).toBeVisible({ - timeout: uiAssertionTimeoutMs, - }); - }).toPass({ timeout: 15_000 }); + await composer.click(); + const scopeOption = page.getByRole("option", { name: /Scope sources/i }); + if (await scopeOption.isVisible({ timeout: 2_000 }).catch(() => false)) { + await scopeOption.click(); + } else { + const actionMenu = page.getByRole("button", { name: "Open answer options" }); + await expect(actionMenu).toBeVisible(); + await waitForReactEventHandler(actionMenu, "onClick"); + await actionMenu.click(); + const actionsMenu = page.getByTestId("daily-actions-menu"); + await expect(actionsMenu).toBeVisible({ timeout: uiAssertionTimeoutMs }); + await actionsMenu.getByRole("menuitem", { name: /^Scope\b/ }).click(); + } + await expect(page.getByTestId("scope-command-popover")).toBeVisible({ timeout: uiAssertionTimeoutMs }); } async function expectMinTouchTarget(locator: Locator, minSize = 44) { @@ -715,18 +714,17 @@ async function openMobileTableFullscreen(page: Page, clinicalTable: Locator) { await scrollMobileTableExpandClearOfFooter(page, clinicalTable); const expandButton = clinicalTable.getByTestId("table-expand-button"); const tableDialog = page.getByTestId("table-fullscreen-dialog"); - await expect(async () => { - if (await tableDialog.isVisible().catch(() => false)) return; - await expect(expandButton).toBeVisible(); - await expandButton.click(); - await expect(tableDialog).toBeVisible({ timeout: 2_000 }); - }).toPass({ timeout: 15_000 }); + await expect(expandButton).toBeVisible(); + await waitForReactEventHandler(expandButton, "onClick"); + await expandButton.click(); + await expect(tableDialog).toBeVisible({ timeout: 15_000 }); return tableDialog; } async function openMobileClinicalGuideMenu(page: Page) { const trigger = page.getByRole("button", { name: "Open Clinical Guide menu" }); await expect(trigger).toBeVisible(); + await waitForReactEventHandler(trigger, "onClick"); await trigger.click(); const menu = page.getByRole("dialog", { name: "Clinical Guide" }); @@ -778,11 +776,9 @@ async function openGuide(page: Page) { const trigger = (await expandedGuide.isVisible().catch(() => false)) ? expandedGuide : railGuide; await expect(trigger).toBeVisible(); await expect(trigger).toBeEnabled(); - await expect(async () => { - if (await dialog.isVisible().catch(() => false)) return; - await trigger.click(); - await expect(dialog).toBeVisible({ timeout: uiAssertionTimeoutMs }); - }).toPass({ timeout: 10_000 }); + await waitForReactEventHandler(trigger, "onClick"); + await trigger.click(); + await expect(dialog).toBeVisible({ timeout: uiAssertionTimeoutMs }); } else { const menu = await openMobileClinicalGuideMenu(page); await menu.getByRole("button", { name: "Guide & help" }).click(); @@ -833,59 +829,15 @@ async function expectAccountSetupSurface(setup: Locator) { await expect(setup).toContainText("Do not enter patient-identifying information."); } -async function openUploadDrawer(page: Page) { - const uploadButton = page.getByRole("button", { name: /Upload document/i }); - const uploadDrawer = page.getByRole("dialog", { name: "Upload and indexing" }); - await expect(uploadButton).toBeVisible(); - - await expect(async () => { - if (await uploadDrawer.isVisible().catch(() => false)) return; - await uploadButton.click(); - await expect(uploadDrawer).toBeVisible({ timeout: uiAssertionTimeoutMs }); - }).toPass({ timeout: 8_000 }); - - return uploadDrawer; -} - -async function openUploadSurface(page: Page) { +async function expectAdminOnlyUploadNotice(page: Page) { const uploadButton = page.getByRole("button", { name: /Upload document/i }); - const uploadDialog = page.getByRole("dialog", { name: "Upload and indexing" }); - const inlineUploadPanel = page.getByRole("tabpanel", { name: "Upload", exact: true }); - const visibleUploadTab = page - .locator('[role="tab"]:visible') - .filter({ hasText: /^Upload\b/ }) - .first(); await expect(uploadButton).toBeVisible(); - await expect(async () => { - if ( - (await uploadDialog.isVisible().catch(() => false)) || - (await inlineUploadPanel.isVisible().catch(() => false)) || - (await visibleUploadTab.isVisible().catch(() => false)) - ) { - return; - } - await uploadButton.click(); - await expect - .poll( - async () => - (await uploadDialog.isVisible().catch(() => false)) || - (await inlineUploadPanel.isVisible().catch(() => false)) || - (await visibleUploadTab.isVisible().catch(() => false)), - ) - .toBe(true); - }).toPass({ timeout: 8_000 }); - - if (await uploadDialog.isVisible().catch(() => false)) { - const uploadTab = uploadDialog.getByRole("tab", { name: /^Upload\b/ }); - if (await uploadTab.isVisible().catch(() => false)) await uploadTab.click(); - const dialogUploadPanel = uploadDialog.getByRole("tabpanel", { name: "Upload", exact: true }); - await expect(dialogUploadPanel).toBeVisible(); - return dialogUploadPanel; - } - - if (!(await inlineUploadPanel.isVisible().catch(() => false))) await visibleUploadTab.click(); - await expect(inlineUploadPanel).toBeVisible(); - return inlineUploadPanel; + await waitForReactEventHandler(uploadButton, "onClick"); + await uploadButton.click(); + await expect(page.getByRole("alert").filter({ hasText: "Upload and indexing tools are admin-only." })).toContainText( + "Use the source library to open indexed documents.", + ); + await expect(page.getByRole("dialog", { name: "Upload and indexing" })).toHaveCount(0); } async function dismissOverlayByHeaderClick(page: Page) { @@ -900,11 +852,9 @@ async function openDailyActions(page: Page, triggerName: string | RegExp = /^Ope await expect(trigger).toBeVisible(); await expect(trigger).toBeEnabled(); - await expect(async () => { - if (await menu.isVisible().catch(() => false)) return; - await trigger.click(); - await expect(menu).toBeVisible({ timeout: uiAssertionTimeoutMs }); - }).toPass({ timeout: 20_000 }); + await waitForReactEventHandler(trigger, "onClick"); + await trigger.click(); + await expect(menu).toBeVisible({ timeout: uiAssertionTimeoutMs }); return menu; } @@ -1153,12 +1103,6 @@ test.describe("Clinical KB UI smoke coverage", () => { await gotoApp(page, route.path); if (route.path.includes("mode=answer")) { await waitForDemoDashboardReady(page); - } else { - await page - .locator("#main-content") - .first() - .waitFor({ state: "visible", timeout: 15_000 }) - .catch(() => undefined); } const activeLink = page.getByRole("link", { name: route.label, exact: true }); @@ -1287,7 +1231,7 @@ test.describe("Clinical KB UI smoke coverage", () => { await expectNoPageHorizontalOverflow(page); }); - test("private mode unauthenticated dashboard gates real-mode search", async ({ page }) => { + test("offline browser gate remains in demo mode when private endpoints are mocked", async ({ page }) => { await page.setViewportSize({ width: 390, height: 820 }); const answerRequests: string[] = []; const unsafeLocalProjectPayload = { @@ -1317,7 +1261,7 @@ test.describe("Clinical KB UI smoke coverage", () => { const questionInput = visibleQuestionInput(page); await questionInput.fill("lithium monitoring"); - await expect(page.getByRole("button", { name: "Generate source-backed answer" })).toBeDisabled(); + await expect(page.getByRole("button", { name: "Generate source-backed answer" })).toBeEnabled(); await expect(page.getByTestId("answer-grounding-chip")).toHaveCount(0); expect(answerRequests).toEqual([]); await expect(page.getByRole("heading", { level: 1, name: "Clinical Guide" })).toBeVisible(); @@ -1739,14 +1683,6 @@ test.describe("Clinical KB UI smoke coverage", () => { await composerPrivacyLink.focus(); await expect(composerPrivacyLink).toBeFocused(); - const uploadSurface = await openUploadSurface(page); - await expect(uploadSurface.getByRole("note")).toBeVisible(); - await expect(uploadSurface.getByText("Do not enter patient-identifiable information.")).toBeVisible(); - const uploadPrivacyLink = uploadSurface.getByRole("link", { name: "Privacy and data processing" }); - await expect(uploadPrivacyLink).toBeVisible(); - await uploadPrivacyLink.focus(); - await expect(uploadPrivacyLink).toBeFocused(); - await page.goto("/privacy", { waitUntil: "domcontentloaded" }); await expect(page.getByRole("main")).toBeVisible(); await expect(page.getByRole("heading", { level: 1, name: "Privacy & data handling" })).toBeVisible(); @@ -2103,16 +2039,9 @@ test.describe("Clinical KB UI smoke coverage", () => { const medicationLink = strip.getByRole("link", { name: "Clozapine", exact: true }); await expect(medicationLink).toHaveAttribute("href", "/medications/clozapine"); - // Re-click on each retry: a single click can be swallowed while the answer - // surface is still hydrating (same guard as the launcher mode switches), and - // app-router URLs only commit after the destination responds — a cold dev - // compile of /medications/[slug] can take ~30s when this spec runs without - // the @critical warm-up journeys. - await expect(async () => { - if (/\/medications\/clozapine/.test(page.url())) return; - await medicationLink.click(); - await expect(page).toHaveURL(/\/medications\/clozapine/, { timeout: 5_000 }); - }).toPass({ timeout: 45_000 }); + await waitForReactEventHandler(medicationLink, "onClick"); + await medicationLink.click(); + await expect(page).toHaveURL(/\/medications\/clozapine/, { timeout: 45_000 }); await expectNoPageHorizontalOverflow(page); }); @@ -2854,9 +2783,14 @@ test.describe("Clinical KB UI smoke coverage", () => { await expect(page.getByText("Source library workspace")).toHaveCount(0); await expect(page.getByText("Document display")).toHaveCount(0); + // The mode switch above is covered independently. Submit from the canonical + // route so a dev-only cross-segment remount cannot abort the mocked POST. + await gotoApp(page, "/documents/search?mode=documents"); const questionInput = visibleQuestionInput(page); + await expect(questionInput).toBeVisible(); + await waitForReactEventHandler(questionInput, "onChange"); await questionInput.fill("lithium monitoring"); - await page.getByRole("button", { name: "Find matching documents" }).click(); + await submitDocumentSearch(page); await expect(page).toHaveURL(/\/documents\/search\?.*q=lithium\+monitoring/); const documentResults = page.getByRole("article").filter({ hasText: "Synthetic Lithium Monitoring Protocol" }); @@ -2907,11 +2841,13 @@ test.describe("Clinical KB UI smoke coverage", () => { await gotoApp(page, "/"); await waitForDemoDashboardReady(page); - await switchToDocumentSearchMode(page); + await gotoApp(page, "/documents/search?mode=documents"); const questionInput = visibleQuestionInput(page); + await expect(questionInput).toBeVisible(); + await waitForReactEventHandler(questionInput, "onChange"); await questionInput.fill("what is the best coffee machine for my kitchen"); - await page.getByRole("button", { name: "Find matching documents" }).click(); + await submitDocumentSearch(page); await expect(page).toHaveURL(/\/documents\/search\?/); await expect(page.locator("body")).not.toContainText(/failed to fetch|Search failed/i); await expect(page.getByRole("heading", { name: "No matching documents" }).first()).toBeVisible(); @@ -3298,20 +3234,17 @@ test.describe("Clinical KB UI smoke coverage", () => { }); await expect(collapseHost).toHaveAttribute("data-scroll-hidden", "true"); - // At the bottom, collapsing the in-flow header grows the scroll viewport - // and clamps scrollTop to the new maximum. That geometry-driven event is - // not an upward user gesture and must not immediately reveal the header. + // At the bottom, collapsing the in-flow header can reflow the scroll + // surface and clamp scrollTop. That geometry-driven event is not an upward + // user gesture and must not immediately reveal the header. await main.evaluate((node) => { node.scrollTop = 0; }); await expect(collapseHost).not.toHaveAttribute("data-scroll-hidden", "true"); - const visibleGeometry = await main.evaluate((node) => ({ - clientHeight: node.clientHeight, - maxOffset: node.scrollHeight - node.clientHeight, - })); + const visibleMaxOffset = await main.evaluate((node) => node.scrollHeight - node.clientHeight); await main.evaluate((node, top) => { node.scrollTop = top; - }, visibleGeometry.maxOffset); + }, visibleMaxOffset); await expect(collapseHost).toHaveAttribute("data-scroll-hidden", "true"); await expect.poll(async () => collapseHost.getAttribute("data-scroll-hidden"), { timeout: 1_000 }).toBe("true"); // The hidden attribute flips before the 240ms grid-row transition has @@ -3325,14 +3258,6 @@ test.describe("Clinical KB UI smoke coverage", () => { }), ) .toBe(0); - const collapsedGeometry = await main.evaluate((node) => ({ - clientHeight: node.clientHeight, - maxOffset: node.scrollHeight - node.clientHeight, - scrollTop: node.scrollTop, - })); - expect(collapsedGeometry.clientHeight).toBeGreaterThan(visibleGeometry.clientHeight); - expect(Math.abs(collapsedGeometry.scrollTop - collapsedGeometry.maxOffset)).toBeLessThanOrEqual(1); - await main.evaluate((node) => { node.scrollTop = Math.max(0, node.scrollTop - 24); }); @@ -3534,58 +3459,26 @@ test.describe("Clinical KB UI smoke coverage", () => { expect(JSON.stringify(payload)).not.toMatch(/sk-|service_role|eyJ/i); }); - test("upload drawer exposes setup checklist and explicit upload labels", async ({ page, request }) => { + test("production upload action remains admin-only for unauthenticated users", async ({ page, request }) => { await page.setViewportSize({ width: 414, height: 820 }); await mockPrivateUnauthenticatedApi(page); - // Upload availability depends on the checkout's env config: env-less servers run in - // read-only demo mode while .env.local local-auth servers accept uploads. The browser - // mocks above do not decide enablement, so read the real server flag and branch the - // enablement assertions on it to keep this test green in both configurations. const setupStatusResponse = await request.get("/api/setup-status"); expect(setupStatusResponse.ok()).toBe(true); - const serverDemoMode = (await setupStatusResponse.json()).demoMode === true; + expect((await setupStatusResponse.json()).demoMode).toBe(true); await gotoApp(page, "/"); await expect(visibleQuestionInput(page)).toBeVisible(); - const uploadDrawer = await openUploadDrawer(page); - - await uploadDrawer.getByRole("tab", { name: /Setup/ }).click(); - await expect(uploadDrawer.getByText("First-run setup checklist")).toBeVisible(); - await expect(uploadDrawer.getByText(".env.local configured")).toBeVisible(); - await expect(uploadDrawer.getByText("Clinical KB Database target")).toBeVisible(); - await expect(uploadDrawer.getByText("supabase/schema.sql applied")).toBeVisible(); - await expect(uploadDrawer.getByText("Search RPC and vector indexes")).toBeVisible(); - await expect(uploadDrawer.getByText("OpenAI API key available")).toBeVisible(); - await expect(uploadDrawer.getByText("npm run worker running")).toBeVisible(); - const uploadTab = uploadDrawer.getByRole("tab", { name: /Upload/ }); - await uploadTab.click(); - await expect(uploadDrawer.getByText("Clinical upload")).toBeVisible(); - await expect(uploadDrawer.getByText("Guideline PDF files")).toBeVisible(); - if (serverDemoMode) { - await expect(uploadTab).toContainText("Locked"); - await expect(uploadDrawer.getByRole("button", { name: "Guideline PDF files" })).toBeDisabled(); - } else { - await expect(uploadTab).toContainText("Ready"); - await expect(uploadDrawer.getByRole("button", { name: "Guideline PDF files" })).toBeEnabled(); - } - await expect(uploadDrawer.getByRole("button", { name: "Upload guidelines" })).toBeVisible(); + await expectAdminOnlyUploadNotice(page); await expectNoPageHorizontalOverflow(page); }); - test("upload drawer disables uploads in demo mode", async ({ page }) => { + test("demo upload action cannot bypass the production admin gate", async ({ page }) => { await page.setViewportSize({ width: 414, height: 820 }); await mockDemoApi(page); await gotoApp(page, "/"); - const uploadDrawer = await openUploadDrawer(page); - - await uploadDrawer.getByRole("tab", { name: /Jobs/ }).click(); - await expect(uploadDrawer.getByText("Indexing progress")).toBeVisible(); - await uploadDrawer.getByRole("tab", { name: /Upload/ }).click(); - await expect(uploadDrawer.getByText("Read-only status")).toBeVisible(); - await expect(uploadDrawer.getByRole("button", { name: "Guideline PDF files" })).toBeDisabled(); - await expect(uploadDrawer.getByRole("button", { name: "Upload guidelines" })).toBeVisible(); + await expectAdminOnlyUploadNotice(page); await expectNoPageHorizontalOverflow(page); }); diff --git a/tests/ui-stress.spec.ts b/tests/ui-stress.spec.ts index 08f4c8a62..c39d247ac 100644 --- a/tests/ui-stress.spec.ts +++ b/tests/ui-stress.spec.ts @@ -331,12 +331,7 @@ test.describe("Clinical KB long-content stress coverage", () => { await mockStressData(page); await page.setViewportSize({ width: viewport.width, height: viewport.height }); await page.goto("/?mode=documents", { waitUntil: "domcontentloaded" }); - // Deterministic app-shell mount wait, not networkidle (persistent fetches). - await page - .locator("#main-content") - .first() - .waitFor({ state: "visible", timeout: 15_000 }) - .catch(() => undefined); + await expect(page.locator("#main-content").first()).toBeVisible({ timeout: 15_000 }); if (viewport.name === "mobile") { const dailyActions = await openDailyActions(page); @@ -344,33 +339,18 @@ test.describe("Clinical KB long-content stress coverage", () => { // the tap lands on Upload rather than an adjacent row mid-animation. await dailyActions.getByRole("menuitem", { name: /Upload(?: PDF)?/ }).click(); await expect(dailyActions).toBeHidden(); - const uploadSurface = page.getByRole("dialog", { name: "Upload and indexing" }); - await expect(uploadSurface).toBeVisible(); - await expect(uploadSurface.getByRole("button", { name: "Show indexed document files" })).toContainText( - "24 indexed", - { timeout: 20_000 }, - ); - const closeUploadSheet = page.getByRole("button", { name: "Close Upload and indexing" }); - if (await closeUploadSheet.isVisible().catch(() => false)) { - await closeUploadSheet.click(); - } + await expect( + page.getByRole("alert").filter({ hasText: "Upload and indexing tools are admin-only." }), + ).toContainText("Use the source library to open indexed documents."); + await expect(page.getByRole("dialog", { name: "Upload and indexing" })).toHaveCount(0); } await expectNoPageHorizontalOverflow(page); - const legacyAnswerModeToggle = page.getByRole("button", { name: "Switch to answer mode" }); - if (await legacyAnswerModeToggle.isVisible().catch(() => false)) { - await legacyAnswerModeToggle.click(); - } else { - const appModeMenu = page.getByRole("button", { name: /^Mode / }); - await expect(appModeMenu).toBeVisible(); - await appModeMenu.click({ force: true }); - const answerMode = page - .getByRole("menu", { name: "Choose app mode" }) - .getByRole("menuitemradio", { name: /^Answer\b/ }); - await expect(answerMode).toBeVisible(); - await answerMode.click({ force: true }); - await expect(page.getByRole("button", { name: "Mode Answer" })).toBeVisible(); - } + // Mode-menu behavior is covered by the launcher suites. This stress test + // owns only dense answer rendering, so enter that route directly rather + // than making its result depend on an unrelated menu transition. + await page.goto("/?mode=answer", { waitUntil: "domcontentloaded" }); + await expect(page.getByRole("button", { name: "Mode Answer" })).toBeVisible(); await page .locator('[aria-label^="Search indexed guidelines by question or keyword"]:visible') diff --git a/tests/ui-tools-collapse.spec.ts b/tests/ui-tools-collapse.spec.ts index a27de46f9..6a1aae73b 100644 --- a/tests/ui-tools-collapse.spec.ts +++ b/tests/ui-tools-collapse.spec.ts @@ -6,13 +6,7 @@ import { expect, test, type Page } from "playwright/test"; async function goto(page: Page, path: string) { await page.goto(path, { waitUntil: "domcontentloaded" }); - // Deterministic app-shell mount wait (networkidle burned the full timeout on - // routes with persistent background fetches; per-test assertions gate readiness). - await page - .locator("#main-content") - .first() - .waitFor({ state: "visible", timeout: 15_000 }) - .catch(() => undefined); + await expect(page.locator("#main-content").first()).toBeVisible({ timeout: 15_000 }); } // The shell's expanded sidebar (now the desktop default) contributes its own diff --git a/tests/ui-tools-task-directory.spec.ts b/tests/ui-tools-task-directory.spec.ts index 6aa899753..f370cc128 100644 --- a/tests/ui-tools-task-directory.spec.ts +++ b/tests/ui-tools-task-directory.spec.ts @@ -7,13 +7,7 @@ const PATH = "/mockups/tools-task-directory"; async function goto(page: Page, path: string) { await page.goto(path, { waitUntil: "domcontentloaded" }); - // Deterministic app-shell mount wait (networkidle burned the full timeout on - // routes with persistent background fetches; per-test assertions gate readiness). - await page - .locator("#main-content") - .first() - .waitFor({ state: "visible", timeout: 15_000 }) - .catch(() => undefined); + await expect(page.locator("#main-content").first()).toBeVisible({ timeout: 15_000 }); } async function expectNoHorizontalOverflow(page: Page) { diff --git a/tests/ui-tools.spec.ts b/tests/ui-tools.spec.ts index ef26aea94..bc2d40788 100644 --- a/tests/ui-tools.spec.ts +++ b/tests/ui-tools.spec.ts @@ -49,6 +49,32 @@ async function blockExternalRequests(page: Page) { }); } +function waitForDifferentialCatalogQuery(page: Page, query: string) { + const expectedQuery = query.trim().toLowerCase(); + return Promise.all( + ["diagnosis", "presentation"].map((kind) => + page.waitForResponse( + (response) => { + const url = new URL(response.url()); + return ( + url.pathname === "/api/differentials" && + url.searchParams.get("kind") === kind && + url.searchParams.get("q")?.trim().toLowerCase() === expectedQuery && + response.ok() + ); + }, + { timeout: 30_000 }, + ), + ), + ); +} + +async function submitDifferentialSearch(page: Page, query: string) { + const submit = page.locator('button[aria-label="Search differential presentations"]:visible'); + await expect(submit).toBeEnabled(); + await Promise.all([waitForDifferentialCatalogQuery(page, query), submit.click()]); +} + async function mockAnswerDashboardApi(page: Page) { await blockExternalRequests(page); await page.route(/\/api\/local-project-id$/, async (route) => { @@ -216,14 +242,22 @@ function launcherLaunchLink(page: Page, title: string) { async function gotoLauncher(page: Page, path = "/?mode=tools") { await page.goto(path, { waitUntil: "domcontentloaded" }); - // Launcher routes keep background fetches alive, so wait for the app shell to - // mount (deterministic) instead of for the network to idle; the per-route - // assertions below still gate real UI readiness. - await page - .locator("#main-content") - .first() - .waitFor({ state: "visible", timeout: 15_000 }) - .catch(() => undefined); + await expect(page.locator("#main-content").first()).toBeVisible({ timeout: 15_000 }); +} + +async function waitForReactEventHandler(locator: Locator, eventName: "onClick" | "onSubmit" = "onClick") { + await expect + .poll( + async () => + locator.evaluate((element, reactEventName) => { + const propsKey = Object.keys(element).find((key) => key.startsWith("__reactProps$")); + if (!propsKey) return false; + const props = (element as unknown as Record<string, Record<string, unknown>>)[propsKey]; + return typeof props?.[reactEventName] === "function"; + }, eventName), + { timeout: 15_000 }, + ) + .toBe(true); } async function expectNoPageHorizontalOverflow(page: Page) { @@ -302,11 +336,9 @@ async function openAppModeMenu(page: Page, currentMode: string) { const menu = page.getByRole("menu", { name: "Choose app mode" }); await expect(trigger).toBeVisible(); - await expect(async () => { - if (await menu.isVisible().catch(() => false)) return; - await trigger.click(); - await expect(menu).toBeVisible({ timeout: 5_000 }); - }).toPass({ timeout: 20_000 }); + await waitForReactEventHandler(trigger); + await trigger.click(); + await expect(menu).toBeVisible({ timeout: 20_000 }); return menu; } @@ -421,15 +453,11 @@ test.describe("Clinical KB tools launcher", () => { await page.addInitScript(() => window.localStorage.setItem("clinical-kb-sidebar-collapsed", "1")); await gotoLauncher(page, "/?mode=answer"); - // Re-open + re-click on each retry: a single click can be swallowed while the - // launcher is still hydrating, leaving the route on /?mode=answer. Mirrors the - // Forms switch below, which already guards against the same flake. - await expect(async () => { - if (/\/services$/.test(page.url())) return; - const menu = await openAppModeMenu(page, "Answer"); - await menu.getByRole("menuitemradio", { name: /^Services\b/ }).click(); - await expect(page).toHaveURL(/\/services$/, { timeout: 3_000 }); - }).toPass({ timeout: 20_000 }); + const answerMenu = await openAppModeMenu(page, "Answer"); + const servicesMode = answerMenu.getByRole("menuitemradio", { name: /^Services\b/ }); + await waitForReactEventHandler(servicesMode); + await servicesMode.click(); + await expect(page).toHaveURL(/\/services$/, { timeout: 20_000 }); await expect(page).toHaveURL(/\/services$/); await expect(page.getByRole("button", { name: "Mode Services" })).toBeVisible(); @@ -455,12 +483,10 @@ test.describe("Clinical KB tools launcher", () => { await expect(servicesMenu.getByRole("menuitemradio", { name: /^Differentials\b/ })).toBeVisible(); await expect(servicesMenu.getByRole("menuitemradio", { name: /^Medication\b/ })).toBeVisible(); await expect(servicesMenu.getByRole("menuitemradio", { name: /^Tools\b/ })).toBeVisible(); - await expect(async () => { - if (/\/forms$/.test(page.url())) return; - const menu = await openAppModeMenu(page, "Services"); - await menu.getByRole("menuitemradio", { name: /^Forms\b/ }).click(); - await expect(page).toHaveURL(/\/forms$/, { timeout: 2_000 }); - }).toPass({ timeout: 20_000 }); + const formsMode = servicesMenu.getByRole("menuitemradio", { name: /^Forms\b/ }); + await waitForReactEventHandler(formsMode); + await formsMode.click(); + await expect(page).toHaveURL(/\/forms$/, { timeout: 20_000 }); await expect(page.getByRole("button", { name: "Mode Forms" })).toBeVisible(); await expect(page.getByTestId("forms-home")).toBeVisible(); await expect(page.getByTestId("form-search-results")).toHaveCount(0); @@ -555,20 +581,19 @@ test.describe("Clinical KB tools launcher", () => { // one dashboard-shell home and one standalone-shell home; the full 5-route // design spec stays in the advisory "mode home routes center the shared // search on mobile" test above. - test("mode home search composer is always present at phone and desktop widths @critical", async ({ page }) => { - test.setTimeout(120_000); - await mockAnswerDashboardApi(page); - - for (const viewport of [ - { name: "phone", width: 390, height: 820 }, - { name: "desktop", width: 1280, height: 900 }, + for (const viewport of [ + { name: "phone", width: 390, height: 820 }, + { name: "desktop", width: 1280, height: 900 }, + ] as const) { + for (const home of [ + { path: "/?mode=answer", testId: "answer-empty-state" }, + { path: "/services", testId: "services-home" }, ] as const) { - await page.setViewportSize({ width: viewport.width, height: viewport.height }); - - for (const home of [ - { path: "/?mode=answer", testId: "answer-empty-state" }, - { path: "/services", testId: "services-home" }, - ] as const) { + test(`mode home search composer is present at ${viewport.name} width on ${home.path} @critical`, async ({ + page, + }) => { + await mockAnswerDashboardApi(page); + await page.setViewportSize({ width: viewport.width, height: viewport.height }); await gotoLauncher(page, home.path); await expect(page.getByTestId(home.testId)).toBeVisible(); // The composer must never vanish: exactly one visible search input. @@ -576,9 +601,9 @@ test.describe("Clinical KB tools launcher", () => { // Hero-centred design: the input lives inside the mode-home hero at // every width, phones included. await expect(page.getByTestId(home.testId).getByTestId("global-search-input")).toBeVisible(); - } + }); } - }); + } test("answer composer keeps the PHI warning visible before submission @critical", async ({ page }) => { await mockAnswerDashboardApi(page); @@ -604,25 +629,18 @@ test.describe("Clinical KB tools launcher", () => { } }); - test("all mode home heroes share identical sizing on mobile", async ({ page }) => { - test.setTimeout(150_000); - await mockAnswerDashboardApi(page); - await page.setViewportSize({ width: 390, height: 820 }); - - // Every mode home renders the shared compact ModeHomeHero, so the icon box - // and type scale must be identical across modes. Baseline: Answer. - let baseline: { iconWidth: number; iconHeight: number; headingFontSize: number; subtitleFontSize: number } | null = - null; - - for (const home of [ - { path: "/?mode=answer", testId: "answer-empty-state", heroTestId: "answer-empty-state" }, - { path: "/?mode=documents", testId: "document-search-empty-state", heroTestId: "document-search-empty-state" }, - { path: "/?mode=prescribing", testId: "medication-home", heroTestId: "medication-home" }, - { path: "/?mode=tools", testId: "tools-home", heroTestId: "tools-home" }, - { path: "/services", testId: "services-home", heroTestId: "services-home-template" }, - { path: "/forms", testId: "forms-home", heroTestId: "forms-home-template" }, - { path: "/differentials", testId: "differentials-home", heroTestId: "differentials-home-template" }, - ] as const) { + for (const home of [ + { path: "/?mode=answer", testId: "answer-empty-state", heroTestId: "answer-empty-state" }, + { path: "/?mode=documents", testId: "document-search-empty-state", heroTestId: "document-search-empty-state" }, + { path: "/?mode=prescribing", testId: "medication-home", heroTestId: "medication-home" }, + { path: "/?mode=tools", testId: "tools-home", heroTestId: "tools-home" }, + { path: "/services", testId: "services-home", heroTestId: "services-home-template" }, + { path: "/forms", testId: "forms-home", heroTestId: "forms-home-template" }, + { path: "/differentials", testId: "differentials-home", heroTestId: "differentials-home-template" }, + ] as const) { + test(`mode home hero uses the shared mobile sizing on ${home.path}`, async ({ page }) => { + await mockAnswerDashboardApi(page); + await page.setViewportSize({ width: 390, height: 820 }); await gotoLauncher(page, home.path); const homeRegion = page.getByTestId(home.testId); await expect(homeRegion).toBeVisible(); @@ -647,20 +665,14 @@ test.describe("Clinical KB tools launcher", () => { headingFontSize, subtitleFontSize, }; - if (!baseline) { - baseline = metrics; - // Compact hero mobile scale: 2.75rem icon, 1.45rem heading, 0.875rem subtitle. - expect(metrics.iconWidth).toBe(44); - expect(metrics.iconHeight).toBe(44); - expect(metrics.headingFontSize).toBeCloseTo(23.2, 1); - expect(metrics.subtitleFontSize).toBeCloseTo(14, 1); - } else { - expect(metrics, `${home.path} hero metrics`).toEqual(baseline); - } + expect(metrics.iconWidth).toBe(44); + expect(metrics.iconHeight).toBe(44); + expect(metrics.headingFontSize).toBeCloseTo(23.2, 1); + expect(metrics.subtitleFontSize).toBeCloseTo(14, 1); await expectNoPageHorizontalOverflow(page); - } - }); + }); + } test("phone bottom-dock search opens the bounded command results sheet", async ({ page }) => { await page.setViewportSize({ width: 390, height: 820 }); @@ -719,30 +731,27 @@ test.describe("Clinical KB tools launcher", () => { await expectNoPageHorizontalOverflow(page); }); - test("mode home routes center the shared search from tablet up", async ({ page }) => { - test.setTimeout(150_000); - await mockAnswerDashboardApi(page); - - for (const viewport of [ - { name: "tablet", width: 768, height: 1024 }, - { name: "desktop", width: 1280, height: 900 }, + for (const viewport of [ + { name: "tablet", width: 768, height: 1024 }, + { name: "desktop", width: 1280, height: 900 }, + ] as const) { + for (const home of [ + { path: "/?mode=answer", testId: "answer-empty-state", heading: "How can I help?", headingLevel: 2 }, + { path: "/?mode=documents", testId: "document-search-empty-state", heading: "Documents", headingLevel: 2 }, + { + path: "/?mode=prescribing", + testId: "medication-home", + heading: "Medication prescribing", + headingLevel: 2, + }, + { path: "/services", testId: "services-home", heading: "Find a service", headingLevel: 1 }, + { path: "/forms", testId: "forms-home", heading: "Forms", headingLevel: 1 }, + { path: "/differentials", testId: "differentials-home", heading: "Differentials", headingLevel: 1 }, + { path: "/tools", testId: "tools-home", heading: "Tools", headingLevel: 1 }, ] as const) { - await page.setViewportSize({ width: viewport.width, height: viewport.height }); - - for (const home of [ - { path: "/?mode=answer", testId: "answer-empty-state", heading: "How can I help?", headingLevel: 2 }, - { path: "/?mode=documents", testId: "document-search-empty-state", heading: "Documents", headingLevel: 2 }, - { - path: "/?mode=prescribing", - testId: "medication-home", - heading: "Medication prescribing", - headingLevel: 2, - }, - { path: "/services", testId: "services-home", heading: "Find a service", headingLevel: 1 }, - { path: "/forms", testId: "forms-home", heading: "Forms", headingLevel: 1 }, - { path: "/differentials", testId: "differentials-home", heading: "Differentials", headingLevel: 1 }, - { path: "/tools", testId: "tools-home", heading: "Tools", headingLevel: 1 }, - ] as const) { + test(`mode home search is centered at ${viewport.name} width on ${home.path}`, async ({ page }) => { + await mockAnswerDashboardApi(page); + await page.setViewportSize({ width: viewport.width, height: viewport.height }); await gotoLauncher(page, home.path); await expect(page.getByTestId(home.testId)).toBeVisible(); await expect(visibleGlobalSearchInput(page)).toHaveCount(1); @@ -773,31 +782,28 @@ test.describe("Clinical KB tools launcher", () => { expect(metrics?.formRight ?? 0).toBeLessThanOrEqual((metrics?.homeRight ?? viewport.width) + 1); expect(Math.abs((metrics?.formCenterX ?? 0) - (metrics?.homeCenterX ?? 0))).toBeLessThanOrEqual(24); await expectNoPageHorizontalOverflow(page); - } + }); } - }); - - test("search result and detail routes keep top search from tablet up", async ({ page }) => { - test.setTimeout(240_000); + } - for (const viewport of [ - { name: "mobile", width: 390, height: 820 }, - { name: "tablet", width: 768, height: 1024 }, - { name: "desktop", width: 1280, height: 900 }, + for (const viewport of [ + { name: "mobile", width: 390, height: 820 }, + { name: "tablet", width: 768, height: 1024 }, + { name: "desktop", width: 1280, height: 900 }, + ] as const) { + for (const route of [ + { path: "/services?q=13YARN&focus=1&run=1", modeButton: "Mode Services", compactBottomSearch: true }, + { path: "/services/13yarn", modeButton: "Mode Services", compactBottomSearch: false }, + { path: "/forms?q=transport&focus=1&run=1", modeButton: "Mode Forms", compactBottomSearch: true }, + { path: "/favourites?q=lithium&focus=1&run=1", modeButton: "Mode Favourites", compactBottomSearch: true }, + { + path: "/differentials?q=acute+confusion&focus=1&run=1", + modeButton: "Mode Differentials", + compactBottomSearch: true, + }, ] as const) { - await page.setViewportSize({ width: viewport.width, height: viewport.height }); - - for (const route of [ - { path: "/services?q=13YARN&focus=1&run=1", modeButton: "Mode Services", compactBottomSearch: true }, - { path: "/services/13yarn", modeButton: "Mode Services", compactBottomSearch: false }, - { path: "/forms?q=transport&focus=1&run=1", modeButton: "Mode Forms", compactBottomSearch: true }, - { path: "/favourites?q=lithium&focus=1&run=1", modeButton: "Mode Favourites", compactBottomSearch: true }, - { - path: "/differentials?q=acute+confusion&focus=1&run=1", - modeButton: "Mode Differentials", - compactBottomSearch: true, - }, - ] as const) { + test(`search route keeps the correct composer at ${viewport.name} width on ${route.path}`, async ({ page }) => { + await page.setViewportSize({ width: viewport.width, height: viewport.height }); await gotoLauncher(page, route.path); await expect(page.getByRole("button", { name: route.modeButton })).toBeVisible({ timeout: 20_000 }); await expect(visibleGlobalSearchInput(page), `${route.path} at ${viewport.name}`).toHaveCount(1, { @@ -823,9 +829,9 @@ test.describe("Clinical KB tools launcher", () => { } await expectNoPageHorizontalOverflow(page); - } + }); } - }); + } test("mode home deep links preserve focus=1 on initial load", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 900 }); @@ -915,7 +921,7 @@ test.describe("Clinical KB tools launcher", () => { test("form detail pages keep the shared forms search wired to form results", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 900 }); await gotoLauncher(page, "/forms/transport-crisis-form"); - await expect(page.getByTestId("form-detail-page")).toBeVisible(); + await expect(page.getByTestId("form-detail-page")).toBeVisible({ timeout: 30_000 }); // Structural coverage — runs on every browser, WebKit included: the form // detail page renders inside the shared shell with the Forms-mode composer @@ -926,25 +932,14 @@ test.describe("Clinical KB tools launcher", () => { const formsSearchInput = page.locator('input[placeholder="Search forms..."]:visible').first(); await expect(formsSearchInput).toBeVisible(); - // The shell now seeds its composer state from the URL and only re-syncs on a - // real navigation, so a programmatic fill on this no-query detail route is no - // longer wiped by a mount-time frame — the race that used to break CI WebKit - // (and could flake Firefox). Drive the fill-and-submit as one retried unit - // regardless, so any residual cross-browser navigation-timing jitter cannot - // flake the route assertion; the assertions below still verify the result. + await expect(page.getByText("Loading your forms registry...")).toBeHidden({ timeout: 30_000 }); const formsSearchButton = page.getByRole("button", { name: "Search forms" }); - await expect(async () => { - // A previous attempt's click may have navigated late — after the inner URL - // wait timed out and triggered a retry. If we have already routed to the - // results page the detail-page input is gone, so re-filling would throw; - // treat the completed navigation as success instead. - if (/\/forms\?/.test(page.url())) return; - await formsSearchInput.fill("transport forms"); - await expect(formsSearchButton).toBeEnabled({ timeout: 1_000 }); - await formsSearchButton.click(); - await expect(page).toHaveURL(/\/forms\?/, { timeout: 2_000 }); - }).toPass({ timeout: 20_000 }); - await expect(page.getByTestId("form-search-results")).toBeVisible(); + await formsSearchInput.fill("transport forms"); + await expect(formsSearchButton).toBeEnabled(); + await waitForReactEventHandler(formsSearchButton.locator("xpath=ancestor::form[1]"), "onSubmit"); + await formsSearchButton.click(); + await expect(page).toHaveURL(/\/forms\?.*\bq=transport(?:\+|%20)forms\b/, { timeout: 20_000 }); + await expect(page.getByTestId("form-search-results")).toBeVisible({ timeout: 20_000 }); await expect(page.getByTestId("form-search-result-transport-crisis-form")).toContainText("Transport order"); await expect( page.getByTestId("form-search-result-transport-crisis-form").getByLabel("Open Transport order"), @@ -1172,8 +1167,7 @@ test.describe("Clinical KB tools launcher", () => { await gotoLauncher(page, "/differentials"); await expect(page.getByRole("button", { name: "Mode Differentials" })).toBeVisible(); await page.locator('input[placeholder="Ask or search a presentation"]:visible').first().fill("acute confusion"); - await expect(page.locator('button[aria-label="Search differential presentations"]:visible')).toBeEnabled(); - await page.locator('button[aria-label="Search differential presentations"]:visible').click(); + await submitDifferentialSearch(page, "acute confusion"); await expect.poll(() => searchRequests.length).toBeGreaterThan(0); expect(searchRequests.at(-1)).toMatchObject({ @@ -1240,7 +1234,7 @@ test.describe("Clinical KB tools launcher", () => { await gotoLauncher(page, "/differentials"); await page.locator('input[placeholder="Ask or search a presentation"]:visible').first().fill("acute confusion"); - await page.locator('button[aria-label="Search differential presentations"]:visible').click(); + await submitDifferentialSearch(page, "acute confusion"); await expect(page.getByTestId("differentials-search-results")).toBeVisible(); const tabs = page.getByTestId("differential-result-type-tabs"); @@ -1336,7 +1330,7 @@ test.describe("Clinical KB tools launcher", () => { await gotoLauncher(page, "/differentials"); await page.locator('input[placeholder="Ask or search a presentation"]:visible').first().fill("acute confusion"); - await page.locator('button[aria-label="Search differential presentations"]:visible').click(); + await submitDifferentialSearch(page, "acute confusion"); await expect(page.getByTestId("differentials-search-results")).toBeVisible(); const tabs = page.getByTestId("differential-result-type-tabs"); diff --git a/tests/ui-visual-artifacts.spec.ts b/tests/ui-visual-artifacts.spec.ts index 23128e11c..88c4c9426 100644 --- a/tests/ui-visual-artifacts.spec.ts +++ b/tests/ui-visual-artifacts.spec.ts @@ -2,8 +2,6 @@ import { expect, test, type Page, type TestInfo } from "playwright/test"; const documentPath = "/documents/11111111-1111-4111-8111-111111111111?page=1&chunk=44444444-4444-4444-8444-444444444442"; -const uiLoadTimeoutMs = 15_000; - async function attachViewportScreenshot( page: Page, testInfo: TestInfo, @@ -13,12 +11,7 @@ async function attachViewportScreenshot( ) { await page.setViewportSize(viewport); await page.goto(path, { waitUntil: "domcontentloaded" }); - // Deterministic app-shell mount wait instead of networkidle (persistent fetches). - await page - .locator("#main-content") - .first() - .waitFor({ state: "visible", timeout: uiLoadTimeoutMs }) - .catch(() => undefined); + await expect(page.locator("#main-content").first()).toBeVisible({ timeout: 15_000 }); await expect(page.locator("body")).toBeVisible(); await testInfo.attach(name, { diff --git a/tests/universal-search-owner-live.test.ts b/tests/universal-search-owner.live.test.ts similarity index 69% rename from tests/universal-search-owner-live.test.ts rename to tests/universal-search-owner.live.test.ts index 156a53452..a87dc4c94 100644 --- a/tests/universal-search-owner-live.test.ts +++ b/tests/universal-search-owner.live.test.ts @@ -1,14 +1,6 @@ import { describe, expect, it } from "vitest"; import { isUsableBrowserSupabaseKey } from "../src/lib/supabase/client"; -// Live owner-auth coverage for universal search. The header sign-in UI is magic-link/OAuth -// only (no password field), so browser-login Playwright coverage is not feasible; instead -// this signs in with the E2E password user via supabase-js and exercises the real route -// handler with a genuine bearer token — the same path the typeahead hook uses. -// -// Skips (never fails) when the live env is absent: demo/offline environments and keys-free -// CI run the mocked coverage in tests/universal-search.test.ts instead. - const liveEnvReady = Boolean( process.env.E2E_USER_EMAIL && process.env.E2E_USER_PASSWORD && @@ -19,7 +11,11 @@ const liveEnvReady = Boolean( process.env.NEXT_PUBLIC_DEMO_MODE !== "true", ); -describe.skipIf(!liveEnvReady)("GET /api/search/universal (live owner auth)", () => { +if (!liveEnvReady) { + throw new Error("Live owner-search tests require complete, non-placeholder E2E and Supabase credentials."); +} + +describe("GET /api/search/universal (live owner auth)", () => { it("serves owner-scoped registry groups through a real session token", { timeout: 45_000 }, async () => { const { createClient } = await import("@supabase/supabase-js"); const supabase = createClient( @@ -31,11 +27,7 @@ describe.skipIf(!liveEnvReady)("GET /api/search/universal (live owner auth)", () email: process.env.E2E_USER_EMAIL!, password: process.env.E2E_USER_PASSWORD!, }); - if (error) { - // Env advertises live credentials but Supabase rejected them (stale secret, wrong project). - console.warn(`Skipping live owner-auth test: ${error.message}`); - return; - } + if (error) throw new Error(`Live owner-auth sign-in failed: ${error.message}`); const token = data.session?.access_token; expect(token).toBeTruthy(); @@ -52,7 +44,6 @@ describe.skipIf(!liveEnvReady)("GET /api/search/universal (live owner auth)", () publicAccess?: boolean; groups: Array<{ kind: string; error?: boolean; items: Array<{ href: string }> }>; }; - // Owner path: neither the demo nor the public-fixture ladder rung served this. expect(payload.demoMode).toBeUndefined(); expect(payload.publicAccess).toBeUndefined(); @@ -64,6 +55,7 @@ describe.skipIf(!liveEnvReady)("GET /api/search/universal (live owner auth)", () const documents = payload.groups.find((group) => group.kind === "documents"); expect(documents?.error).toBeUndefined(); - await supabase.auth.signOut(); + const signOut = await supabase.auth.signOut(); + if (signOut.error) throw new Error(`Live owner-auth sign-out failed: ${signOut.error.message}`); }); }); diff --git a/tests/universal-search.test.ts b/tests/universal-search.test.ts index 897f49994..d5c84b80f 100644 --- a/tests/universal-search.test.ts +++ b/tests/universal-search.test.ts @@ -1,8 +1,29 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +let resetModulesAfterTest = false; +const mockedModuleSpecifiers = [ + "@/lib/demo-data", + "@/lib/differentials", + "@/lib/document-enrichment", + "@/lib/env", + "@/lib/rag", + "@/lib/supabase/admin", + "@/lib/tools-catalog", + "@/lib/universal-search", +] as const; + +function isolateNextModuleImport() { + vi.resetModules(); + resetModulesAfterTest = true; +} + afterEach(() => { vi.unstubAllEnvs(); - vi.resetModules(); + if (resetModulesAfterTest) { + for (const specifier of mockedModuleSpecifiers) vi.doUnmock(specifier); + vi.resetModules(); + resetModulesAfterTest = false; + } }); async function loadUniversalSearch() { @@ -98,6 +119,7 @@ describe("runUniversalSearch (demo/fixtures path)", () => { }); it("isolates a failing domain instead of blanking the response", async () => { + isolateNextModuleImport(); vi.doMock("@/lib/tools-catalog", async (importOriginal) => { const actual = await importOriginal<typeof import("../src/lib/tools-catalog")>(); return { @@ -118,6 +140,7 @@ describe("runUniversalSearch (demo/fixtures path)", () => { }); it("isolates a failing presentations adapter without touching the differentials group", async () => { + isolateNextModuleImport(); vi.doMock("@/lib/differentials", async (importOriginal) => { const actual = await importOriginal<typeof import("../src/lib/differentials")>(); return { @@ -136,10 +159,6 @@ describe("runUniversalSearch (demo/fixtures path)", () => { const differentials = response.groups.find((group) => group.kind === "differentials"); expect(differentials?.error).toBeUndefined(); expect(differentials?.items.length ?? 0).toBeGreaterThan(0); - - // doMock registrations survive resetModules, so drop it here or every later test in this - // file would import the throwing presentations ranker. - vi.doUnmock("@/lib/differentials"); }); it("uses demo document search when no Supabase client is supplied", async () => { @@ -151,6 +170,7 @@ describe("runUniversalSearch (demo/fixtures path)", () => { }); it("keeps registry hrefs when document search uses related-document mapping", async () => { + isolateNextModuleImport(); vi.doMock("@/lib/rag", async (importOriginal) => { const actual = await importOriginal<typeof import("../src/lib/rag")>(); return { @@ -213,6 +233,7 @@ describe("runUniversalSearch (demo/fixtures path)", () => { describe("GET /api/search/universal (demo mode)", () => { it("serves fixture-backed groups with demoMode flagged", async () => { + isolateNextModuleImport(); vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "true"); const { GET } = await import("../src/app/api/search/universal/route"); const response = await GET(new Request("http://localhost/api/search/universal?q=acamprosate&limit=3")); @@ -228,6 +249,7 @@ describe("GET /api/search/universal (demo mode)", () => { }); it("rejects queries under the minimum length", async () => { + isolateNextModuleImport(); vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "true"); const { GET } = await import("../src/app/api/search/universal/route"); const response = await GET(new Request("http://localhost/api/search/universal?q=a")); @@ -235,6 +257,7 @@ describe("GET /api/search/universal (demo mode)", () => { }); it("ignores unknown domains in the CSV filter", async () => { + isolateNextModuleImport(); vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "true"); const { GET } = await import("../src/app/api/search/universal/route"); const response = await GET(new Request("http://localhost/api/search/universal?q=monitoring&domains=tools,bogus")); @@ -244,6 +267,7 @@ describe("GET /api/search/universal (demo mode)", () => { }); it("serves the presentations domain through the CSV filter", async () => { + isolateNextModuleImport(); vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "true"); const { GET } = await import("../src/app/api/search/universal/route"); const response = await GET( @@ -256,6 +280,7 @@ describe("GET /api/search/universal (demo mode)", () => { }); it("accepts a mode context and rejects unknown modes", async () => { + isolateNextModuleImport(); vi.stubEnv("NEXT_PUBLIC_DEMO_MODE", "true"); const { GET } = await import("../src/app/api/search/universal/route"); const response = await GET( @@ -338,6 +363,7 @@ describe("runUniversalSearch (query intelligence & ranking)", () => { it("passes the ORIGINAL (uncorrected) query to the documents domain", async () => { const captured: string[] = []; + isolateNextModuleImport(); vi.doMock("@/lib/demo-data", async (importOriginal) => { const actual = await importOriginal<typeof import("../src/lib/demo-data")>(); return { @@ -470,6 +496,7 @@ describe("GET /api/search/universal (live public/owner path)", () => { client: ReturnType<typeof createSupabaseMock>, runUniversalSearch: ReturnType<typeof createRunMock>, ) { + isolateNextModuleImport(); // env:{} keeps isDemoMode false (forcing the live path) while leaving the Supabase public // keys unset, so the cookie-session probe resolves anonymous without a network call. vi.doMock("@/lib/env", () => ({ diff --git a/tests/verify-pr-local.test.ts b/tests/verify-pr-local.test.ts index eef8010ef..092a7d990 100644 --- a/tests/verify-pr-local.test.ts +++ b/tests/verify-pr-local.test.ts @@ -17,6 +17,7 @@ describe("verify-pr-local CLI", () => { expect(output).toContain("Changed files: docs/frontend-architecture.md"); expect(output).toContain("PR-local verification plan (dry run)"); expect(output).toContain("- npm run check:runtime"); + expect(output).toContain("- npm run format:changed"); expect(output).toContain("- build skipped"); expect(output).not.toContain("\n> npm run "); }); @@ -25,7 +26,8 @@ describe("verify-pr-local CLI", () => { const output = dryRun("src/app/api/answer/stream/route.ts", "--extended"); expect(output).toContain("- npm run build"); - expect(output).toContain("- npm run eval:rag:offline"); + expect(output).toContain("- npm run check:rag:fixtures"); + expect(output).not.toContain("- npm run eval:rag:offline"); expect(output).toContain("- npm run verify:ui"); }); diff --git a/tests/worker-bundle.test.ts b/tests/worker-bundle.test.ts index 4b7d636e1..3be7d234a 100644 --- a/tests/worker-bundle.test.ts +++ b/tests/worker-bundle.test.ts @@ -20,7 +20,7 @@ const repoRoot = fileURLToPath(new URL("..", import.meta.url)); * both, so the class of bug is caught in CI instead of at deploy. */ describe("worker production bundle", () => { - it("keeps every external import resolvable under plain node with prod-only deps", async () => { + it("keeps every external import resolvable under plain node with prod-only deps", { timeout: 60_000 }, async () => { const result = await build({ ...workerBuildOptions, write: false, @@ -72,6 +72,7 @@ describe("worker production bundle", () => { cwd: repoRoot, env: { ...process.env, WORKER_BUNDLE_SPECS: JSON.stringify(bareSpecs) }, encoding: "utf8", + timeout: 45_000, }); expect(JSON.parse(out), "externals plain `node` cannot resolve at boot").toEqual([]); }); diff --git a/vitest.config.mts b/vitest.config.mts index e0a7a8016..adb5f5182 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -1,3 +1,5 @@ +const liveProviderTests = process.env.ALLOW_PROVIDER_TESTS === "true"; + const config = { test: { // Route and RAG tests cold-import large Next.js module graphs inside the test @@ -40,21 +42,26 @@ const config = { // Node environment, unchanged glob — existing tests behave exactly as before. name: "node", environment: "node", - include: ["tests/**/*.test.ts"], - }, - }, - { - extends: true, - test: { - // Interactive component tier: @testing-library/react under jsdom. Kept on a - // distinct `*.dom.test.tsx` glob so it can never collect the node suite's - // `*.test.ts` files (and vice versa). - name: "jsdom", - environment: "jsdom", - include: ["tests/**/*.dom.test.tsx"], - setupFiles: ["tests/setup/jsdom.setup.ts"], + include: liveProviderTests ? ["tests/**/*.live.test.ts"] : ["tests/**/*.test.ts"], + exclude: liveProviderTests ? [] : ["tests/**/*.live.test.ts"], }, }, + ...(!liveProviderTests + ? [ + { + extends: true, + test: { + // Interactive component tier: @testing-library/react under jsdom. Kept on a + // distinct `*.dom.test.tsx` glob so it can never collect the node suite's + // `*.test.ts` files (and vice versa). + name: "jsdom", + environment: "jsdom", + include: ["tests/**/*.dom.test.tsx"], + setupFiles: ["tests/setup/jsdom.setup.ts"], + }, + }, + ] + : []), ], }, resolve: {