diff --git a/.env.example b/.env.example index 0b31c1105..d8fbcaece 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,21 @@ INDEXING_V3_AGENT_SECRET=your-long-random-cron-shared-secret # Secret required for /api/health?deep=1 Supabase probe (x-health-deep-token header). HEALTH_DEEP_PROBE_SECRET=your-long-random-health-deep-probe-secret +# Dedicated staging-only cross-tenant regression harness. These values are read +# only by `npm run test:cross-tenant:staging`; never point them at production. +# The two accounts must be distinct, non-human staging test users. The staging +# app must use the same Supabase project and RAG_PROVIDER_MODE=offline. +#CROSS_TENANT_STAGING_APP_URL=https://clinical-kb-staging.example.org +#CROSS_TENANT_SUPABASE_URL=https://your-staging-project-ref.supabase.co +#CROSS_TENANT_PROJECT_REF=your-staging-project-ref +#CROSS_TENANT_PUBLISHABLE_KEY=your-staging-publishable-key +#CROSS_TENANT_SERVICE_ROLE_KEY=your-staging-service-role-key +#CROSS_TENANT_USER_A_EMAIL=tenancy-a@example.org +#CROSS_TENANT_USER_A_PASSWORD=replace-with-staging-user-a-password +#CROSS_TENANT_USER_B_EMAIL=tenancy-b@example.org +#CROSS_TENANT_USER_B_PASSWORD=replace-with-staging-user-b-password +#CROSS_TENANT_DOCUMENT_BUCKET=clinical-documents +#CROSS_TENANT_EVIDENCE_PATH=artifacts/staging-tenancy-evidence.json # Local-only no-auth mode (development only). # Enable one or both of these to load real data without per-request browser auth: # - NEXT_PUBLIC_LOCAL_NO_AUTH (client + server) diff --git a/.github/workflows/staging-tenancy.yml b/.github/workflows/staging-tenancy.yml new file mode 100644 index 000000000..087c6d95c --- /dev/null +++ b/.github/workflows/staging-tenancy.yml @@ -0,0 +1,60 @@ +name: Staging tenancy isolation + +on: + workflow_dispatch: + schedule: + - cron: "30 18 * * *" + +concurrency: + group: staging-tenancy-isolation + cancel-in-progress: false + +permissions: + contents: read + +jobs: + cross-tenant: + name: Cross-tenant staging harness + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run cross-tenant staging checks + env: + CROSS_TENANT_STAGING_APP_URL: ${{ secrets.CROSS_TENANT_STAGING_APP_URL }} + CROSS_TENANT_SUPABASE_URL: ${{ secrets.CROSS_TENANT_SUPABASE_URL }} + CROSS_TENANT_PROJECT_REF: ${{ secrets.CROSS_TENANT_PROJECT_REF }} + CROSS_TENANT_PUBLISHABLE_KEY: ${{ secrets.CROSS_TENANT_PUBLISHABLE_KEY }} + CROSS_TENANT_SERVICE_ROLE_KEY: ${{ secrets.CROSS_TENANT_SERVICE_ROLE_KEY }} + CROSS_TENANT_USER_A_EMAIL: ${{ secrets.CROSS_TENANT_USER_A_EMAIL }} + CROSS_TENANT_USER_A_PASSWORD: ${{ secrets.CROSS_TENANT_USER_A_PASSWORD }} + CROSS_TENANT_USER_B_EMAIL: ${{ secrets.CROSS_TENANT_USER_B_EMAIL }} + CROSS_TENANT_USER_B_PASSWORD: ${{ secrets.CROSS_TENANT_USER_B_PASSWORD }} + CROSS_TENANT_DOCUMENT_BUCKET: ${{ vars.CROSS_TENANT_DOCUMENT_BUCKET }} + CROSS_TENANT_EVIDENCE_PATH: artifacts/staging-tenancy-evidence.json + CROSS_TENANT_COMMIT_SHA: ${{ github.sha }} + CROSS_TENANT_WORKFLOW_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: npm run test:cross-tenant:staging + + - name: Upload tenancy evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: staging-tenancy-evidence-${{ github.run_id }} + path: artifacts/staging-tenancy-evidence.json + if-no-files-found: error + retention-days: 14 diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index b12ebd09d..94d532865 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -408,6 +408,7 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-14 | codex/specifiers-design | c3e7024e15d2157bbc0b989276324cc6ad3eea5c | Specifiers UI, clinical decision-support, accessibility, and release-readiness review | One P2 WCAG contrast defect was reproduced and fixed across the builder, comparison, and map accent eyebrows. No remaining high-confidence defect was found in the changed scope; residual risk is clinical/manual governance of the original specifier summaries. | Focused Vitest 18/18; focused Chromium desktop/mobile 2/2 with serious/critical axe WCAG A/AA scanning and overflow checks; `npm run check:production-readiness:ci` READY; `npm run verify:cheap` 2,210 passed/1 skipped; `npm run verify:pr-local` 2,213 passed/1 skipped plus production build and client-bundle secret scan; `git diff --check`. Full advisory `verify:ui` exceeded the local 10-minute execution window; provider-backed checks were not run. | | 2026-07-14 | PR #633 / codex/specifiers-design | 03daca3bfdf86517b9956f3c7d91be27d9f1a751 | Review follow-up and failed UI regression | Eight distinct P2 behaviors across nine review threads were confirmed and fixed: initial selection normalization, duplicate query removal, clinical applicability mapping, canonical Specifiers routing, severity-neutral base wording, diagnostic-section ordering, base/specifier compatibility, and severe psychotic-features wording. The failed app-menu keyboard test was a stale order assertion and now covers the inserted Specifiers item. | Focused Vitest 6/6; focused Chromium 4/4; scoped ESLint and TypeScript; CI-mode production readiness READY; production build and client-bundle secret scan passed. Hosted required checks passed on the refreshed `main` merge before the final compatibility fixes; provider-backed Supabase/OpenAI checks were not run. | | 2026-07-14 | PR #634 / codex/global-answer-reliability | b411329ec5f181661e5d49276c398440aa928fa2 | review-followup | One late P2 fast-context defect was confirmed: Australian tier ordering could push a higher-ranked supplementary passage outside the four-chunk routine fast budget. Fixed by preserving the retrieval-ranked, crowding-capped candidate budget before applying the order-only Australian preference within that set. | GitHub connector review-thread inspection; focused RAG context-budget suite 22/22; ESLint; TypeScript; Prettier; `git diff --check`. | +| 2026-07-14 | codex/release-blocker-remediation | 550e0588866c38583bd9445fc109ea7832a98211 | working-tree release remediation review | Reconciled the remediation onto current main after three retained-stash fast-forward syncs, preserving the extracted RAG and document-viewer architecture and main's Sentry removal. Review confirmed and fixed the Windows offline-release launcher failure, the offline Railway health-check blocker, and shared staging-test passwords. No remaining high-confidence issue was found in the changed local scope. Provider-backed production and staging evidence remains a post-PR gate. | `npm ci` and `npm ls --depth=0` passed; focused registry/offline/RAG/viewer/tenancy Vitest 207/207 plus health/config follow-up 15/15; `npm run format:check`; `npm run check:github-actions`; `npm run check:ci-scope`; `npm run verify:cheap` passed runtime, generated guards, lint, TypeScript, and full Vitest (254 files passed, 1 skipped; 2,325 tests passed, 1 skipped); `git diff --check`. | | 2026-07-14 | claude/audit-ci-browser-gate-2026-07-13 | 09be064a09e8199e9359c50ea0dfceb25847cacf | branch-cleanup | Deleted local ref after confirming no patch-unique content remained; remote ref was left untouched. | Local cherry-pick-aware comparison to current `origin/main`; worktree/activity scan. | | 2026-07-14 | claude/beautiful-hamilton-5df54c | 5e2e90f0a3af4039c7e15515151569228476a60c | branch-cleanup | Deleted local redundant ref; exact head was already represented on `origin/main`. | Local cherry-pick-aware comparison and prior exact-head ledger evidence. | | 2026-07-14 | claude/canary-gate-fixes | 85411f5db736e111fdb278468787dc8b32bb5ebe | branch-cleanup | Deleted local ref after confirming no patch-unique content remained; remote ref was left untouched. | Local cherry-pick-aware comparison to current `origin/main`. | @@ -479,3 +480,8 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-14 | HEAD (detached) 570e6ba56ae60bea56a32801b9cc96c5a8dfde4f | 570e6ba56ae60bea56a32801b9cc96c5a8dfde4f | RAG retrieval/ranking/selection/answer-generation audit (fresh scoped pass; PR #649 D4/D5 governance levers safe-by-default focus, token/effort waste, provider routing) | No P0/P1. Both #649 levers verified safe-by-default and fail-safe: D4 `unknownCurrentnessPenalty` default 0 (no-op, clamped non-negative, activated only via `RAG_RANKING_CONFIG`); D5 `NEXT_PUBLIC_RAG_TRUST_CAP_ALL_CLAIMS` unset=false (only tightens display trust high→medium, never exposes more; `NEXT_PUBLIC` correct as `buildAnswerRenderModel` runs client-side in `ClinicalDashboard.tsx`). Reasoning-effort defaults correct (`OPENAI_STRONG_REASONING_EFFORT`=medium, fast=low; `strongReasoningEffortForQueryClass` never raises, caps routine at medium, keeps dose/threshold at configured). Provider mode default `auto`. P3 (reaffirmed): (1) `Math.max(hybrid_score, boosted)` floor at rag.ts:663 can nullify demotion penalties (outdated/unknown/poor/lowIndex) at the list tail, making D4 partly inert when activated; (2) `document_status` defaults to `"unknown"` (source-metadata.ts:34) for unenriched docs, so activating D4 penalizes the corpus-wide fallback status, not a curated signal — same mechanism that dropped selection doc-recall@5 1.0→0.76 (retrieval-selection.ts:340) — eval gate is the safeguard. | Pure review, no mutations except this ledger append. Offline focused Vitest: answer-render-policy + ranking-config + answer-responsiveness-gate 54/54 passed. Provider-backed (Supabase/OpenAI), `eval:retrieval:quality`, browser, and release checks not run (confirmation boundary). | | 2026-07-14 | HEAD (detached) 570e6ba56ae60bea56a32801b9cc96c5a8dfde4f | 570e6ba56ae60bea56a32801b9cc96c5a8dfde4f | frontend/UI/accessibility audit — global-search-shell, master-search-header, composer, answer surfaces, document viewer, clinical dashboard modules; design-token usage, reduced-motion/forced-colors, icon aria, focus traps, composer/header placement | No P0. P2: (1) `aria-describedby`+`aria-hidden="true"` conflict in `mode-action-popup.tsx:622,651` makes menu descriptions invisible to AT; (2) ~25 dynamic `` render sites missing `aria-hidden` across dashboard modules — ESLint `require-lucide-icon-aria` rule gap for LucideIcon-typed variables; (3) Mode menu (`role="menu"` in header) does not close on Tab — keyboard users can Tab away from an open menu without dismissing it; (4) No live region on streaming `NaturalLanguageAnswer` — screen reader users not notified of incremental answer content. P3: (5) `--surface-glass`/`--panel-gloss` not remapped in `@media (forced-colors: active)` block — image-lightbox and PDF toolbar control bars could become invisible in high-contrast; (6) `bg-black/45` on Sheet backdrop instead of `var(--overlay-backdrop)` token; (7) `active:scale-[0.99]` on action-popup buttons without `motion-safe:` — still fires as a visual jump under reduced-motion; (8) Microsoft/Google brand hex squares not `forced-color-adjust:none` — lose brand identity in high-contrast mode. | Pure static review, no mutations. Files read: `master-search-header.tsx`, `global-search-shell.tsx`, `globals.css`, `sheet.tsx`, `mode-action-popup.tsx`, `image-lightbox.tsx`, `answer-content.tsx`, `ClinicalDashboard.tsx` (partial), `use-dismissable-layer.ts`, `layout.tsx`, `eslint-rules/require-lucide-icon-aria.mjs`, `ui-accessibility.spec.ts`, `process-hardening.md`. Browser/live checks not run (confirmation boundary). | | 2026-07-14 | main / 570e6ba56ae60bea56a32801b9cc96c5a8dfde4f | 570e6ba56ae60bea56a32801b9cc96c5a8dfde4f | repo-wide multi-skill audit (repo-auditor, security, clinical-governance, RAG, ingestion-worker, API, frontend-ui, release-readiness, testing/code-quality) | Highest code P1: anonymous public catalogs bypass rate limits while serving multi-MB payloads (medications 3.4 MB, services 894 KB, differentials 1.2 MB; `shouldResolvePublicCatalogAccess()` early-return in registry/medications/differentials routes skips `consumeSubjectApiRateLimit()` for requests without session cookie or bearer token). Active OPERATOR/LEGAL launch blockers: PIA-1 APP 8 overseas processing (Railway SG + OpenAI US), PIA-2 Railway `RAG_QUERY_HASH_SECRET` verify, unrun `verify:release`/golden evals, staging soak, Eval Canary trust, operator-backlog staleness vs runbooks. Confirmed code P2 cluster: public-doc DTO leaks (`storage_path`), single-layer service-role tenancy, commit-RPC unreachable fallback (`worker/main.ts:545-547`), recovery plan pending+failed unique-index crash, unwired `decideReindexGate`, CI scope misses (`src/lib/app-modes.ts`/`clinical-safety.ts` skip UI/RAG gates), a11y describedby/icon/Tab/live-region gaps, soft `@critical` safety UI assert, unenforced bundle budget, `.env.example` weak-OR flag. Residual risk: OCR quality upstream labels + hybrid-RPC latency tail. | Specialist audits + `ci-change-scope` probe; structure `verify:cheap` (2,290/2 skipped); focused Vitest governance/RAG/ingestion suites; no provider/live Supabase/OpenAI/`verify:release`/`check:drift` (confirmation boundary). | +| 2026-07-14 | PR #655 / codex/release-blocker-remediation | 1ca2f9f372e23f563de0bc4f823cd341d82a562a | review-followup | One P1 offline-startup defect was confirmed: production instrumentation still required OpenAI after readiness and health accepted explicit offline mode. Fixed the boot guard so only explicit `offline` may omit OpenAI while `auto` and `openai` remain fail-closed. | GitHub review-thread inspection; focused instrumentation/readiness/health Vitest 21/21; scoped ESLint; Prettier; `git diff --check`. Hosted required checks on the reviewed head were green, including Gitleaks, PR required, unit coverage, build, UI, SAST, Docker images, and migration replay. | +| 2026-07-14 | PR #655 / codex/release-blocker-remediation | c8a3dd118b8bf802418598c7cb48083893625004 | production governance preflight follow-up | The live dry-run correctly stopped because 554 registry projections are deliberately public (`owner_id = null`) while 232 differentials remain owner-scoped. No second tenant exists. Updated reconciliation to preserve public/owner scope, reject any foreign owner or label-owner mismatch, and scope generated intent labels to the existing document owner without mutating ownership. | Read-only production ownership and label topology probes; focused registry/governance Vitest 18/18; scoped ESLint; Prettier; full TypeScript; `git diff --check`; production dry-run inspected 786 documents (554 public, 232 owner-scoped), planned 786 metadata updates and 786 intent-label inserts, and reported zero chunk rows/OpenAI calls. | +| 2026-07-14 | PR #655 / codex/release-blocker-remediation | 3b152ed1f2f4f08b5672adaf0dc3b433f8ba8db1 + reviewed follow-up diff | final review-thread and release-readiness follow-up | Confirmed and fixed one P1 maintenance-path tenancy defect: registry embedding metadata refreshes could re-private public registry documents. The refresh now preserves public/owner scope, keeps generated intent-label ownership aligned, is idempotent, and rejects foreign-owner documents. Three scoped P2 review items were also resolved: answer-owner ref mutation moved out of render, PDF page changes use router navigation without scroll reset, and the worker-free staging harness no longer enqueues a reindex job before cleanup. No other high-confidence issue remained in the reviewed follow-up diff. | GitHub review-thread inspection; bundled Next.js navigation guide; focused Vitest 38/38; scoped ESLint; Prettier; full TypeScript; `git diff --check`. Final-head hosted CI, staging evidence, and provider-free production governance gates remain required after push. | +| 2026-07-14 | PR #655 / codex/release-blocker-remediation | 978d4f462fcdd4f665060bfc86ed62d8617751cb + reviewed follow-up diff | final automated-review disposition | Fixed the remaining valid review findings: offline evaluation now excludes forced-vector fixtures and owns provider-mode selection; registry detection is shared; staging Supabase calls are bounded; retrieval is covered by a request-start deadline; deadline-expired answers are not cached; and registry label reconciliation preserves reviewer metadata and confidence while refreshing generator-owned metadata. The unsupported-related-document deadline finding was not applicable because the configured unsupported route budget is intentionally `0` and creates no deadline. | GitHub review-thread inspection; focused Vitest 58/58; scoped ESLint; Prettier; full TypeScript; `git diff --check`. Flaky aggregate browser/local suites intentionally not repeated; final-head hosted CI and staging evidence remain required. | +| 2026-07-14 | PR #655 / codex/release-blocker-remediation | dedb38a4a1bb05f87b94a89f5cade7b4a8109c99 + reviewed follow-up diff | late automated-review safety follow-up | Fixed two newly raised scoped issues: provider-free governance now rejects public differential projections while continuing to allow the three intentionally public registry kinds, and owner-scoped answer-thread clearing also removes the unscoped legacy session/local key so old clinical text is not retained. | GitHub review-thread inspection; focused Vitest 13/13; scoped ESLint; Prettier; `git diff --check`. | diff --git a/docs/process-hardening.md b/docs/process-hardening.md index 7ddf2224a..f9a4c19ed 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -2,6 +2,13 @@ This document turns the current process review into phased, durable repo practice. It separates changes that already take effect from work that should stay explicit until it is implemented. +## Staging tenancy evidence + +The provider-backed A/B tenancy regression is intentionally outside local and PR +gates. Run the standalone manual/nightly workflow and attach a recent green evidence +artifact before release; see +[`docs/staging-tenancy-release-evidence.md`](staging-tenancy-release-evidence.md). + ## Phase 1 - Active now - `npm run verify:cheap` is the default broad local gate for source/config/test changes: `check:runtime`, `sitemap:check`, lint, typecheck, and unit tests. diff --git a/docs/staging-setup.md b/docs/staging-setup.md index 26bf9ea7d..14ddcf9ec 100644 --- a/docs/staging-setup.md +++ b/docs/staging-setup.md @@ -33,17 +33,11 @@ those vars are unset. Then confirm health: `npm run check:indexing` (runs `search_schema_health()` over the hybrid RPCs) should report ok. -3. **Seed a small corpus (~50 docs).** Use synthetic/public content only — do - **not** copy clinical production documents into staging. - - ```bash - npm run samples # generate synthetic sample documents - npm run import:docs # queue them for ingestion - # (run the worker — section B4 — to process the queue) - npm run registry:seed -- --owner-id --write --confirm - npm run differentials:seed - npm run medications:seed - ``` +3. **Keep the tenancy profile synthetic and worker-free.** Do **not** copy + clinical production documents into staging and do not start an ingestion + worker. The cross-tenant harness creates the minimal private documents, + lexical chunks, and storage objects it needs, then removes them in `finally` + cleanup. This keeps the release proof deterministic and provider-free. 4. **Capture the keys** for the staging project (dashboard → API): the publishable key (`sb_publishable_…`) and the service-role secret @@ -52,11 +46,11 @@ those vars are unset. ## B. Staging app host (compute tier) Host: **Railway**, same as production (see `docs/deployment-architecture.md` §2). -Stand staging up as a **second environment** in the `clinical-kb` Railway project -(e.g. a `staging` environment) or a separate project, with its own `app` (+ -optional `worker`) service pinned to **Southeast Asia (`asia-southeast1-eqsg3a`, -Singapore)** — the closest region to the staging Supabase project in Sydney. -Reuse the same images; only the environment variables differ. +Stand staging up as a **second environment** in the `clinical-kb` Railway project, +with one `app` service pinned to **Southeast Asia +(`asia-southeast1-eqsg3a`, Singapore)** — the closest region to the staging +Supabase project in Sydney. Do not create a worker for this release profile. +Reuse the app image; only the environment variables differ. 1. **Build the image** — Railway builds it remotely from the committed `Dockerfile` on deploy, so there is no local `docker build` and no local @@ -72,16 +66,19 @@ Reuse the same images; only the environment variables differ. 2. **Runtime secrets** (injected at deploy, never baked into the image) — all with **staging** values, distinct from production: - | Var | Value | - | ------------------------------- | --------------------------------------------------------- | - | `SUPABASE_SERVICE_ROLE_KEY` | staging `sb_secret_…` | - | `OPENAI_API_KEY` | a staging-only OpenAI key; never reuse the production key | - | `SUPABASE_PROJECT_REF` | `` | - | `SUPABASE_PROJECT_NAME` | `Clinical KB Staging` | - | `SUPABASE_STAGING_PROJECT_REF` | `` | - | `SUPABASE_STAGING_PROJECT_NAME` | `Clinical KB Staging` | - | `RAG_QUERY_HASH_SECRET` | a staging secret | - | `RAG_PROVIDER_MODE` | `auto` | + | Var | Value | + | ------------------------------- | -------------------------- | + | `SUPABASE_SERVICE_ROLE_KEY` | staging `sb_secret_…` | + | `SUPABASE_PROJECT_REF` | `` | + | `SUPABASE_PROJECT_NAME` | `Clinical KB Staging` | + | `SUPABASE_STAGING_PROJECT_REF` | `` | + | `SUPABASE_STAGING_PROJECT_NAME` | `Clinical KB Staging` | + | `RAG_QUERY_HASH_SECRET` | unique staging-only secret | + | `RAG_PROVIDER_MODE` | `offline` | + + Do not set `OPENAI_API_KEY`, `OPENAI_ORG_ID`, or `OPENAI_PROJECT_ID` in the + staging environment. Offline mode uses lexical retrieval and deterministic, + cited source-only answers; it performs no embedding or generation request. The two `SUPABASE_STAGING_PROJECT_*` vars are what make the identity guard accept the staging project. Setting `NEXT_PUBLIC_SUPABASE_URL` + @@ -93,14 +90,19 @@ Reuse the same images; only the environment variables differ. policy to `ON_FAILURE`, and one replica pinned to `southeast-asia` with no scale-to-zero (mirror `railway.app.json`). -4. **Worker (optional, for ingestion in staging):** build `Dockerfile.worker` - and run one instance with the same staging secrets. Required to process the - seed queue from A3. +4. **No worker:** this staging environment intentionally has no ingestion + service. The tenancy harness inserts and removes its synthetic lexical rows + directly through its dedicated staging credentials. ## C. Validate staging 1. Boot check: `GET /api/health` → `{"status":"ok"}` with the staging project. -2. Load check — the soak test is hard-guarded against production: +2. Tenancy isolation: configure the dedicated A/B test accounts and standalone + workflow described in + [`staging-tenancy-release-evidence.md`](staging-tenancy-release-evidence.md). + The harness requires an app deployment with `RAG_PROVIDER_MODE=offline` and is + hard-guarded against the production project. +3. Load check — the soak test is hard-guarded against production: ```bash npx tsx scripts/soak-test.ts --target https:// \ diff --git a/docs/staging-tenancy-release-evidence.md b/docs/staging-tenancy-release-evidence.md new file mode 100644 index 000000000..6516a8940 --- /dev/null +++ b/docs/staging-tenancy-release-evidence.md @@ -0,0 +1,79 @@ +# Staging tenancy release evidence + +The cross-tenant staging harness is the executable proof that the service-role API +layer preserves private owner boundaries. It is intentionally separate from PR and +local verification because it signs in to a hosted staging app, writes synthetic +fixtures to a dedicated staging Supabase project, and then removes them. + +## Safety contract + +Run `npm run test:cross-tenant:staging` only against a dedicated staging deployment. +The harness exits before creating clients or writing data when any required value is +missing or placeholder-like, the project ref is the production ref +`sjrfecxgysukkwxsowpy`, or the Supabase URL does not match the declared project ref. +It signs both test accounts in before creating the service-role client and refuses to +write fixtures if both credentials resolve to the same user. + +The staging app used by this check must: + +- target the same dedicated staging Supabase project; +- set `RAG_PROVIDER_MODE=offline`, which the harness proves by requiring the exact + `source_only_offline_mode` answer fallback; +- use two distinct, non-human test accounts that are not used interactively; and +- avoid running an ingestion worker during the short check, because full reindex is + exercised last and the harness owns cleanup. + +Configure these repository secrets for `.github/workflows/staging-tenancy.yml`: + +| Secret | Purpose | +| ------------------------------------------------------------ | ----------------------------------------------------------- | +| `CROSS_TENANT_STAGING_APP_URL` | HTTPS origin of the offline staging app | +| `CROSS_TENANT_SUPABASE_URL` | Dedicated staging Supabase HTTPS origin | +| `CROSS_TENANT_PROJECT_REF` | Dedicated staging project ref | +| `CROSS_TENANT_PUBLISHABLE_KEY` | Staging publishable/anon key | +| `CROSS_TENANT_SERVICE_ROLE_KEY` | Staging service-role key used only for fixtures and cleanup | +| `CROSS_TENANT_USER_A_EMAIL` / `CROSS_TENANT_USER_A_PASSWORD` | First staging test user | +| `CROSS_TENANT_USER_B_EMAIL` / `CROSS_TENANT_USER_B_PASSWORD` | Second staging test user | + +`CROSS_TENANT_DOCUMENT_BUCKET` is an optional repository variable and defaults to +`clinical-documents`. The harness reads only the `CROSS_TENANT_*` namespace; it does +not fall back to production application credentials. + +## Proof matrix + +Each run creates one private document, page, lexical chunk, and storage object for +each user under a unique run ID. It then proves: + +| Surface | Owner proof | Cross-tenant proof | +| ------------------ | ----------------------------------- | -------------------------------------------- | +| Document list | A and B each find their own fixture | B cannot list A's fixture | +| Document detail | A receives A's document | B receives 404 for A's document | +| Signed URL | A receives a signed URL | B receives 404 | +| Labels | A creates a manual label | B receives 404 | +| Mutation | A renames A's document | B receives 404 | +| Universal search | A finds A's document | B gets no A result | +| Offline retrieval | A retrieves A's chunk | B gets an empty scope and result set | +| Source-only answer | A gets cited source-only evidence | B gets unsupported with no sources/citations | +| Reindex | A can queue full reindex | B receives 404 | + +Cleanup runs in `finally` semantics in reverse dependency order, including run-time +query telemetry for the two dedicated test users, reindex jobs/stages, labels, +chunks, pages, documents, and storage objects. Cleanup failure makes the run fail. +The test never skips a scenario when explicitly invoked. + +## Release evidence + +The standalone workflow runs nightly and on manual dispatch. It is not included in +`verify:cheap`, `verify:pr-local`, PR-required checks, or local release gates. + +Before a release, record a successful workflow run that: + +1. completed within the previous seven days; +2. tested the release commit SHA (or a commit proven to be its ancestor with no + later tenancy/API changes); and +3. retained the `staging-tenancy-evidence-` artifact. + +The JSON artifact records the tested commit, workflow URL, staging project identity, +completed scenario checkpoints, final status, and cleanup status without including +credentials. A missing artifact, failed cleanup, stale run, different project, or +non-offline answer is not acceptable release evidence. diff --git a/package.json b/package.json index 020e2e6d9..42e168f9d 100644 --- a/package.json +++ b/package.json @@ -33,10 +33,12 @@ "test:e2e:mockups": "node scripts/run-playwright.mjs --project=chromium-mockups", "test:e2e:chromium": "node scripts/run-playwright.mjs --project=chromium --project=chromium-mockups", "test:e2e:visual": "node scripts/run-playwright.mjs --config=playwright.visual.config.ts", + "test:cross-tenant:staging": "node scripts/run-tsx.mjs scripts/test-cross-tenant-staging.ts", "verify:cheap": "npm run check:runtime && npm run check:github-actions && npm run sitemap:check && npm run brand:check && npm run check:type-scale && npm run check:icon-scale && npm run lint && npm run typecheck && npm run test", "verify:pr-local": "node scripts/verify-pr-local.mjs", "verify:ui": "npm run check:runtime && npm run test:e2e:chromium", "verify:release": "npm run check:runtime && npm run lint && npm run typecheck && npm run test && npm run build && npm run test:e2e && npm run check:production-readiness && npm run governance:release && npm run eval:quality:release", + "verify:release:offline": "node scripts/verify-release-offline.mjs", "ci:env-check": "node scripts/check-ci-env.mjs", "check:github-actions": "node scripts/check-github-action-pins.mjs", "check:client-bundle-secrets": "node scripts/check-client-bundle-secrets.mjs", @@ -92,6 +94,7 @@ "recover:ingestion": "node scripts/run-tsx.mjs scripts/recover-ingestion-queue.ts", "registry:seed": "node scripts/run-tsx.mjs scripts/seed-registry-records.ts", "registry:embed": "node scripts/run-tsx.mjs scripts/embed-registry-records.ts", + "registry:reconcile-governance": "node scripts/run-tsx.mjs scripts/reconcile-registry-governance.ts", "services:import": "node scripts/run-tsx.mjs scripts/import-services-export.ts", "differentials:import": "node scripts/run-tsx.mjs scripts/import-differentials-export.ts", "differentials:seed": "node scripts/run-tsx.mjs scripts/seed-differential-records.ts", @@ -112,6 +115,7 @@ "eval:answer-quality": "node scripts/run-eval-safe.mjs scripts/eval-answer-quality.ts", "eval:quality": "node scripts/run-eval-safe.mjs scripts/eval-quality.ts", "eval:quality:release": "npm run eval:quality -- --fail-on-threshold --source-metadata-debt docs/release-source-metadata-debt-2026-06-30.json", + "eval:quality:release:offline": "npm run eval:quality -- --provider-mode offline --fail-on-threshold --source-metadata-debt docs/release-source-metadata-debt-2026-06-30.json", "eval:retrieval": "node scripts/run-eval-safe.mjs scripts/eval-retrieval.ts", "eval:retrieval:quality": "node scripts/run-eval-safe.mjs scripts/eval-retrieval.ts --mode quality", "eval:retrieval:index-unit-vector": "node scripts/run-eval-safe.mjs scripts/eval-retrieval.ts --mode quality --force-embedding --json-out artifacts/retrieval/index-unit-vector.json", diff --git a/scripts/build-clinical-review-queue.ts b/scripts/build-clinical-review-queue.ts new file mode 100644 index 000000000..cfa91d52e --- /dev/null +++ b/scripts/build-clinical-review-queue.ts @@ -0,0 +1,56 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; + +import { buildClinicalReviewQueue } from "@/lib/clinical-review-queue"; + +type Args = { input?: string; output?: string; help: boolean }; + +function parseArgs(argv: string[]): Args { + const args: Args = { help: false }; + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--help" || token === "-h") { + args.help = true; + continue; + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); + if (token === "--input") args.input = value; + else if (token === "--output") args.output = value; + else throw new Error(`Unknown option: ${token}`); + index += 1; + } + return args; +} + +function usage() { + return [ + "Usage: node scripts/run-tsx.mjs scripts/build-clinical-review-queue.ts --input [--output ]", + "", + "Builds a deterministic, deduplicated review queue without changing source governance statuses.", + ].join("\n"); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + return; + } + if (!args.input) throw new Error("--input is required."); + const input = JSON.parse(await readFile(args.input, "utf8")) as unknown; + const output = `${JSON.stringify(buildClinicalReviewQueue(input), null, 2)}\n`; + if (args.output) { + await writeFile(args.output, output, "utf8"); + console.log(`Wrote clinical review queue to ${args.output}`); + } else { + process.stdout.write(output); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/check-document-label-coverage.ts b/scripts/check-document-label-coverage.ts index d49df1346..05ba3b47b 100644 --- a/scripts/check-document-label-coverage.ts +++ b/scripts/check-document-label-coverage.ts @@ -1,8 +1,10 @@ import * as nextEnv from "@next/env"; import { promises as fs } from "node:fs"; import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; import { reviewDocumentTagQuality } from "@/lib/document-tags"; import type { DocumentLabel } from "@/lib/types"; +import { isRegistryProjectionDocument } from "./lib/registry-projection-document"; const loadEnvConfig = nextEnv.loadEnvConfig ?? @@ -20,13 +22,16 @@ type CoverageArgs = { type SupabaseAdmin = Awaited>; -type DocumentRow = { +export type DocumentLabelCoverageDocument = { id: string; title: string; file_name: string; + file_type: string | null; + source_path: string | null; + metadata: Record | null; }; -type LabelRow = { +export type DocumentLabelCoverageLabel = { id: string; document_id: string; label: string; @@ -88,7 +93,7 @@ function usage() { return [ "Usage: npm run check:document-label-coverage -- [options]", "", - "Checks every indexed document has generated site and document_type labels.", + "Checks physical-document labels and registry metadata/smart-v2 contracts.", "", "Options:", " --json Print machine-readable JSON.", @@ -161,7 +166,7 @@ async function fetchAll( return rows; } -function countByLabelType(labels: LabelRow[]) { +function countByLabelType(labels: DocumentLabelCoverageLabel[]) { const counts = new Map(); for (const label of labels) { counts.set(label.label_type, (counts.get(label.label_type) ?? 0) + 1); @@ -170,6 +175,40 @@ function countByLabelType(labels: LabelRow[]) { } const smartV2LabelTypes = new Set(["clinical_action", "care_phase", "document_intent", "content_feature"]); +const registryKinds = new Set(["service", "form", "medication", "differential"]); +const registryIntentByKind = { + service: "operational-process", + form: "documentation-requirement", + medication: "medication-instruction", + differential: "decision-support", +} as const; +const requiredRegistryMetadataKeys = [ + "source_kind", + "registry_record_kind", + "registry_record_id", + "registry_record_slug", + "publisher", + "document_status", + "clinical_validation_status", + "clinical_validation_evidence", + "extraction_quality", +] as const; + +function metadataRecord(value: unknown) { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function nonEmptyMetadataValue(value: unknown) { + if (typeof value === "string") return value.trim().length > 0; + if (value && typeof value === "object" && !Array.isArray(value)) return Object.keys(value).length > 0; + return value !== undefined && value !== null; +} + +function registryKindFor(document: DocumentLabelCoverageDocument) { + const metadata = metadataRecord(document.metadata); + const kind = typeof metadata.registry_record_kind === "string" ? metadata.registry_record_kind : ""; + return registryKinds.has(kind) ? (kind as keyof typeof registryIntentByKind) : null; +} function countQualityIssues(issues: ReturnType) { const counts = new Map(); @@ -177,27 +216,18 @@ function countQualityIssues(issues: ReturnType) return Object.fromEntries([...counts.entries()].sort()); } -async function main() { - const args = parseArgs(process.argv.slice(2)); - if (args.help) { - console.log(usage()); - return; - } - - const supabase = await loadAdminClient(); - const allowedSiteMissing = await loadAllowlist(args.allowedSiteMissingPath); - const allowedDocumentTypeMissing = await loadAllowlist(args.allowedDocumentTypeMissingPath); - const documents = await fetchAll(supabase, "documents", "id,title,file_name", (query) => - query.eq("status", "indexed"), - ); - const labels = await fetchAll( - supabase, - "document_labels", - "id,document_id,label,label_type,source,confidence", - (query) => query.eq("source", "generated"), - ); - +export function buildDocumentLabelCoverageReport(args: { + documents: DocumentLabelCoverageDocument[]; + labels: DocumentLabelCoverageLabel[]; + allowedSiteMissing?: Set; + allowedDocumentTypeMissing?: Set; +}) { + const { documents, labels } = args; + const allowedSiteMissing = args.allowedSiteMissing ?? new Set(); + const allowedDocumentTypeMissing = args.allowedDocumentTypeMissing ?? new Set(); const documentIds = new Set(documents.map((document) => document.id)); + const physicalDocuments = documents.filter((document) => !isRegistryProjectionDocument(document)); + const registryDocuments = documents.filter(isRegistryProjectionDocument); const generatedDocumentIds = new Set(labels.map((label) => label.document_id)); const siteDocumentIds = new Set( labels.filter((label) => label.label_type === "site").map((label) => label.document_id), @@ -207,35 +237,68 @@ async function main() { ); const smartV2Labels = labels.filter((label) => smartV2LabelTypes.has(label.label_type)); const smartV2DocumentIds = new Set(smartV2Labels.map((label) => label.document_id)); + const labelsByDocument = new Map(); + for (const label of labels) { + labelsByDocument.set(label.document_id, [...(labelsByDocument.get(label.document_id) ?? []), label]); + } const missingGenerated = [...documentIds].filter((id) => !generatedDocumentIds.has(id)); const missingSmartV2 = [...documentIds].filter((id) => !smartV2DocumentIds.has(id)); const allowedSiteMissingDocs = [...allowedSiteMissing].filter( - (id) => !siteDocumentIds.has(id) && documentIds.has(id), + (id) => !siteDocumentIds.has(id) && physicalDocuments.some((document) => document.id === id), ); const allowedDocumentTypeMissingDocs = [...allowedDocumentTypeMissing].filter( - (id) => !documentTypeDocumentIds.has(id) && documentIds.has(id), - ); - const missingSite = [...documentIds].filter((id) => !siteDocumentIds.has(id) && !allowedSiteMissing.has(id)); - const missingDocumentType = [...documentIds].filter( - (id) => !documentTypeDocumentIds.has(id) && !allowedDocumentTypeMissing.has(id), + (id) => !documentTypeDocumentIds.has(id) && physicalDocuments.some((document) => document.id === id), ); - const labelsByDocument = new Map(); - for (const label of labels) { - labelsByDocument.set(label.document_id, [...(labelsByDocument.get(label.document_id) ?? []), label]); - } + const missingSite = physicalDocuments + .map((document) => document.id) + .filter((id) => !siteDocumentIds.has(id) && !allowedSiteMissing.has(id)); + const missingDocumentType = physicalDocuments + .map((document) => document.id) + .filter((id) => !documentTypeDocumentIds.has(id) && !allowedDocumentTypeMissing.has(id)); + + const registryContractGaps = registryDocuments.flatMap((document) => { + const metadata = metadataRecord(document.metadata); + const missingKeys: string[] = requiredRegistryMetadataKeys.filter((key) => { + if (key === "source_kind") return metadata.source_kind !== "registry_record"; + if (key === "registry_record_kind") return registryKindFor(document) === null; + return !nonEmptyMetadataValue(metadata[key]); + }); + const kind = registryKindFor(document); + const documentLabels = labelsByDocument.get(document.id) ?? []; + const expectedIntent = kind ? registryIntentByKind[kind] : null; + const hasExpectedIntent = + expectedIntent !== null && + documentLabels.some( + (label) => + label.source === "generated" && label.label_type === "document_intent" && label.label === expectedIntent, + ); + const hasGeneratedSite = documentLabels.some( + (label) => label.source === "generated" && label.label_type === "site", + ); + if (!hasExpectedIntent) missingKeys.push("smart_v2_document_intent"); + if (hasGeneratedSite) missingKeys.push("generated_site_label"); + return missingKeys.length > 0 + ? [{ id: document.id, title: document.title, file_name: document.file_name, missing_keys: [...missingKeys] }] + : []; + }); + const qualityIssues = reviewDocumentTagQuality( documents.map((document) => ({ ...document, labels: labelsByDocument.get(document.id) ?? [], })), - { overusedThreshold: Math.max(100, Math.ceil(documents.length * 0.35)) }, ); + const passed = + missingGenerated.length === 0 && + missingSite.length === 0 && + missingDocumentType.length === 0 && + registryContractGaps.length === 0; - const passed = missingGenerated.length === 0 && missingSite.length === 0 && missingDocumentType.length === 0; - - const report = { + return { indexed_documents: documents.length, + physical_documents: physicalDocuments.length, + registry_documents: registryDocuments.length, generated_label_rows: labels.length, generated_documents: generatedDocumentIds.size, indexed_without_generated: missingGenerated.length, @@ -258,6 +321,18 @@ async function main() { examples: issue.examples, document_titles: issue.documentTitles, })), + physical_contract: { + indexed_documents: physicalDocuments.length, + without_site: missingSite.length, + without_document_type: missingDocumentType.length, + passed: missingSite.length === 0 && missingDocumentType.length === 0, + }, + registry_contract: { + indexed_documents: registryDocuments.length, + documents_with_gaps: registryContractGaps.length, + sample_gaps: registryContractGaps.slice(0, 25), + passed: registryContractGaps.length === 0, + }, sample_missing_generated: missingGenerated.slice(0, 10), sample_missing_site: missingSite.slice(0, 10), sample_missing_document_type: missingDocumentType.slice(0, 10), @@ -268,12 +343,45 @@ async function main() { allowed_document_type_missing_docs: allowedDocumentTypeMissingDocs, passed, }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + return; + } + + const supabase = await loadAdminClient(); + const allowedSiteMissing = await loadAllowlist(args.allowedSiteMissingPath); + const allowedDocumentTypeMissing = await loadAllowlist(args.allowedDocumentTypeMissingPath); + const documents = await fetchAll( + supabase, + "documents", + "id,title,file_name,file_type,source_path,metadata", + (query) => query.eq("status", "indexed"), + ); + const labels = await fetchAll( + supabase, + "document_labels", + "id,document_id,label,label_type,source,confidence", + (query) => query.eq("source", "generated"), + ); + + const report = buildDocumentLabelCoverageReport({ + documents, + labels, + allowedSiteMissing, + allowedDocumentTypeMissing, + }); if (args.json) { console.log(JSON.stringify(report, null, 2)); } else { console.log("[Document Label Coverage]"); console.log(`Indexed documents: ${report.indexed_documents}`); + console.log(`Physical documents: ${report.physical_documents}`); + console.log(`Registry documents: ${report.registry_documents}`); console.log(`Generated label rows: ${report.generated_label_rows}`); console.log(`Documents with generated labels: ${report.generated_documents}`); console.log(`Indexed without generated labels: ${report.indexed_without_generated}`); @@ -298,21 +406,28 @@ async function main() { .map(([type, count]) => `${type}=${count}`) .join(", ")}`, ); - if (allowedSiteMissingDocs.length) { - console.log(`Allowed indexed docs without site labels (from allowlist): ${allowedSiteMissingDocs.length}`); + console.log(`Registry contract gaps: ${report.registry_contract.documents_with_gaps}`); + if (report.allowed_site_missing_docs.length) { + console.log( + `Allowed indexed docs without site labels (from allowlist): ${report.allowed_site_missing_docs.length}`, + ); } - if (allowedDocumentTypeMissingDocs.length) { + if (report.allowed_document_type_missing_docs.length) { console.log( - `Allowed indexed docs without document_type labels (from allowlist): ${allowedDocumentTypeMissingDocs.length}`, + `Allowed indexed docs without document_type labels (from allowlist): ${report.allowed_document_type_missing_docs.length}`, ); } - console.log(passed ? "PASS: generated label coverage is complete." : "FAIL: generated label coverage has gaps."); + console.log( + report.passed ? "PASS: generated label coverage is complete." : "FAIL: generated label coverage has gaps.", + ); } - if (!passed) process.exitCode = 1; + if (!report.passed) process.exitCode = 1; } -main().catch((error) => { - console.error(error instanceof Error ? error.message : error); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/classify-documents.ts b/scripts/classify-documents.ts index fe9d1bde4..e2880afa0 100644 --- a/scripts/classify-documents.ts +++ b/scripts/classify-documents.ts @@ -1,5 +1,9 @@ import * as nextEnv from "@next/env"; +import { pathToFileURL } from "node:url"; import { documentLabelTier, normalizeDocumentLabelForStorage } from "@/lib/document-tags"; +import { isRegistryProjectionDocument } from "./lib/registry-projection-document"; + +export { isRegistryProjectionDocument } from "./lib/registry-projection-document"; const loadEnvConfig = nextEnv.loadEnvConfig ?? @@ -534,9 +538,16 @@ async function main() { } const supabase = await loadAdminClient(); const loadedDocuments = await loadDocuments(supabase, args); + const registryDocuments = loadedDocuments.filter(isRegistryProjectionDocument); + const classifiableDocuments = loadedDocuments.filter((document) => !isRegistryProjectionDocument(document)); + if (registryDocuments.length > 0) { + console.log( + `[classify:documents] Skipped ${registryDocuments.length} registry projection(s); use registry:reconcile-governance to repair deterministic governance metadata and labels without embeddings.`, + ); + } const documents = args.onlyMissingSmartV2 - ? await filterDocumentsMissingSmartV2(supabase, loadedDocuments) - : loadedDocuments; + ? await filterDocumentsMissingSmartV2(supabase, classifiableDocuments) + : classifiableDocuments; const plans = []; for (const document of documents) { @@ -550,7 +561,9 @@ async function main() { console.log(`\nUpdated ${plans.length} document organization profile(s).`); } -main().catch((error) => { - console.error(error instanceof Error ? error.message : error); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 206a62efa..67a8a0b0b 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -27,10 +27,14 @@ import { type SupabaseEvalCaseClient, } from "@/lib/rag-eval-cases"; import { sourceGovernanceWarnings } from "@/lib/source-governance"; +import { answerRouteBudgetMs } from "@/lib/rag-route-budget"; +import type { AnswerRouteMode } from "@/lib/rag-routing"; import type { RagAnswer } from "@/lib/types"; loadEnvConfig(process.cwd()); +export type EvalQualityProviderMode = "openai" | "offline"; + type EvalQualityArgs = { fixture: string; ownerEmail?: string; @@ -46,6 +50,7 @@ type EvalQualityArgs = { ragOnly: boolean; skipPreflight: boolean; forceEmbedding: boolean; + providerMode: EvalQualityProviderMode; }; export type RagQualityResult = { @@ -76,7 +81,23 @@ export type RagQualityResult = { unverifiedNumericTokenCount: number; hasFaithfulnessWarning: boolean; routingReason?: string; + timings?: { + retrievalMs: number; + routingMs: number; + generationMs: number; + verificationMs: number; + totalMs: number; + routeBudgetMs: number; + routeDeadlineExceeded: boolean; + }; + routeCeilingExceeded?: boolean; estimatedCostUsd: number | null; + openAIRequestIds?: string[]; + openAIUsage?: { + inputTokens: number; + cachedInputTokens: number; + outputTokens: number; + }; }; export type QualityFailureCategory = @@ -129,6 +150,28 @@ export function deliveredGroundedAfterSourceGovernancePolicy( const crossRegionRunnerLatencyContext = process.env.EVAL_LATENCY_CONTEXT === "cross-region-runner"; +export function ragAnswerTimingDiagnostics(answer: Pick) { + const latencyTimings = answer.latencyTimings; + const answerRoute = answer.routingMode ?? "unsupported"; + const defaultRouteBudgetMs = answerRouteBudgetMs[answerRoute as AnswerRouteMode] ?? 0; + const routeBudgetMs = latencyTimings?.route_budget_ms ?? defaultRouteBudgetMs; + const totalMs = latencyTimings?.total_latency_ms ?? 0; + const generationMs = latencyTimings?.generation_latency_ms ?? 0; + const routeDeadlineExceeded = latencyTimings?.route_deadline_exceeded ?? false; + return { + timings: { + retrievalMs: latencyTimings?.retrieval_latency_ms ?? latencyTimings?.search_latency_ms ?? 0, + routingMs: latencyTimings?.routing_latency_ms ?? 0, + generationMs, + verificationMs: latencyTimings?.verification_latency_ms ?? 0, + totalMs, + routeBudgetMs, + routeDeadlineExceeded, + }, + routeCeilingExceeded: routeDeadlineExceeded || (routeBudgetMs === 0 ? generationMs > 0 : totalMs > routeBudgetMs), + }; +} + const qualityThresholds = { retrievalTopKHitRate: 0.8, retrievalDocumentRecallAt5: 0.8, @@ -181,6 +224,7 @@ function parseArgs(argv: string[]): EvalQualityArgs { ragOnly: false, skipPreflight: false, forceEmbedding: false, + providerMode: "openai", }; for (let index = 0; index < argv.length; index += 1) { @@ -224,6 +268,12 @@ function parseArgs(argv: string[]): EvalQualityArgs { if (token === "--question") args.question = value; if (token === "--output-dir") args.outputDir = value; if (token === "--source-metadata-debt") args.sourceMetadataDebt = value; + if (token === "--provider-mode") { + if (value !== "openai" && value !== "offline") { + throw new Error("--provider-mode must be openai or offline."); + } + args.providerMode = value; + } } if (args.retrievalOnly && args.ragOnly) throw new Error("Use only one of --retrieval-only or --rag-only."); @@ -234,6 +284,21 @@ function parseArgs(argv: string[]): EvalQualityArgs { return args; } +export function configureEvalProviderEnvironment(providerMode: EvalQualityProviderMode) { + process.env.RAG_PROVIDER_MODE = providerMode; + if (providerMode !== "offline") return; + delete process.env.OPENAI_API_KEY; + delete process.env.OPENAI_ORG_ID; + delete process.env.OPENAI_PROJECT_ID; +} + +export function retrievalCasesForProviderMode( + cases: T[], + providerMode: EvalQualityProviderMode, +) { + return providerMode === "offline" ? cases.filter((testCase) => !testCase.forceEmbedding) : cases; +} + export function qualityFailureCategory(message: string): QualityFailureCategory { const normalized = message.toLowerCase(); if (normalized.includes("query class")) return "query_class"; @@ -426,7 +491,7 @@ function topResultGovernanceCounts(results: GoldenRetrievalResult[]) { }; } -function summarizeRagQualityResults(results: RagQualityResult[]) { +function summarizeRagQualityResults(results: RagQualityResult[], providerMode: EvalQualityProviderMode) { const supported = results.filter((result) => result.supported); const unsupported = results.filter((result) => !result.supported); // A supported case counts as grounded-supported when it grounds, OR — for @@ -435,9 +500,12 @@ function summarizeRagQualityResults(results: RagQualityResult[]) { // Requiring expectedHit keeps the guard honest: a real retrieval regression that // stops surfacing the expected docs is NOT accepted and still drags the rate below // threshold, hard-failing the canary. - const groundedSupported = supported.filter( - (result) => result.grounded || (result.acceptSourceOnly && result.expectedHit && result.citations > 0), - ).length; + const groundedSupported = supported.filter((result) => { + if (providerMode === "offline") { + return result.model === null && result.expectedHit && result.citations > 0; + } + return result.grounded || (result.acceptSourceOnly && result.expectedHit && result.citations > 0); + }).length; const unsupportedCorrect = unsupported.filter((result) => !result.grounded).length; const citationFailures = results.filter((result) => result.failures.some((failure) => qualityFailureCategory(failure) === "citation"), @@ -455,6 +523,7 @@ function summarizeRagQualityResults(results: RagQualityResult[]) { const expectedDangerWarningMissing = results.filter((result) => result.failures.includes("expected danger source governance warning missing"), ).length; + const routeCeilingFailures = results.filter((result) => result.routeCeilingExceeded).length; const latencies = results.map((result) => result.latencyMs); const routeLatencyP95 = Object.fromEntries( Array.from( @@ -483,6 +552,7 @@ function summarizeRagQualityResults(results: RagQualityResult[]) { source_governance_warning_rate: rate(sourceGovernanceWarnings.length, results.length), source_governance_danger_failure_rate: rate(sourceGovernanceDangerFailures.length, results.length), expected_danger_warning_missing_count: expectedDangerWarningMissing, + route_ceiling_failure_count: routeCeilingFailures, median_latency_ms: percentile(latencies, 50), p95_latency_ms: percentile(latencies, 95), route_p95_latency_ms: routeLatencyP95, @@ -497,11 +567,32 @@ export function buildEvalQualityReport(args: { retrievalResults: GoldenRetrievalResult[]; ragResults: RagQualityResult[]; sourceMetadataDebtAcceptance?: SourceMetadataDebtAcceptance; + providerMode?: EvalQualityProviderMode; }) { + const providerMode = args.providerMode ?? "openai"; const retrievalSummary = summarizeGoldenRetrievalResults(args.retrievalResults); - const ragSummary = summarizeRagQualityResults(args.ragResults); + const ragSummary = summarizeRagQualityResults(args.ragResults, providerMode); const governance = topResultGovernanceCounts(args.retrievalResults); const thresholdFailures: string[] = []; + const providerEvidence = { + mode: providerMode, + model_case_count: args.ragResults.filter((result) => Boolean(result.model)).length, + openai_request_id_count: args.ragResults.reduce((sum, result) => sum + (result.openAIRequestIds?.length ?? 0), 0), + token_usage_case_count: args.ragResults.filter((result) => { + const usage = result.openAIUsage; + return Boolean(usage && usage.inputTokens + usage.cachedInputTokens + usage.outputTokens > 0); + }).length, + nonzero_cost_case_count: args.ragResults.filter((result) => (result.estimatedCostUsd ?? 0) > 0).length, + generation_latency_case_count: args.ragResults.filter( + (result) => (result.generationLatencyMs ?? result.timings?.generationMs ?? 0) > 0, + ).length, + }; + if (providerMode === "offline") { + for (const [key, count] of Object.entries(providerEvidence)) { + if (key === "mode" || count === 0) continue; + thresholdFailures.push(`offline provider invariant ${key} ${count} above 0`); + } + } if (args.retrievalResults.length > 0) { if (retrievalSummary.top_k_hit_rate < qualityThresholds.retrievalTopKHitRate) { @@ -569,6 +660,9 @@ export function buildEvalQualityReport(args: { `RAG expected_danger_warning_missing_count ${ragSummary.expected_danger_warning_missing_count} above 0`, ); } + if (ragSummary.route_ceiling_failure_count > 0) { + thresholdFailures.push(`RAG route_ceiling_failure_count ${ragSummary.route_ceiling_failure_count} above 0`); + } if (ragSummary.p95_latency_ms > qualityThresholds.ragP95LatencyMs) { thresholdFailures.push( `RAG p95_latency_ms ${ragSummary.p95_latency_ms} above ${qualityThresholds.ragP95LatencyMs}`, @@ -593,6 +687,12 @@ export function buildEvalQualityReport(args: { return { generated_at: args.generatedAt ?? new Date().toISOString(), + provider: { + ...providerEvidence, + passed: + providerMode !== "offline" || + Object.entries(providerEvidence).every(([key, value]) => key === "mode" || value === 0), + }, thresholds: qualityThresholds, retrieval: { summary: retrievalSummary, @@ -634,11 +734,17 @@ function ragCaseDiagnosticsTable(results: RagQualityResult[]) { result.id, result.route, result.latencyRoute, + result.routingReason, result.latencyMs, + result.timings?.retrievalMs, + result.timings?.routingMs, result.searchLatencyMs, result.generationLatencyMs, + result.timings?.verificationMs, result.rpcLatencyMs, result.embeddingLatencyMs, + result.timings?.routeBudgetMs, + result.timings?.routeDeadlineExceeded ? "yes" : "no", result.model, result.failures.length > 0 ? `failed (${result.failures.length})` : "passed", ] @@ -646,8 +752,8 @@ function ragCaseDiagnosticsTable(results: RagQualityResult[]) { .join(" | "), ); return [ - "| Case | Route | Latency SLO | Total ms | Search ms | Generation ms | RPC ms | Embedding ms | Model | Result |", - "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- | --- |", + "| Case | Route | Latency SLO | Reason | Total ms | Retrieval ms | Routing ms | Search ms | Generation ms | Verification ms | RPC ms | Embedding ms | Budget ms | Deadline | Model | Result |", + "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- |", ...rows.map((row) => `| ${row} |`), ].join("\n"); } @@ -727,14 +833,35 @@ export function renderEvalQualityMarkdown(report: EvalQualityReport) { item.topFiles.join(" | ") || "none" }\n route=${item.route} grounded=${item.grounded} citations=${item.citations} numericWarnings=${ item.unverifiedNumericTokenCount - } faithfulnessWarning=${item.hasFaithfulnessWarning ? "yes" : "no"} sourceWarnings=${item.sourceWarningCount}`, + } faithfulnessWarning=${item.hasFaithfulnessWarning ? "yes" : "no"} sourceWarnings=${ + item.sourceWarningCount + }\n reason=${item.routingReason ?? "none"}\n timings retrieval=${ + item.timings?.retrievalMs ?? "n/a" + }ms routing=${item.timings?.routingMs ?? "n/a"}ms generation=${ + item.timings?.generationMs ?? "n/a" + }ms verification=${item.timings?.verificationMs ?? "n/a"}ms total=${ + item.timings?.totalMs ?? item.latencyMs + }ms budget=${ + item.timings?.routeBudgetMs ?? "n/a" + }ms deadline=${item.timings?.routeDeadlineExceeded ? "yes" : "no"}`, ) .join("\n"); - return `# Retrieval Quality Report Generated: ${report.generated_at} +## Provider Profile + +${markdownTable([ + ["Mode", report.provider.mode], + ["Model cases", report.provider.model_case_count], + ["OpenAI request IDs", report.provider.openai_request_id_count], + ["Token-usage cases", report.provider.token_usage_case_count], + ["Nonzero-cost cases", report.provider.nonzero_cost_case_count], + ["Generation-latency cases", report.provider.generation_latency_case_count], + ["Provider-free invariants", report.provider.passed ? "passed" : "failed"], +])} + ## Threshold Status Blocking failures: @@ -813,6 +940,7 @@ ${markdownTable([ ["Numeric grounding failure rate", rag.numeric_grounding_failure_rate], ["Source governance warning rate", rag.source_governance_warning_rate], ["Source governance danger failure rate", rag.source_governance_danger_failure_rate], + ["Route ceiling failures", rag.route_ceiling_failure_count], ["P95 latency ms", rag.p95_latency_ms], ["Estimated cost USD", rag.estimated_cost_usd], ])} @@ -845,6 +973,7 @@ async function runRetrievalQualityCases(args: { limit?: number; query?: string; forceEmbedding?: boolean; + providerMode: EvalQualityProviderMode; supabase: Awaited>; }) { const [{ searchChunksWithTelemetry }, capturedCases] = await Promise.all([ @@ -855,7 +984,10 @@ async function runRetrievalQualityCases(args: { limit: args.limit, }), ]); - const allCases = [...capturedCases.map(capturedRagCaseToGoldenCase), ...loadGoldenRetrievalCases(args.fixture)]; + const allCases = retrievalCasesForProviderMode( + [...capturedCases.map(capturedRagCaseToGoldenCase), ...loadGoldenRetrievalCases(args.fixture)], + args.providerMode, + ); const filtered = args.query ? allCases.filter((item) => item.id === args.query || item.query.toLowerCase().includes(args.query!.toLowerCase())) : allCases; @@ -898,6 +1030,7 @@ async function runRagQualityCases(args: { ownerId?: string; limit?: number; question?: string; + providerMode: EvalQualityProviderMode; supabase: Awaited>; }) { const [{ answerQuestionWithScope }, capturedCases] = await Promise.all([ @@ -934,6 +1067,20 @@ async function runRagQualityCases(args: { expectsDangerWarning: testCase.expectsSourceDangerWarning, }), ); + const { timings, routeCeilingExceeded } = ragAnswerTimingDiagnostics(answer); + if (routeCeilingExceeded) { + failures.push( + `route latency ceiling exceeded: ${timings.totalMs}ms total, ${timings.generationMs}ms generation, ${timings.routeBudgetMs}ms budget`, + ); + } + const openAIUsage = { + inputTokens: answer.openAIUsage?.input_tokens ?? 0, + cachedInputTokens: answer.openAIUsage?.cached_input_tokens ?? 0, + outputTokens: answer.openAIUsage?.output_tokens ?? 0, + }; + const hasOpenAIUsage = openAIUsage.inputTokens + openAIUsage.cachedInputTokens + openAIUsage.outputTokens > 0; + const openAIRequestIds = answer.openAIRequestIds?.filter(Boolean) ?? []; + const generationLatencyMs = answer.latencyTimings?.generation_latency_ms ?? 0; results.push({ id: testCase.id, @@ -949,7 +1096,7 @@ async function runRagQualityCases(args: { acceptSourceOnly: testCase.acceptSourceOnly, latencyMs: answer.latencyTimings?.total_latency_ms ?? 0, searchLatencyMs: answer.latencyTimings?.search_latency_ms, - generationLatencyMs: answer.latencyTimings?.generation_latency_ms, + generationLatencyMs: generationLatencyMs > 0 ? generationLatencyMs : undefined, rpcLatencyMs: answer.latencyTimings?.supabase_rpc_latency_ms, embeddingLatencyMs: answer.latencyTimings?.embedding_latency_ms, route: answer.routingMode ?? "none", @@ -963,11 +1110,11 @@ async function runRagQualityCases(args: { unverifiedNumericTokenCount: answer.unverifiedNumericTokens?.length ?? 0, hasFaithfulnessWarning: Boolean(answer.faithfulnessWarning), routingReason: answer.routingReason, - estimatedCostUsd: estimateCostUsd({ - inputTokens: answer.openAIUsage?.input_tokens ?? 0, - cachedInputTokens: answer.openAIUsage?.cached_input_tokens ?? 0, - outputTokens: answer.openAIUsage?.output_tokens ?? 0, - }), + timings, + routeCeilingExceeded, + estimatedCostUsd: hasOpenAIUsage ? estimateCostUsd(openAIUsage) : null, + openAIRequestIds: openAIRequestIds.length > 0 ? openAIRequestIds : undefined, + openAIUsage: hasOpenAIUsage ? openAIUsage : undefined, }); } @@ -1039,13 +1186,17 @@ async function loadSourceMetadataDebtAcceptance(path: string): Promise) : {}; +} + +export function isRegistryProjectionDocument(document: RegistryProjectionDocument) { + const metadata = metadataRecord(document.metadata); + return ( + metadata.source_kind === "registry_record" || + document.file_type === "application/vnd.clinical-kb.registry+json" || + document.source_path?.startsWith("registry://") === true || + document.file_name?.endsWith(".registry.json") === true + ); +} diff --git a/scripts/production-readiness.ts b/scripts/production-readiness.ts index 4d643c96a..85e84be85 100644 --- a/scripts/production-readiness.ts +++ b/scripts/production-readiness.ts @@ -1,6 +1,7 @@ import { access, readFile } from "node:fs/promises"; import { constants } from "node:fs"; import path from "node:path"; +import { pathToFileURL } from "node:url"; import { loadEnvConfig } from "@next/env"; import { checkSupabaseProjectConfig } from "@/lib/supabase/project"; @@ -38,6 +39,11 @@ function placeholderLooksLikeExample(value: string) { return /replace-with|your-|example|-example-|\{\w+\}|xxxx|todo|placeholder/i.test(value); } +export function openAIReadinessPolicy(providerMode: "auto" | "openai" | "offline", apiKey?: string) { + if (providerMode === "offline") return { required: false, ready: true } as const; + return { required: true, ready: Boolean(apiKey) } as const; +} + async function checkRequiredFile(filePath: string, message: string) { try { await access(filePath, constants.F_OK); @@ -214,18 +220,23 @@ async function main() { } } - try { - envModule.requireOpenAIEnv(); - result.passes.push("OpenAI API key is configured."); - if (placeholderLooksLikeExample(envModule.env.OPENAI_API_KEY ?? "")) { - result.failures.push("OPENAI_API_KEY still looks like a placeholder."); - } - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - if (isMissingEnvError(message)) { - recordIssue(`OpenAI configuration issue: ${message}`, { downgradeToWarningInCi: true }); - } else { - result.failures.push(`OpenAI configuration issue: ${message}`); + const openAIReadiness = openAIReadinessPolicy(envModule.env.RAG_PROVIDER_MODE, envModule.env.OPENAI_API_KEY); + if (!openAIReadiness.required) { + result.passes.push("OpenAI API key is not required because RAG_PROVIDER_MODE is explicitly offline."); + } else { + try { + envModule.requireOpenAIEnv(); + result.passes.push("OpenAI API key is configured."); + if (placeholderLooksLikeExample(envModule.env.OPENAI_API_KEY ?? "")) { + result.failures.push("OPENAI_API_KEY still looks like a placeholder."); + } + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (isMissingEnvError(message)) { + recordIssue(`OpenAI configuration issue: ${message}`, { downgradeToWarningInCi: true }); + } else { + result.failures.push(`OpenAI configuration issue: ${message}`); + } } } @@ -305,7 +316,9 @@ async function main() { } } -main().catch((error) => { - result.failures.push(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + result.failures.push(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/reconcile-registry-governance.ts b/scripts/reconcile-registry-governance.ts new file mode 100644 index 000000000..7526a618b --- /dev/null +++ b/scripts/reconcile-registry-governance.ts @@ -0,0 +1,507 @@ +import { pathToFileURL } from "node:url"; +import { loadEnvConfig } from "@next/env"; + +import type { DifferentialRecordRow } from "@/lib/differential-records"; +import type { MedicationRecordRow } from "@/lib/medication-records"; +import type { RegistryRecordRow } from "@/lib/registry-records"; +import { mergeRegistryGeneratedLabelMetadata } from "@/lib/registry-corpus"; +import type { Json, TablesInsert } from "@/lib/supabase/database.types"; +import type { RegistryCorpusKind, RegistryGovernanceProjection } from "@/lib/registry-corpus"; + +loadEnvConfig(process.cwd()); + +const PRODUCTION_PROJECT_REF = "sjrfecxgysukkwxsowpy"; +const READ_BATCH_SIZE = 100; +const WRITE_CONCURRENCY = 8; + +export const expectedRegistryProjectionCounts = { + service: 222, + form: 4, + medication: 328, + differential: 232, +} as const satisfies Record; + +type Args = { + ownerId?: string; + expectedProjectRef?: string; + write: boolean; + confirmed: boolean; + json: boolean; +}; + +export type RegistryGovernanceDocument = { + id: string; + owner_id: string | null; + metadata: Json | null; +}; + +export type RegistryGovernanceLabel = { + id: string; + document_id: string; + owner_id: string | null; + label: string; + label_type: string; + source: string; + confidence: number | null; + metadata: Json | null; +}; + +type RegistryGovernanceDocumentUpdate = { + id: string; + ownerId: string | null; + metadata: Record; +}; + +export type RegistryGovernancePlan = { + ownerId: string; + projections: RegistryGovernanceProjection[]; + documentUpdates: RegistryGovernanceDocumentUpdate[]; + labelsToInsert: TablesInsert<"document_labels">[]; + labelsToUpdate: TablesInsert<"document_labels">[]; + labelIdsToDelete: string[]; + publicDocumentCount: number; + ownerScopedDocumentCount: number; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { write: false, confirmed: false, json: false }; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--write") { + args.write = true; + continue; + } + if (token === "--confirm") { + args.confirmed = true; + continue; + } + if (token === "--json") { + args.json = true; + continue; + } + + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); + if (token === "--owner-id") args.ownerId = value; + else if (token === "--expected-project-ref") args.expectedProjectRef = value; + else throw new Error(`Unknown argument: ${token}`); + index += 1; + } + + if (args.write && !args.confirmed) { + throw new Error("Refusing to write without both --write and --confirm."); + } + if (args.write && args.expectedProjectRef !== PRODUCTION_PROJECT_REF) { + throw new Error( + `Production writes require --expected-project-ref ${PRODUCTION_PROJECT_REF}; no other project is authorized.`, + ); + } + return args; +} + +function metadataRecord(value: Json | null): Record { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function stableJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableJson((value as Record)[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +function duplicates(values: string[]) { + const seen = new Set(); + const duplicateValues = new Set(); + for (const value of values) { + if (seen.has(value)) duplicateValues.add(value); + seen.add(value); + } + return [...duplicateValues].sort(); +} + +function expectedLabelCurrent(existing: RegistryGovernanceLabel, expected: TablesInsert<"document_labels">) { + return ( + existing.owner_id === expected.owner_id && + existing.confidence === expected.confidence && + stableJson(metadataRecord(existing.metadata)) === stableJson(expected.metadata ?? {}) + ); +} + +export function buildRegistryGovernancePlan(args: { + projections: RegistryGovernanceProjection[]; + documents: RegistryGovernanceDocument[]; + labels: RegistryGovernanceLabel[]; + expectedOwnerId?: string; + expectedCounts?: Record; +}): RegistryGovernancePlan { + const expectedCounts = args.expectedCounts ?? expectedRegistryProjectionCounts; + const observedCounts = { service: 0, form: 0, medication: 0, differential: 0 } satisfies Record< + RegistryCorpusKind, + number + >; + for (const projection of args.projections) observedCounts[projection.kind] += 1; + for (const kind of Object.keys(expectedCounts) as RegistryCorpusKind[]) { + if (observedCounts[kind] !== expectedCounts[kind]) { + throw new Error( + `Registry projection count mismatch for ${kind}: expected ${expectedCounts[kind]}, found ${observedCounts[kind]}.`, + ); + } + } + + const ownerIds = new Set(args.projections.map((projection) => projection.ownerId)); + if (ownerIds.size !== 1) { + throw new Error(`Registry projections must resolve to one owner; found ${ownerIds.size}.`); + } + const ownerId = [...ownerIds][0]; + if (!ownerId) throw new Error("Registry projections did not resolve an owner."); + if (args.expectedOwnerId && ownerId !== args.expectedOwnerId) { + throw new Error(`Registry owner mismatch: expected ${args.expectedOwnerId}, found ${ownerId}.`); + } + + const duplicateSourceIdentities = duplicates( + args.projections.map((projection) => `${projection.kind}:${projection.recordId}`), + ); + const duplicateProjectionIds = duplicates(args.projections.map((projection) => projection.documentId)); + const duplicateDocumentIds = duplicates(args.documents.map((document) => document.id)); + const duplicateGeneratedLabelKeys = duplicates( + args.labels + .filter((label) => label.source === "generated") + .map((label) => `${label.document_id}:${label.label_type}:${label.label}:${label.source}`), + ); + const duplicateProblems = [ + ...duplicateSourceIdentities.map((value) => `source ${value}`), + ...duplicateProjectionIds.map((value) => `projection ${value}`), + ...duplicateDocumentIds.map((value) => `document ${value}`), + ...duplicateGeneratedLabelKeys.map((value) => `generated label ${value}`), + ]; + if (duplicateProblems.length > 0) { + throw new Error(`Duplicate registry governance identities: ${duplicateProblems.slice(0, 10).join(", ")}.`); + } + + const documentById = new Map(args.documents.map((document) => [document.id, document])); + const missingDocumentIds = args.projections + .map((projection) => projection.documentId) + .filter((documentId) => !documentById.has(documentId)); + if (missingDocumentIds.length > 0) { + throw new Error( + `Missing ${missingDocumentIds.length} deterministic registry document projection(s): ${missingDocumentIds.slice(0, 10).join(", ")}.`, + ); + } + + // Registry source records remain owner-scoped, but production deliberately + // promotes the public service/form/medication projections to owner_id = null. + // Accept that public scope while continuing to reject any different tenant. + const publicProjectionKinds = new Set(["service", "form", "medication"]); + const ownerMismatches = args.projections.filter((projection) => { + const documentOwnerId = documentById.get(projection.documentId)?.owner_id; + return ( + documentOwnerId !== projection.ownerId && + !(documentOwnerId === null && publicProjectionKinds.has(projection.kind)) + ); + }); + if (ownerMismatches.length > 0) { + throw new Error( + `Registry document foreign-owner mismatch for ${ownerMismatches.length} projection(s): ${ownerMismatches + .slice(0, 10) + .map((projection) => projection.documentId) + .join(", ")}.`, + ); + } + + const labelOwnerMismatches = args.labels.filter((label) => { + const document = documentById.get(label.document_id); + return document && label.owner_id !== document.owner_id; + }); + if (labelOwnerMismatches.length > 0) { + throw new Error( + `Registry label owner mismatch for ${labelOwnerMismatches.length} label(s): ${labelOwnerMismatches + .slice(0, 10) + .map((label) => label.id) + .join(", ")}.`, + ); + } + + const labelsByDocument = new Map(); + for (const label of args.labels) { + labelsByDocument.set(label.document_id, [...(labelsByDocument.get(label.document_id) ?? []), label]); + } + + const documentUpdates: RegistryGovernanceDocumentUpdate[] = []; + const labelsToInsert: TablesInsert<"document_labels">[] = []; + const labelsToUpdate: TablesInsert<"document_labels">[] = []; + const labelIdsToDelete: string[] = []; + + for (const projection of args.projections) { + const document = documentById.get(projection.documentId)!; + const mergedMetadata = { ...metadataRecord(document.metadata), ...projection.requiredMetadata }; + if (stableJson(mergedMetadata) !== stableJson(metadataRecord(document.metadata))) { + documentUpdates.push({ id: document.id, ownerId: document.owner_id, metadata: mergedMetadata }); + } + + const documentLabels = labelsByDocument.get(projection.documentId) ?? []; + for (const label of documentLabels) { + if (label.source !== "generated") continue; + if (label.label_type === "site") labelIdsToDelete.push(label.id); + if (label.label_type === "document_intent" && label.label !== projection.intentLabel.label) { + labelIdsToDelete.push(label.id); + } + } + + const expectedLabel = documentLabels.find( + (label) => + label.source === "generated" && + label.label_type === "document_intent" && + label.label === projection.intentLabel.label, + ); + const intentLabel = { + ...projection.intentLabel, + owner_id: document.owner_id, + confidence: expectedLabel?.confidence ?? projection.intentLabel.confidence, + metadata: mergeRegistryGeneratedLabelMetadata( + expectedLabel?.metadata ?? null, + projection.intentLabel.metadata ?? null, + ), + }; + if (!expectedLabel) labelsToInsert.push(intentLabel); + else if (!expectedLabelCurrent(expectedLabel, intentLabel)) labelsToUpdate.push(intentLabel); + } + + return { + ownerId, + projections: args.projections, + documentUpdates, + labelsToInsert, + labelsToUpdate, + labelIdsToDelete: [...new Set(labelIdsToDelete)].sort(), + publicDocumentCount: args.documents.filter((document) => document.owner_id === null).length, + ownerScopedDocumentCount: args.documents.filter((document) => document.owner_id === ownerId).length, + }; +} + +export function assertExpectedRegistryProjectRef(args: { + expectedProjectRef: string; + configuredProjectRef?: string; + supabaseUrl?: string; +}) { + const configuredProjectRef = args.configuredProjectRef?.trim(); + if (configuredProjectRef !== args.expectedProjectRef) { + throw new Error( + `Supabase project ref mismatch: expected ${args.expectedProjectRef}, configured ${configuredProjectRef || "not set"}.`, + ); + } + let urlProjectRef = ""; + try { + urlProjectRef = args.supabaseUrl ? (new URL(args.supabaseUrl).hostname.split(".")[0] ?? "") : ""; + } catch { + throw new Error("NEXT_PUBLIC_SUPABASE_URL is not a valid URL."); + } + if (urlProjectRef !== args.expectedProjectRef) { + throw new Error( + `Supabase URL project mismatch: expected ${args.expectedProjectRef}, observed ${urlProjectRef || "not set"}.`, + ); + } +} + +function chunks(values: T[], size: number) { + const output: T[][] = []; + for (let index = 0; index < values.length; index += size) output.push(values.slice(index, index + size)); + return output; +} + +async function loadAdminClient() { + const { createAdminClient } = await import("@/lib/supabase/admin"); + return createAdminClient(); +} + +type AdminClient = Awaited>; + +async function loadAllRows( + loadPage: (from: number, to: number) => PromiseLike<{ data: Row[] | null; error: { message: string } | null }>, +) { + const rows: Row[] = []; + const pageSize = 1000; + while (true) { + const { data, error } = await loadPage(rows.length, rows.length + pageSize - 1); + if (error) throw new Error(error.message); + const page = data ?? []; + rows.push(...page); + if (page.length < pageSize) return rows; + } +} + +type OwnerCounts = Record; + +async function resolveProductionOwner(supabase: AdminClient, requestedOwnerId?: string) { + const [registryRows, medicationRows, differentialRows] = await Promise.all([ + loadAllRows<{ owner_id: string; kind: string }>((from, to) => + supabase.from("clinical_registry_records").select("owner_id,kind").range(from, to), + ), + loadAllRows<{ owner_id: string }>((from, to) => + supabase.from("medication_records").select("owner_id").range(from, to), + ), + loadAllRows<{ owner_id: string }>((from, to) => + supabase.from("differential_records").select("owner_id").range(from, to), + ), + ]); + const counts = new Map(); + const ownerCounts = (ownerId: string) => { + const current = counts.get(ownerId) ?? { service: 0, form: 0, medication: 0, differential: 0 }; + counts.set(ownerId, current); + return current; + }; + for (const row of registryRows) ownerCounts(row.owner_id)[row.kind === "form" ? "form" : "service"] += 1; + for (const row of medicationRows) ownerCounts(row.owner_id).medication += 1; + for (const row of differentialRows) ownerCounts(row.owner_id).differential += 1; + + const candidates = [...counts.entries()].filter(([, count]) => + (Object.keys(expectedRegistryProjectionCounts) as RegistryCorpusKind[]).every( + (kind) => count[kind] === expectedRegistryProjectionCounts[kind], + ), + ); + if (candidates.length !== 1) { + throw new Error(`Expected exactly one owner with the production registry profile; found ${candidates.length}.`); + } + const ownerId = candidates[0]![0]; + if (requestedOwnerId && ownerId !== requestedOwnerId) { + throw new Error(`Resolved production owner ${ownerId} does not match --owner-id ${requestedOwnerId}.`); + } + return ownerId; +} + +async function loadProjections(supabase: AdminClient, ownerId: string) { + const [registryResult, medicationResult, differentialResult] = await Promise.all([ + supabase.from("clinical_registry_records").select("*").eq("owner_id", ownerId).order("kind").order("title"), + supabase.from("medication_records").select("*").eq("owner_id", ownerId).order("name"), + supabase.from("differential_records").select("*").eq("owner_id", ownerId).order("kind").order("title"), + ]); + if (registryResult.error) throw new Error(`Could not load registry records: ${registryResult.error.message}`); + if (medicationResult.error) throw new Error(`Could not load medication records: ${medicationResult.error.message}`); + if (differentialResult.error) + throw new Error(`Could not load differential records: ${differentialResult.error.message}`); + + const { + clinicalRegistryRowsToCorpusEntries, + differentialRowsToCorpusEntries, + medicationRowsToCorpusEntries, + registryGovernanceProjection, + } = await import("@/lib/registry-corpus"); + return [ + ...clinicalRegistryRowsToCorpusEntries((registryResult.data ?? []) as RegistryRecordRow[]), + ...medicationRowsToCorpusEntries((medicationResult.data ?? []) as MedicationRecordRow[]), + ...differentialRowsToCorpusEntries((differentialResult.data ?? []) as DifferentialRecordRow[]), + ].map(registryGovernanceProjection); +} + +async function loadExistingGovernance(supabase: AdminClient, documentIds: string[]) { + const documents: RegistryGovernanceDocument[] = []; + const labels: RegistryGovernanceLabel[] = []; + for (const documentIdBatch of chunks(documentIds, READ_BATCH_SIZE)) { + const [documentResult, labelResult] = await Promise.all([ + supabase.from("documents").select("id,owner_id,metadata").in("id", documentIdBatch), + supabase + .from("document_labels") + .select("id,document_id,owner_id,label,label_type,source,confidence,metadata") + .in("document_id", documentIdBatch), + ]); + if (documentResult.error) throw new Error(`Registry document preflight failed: ${documentResult.error.message}`); + if (labelResult.error) throw new Error(`Registry label preflight failed: ${labelResult.error.message}`); + documents.push(...((documentResult.data ?? []) as RegistryGovernanceDocument[])); + labels.push(...((labelResult.data ?? []) as RegistryGovernanceLabel[])); + } + return { documents, labels }; +} + +async function runWithConcurrency(items: T[], concurrency: number, worker: (item: T) => Promise) { + let nextIndex = 0; + async function runWorker() { + while (nextIndex < items.length) { + const item = items[nextIndex++]; + if (item) await worker(item); + } + } + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => runWorker())); +} + +async function applyPlan(supabase: AdminClient, plan: RegistryGovernancePlan) { + await runWithConcurrency(plan.documentUpdates, WRITE_CONCURRENCY, async (update) => { + const updateQuery = supabase.from("documents").update({ metadata: update.metadata }).eq("id", update.id); + const scopedUpdate = + update.ownerId === null ? updateQuery.is("owner_id", null) : updateQuery.eq("owner_id", update.ownerId); + const { data, error } = await scopedUpdate.select("id"); + if (error) throw new Error(`Registry metadata update failed for ${update.id}: ${error.message}`); + if (data?.length !== 1) throw new Error(`Registry metadata update did not match exactly one row for ${update.id}.`); + }); + + for (const labelIdBatch of chunks(plan.labelIdsToDelete, READ_BATCH_SIZE)) { + const { error } = await supabase.from("document_labels").delete().in("id", labelIdBatch); + if (error) throw new Error(`Registry generated-label cleanup failed: ${error.message}`); + } + for (const labelBatch of chunks([...plan.labelsToInsert, ...plan.labelsToUpdate], READ_BATCH_SIZE)) { + const { error } = await supabase + .from("document_labels") + .upsert(labelBatch, { onConflict: "document_id,label_type,label,source" }); + if (error) throw new Error(`Registry intent-label upsert failed: ${error.message}`); + } +} + +function reportForPlan(plan: RegistryGovernancePlan, write: boolean) { + return { + mode: write ? "write" : "dry-run", + owner_id: plan.ownerId, + documents_inspected: plan.projections.length, + documents_updated: plan.documentUpdates.length, + labels_inserted: plan.labelsToInsert.length, + labels_updated: plan.labelsToUpdate.length, + labels_deleted: plan.labelIdsToDelete.length, + public_documents: plan.publicDocumentCount, + owner_scoped_documents: plan.ownerScopedDocumentCount, + chunk_rows_touched: 0, + openai_calls: 0, + }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const expectedProjectRef = args.expectedProjectRef ?? PRODUCTION_PROJECT_REF; + assertExpectedRegistryProjectRef({ + expectedProjectRef, + configuredProjectRef: process.env.SUPABASE_PROJECT_REF, + supabaseUrl: process.env.NEXT_PUBLIC_SUPABASE_URL, + }); + const { requireServerEnv } = await import("@/lib/env"); + requireServerEnv(); + const supabase = await loadAdminClient(); + const ownerId = await resolveProductionOwner(supabase, args.ownerId); + const projections = await loadProjections(supabase, ownerId); + const existing = await loadExistingGovernance( + supabase, + projections.map((projection) => projection.documentId), + ); + const plan = buildRegistryGovernancePlan({ ...existing, projections, expectedOwnerId: ownerId }); + const report = reportForPlan(plan, args.write); + + if (args.write) await applyPlan(supabase, plan); + if (args.json) console.log(JSON.stringify(report, null, 2)); + else { + console.log("[registry:reconcile-governance]"); + for (const [key, value] of Object.entries(report)) console.log(` ${key}: ${value}`); + if (!args.write) { + console.log( + `Dry run only. Re-run with --write --confirm --expected-project-ref ${PRODUCTION_PROJECT_REF} to apply this exact scope.`, + ); + } + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(`[registry:reconcile-governance] ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + }); +} diff --git a/scripts/run-eval-safe.mjs b/scripts/run-eval-safe.mjs index 3f3c9f371..69f3c810e 100644 --- a/scripts/run-eval-safe.mjs +++ b/scripts/run-eval-safe.mjs @@ -7,6 +7,9 @@ import { fileURLToPath } from "node:url"; const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const isWindows = process.platform === "win32"; const [targetScript, ...forwardArgs] = process.argv.slice(2); +const offlineProviderRequested = forwardArgs.some( + (token, index) => token === "--provider-mode" && forwardArgs[index + 1] === "offline", +); if (!targetScript) { console.error("Usage: node scripts/run-eval-safe.mjs