From 881c51eb243d45679ace1d60fb1d5b2773d78ba0 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 14 Jul 2026 18:57:57 +0800 Subject: [PATCH 1/8] fix: close release governance and offline readiness gaps --- .env.example | 15 + .github/workflows/staging-tenancy.yml | 60 ++ docs/branch-review-ledger.md | 1 + docs/process-hardening.md | 7 + docs/staging-setup.md | 62 +- docs/staging-tenancy-release-evidence.md | 79 +++ package.json | 4 + scripts/build-clinical-review-queue.ts | 56 ++ scripts/check-document-label-coverage.ts | 219 ++++-- scripts/classify-documents.ts | 34 +- scripts/eval-quality.ts | 176 ++++- scripts/production-readiness.ts | 45 +- scripts/reconcile-registry-governance.ts | 472 +++++++++++++ scripts/run-eval-safe.mjs | 14 + scripts/test-cross-tenant-staging.ts | 644 ++++++++++++++++++ scripts/verify-release-offline.mjs | 55 ++ src/app/page.tsx | 6 + src/components/ClinicalDashboard.tsx | 43 +- src/components/DocumentViewer.tsx | 8 +- .../answer-result-surface.tsx | 39 +- .../global-search-shell.tsx | 7 +- .../clinical-dashboard/use-hide-on-scroll.ts | 7 + .../document-viewer/pdf-canvas-viewer.tsx | 4 + src/lib/answer-thread-storage.ts | 6 +- src/lib/clinical-review-queue.ts | 160 +++++ src/lib/health-response.ts | 3 +- src/lib/rag-route-budget.ts | 121 ++++ src/lib/rag.ts | 167 +++-- src/lib/registry-corpus.ts | 205 ++++-- src/lib/types.ts | 5 + tests/answer-progress-ui-smoke.spec.ts | 13 +- tests/answer-thread-storage.test.ts | 10 + tests/classify-documents.test.ts | 23 + tests/clinical-review-queue.test.ts | 103 +++ tests/cross-tenant-staging-config.test.ts | 58 ++ tests/document-label-coverage.test.ts | 105 +++ tests/eval-quality.test.ts | 134 +++- tests/health-route.test.ts | 34 +- tests/offline-release-profile.test.ts | 30 + tests/playwright-scroll.ts | 20 +- tests/production-readiness-offline.test.ts | 15 + tests/rag-answer-fallback.test.ts | 24 + tests/rag-route-budget.test.ts | 54 ++ tests/registry-corpus.test.ts | 122 +++- ...registry-governance-reconciliation.test.ts | 185 +++++ tests/retrieval-owner-filter-guard.test.ts | 200 ++++++ tests/ui-smoke.spec.ts | 179 +++-- tests/ui-tools.spec.ts | 27 +- 48 files changed, 3708 insertions(+), 352 deletions(-) create mode 100644 .github/workflows/staging-tenancy.yml create mode 100644 docs/staging-tenancy-release-evidence.md create mode 100644 scripts/build-clinical-review-queue.ts create mode 100644 scripts/reconcile-registry-governance.ts create mode 100644 scripts/test-cross-tenant-staging.ts create mode 100644 scripts/verify-release-offline.mjs create mode 100644 src/lib/clinical-review-queue.ts create mode 100644 src/lib/rag-route-budget.ts create mode 100644 tests/classify-documents.test.ts create mode 100644 tests/clinical-review-queue.test.ts create mode 100644 tests/cross-tenant-staging-config.test.ts create mode 100644 tests/document-label-coverage.test.ts create mode 100644 tests/offline-release-profile.test.ts create mode 100644 tests/production-readiness-offline.test.ts create mode 100644 tests/rag-route-budget.test.ts create mode 100644 tests/registry-governance-reconciliation.test.ts diff --git a/.env.example b/.env.example index 0b31c1105..d8fbcaece 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,21 @@ INDEXING_V3_AGENT_SECRET=your-long-random-cron-shared-secret # Secret required for /api/health?deep=1 Supabase probe (x-health-deep-token header). HEALTH_DEEP_PROBE_SECRET=your-long-random-health-deep-probe-secret +# Dedicated staging-only cross-tenant regression harness. These values are read +# only by `npm run test:cross-tenant:staging`; never point them at production. +# The two accounts must be distinct, non-human staging test users. The staging +# app must use the same Supabase project and RAG_PROVIDER_MODE=offline. +#CROSS_TENANT_STAGING_APP_URL=https://clinical-kb-staging.example.org +#CROSS_TENANT_SUPABASE_URL=https://your-staging-project-ref.supabase.co +#CROSS_TENANT_PROJECT_REF=your-staging-project-ref +#CROSS_TENANT_PUBLISHABLE_KEY=your-staging-publishable-key +#CROSS_TENANT_SERVICE_ROLE_KEY=your-staging-service-role-key +#CROSS_TENANT_USER_A_EMAIL=tenancy-a@example.org +#CROSS_TENANT_USER_A_PASSWORD=replace-with-staging-user-a-password +#CROSS_TENANT_USER_B_EMAIL=tenancy-b@example.org +#CROSS_TENANT_USER_B_PASSWORD=replace-with-staging-user-b-password +#CROSS_TENANT_DOCUMENT_BUCKET=clinical-documents +#CROSS_TENANT_EVIDENCE_PATH=artifacts/staging-tenancy-evidence.json # Local-only no-auth mode (development only). # Enable one or both of these to load real data without per-request browser auth: # - NEXT_PUBLIC_LOCAL_NO_AUTH (client + server) diff --git a/.github/workflows/staging-tenancy.yml b/.github/workflows/staging-tenancy.yml new file mode 100644 index 000000000..087c6d95c --- /dev/null +++ b/.github/workflows/staging-tenancy.yml @@ -0,0 +1,60 @@ +name: Staging tenancy isolation + +on: + workflow_dispatch: + schedule: + - cron: "30 18 * * *" + +concurrency: + group: staging-tenancy-isolation + cancel-in-progress: false + +permissions: + contents: read + +jobs: + cross-tenant: + name: Cross-tenant staging harness + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Run cross-tenant staging checks + env: + CROSS_TENANT_STAGING_APP_URL: ${{ secrets.CROSS_TENANT_STAGING_APP_URL }} + CROSS_TENANT_SUPABASE_URL: ${{ secrets.CROSS_TENANT_SUPABASE_URL }} + CROSS_TENANT_PROJECT_REF: ${{ secrets.CROSS_TENANT_PROJECT_REF }} + CROSS_TENANT_PUBLISHABLE_KEY: ${{ secrets.CROSS_TENANT_PUBLISHABLE_KEY }} + CROSS_TENANT_SERVICE_ROLE_KEY: ${{ secrets.CROSS_TENANT_SERVICE_ROLE_KEY }} + CROSS_TENANT_USER_A_EMAIL: ${{ secrets.CROSS_TENANT_USER_A_EMAIL }} + CROSS_TENANT_USER_A_PASSWORD: ${{ secrets.CROSS_TENANT_USER_A_PASSWORD }} + CROSS_TENANT_USER_B_EMAIL: ${{ secrets.CROSS_TENANT_USER_B_EMAIL }} + CROSS_TENANT_USER_B_PASSWORD: ${{ secrets.CROSS_TENANT_USER_B_PASSWORD }} + CROSS_TENANT_DOCUMENT_BUCKET: ${{ vars.CROSS_TENANT_DOCUMENT_BUCKET }} + CROSS_TENANT_EVIDENCE_PATH: artifacts/staging-tenancy-evidence.json + CROSS_TENANT_COMMIT_SHA: ${{ github.sha }} + CROSS_TENANT_WORKFLOW_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: npm run test:cross-tenant:staging + + - name: Upload tenancy evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: staging-tenancy-evidence-${{ github.run_id }} + path: artifacts/staging-tenancy-evidence.json + if-no-files-found: error + retention-days: 14 diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index 18fd426aa..58eb83547 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -408,3 +408,4 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-14 | codex/specifiers-design | c3e7024e15d2157bbc0b989276324cc6ad3eea5c | Specifiers UI, clinical decision-support, accessibility, and release-readiness review | One P2 WCAG contrast defect was reproduced and fixed across the builder, comparison, and map accent eyebrows. No remaining high-confidence defect was found in the changed scope; residual risk is clinical/manual governance of the original specifier summaries. | Focused Vitest 18/18; focused Chromium desktop/mobile 2/2 with serious/critical axe WCAG A/AA scanning and overflow checks; `npm run check:production-readiness:ci` READY; `npm run verify:cheap` 2,210 passed/1 skipped; `npm run verify:pr-local` 2,213 passed/1 skipped plus production build and client-bundle secret scan; `git diff --check`. Full advisory `verify:ui` exceeded the local 10-minute execution window; provider-backed checks were not run. | | 2026-07-14 | PR #633 / codex/specifiers-design | 03daca3bfdf86517b9956f3c7d91be27d9f1a751 | Review follow-up and failed UI regression | Eight distinct P2 behaviors across nine review threads were confirmed and fixed: initial selection normalization, duplicate query removal, clinical applicability mapping, canonical Specifiers routing, severity-neutral base wording, diagnostic-section ordering, base/specifier compatibility, and severe psychotic-features wording. The failed app-menu keyboard test was a stale order assertion and now covers the inserted Specifiers item. | Focused Vitest 6/6; focused Chromium 4/4; scoped ESLint and TypeScript; CI-mode production readiness READY; production build and client-bundle secret scan passed. Hosted required checks passed on the refreshed `main` merge before the final compatibility fixes; provider-backed Supabase/OpenAI checks were not run. | | 2026-07-14 | PR #634 / codex/global-answer-reliability | b411329ec5f181661e5d49276c398440aa928fa2 | review-followup | One late P2 fast-context defect was confirmed: Australian tier ordering could push a higher-ranked supplementary passage outside the four-chunk routine fast budget. Fixed by preserving the retrieval-ranked, crowding-capped candidate budget before applying the order-only Australian preference within that set. | GitHub connector review-thread inspection; focused RAG context-budget suite 22/22; ESLint; TypeScript; Prettier; `git diff --check`. | +| 2026-07-14 | codex/release-blocker-remediation | 550e0588866c38583bd9445fc109ea7832a98211 | working-tree release remediation review | Reconciled the remediation onto current main after three retained-stash fast-forward syncs, preserving the extracted RAG and document-viewer architecture and main's Sentry removal. Review confirmed and fixed the Windows offline-release launcher failure, the offline Railway health-check blocker, and shared staging-test passwords. No remaining high-confidence issue was found in the changed local scope. Provider-backed production and staging evidence remains a post-PR gate. | `npm ci` and `npm ls --depth=0` passed; focused registry/offline/RAG/viewer/tenancy Vitest 207/207 plus health/config follow-up 15/15; `npm run format:check`; `npm run check:github-actions`; `npm run check:ci-scope`; `npm run verify:cheap` passed runtime, generated guards, lint, TypeScript, and full Vitest (254 files passed, 1 skipped; 2,325 tests passed, 1 skipped); `git diff --check`. | diff --git a/docs/process-hardening.md b/docs/process-hardening.md index 7ddf2224a..f9a4c19ed 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -2,6 +2,13 @@ This document turns the current process review into phased, durable repo practice. It separates changes that already take effect from work that should stay explicit until it is implemented. +## Staging tenancy evidence + +The provider-backed A/B tenancy regression is intentionally outside local and PR +gates. Run the standalone manual/nightly workflow and attach a recent green evidence +artifact before release; see +[`docs/staging-tenancy-release-evidence.md`](staging-tenancy-release-evidence.md). + ## Phase 1 - Active now - `npm run verify:cheap` is the default broad local gate for source/config/test changes: `check:runtime`, `sitemap:check`, lint, typecheck, and unit tests. diff --git a/docs/staging-setup.md b/docs/staging-setup.md index 26bf9ea7d..14ddcf9ec 100644 --- a/docs/staging-setup.md +++ b/docs/staging-setup.md @@ -33,17 +33,11 @@ those vars are unset. Then confirm health: `npm run check:indexing` (runs `search_schema_health()` over the hybrid RPCs) should report ok. -3. **Seed a small corpus (~50 docs).** Use synthetic/public content only — do - **not** copy clinical production documents into staging. - - ```bash - npm run samples # generate synthetic sample documents - npm run import:docs # queue them for ingestion - # (run the worker — section B4 — to process the queue) - npm run registry:seed -- --owner-id --write --confirm - npm run differentials:seed - npm run medications:seed - ``` +3. **Keep the tenancy profile synthetic and worker-free.** Do **not** copy + clinical production documents into staging and do not start an ingestion + worker. The cross-tenant harness creates the minimal private documents, + lexical chunks, and storage objects it needs, then removes them in `finally` + cleanup. This keeps the release proof deterministic and provider-free. 4. **Capture the keys** for the staging project (dashboard → API): the publishable key (`sb_publishable_…`) and the service-role secret @@ -52,11 +46,11 @@ those vars are unset. ## B. Staging app host (compute tier) Host: **Railway**, same as production (see `docs/deployment-architecture.md` §2). -Stand staging up as a **second environment** in the `clinical-kb` Railway project -(e.g. a `staging` environment) or a separate project, with its own `app` (+ -optional `worker`) service pinned to **Southeast Asia (`asia-southeast1-eqsg3a`, -Singapore)** — the closest region to the staging Supabase project in Sydney. -Reuse the same images; only the environment variables differ. +Stand staging up as a **second environment** in the `clinical-kb` Railway project, +with one `app` service pinned to **Southeast Asia +(`asia-southeast1-eqsg3a`, Singapore)** — the closest region to the staging +Supabase project in Sydney. Do not create a worker for this release profile. +Reuse the app image; only the environment variables differ. 1. **Build the image** — Railway builds it remotely from the committed `Dockerfile` on deploy, so there is no local `docker build` and no local @@ -72,16 +66,19 @@ Reuse the same images; only the environment variables differ. 2. **Runtime secrets** (injected at deploy, never baked into the image) — all with **staging** values, distinct from production: - | Var | Value | - | ------------------------------- | --------------------------------------------------------- | - | `SUPABASE_SERVICE_ROLE_KEY` | staging `sb_secret_…` | - | `OPENAI_API_KEY` | a staging-only OpenAI key; never reuse the production key | - | `SUPABASE_PROJECT_REF` | `` | - | `SUPABASE_PROJECT_NAME` | `Clinical KB Staging` | - | `SUPABASE_STAGING_PROJECT_REF` | `` | - | `SUPABASE_STAGING_PROJECT_NAME` | `Clinical KB Staging` | - | `RAG_QUERY_HASH_SECRET` | a staging secret | - | `RAG_PROVIDER_MODE` | `auto` | + | Var | Value | + | ------------------------------- | -------------------------- | + | `SUPABASE_SERVICE_ROLE_KEY` | staging `sb_secret_…` | + | `SUPABASE_PROJECT_REF` | `` | + | `SUPABASE_PROJECT_NAME` | `Clinical KB Staging` | + | `SUPABASE_STAGING_PROJECT_REF` | `` | + | `SUPABASE_STAGING_PROJECT_NAME` | `Clinical KB Staging` | + | `RAG_QUERY_HASH_SECRET` | unique staging-only secret | + | `RAG_PROVIDER_MODE` | `offline` | + + Do not set `OPENAI_API_KEY`, `OPENAI_ORG_ID`, or `OPENAI_PROJECT_ID` in the + staging environment. Offline mode uses lexical retrieval and deterministic, + cited source-only answers; it performs no embedding or generation request. The two `SUPABASE_STAGING_PROJECT_*` vars are what make the identity guard accept the staging project. Setting `NEXT_PUBLIC_SUPABASE_URL` + @@ -93,14 +90,19 @@ Reuse the same images; only the environment variables differ. policy to `ON_FAILURE`, and one replica pinned to `southeast-asia` with no scale-to-zero (mirror `railway.app.json`). -4. **Worker (optional, for ingestion in staging):** build `Dockerfile.worker` - and run one instance with the same staging secrets. Required to process the - seed queue from A3. +4. **No worker:** this staging environment intentionally has no ingestion + service. The tenancy harness inserts and removes its synthetic lexical rows + directly through its dedicated staging credentials. ## C. Validate staging 1. Boot check: `GET /api/health` → `{"status":"ok"}` with the staging project. -2. Load check — the soak test is hard-guarded against production: +2. Tenancy isolation: configure the dedicated A/B test accounts and standalone + workflow described in + [`staging-tenancy-release-evidence.md`](staging-tenancy-release-evidence.md). + The harness requires an app deployment with `RAG_PROVIDER_MODE=offline` and is + hard-guarded against the production project. +3. Load check — the soak test is hard-guarded against production: ```bash npx tsx scripts/soak-test.ts --target https:// \ diff --git a/docs/staging-tenancy-release-evidence.md b/docs/staging-tenancy-release-evidence.md new file mode 100644 index 000000000..6516a8940 --- /dev/null +++ b/docs/staging-tenancy-release-evidence.md @@ -0,0 +1,79 @@ +# Staging tenancy release evidence + +The cross-tenant staging harness is the executable proof that the service-role API +layer preserves private owner boundaries. It is intentionally separate from PR and +local verification because it signs in to a hosted staging app, writes synthetic +fixtures to a dedicated staging Supabase project, and then removes them. + +## Safety contract + +Run `npm run test:cross-tenant:staging` only against a dedicated staging deployment. +The harness exits before creating clients or writing data when any required value is +missing or placeholder-like, the project ref is the production ref +`sjrfecxgysukkwxsowpy`, or the Supabase URL does not match the declared project ref. +It signs both test accounts in before creating the service-role client and refuses to +write fixtures if both credentials resolve to the same user. + +The staging app used by this check must: + +- target the same dedicated staging Supabase project; +- set `RAG_PROVIDER_MODE=offline`, which the harness proves by requiring the exact + `source_only_offline_mode` answer fallback; +- use two distinct, non-human test accounts that are not used interactively; and +- avoid running an ingestion worker during the short check, because full reindex is + exercised last and the harness owns cleanup. + +Configure these repository secrets for `.github/workflows/staging-tenancy.yml`: + +| Secret | Purpose | +| ------------------------------------------------------------ | ----------------------------------------------------------- | +| `CROSS_TENANT_STAGING_APP_URL` | HTTPS origin of the offline staging app | +| `CROSS_TENANT_SUPABASE_URL` | Dedicated staging Supabase HTTPS origin | +| `CROSS_TENANT_PROJECT_REF` | Dedicated staging project ref | +| `CROSS_TENANT_PUBLISHABLE_KEY` | Staging publishable/anon key | +| `CROSS_TENANT_SERVICE_ROLE_KEY` | Staging service-role key used only for fixtures and cleanup | +| `CROSS_TENANT_USER_A_EMAIL` / `CROSS_TENANT_USER_A_PASSWORD` | First staging test user | +| `CROSS_TENANT_USER_B_EMAIL` / `CROSS_TENANT_USER_B_PASSWORD` | Second staging test user | + +`CROSS_TENANT_DOCUMENT_BUCKET` is an optional repository variable and defaults to +`clinical-documents`. The harness reads only the `CROSS_TENANT_*` namespace; it does +not fall back to production application credentials. + +## Proof matrix + +Each run creates one private document, page, lexical chunk, and storage object for +each user under a unique run ID. It then proves: + +| Surface | Owner proof | Cross-tenant proof | +| ------------------ | ----------------------------------- | -------------------------------------------- | +| Document list | A and B each find their own fixture | B cannot list A's fixture | +| Document detail | A receives A's document | B receives 404 for A's document | +| Signed URL | A receives a signed URL | B receives 404 | +| Labels | A creates a manual label | B receives 404 | +| Mutation | A renames A's document | B receives 404 | +| Universal search | A finds A's document | B gets no A result | +| Offline retrieval | A retrieves A's chunk | B gets an empty scope and result set | +| Source-only answer | A gets cited source-only evidence | B gets unsupported with no sources/citations | +| Reindex | A can queue full reindex | B receives 404 | + +Cleanup runs in `finally` semantics in reverse dependency order, including run-time +query telemetry for the two dedicated test users, reindex jobs/stages, labels, +chunks, pages, documents, and storage objects. Cleanup failure makes the run fail. +The test never skips a scenario when explicitly invoked. + +## Release evidence + +The standalone workflow runs nightly and on manual dispatch. It is not included in +`verify:cheap`, `verify:pr-local`, PR-required checks, or local release gates. + +Before a release, record a successful workflow run that: + +1. completed within the previous seven days; +2. tested the release commit SHA (or a commit proven to be its ancestor with no + later tenancy/API changes); and +3. retained the `staging-tenancy-evidence-` artifact. + +The JSON artifact records the tested commit, workflow URL, staging project identity, +completed scenario checkpoints, final status, and cleanup status without including +credentials. A missing artifact, failed cleanup, stale run, different project, or +non-offline answer is not acceptable release evidence. diff --git a/package.json b/package.json index 020e2e6d9..42e168f9d 100644 --- a/package.json +++ b/package.json @@ -33,10 +33,12 @@ "test:e2e:mockups": "node scripts/run-playwright.mjs --project=chromium-mockups", "test:e2e:chromium": "node scripts/run-playwright.mjs --project=chromium --project=chromium-mockups", "test:e2e:visual": "node scripts/run-playwright.mjs --config=playwright.visual.config.ts", + "test:cross-tenant:staging": "node scripts/run-tsx.mjs scripts/test-cross-tenant-staging.ts", "verify:cheap": "npm run check:runtime && npm run check:github-actions && npm run sitemap:check && npm run brand:check && npm run check:type-scale && npm run check:icon-scale && npm run lint && npm run typecheck && npm run test", "verify:pr-local": "node scripts/verify-pr-local.mjs", "verify:ui": "npm run check:runtime && npm run test:e2e:chromium", "verify:release": "npm run check:runtime && npm run lint && npm run typecheck && npm run test && npm run build && npm run test:e2e && npm run check:production-readiness && npm run governance:release && npm run eval:quality:release", + "verify:release:offline": "node scripts/verify-release-offline.mjs", "ci:env-check": "node scripts/check-ci-env.mjs", "check:github-actions": "node scripts/check-github-action-pins.mjs", "check:client-bundle-secrets": "node scripts/check-client-bundle-secrets.mjs", @@ -92,6 +94,7 @@ "recover:ingestion": "node scripts/run-tsx.mjs scripts/recover-ingestion-queue.ts", "registry:seed": "node scripts/run-tsx.mjs scripts/seed-registry-records.ts", "registry:embed": "node scripts/run-tsx.mjs scripts/embed-registry-records.ts", + "registry:reconcile-governance": "node scripts/run-tsx.mjs scripts/reconcile-registry-governance.ts", "services:import": "node scripts/run-tsx.mjs scripts/import-services-export.ts", "differentials:import": "node scripts/run-tsx.mjs scripts/import-differentials-export.ts", "differentials:seed": "node scripts/run-tsx.mjs scripts/seed-differential-records.ts", @@ -112,6 +115,7 @@ "eval:answer-quality": "node scripts/run-eval-safe.mjs scripts/eval-answer-quality.ts", "eval:quality": "node scripts/run-eval-safe.mjs scripts/eval-quality.ts", "eval:quality:release": "npm run eval:quality -- --fail-on-threshold --source-metadata-debt docs/release-source-metadata-debt-2026-06-30.json", + "eval:quality:release:offline": "npm run eval:quality -- --provider-mode offline --fail-on-threshold --source-metadata-debt docs/release-source-metadata-debt-2026-06-30.json", "eval:retrieval": "node scripts/run-eval-safe.mjs scripts/eval-retrieval.ts", "eval:retrieval:quality": "node scripts/run-eval-safe.mjs scripts/eval-retrieval.ts --mode quality", "eval:retrieval:index-unit-vector": "node scripts/run-eval-safe.mjs scripts/eval-retrieval.ts --mode quality --force-embedding --json-out artifacts/retrieval/index-unit-vector.json", diff --git a/scripts/build-clinical-review-queue.ts b/scripts/build-clinical-review-queue.ts new file mode 100644 index 000000000..cfa91d52e --- /dev/null +++ b/scripts/build-clinical-review-queue.ts @@ -0,0 +1,56 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; + +import { buildClinicalReviewQueue } from "@/lib/clinical-review-queue"; + +type Args = { input?: string; output?: string; help: boolean }; + +function parseArgs(argv: string[]): Args { + const args: Args = { help: false }; + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--help" || token === "-h") { + args.help = true; + continue; + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); + if (token === "--input") args.input = value; + else if (token === "--output") args.output = value; + else throw new Error(`Unknown option: ${token}`); + index += 1; + } + return args; +} + +function usage() { + return [ + "Usage: node scripts/run-tsx.mjs scripts/build-clinical-review-queue.ts --input [--output ]", + "", + "Builds a deterministic, deduplicated review queue without changing source governance statuses.", + ].join("\n"); +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + return; + } + if (!args.input) throw new Error("--input is required."); + const input = JSON.parse(await readFile(args.input, "utf8")) as unknown; + const output = `${JSON.stringify(buildClinicalReviewQueue(input), null, 2)}\n`; + if (args.output) { + await writeFile(args.output, output, "utf8"); + console.log(`Wrote clinical review queue to ${args.output}`); + } else { + process.stdout.write(output); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/check-document-label-coverage.ts b/scripts/check-document-label-coverage.ts index d49df1346..01c2b1e50 100644 --- a/scripts/check-document-label-coverage.ts +++ b/scripts/check-document-label-coverage.ts @@ -1,6 +1,7 @@ import * as nextEnv from "@next/env"; import { promises as fs } from "node:fs"; import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; import { reviewDocumentTagQuality } from "@/lib/document-tags"; import type { DocumentLabel } from "@/lib/types"; @@ -20,13 +21,16 @@ type CoverageArgs = { type SupabaseAdmin = Awaited>; -type DocumentRow = { +export type DocumentLabelCoverageDocument = { id: string; title: string; file_name: string; + file_type: string | null; + source_path: string | null; + metadata: Record | null; }; -type LabelRow = { +export type DocumentLabelCoverageLabel = { id: string; document_id: string; label: string; @@ -88,7 +92,7 @@ function usage() { return [ "Usage: npm run check:document-label-coverage -- [options]", "", - "Checks every indexed document has generated site and document_type labels.", + "Checks physical-document labels and registry metadata/smart-v2 contracts.", "", "Options:", " --json Print machine-readable JSON.", @@ -161,7 +165,7 @@ async function fetchAll( return rows; } -function countByLabelType(labels: LabelRow[]) { +function countByLabelType(labels: DocumentLabelCoverageLabel[]) { const counts = new Map(); for (const label of labels) { counts.set(label.label_type, (counts.get(label.label_type) ?? 0) + 1); @@ -170,6 +174,49 @@ function countByLabelType(labels: LabelRow[]) { } const smartV2LabelTypes = new Set(["clinical_action", "care_phase", "document_intent", "content_feature"]); +const registryKinds = new Set(["service", "form", "medication", "differential"]); +const registryIntentByKind = { + service: "operational-process", + form: "documentation-requirement", + medication: "medication-instruction", + differential: "decision-support", +} as const; +const requiredRegistryMetadataKeys = [ + "source_kind", + "registry_record_kind", + "registry_record_id", + "registry_record_slug", + "publisher", + "document_status", + "clinical_validation_status", + "clinical_validation_evidence", + "extraction_quality", +] as const; + +function metadataRecord(value: unknown) { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function nonEmptyMetadataValue(value: unknown) { + if (typeof value === "string") return value.trim().length > 0; + if (value && typeof value === "object" && !Array.isArray(value)) return Object.keys(value).length > 0; + return value !== undefined && value !== null; +} + +function registryKindFor(document: DocumentLabelCoverageDocument) { + const metadata = metadataRecord(document.metadata); + const kind = typeof metadata.registry_record_kind === "string" ? metadata.registry_record_kind : ""; + return registryKinds.has(kind) ? (kind as keyof typeof registryIntentByKind) : null; +} + +function isRegistryDocument(document: DocumentLabelCoverageDocument) { + const metadata = metadataRecord(document.metadata); + return ( + metadata.source_kind === "registry_record" || + document.file_type === "application/vnd.clinical-kb.registry+json" || + document.source_path?.startsWith("registry://") === true + ); +} function countQualityIssues(issues: ReturnType) { const counts = new Map(); @@ -177,27 +224,18 @@ function countQualityIssues(issues: ReturnType) return Object.fromEntries([...counts.entries()].sort()); } -async function main() { - const args = parseArgs(process.argv.slice(2)); - if (args.help) { - console.log(usage()); - return; - } - - const supabase = await loadAdminClient(); - const allowedSiteMissing = await loadAllowlist(args.allowedSiteMissingPath); - const allowedDocumentTypeMissing = await loadAllowlist(args.allowedDocumentTypeMissingPath); - const documents = await fetchAll(supabase, "documents", "id,title,file_name", (query) => - query.eq("status", "indexed"), - ); - const labels = await fetchAll( - supabase, - "document_labels", - "id,document_id,label,label_type,source,confidence", - (query) => query.eq("source", "generated"), - ); - +export function buildDocumentLabelCoverageReport(args: { + documents: DocumentLabelCoverageDocument[]; + labels: DocumentLabelCoverageLabel[]; + allowedSiteMissing?: Set; + allowedDocumentTypeMissing?: Set; +}) { + const { documents, labels } = args; + const allowedSiteMissing = args.allowedSiteMissing ?? new Set(); + const allowedDocumentTypeMissing = args.allowedDocumentTypeMissing ?? new Set(); const documentIds = new Set(documents.map((document) => document.id)); + const physicalDocuments = documents.filter((document) => !isRegistryDocument(document)); + const registryDocuments = documents.filter(isRegistryDocument); const generatedDocumentIds = new Set(labels.map((label) => label.document_id)); const siteDocumentIds = new Set( labels.filter((label) => label.label_type === "site").map((label) => label.document_id), @@ -207,35 +245,68 @@ async function main() { ); const smartV2Labels = labels.filter((label) => smartV2LabelTypes.has(label.label_type)); const smartV2DocumentIds = new Set(smartV2Labels.map((label) => label.document_id)); + const labelsByDocument = new Map(); + for (const label of labels) { + labelsByDocument.set(label.document_id, [...(labelsByDocument.get(label.document_id) ?? []), label]); + } const missingGenerated = [...documentIds].filter((id) => !generatedDocumentIds.has(id)); const missingSmartV2 = [...documentIds].filter((id) => !smartV2DocumentIds.has(id)); const allowedSiteMissingDocs = [...allowedSiteMissing].filter( - (id) => !siteDocumentIds.has(id) && documentIds.has(id), + (id) => !siteDocumentIds.has(id) && physicalDocuments.some((document) => document.id === id), ); const allowedDocumentTypeMissingDocs = [...allowedDocumentTypeMissing].filter( - (id) => !documentTypeDocumentIds.has(id) && documentIds.has(id), - ); - const missingSite = [...documentIds].filter((id) => !siteDocumentIds.has(id) && !allowedSiteMissing.has(id)); - const missingDocumentType = [...documentIds].filter( - (id) => !documentTypeDocumentIds.has(id) && !allowedDocumentTypeMissing.has(id), + (id) => !documentTypeDocumentIds.has(id) && physicalDocuments.some((document) => document.id === id), ); - const labelsByDocument = new Map(); - for (const label of labels) { - labelsByDocument.set(label.document_id, [...(labelsByDocument.get(label.document_id) ?? []), label]); - } + const missingSite = physicalDocuments + .map((document) => document.id) + .filter((id) => !siteDocumentIds.has(id) && !allowedSiteMissing.has(id)); + const missingDocumentType = physicalDocuments + .map((document) => document.id) + .filter((id) => !documentTypeDocumentIds.has(id) && !allowedDocumentTypeMissing.has(id)); + + const registryContractGaps = registryDocuments.flatMap((document) => { + const metadata = metadataRecord(document.metadata); + const missingKeys: string[] = requiredRegistryMetadataKeys.filter((key) => { + if (key === "source_kind") return metadata.source_kind !== "registry_record"; + if (key === "registry_record_kind") return registryKindFor(document) === null; + return !nonEmptyMetadataValue(metadata[key]); + }); + const kind = registryKindFor(document); + const documentLabels = labelsByDocument.get(document.id) ?? []; + const expectedIntent = kind ? registryIntentByKind[kind] : null; + const hasExpectedIntent = + expectedIntent !== null && + documentLabels.some( + (label) => + label.source === "generated" && label.label_type === "document_intent" && label.label === expectedIntent, + ); + const hasGeneratedSite = documentLabels.some( + (label) => label.source === "generated" && label.label_type === "site", + ); + if (!hasExpectedIntent) missingKeys.push("smart_v2_document_intent"); + if (hasGeneratedSite) missingKeys.push("generated_site_label"); + return missingKeys.length > 0 + ? [{ id: document.id, title: document.title, file_name: document.file_name, missing_keys: [...missingKeys] }] + : []; + }); + const qualityIssues = reviewDocumentTagQuality( documents.map((document) => ({ ...document, labels: labelsByDocument.get(document.id) ?? [], })), - { overusedThreshold: Math.max(100, Math.ceil(documents.length * 0.35)) }, ); + const passed = + missingGenerated.length === 0 && + missingSite.length === 0 && + missingDocumentType.length === 0 && + registryContractGaps.length === 0; - const passed = missingGenerated.length === 0 && missingSite.length === 0 && missingDocumentType.length === 0; - - const report = { + return { indexed_documents: documents.length, + physical_documents: physicalDocuments.length, + registry_documents: registryDocuments.length, generated_label_rows: labels.length, generated_documents: generatedDocumentIds.size, indexed_without_generated: missingGenerated.length, @@ -258,6 +329,18 @@ async function main() { examples: issue.examples, document_titles: issue.documentTitles, })), + physical_contract: { + indexed_documents: physicalDocuments.length, + without_site: missingSite.length, + without_document_type: missingDocumentType.length, + passed: missingSite.length === 0 && missingDocumentType.length === 0, + }, + registry_contract: { + indexed_documents: registryDocuments.length, + documents_with_gaps: registryContractGaps.length, + sample_gaps: registryContractGaps.slice(0, 25), + passed: registryContractGaps.length === 0, + }, sample_missing_generated: missingGenerated.slice(0, 10), sample_missing_site: missingSite.slice(0, 10), sample_missing_document_type: missingDocumentType.slice(0, 10), @@ -268,12 +351,45 @@ async function main() { allowed_document_type_missing_docs: allowedDocumentTypeMissingDocs, passed, }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + return; + } + + const supabase = await loadAdminClient(); + const allowedSiteMissing = await loadAllowlist(args.allowedSiteMissingPath); + const allowedDocumentTypeMissing = await loadAllowlist(args.allowedDocumentTypeMissingPath); + const documents = await fetchAll( + supabase, + "documents", + "id,title,file_name,file_type,source_path,metadata", + (query) => query.eq("status", "indexed"), + ); + const labels = await fetchAll( + supabase, + "document_labels", + "id,document_id,label,label_type,source,confidence", + (query) => query.eq("source", "generated"), + ); + + const report = buildDocumentLabelCoverageReport({ + documents, + labels, + allowedSiteMissing, + allowedDocumentTypeMissing, + }); if (args.json) { console.log(JSON.stringify(report, null, 2)); } else { console.log("[Document Label Coverage]"); console.log(`Indexed documents: ${report.indexed_documents}`); + console.log(`Physical documents: ${report.physical_documents}`); + console.log(`Registry documents: ${report.registry_documents}`); console.log(`Generated label rows: ${report.generated_label_rows}`); console.log(`Documents with generated labels: ${report.generated_documents}`); console.log(`Indexed without generated labels: ${report.indexed_without_generated}`); @@ -298,21 +414,28 @@ async function main() { .map(([type, count]) => `${type}=${count}`) .join(", ")}`, ); - if (allowedSiteMissingDocs.length) { - console.log(`Allowed indexed docs without site labels (from allowlist): ${allowedSiteMissingDocs.length}`); + console.log(`Registry contract gaps: ${report.registry_contract.documents_with_gaps}`); + if (report.allowed_site_missing_docs.length) { + console.log( + `Allowed indexed docs without site labels (from allowlist): ${report.allowed_site_missing_docs.length}`, + ); } - if (allowedDocumentTypeMissingDocs.length) { + if (report.allowed_document_type_missing_docs.length) { console.log( - `Allowed indexed docs without document_type labels (from allowlist): ${allowedDocumentTypeMissingDocs.length}`, + `Allowed indexed docs without document_type labels (from allowlist): ${report.allowed_document_type_missing_docs.length}`, ); } - console.log(passed ? "PASS: generated label coverage is complete." : "FAIL: generated label coverage has gaps."); + console.log( + report.passed ? "PASS: generated label coverage is complete." : "FAIL: generated label coverage has gaps.", + ); } - if (!passed) process.exitCode = 1; + if (!report.passed) process.exitCode = 1; } -main().catch((error) => { - console.error(error instanceof Error ? error.message : error); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/classify-documents.ts b/scripts/classify-documents.ts index fe9d1bde4..f1cd878ac 100644 --- a/scripts/classify-documents.ts +++ b/scripts/classify-documents.ts @@ -1,4 +1,5 @@ import * as nextEnv from "@next/env"; +import { pathToFileURL } from "node:url"; import { documentLabelTier, normalizeDocumentLabelForStorage } from "@/lib/document-tags"; const loadEnvConfig = @@ -154,6 +155,18 @@ function metadataRecord(value: unknown) { return value && typeof value === "object" && !Array.isArray(value) ? { ...(value as Record) } : {}; } +/** Registry projections have deterministic labels owned by the registry corpus + * producer. Sending them through the generic classifier can replace those + * labels with text-derived fallbacks and break the registry contract. */ +export function isRegistryProjectionDocument(document: Pick) { + const metadata = metadataRecord(document.metadata); + return ( + metadata.source_kind === "registry_record" || + document.source_path?.startsWith("registry://") === true || + document.file_name.endsWith(".registry.json") + ); +} + type ExistingGeneratedLabelRow = { id: string; document_id: string; @@ -534,9 +547,16 @@ async function main() { } const supabase = await loadAdminClient(); const loadedDocuments = await loadDocuments(supabase, args); + const registryDocuments = loadedDocuments.filter(isRegistryProjectionDocument); + const classifiableDocuments = loadedDocuments.filter((document) => !isRegistryProjectionDocument(document)); + if (registryDocuments.length > 0) { + console.log( + `[classify:documents] Skipped ${registryDocuments.length} registry projection(s); use registry:reconcile-governance to repair deterministic governance metadata and labels without embeddings.`, + ); + } const documents = args.onlyMissingSmartV2 - ? await filterDocumentsMissingSmartV2(supabase, loadedDocuments) - : loadedDocuments; + ? await filterDocumentsMissingSmartV2(supabase, classifiableDocuments) + : classifiableDocuments; const plans = []; for (const document of documents) { @@ -550,7 +570,9 @@ async function main() { console.log(`\nUpdated ${plans.length} document organization profile(s).`); } -main().catch((error) => { - console.error(error instanceof Error ? error.message : error); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 206a62efa..85efd84a3 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -27,10 +27,14 @@ import { type SupabaseEvalCaseClient, } from "@/lib/rag-eval-cases"; import { sourceGovernanceWarnings } from "@/lib/source-governance"; +import { answerRouteBudgetMs } from "@/lib/rag-route-budget"; +import type { AnswerRouteMode } from "@/lib/rag-routing"; import type { RagAnswer } from "@/lib/types"; loadEnvConfig(process.cwd()); +export type EvalQualityProviderMode = "openai" | "offline"; + type EvalQualityArgs = { fixture: string; ownerEmail?: string; @@ -46,6 +50,7 @@ type EvalQualityArgs = { ragOnly: boolean; skipPreflight: boolean; forceEmbedding: boolean; + providerMode: EvalQualityProviderMode; }; export type RagQualityResult = { @@ -76,7 +81,23 @@ export type RagQualityResult = { unverifiedNumericTokenCount: number; hasFaithfulnessWarning: boolean; routingReason?: string; + timings?: { + retrievalMs: number; + routingMs: number; + generationMs: number; + verificationMs: number; + totalMs: number; + routeBudgetMs: number; + routeDeadlineExceeded: boolean; + }; + routeCeilingExceeded?: boolean; estimatedCostUsd: number | null; + openAIRequestIds?: string[]; + openAIUsage?: { + inputTokens: number; + cachedInputTokens: number; + outputTokens: number; + }; }; export type QualityFailureCategory = @@ -129,6 +150,28 @@ export function deliveredGroundedAfterSourceGovernancePolicy( const crossRegionRunnerLatencyContext = process.env.EVAL_LATENCY_CONTEXT === "cross-region-runner"; +export function ragAnswerTimingDiagnostics(answer: Pick) { + const latencyTimings = answer.latencyTimings; + const answerRoute = answer.routingMode ?? "unsupported"; + const defaultRouteBudgetMs = answerRouteBudgetMs[answerRoute as AnswerRouteMode] ?? 0; + const routeBudgetMs = latencyTimings?.route_budget_ms ?? defaultRouteBudgetMs; + const totalMs = latencyTimings?.total_latency_ms ?? 0; + const generationMs = latencyTimings?.generation_latency_ms ?? 0; + const routeDeadlineExceeded = latencyTimings?.route_deadline_exceeded ?? false; + return { + timings: { + retrievalMs: latencyTimings?.retrieval_latency_ms ?? latencyTimings?.search_latency_ms ?? 0, + routingMs: latencyTimings?.routing_latency_ms ?? 0, + generationMs, + verificationMs: latencyTimings?.verification_latency_ms ?? 0, + totalMs, + routeBudgetMs, + routeDeadlineExceeded, + }, + routeCeilingExceeded: routeDeadlineExceeded || (routeBudgetMs === 0 ? generationMs > 0 : totalMs > routeBudgetMs), + }; +} + const qualityThresholds = { retrievalTopKHitRate: 0.8, retrievalDocumentRecallAt5: 0.8, @@ -181,6 +224,7 @@ function parseArgs(argv: string[]): EvalQualityArgs { ragOnly: false, skipPreflight: false, forceEmbedding: false, + providerMode: "openai", }; for (let index = 0; index < argv.length; index += 1) { @@ -224,6 +268,12 @@ function parseArgs(argv: string[]): EvalQualityArgs { if (token === "--question") args.question = value; if (token === "--output-dir") args.outputDir = value; if (token === "--source-metadata-debt") args.sourceMetadataDebt = value; + if (token === "--provider-mode") { + if (value !== "openai" && value !== "offline") { + throw new Error("--provider-mode must be openai or offline."); + } + args.providerMode = value; + } } if (args.retrievalOnly && args.ragOnly) throw new Error("Use only one of --retrieval-only or --rag-only."); @@ -234,6 +284,14 @@ function parseArgs(argv: string[]): EvalQualityArgs { return args; } +export function configureEvalProviderEnvironment(providerMode: EvalQualityProviderMode) { + if (providerMode !== "offline") return; + process.env.RAG_PROVIDER_MODE = "offline"; + delete process.env.OPENAI_API_KEY; + delete process.env.OPENAI_ORG_ID; + delete process.env.OPENAI_PROJECT_ID; +} + export function qualityFailureCategory(message: string): QualityFailureCategory { const normalized = message.toLowerCase(); if (normalized.includes("query class")) return "query_class"; @@ -426,7 +484,7 @@ function topResultGovernanceCounts(results: GoldenRetrievalResult[]) { }; } -function summarizeRagQualityResults(results: RagQualityResult[]) { +function summarizeRagQualityResults(results: RagQualityResult[], providerMode: EvalQualityProviderMode) { const supported = results.filter((result) => result.supported); const unsupported = results.filter((result) => !result.supported); // A supported case counts as grounded-supported when it grounds, OR — for @@ -435,9 +493,12 @@ function summarizeRagQualityResults(results: RagQualityResult[]) { // Requiring expectedHit keeps the guard honest: a real retrieval regression that // stops surfacing the expected docs is NOT accepted and still drags the rate below // threshold, hard-failing the canary. - const groundedSupported = supported.filter( - (result) => result.grounded || (result.acceptSourceOnly && result.expectedHit && result.citations > 0), - ).length; + const groundedSupported = supported.filter((result) => { + if (providerMode === "offline") { + return result.model === null && result.expectedHit && result.citations > 0; + } + return result.grounded || (result.acceptSourceOnly && result.expectedHit && result.citations > 0); + }).length; const unsupportedCorrect = unsupported.filter((result) => !result.grounded).length; const citationFailures = results.filter((result) => result.failures.some((failure) => qualityFailureCategory(failure) === "citation"), @@ -455,6 +516,7 @@ function summarizeRagQualityResults(results: RagQualityResult[]) { const expectedDangerWarningMissing = results.filter((result) => result.failures.includes("expected danger source governance warning missing"), ).length; + const routeCeilingFailures = results.filter((result) => result.routeCeilingExceeded).length; const latencies = results.map((result) => result.latencyMs); const routeLatencyP95 = Object.fromEntries( Array.from( @@ -483,6 +545,7 @@ function summarizeRagQualityResults(results: RagQualityResult[]) { source_governance_warning_rate: rate(sourceGovernanceWarnings.length, results.length), source_governance_danger_failure_rate: rate(sourceGovernanceDangerFailures.length, results.length), expected_danger_warning_missing_count: expectedDangerWarningMissing, + route_ceiling_failure_count: routeCeilingFailures, median_latency_ms: percentile(latencies, 50), p95_latency_ms: percentile(latencies, 95), route_p95_latency_ms: routeLatencyP95, @@ -497,11 +560,32 @@ export function buildEvalQualityReport(args: { retrievalResults: GoldenRetrievalResult[]; ragResults: RagQualityResult[]; sourceMetadataDebtAcceptance?: SourceMetadataDebtAcceptance; + providerMode?: EvalQualityProviderMode; }) { + const providerMode = args.providerMode ?? "openai"; const retrievalSummary = summarizeGoldenRetrievalResults(args.retrievalResults); - const ragSummary = summarizeRagQualityResults(args.ragResults); + const ragSummary = summarizeRagQualityResults(args.ragResults, providerMode); const governance = topResultGovernanceCounts(args.retrievalResults); const thresholdFailures: string[] = []; + const providerEvidence = { + mode: providerMode, + model_case_count: args.ragResults.filter((result) => Boolean(result.model)).length, + openai_request_id_count: args.ragResults.reduce((sum, result) => sum + (result.openAIRequestIds?.length ?? 0), 0), + token_usage_case_count: args.ragResults.filter((result) => { + const usage = result.openAIUsage; + return Boolean(usage && usage.inputTokens + usage.cachedInputTokens + usage.outputTokens > 0); + }).length, + nonzero_cost_case_count: args.ragResults.filter((result) => (result.estimatedCostUsd ?? 0) > 0).length, + generation_latency_case_count: args.ragResults.filter( + (result) => (result.generationLatencyMs ?? result.timings?.generationMs ?? 0) > 0, + ).length, + }; + if (providerMode === "offline") { + for (const [key, count] of Object.entries(providerEvidence)) { + if (key === "mode" || count === 0) continue; + thresholdFailures.push(`offline provider invariant ${key} ${count} above 0`); + } + } if (args.retrievalResults.length > 0) { if (retrievalSummary.top_k_hit_rate < qualityThresholds.retrievalTopKHitRate) { @@ -569,6 +653,9 @@ export function buildEvalQualityReport(args: { `RAG expected_danger_warning_missing_count ${ragSummary.expected_danger_warning_missing_count} above 0`, ); } + if (ragSummary.route_ceiling_failure_count > 0) { + thresholdFailures.push(`RAG route_ceiling_failure_count ${ragSummary.route_ceiling_failure_count} above 0`); + } if (ragSummary.p95_latency_ms > qualityThresholds.ragP95LatencyMs) { thresholdFailures.push( `RAG p95_latency_ms ${ragSummary.p95_latency_ms} above ${qualityThresholds.ragP95LatencyMs}`, @@ -593,6 +680,12 @@ export function buildEvalQualityReport(args: { return { generated_at: args.generatedAt ?? new Date().toISOString(), + provider: { + ...providerEvidence, + passed: + providerMode !== "offline" || + Object.entries(providerEvidence).every(([key, value]) => key === "mode" || value === 0), + }, thresholds: qualityThresholds, retrieval: { summary: retrievalSummary, @@ -634,11 +727,17 @@ function ragCaseDiagnosticsTable(results: RagQualityResult[]) { result.id, result.route, result.latencyRoute, + result.routingReason, result.latencyMs, + result.timings?.retrievalMs, + result.timings?.routingMs, result.searchLatencyMs, result.generationLatencyMs, + result.timings?.verificationMs, result.rpcLatencyMs, result.embeddingLatencyMs, + result.timings?.routeBudgetMs, + result.timings?.routeDeadlineExceeded ? "yes" : "no", result.model, result.failures.length > 0 ? `failed (${result.failures.length})` : "passed", ] @@ -646,8 +745,8 @@ function ragCaseDiagnosticsTable(results: RagQualityResult[]) { .join(" | "), ); return [ - "| Case | Route | Latency SLO | Total ms | Search ms | Generation ms | RPC ms | Embedding ms | Model | Result |", - "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- | --- |", + "| Case | Route | Latency SLO | Reason | Total ms | Retrieval ms | Routing ms | Search ms | Generation ms | Verification ms | RPC ms | Embedding ms | Budget ms | Deadline | Model | Result |", + "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- |", ...rows.map((row) => `| ${row} |`), ].join("\n"); } @@ -727,14 +826,35 @@ export function renderEvalQualityMarkdown(report: EvalQualityReport) { item.topFiles.join(" | ") || "none" }\n route=${item.route} grounded=${item.grounded} citations=${item.citations} numericWarnings=${ item.unverifiedNumericTokenCount - } faithfulnessWarning=${item.hasFaithfulnessWarning ? "yes" : "no"} sourceWarnings=${item.sourceWarningCount}`, + } faithfulnessWarning=${item.hasFaithfulnessWarning ? "yes" : "no"} sourceWarnings=${ + item.sourceWarningCount + }\n reason=${item.routingReason ?? "none"}\n timings retrieval=${ + item.timings?.retrievalMs ?? "n/a" + }ms routing=${item.timings?.routingMs ?? "n/a"}ms generation=${ + item.timings?.generationMs ?? "n/a" + }ms verification=${item.timings?.verificationMs ?? "n/a"}ms total=${ + item.timings?.totalMs ?? item.latencyMs + }ms budget=${ + item.timings?.routeBudgetMs ?? "n/a" + }ms deadline=${item.timings?.routeDeadlineExceeded ? "yes" : "no"}`, ) .join("\n"); - return `# Retrieval Quality Report Generated: ${report.generated_at} +## Provider Profile + +${markdownTable([ + ["Mode", report.provider.mode], + ["Model cases", report.provider.model_case_count], + ["OpenAI request IDs", report.provider.openai_request_id_count], + ["Token-usage cases", report.provider.token_usage_case_count], + ["Nonzero-cost cases", report.provider.nonzero_cost_case_count], + ["Generation-latency cases", report.provider.generation_latency_case_count], + ["Provider-free invariants", report.provider.passed ? "passed" : "failed"], +])} + ## Threshold Status Blocking failures: @@ -813,6 +933,7 @@ ${markdownTable([ ["Numeric grounding failure rate", rag.numeric_grounding_failure_rate], ["Source governance warning rate", rag.source_governance_warning_rate], ["Source governance danger failure rate", rag.source_governance_danger_failure_rate], + ["Route ceiling failures", rag.route_ceiling_failure_count], ["P95 latency ms", rag.p95_latency_ms], ["Estimated cost USD", rag.estimated_cost_usd], ])} @@ -898,6 +1019,7 @@ async function runRagQualityCases(args: { ownerId?: string; limit?: number; question?: string; + providerMode: EvalQualityProviderMode; supabase: Awaited>; }) { const [{ answerQuestionWithScope }, capturedCases] = await Promise.all([ @@ -934,6 +1056,20 @@ async function runRagQualityCases(args: { expectsDangerWarning: testCase.expectsSourceDangerWarning, }), ); + const { timings, routeCeilingExceeded } = ragAnswerTimingDiagnostics(answer); + if (routeCeilingExceeded) { + failures.push( + `route latency ceiling exceeded: ${timings.totalMs}ms total, ${timings.generationMs}ms generation, ${timings.routeBudgetMs}ms budget`, + ); + } + const openAIUsage = { + inputTokens: answer.openAIUsage?.input_tokens ?? 0, + cachedInputTokens: answer.openAIUsage?.cached_input_tokens ?? 0, + outputTokens: answer.openAIUsage?.output_tokens ?? 0, + }; + const hasOpenAIUsage = openAIUsage.inputTokens + openAIUsage.cachedInputTokens + openAIUsage.outputTokens > 0; + const openAIRequestIds = answer.openAIRequestIds?.filter(Boolean) ?? []; + const generationLatencyMs = answer.latencyTimings?.generation_latency_ms ?? 0; results.push({ id: testCase.id, @@ -949,7 +1085,7 @@ async function runRagQualityCases(args: { acceptSourceOnly: testCase.acceptSourceOnly, latencyMs: answer.latencyTimings?.total_latency_ms ?? 0, searchLatencyMs: answer.latencyTimings?.search_latency_ms, - generationLatencyMs: answer.latencyTimings?.generation_latency_ms, + generationLatencyMs: generationLatencyMs > 0 ? generationLatencyMs : undefined, rpcLatencyMs: answer.latencyTimings?.supabase_rpc_latency_ms, embeddingLatencyMs: answer.latencyTimings?.embedding_latency_ms, route: answer.routingMode ?? "none", @@ -963,11 +1099,11 @@ async function runRagQualityCases(args: { unverifiedNumericTokenCount: answer.unverifiedNumericTokens?.length ?? 0, hasFaithfulnessWarning: Boolean(answer.faithfulnessWarning), routingReason: answer.routingReason, - estimatedCostUsd: estimateCostUsd({ - inputTokens: answer.openAIUsage?.input_tokens ?? 0, - cachedInputTokens: answer.openAIUsage?.cached_input_tokens ?? 0, - outputTokens: answer.openAIUsage?.output_tokens ?? 0, - }), + timings, + routeCeilingExceeded, + estimatedCostUsd: hasOpenAIUsage ? estimateCostUsd(openAIUsage) : null, + openAIRequestIds: openAIRequestIds.length > 0 ? openAIRequestIds : undefined, + openAIUsage: hasOpenAIUsage ? openAIUsage : undefined, }); } @@ -1039,13 +1175,14 @@ async function loadSourceMetadataDebtAcceptance(path: string): Promise { - result.failures.push(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + result.failures.push(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/reconcile-registry-governance.ts b/scripts/reconcile-registry-governance.ts new file mode 100644 index 000000000..b506b764f --- /dev/null +++ b/scripts/reconcile-registry-governance.ts @@ -0,0 +1,472 @@ +import { pathToFileURL } from "node:url"; +import { loadEnvConfig } from "@next/env"; + +import type { DifferentialRecordRow } from "@/lib/differential-records"; +import type { MedicationRecordRow } from "@/lib/medication-records"; +import type { RegistryRecordRow } from "@/lib/registry-records"; +import type { Json, TablesInsert } from "@/lib/supabase/database.types"; +import type { RegistryCorpusKind, RegistryGovernanceProjection } from "@/lib/registry-corpus"; + +loadEnvConfig(process.cwd()); + +const PRODUCTION_PROJECT_REF = "sjrfecxgysukkwxsowpy"; +const READ_BATCH_SIZE = 100; +const WRITE_CONCURRENCY = 8; + +export const expectedRegistryProjectionCounts = { + service: 222, + form: 4, + medication: 328, + differential: 232, +} as const satisfies Record; + +type Args = { + ownerId?: string; + expectedProjectRef?: string; + write: boolean; + confirmed: boolean; + json: boolean; +}; + +export type RegistryGovernanceDocument = { + id: string; + owner_id: string; + metadata: Json | null; +}; + +export type RegistryGovernanceLabel = { + id: string; + document_id: string; + owner_id: string; + label: string; + label_type: string; + source: string; + confidence: number | null; + metadata: Json | null; +}; + +type RegistryGovernanceDocumentUpdate = { + id: string; + ownerId: string; + metadata: Record; +}; + +export type RegistryGovernancePlan = { + ownerId: string; + projections: RegistryGovernanceProjection[]; + documentUpdates: RegistryGovernanceDocumentUpdate[]; + labelsToInsert: TablesInsert<"document_labels">[]; + labelsToUpdate: TablesInsert<"document_labels">[]; + labelIdsToDelete: string[]; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { write: false, confirmed: false, json: false }; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--write") { + args.write = true; + continue; + } + if (token === "--confirm") { + args.confirmed = true; + continue; + } + if (token === "--json") { + args.json = true; + continue; + } + + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); + if (token === "--owner-id") args.ownerId = value; + else if (token === "--expected-project-ref") args.expectedProjectRef = value; + else throw new Error(`Unknown argument: ${token}`); + index += 1; + } + + if (args.write && !args.confirmed) { + throw new Error("Refusing to write without both --write and --confirm."); + } + if (args.write && args.expectedProjectRef !== PRODUCTION_PROJECT_REF) { + throw new Error( + `Production writes require --expected-project-ref ${PRODUCTION_PROJECT_REF}; no other project is authorized.`, + ); + } + return args; +} + +function metadataRecord(value: Json | null): Record { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function stableJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableJson((value as Record)[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "null"; +} + +function duplicates(values: string[]) { + const seen = new Set(); + const duplicateValues = new Set(); + for (const value of values) { + if (seen.has(value)) duplicateValues.add(value); + seen.add(value); + } + return [...duplicateValues].sort(); +} + +function expectedLabelCurrent(existing: RegistryGovernanceLabel, expected: TablesInsert<"document_labels">) { + return ( + existing.owner_id === expected.owner_id && + existing.confidence === expected.confidence && + stableJson(metadataRecord(existing.metadata)) === stableJson(expected.metadata ?? {}) + ); +} + +export function buildRegistryGovernancePlan(args: { + projections: RegistryGovernanceProjection[]; + documents: RegistryGovernanceDocument[]; + labels: RegistryGovernanceLabel[]; + expectedOwnerId?: string; + expectedCounts?: Record; +}): RegistryGovernancePlan { + const expectedCounts = args.expectedCounts ?? expectedRegistryProjectionCounts; + const observedCounts = { service: 0, form: 0, medication: 0, differential: 0 } satisfies Record< + RegistryCorpusKind, + number + >; + for (const projection of args.projections) observedCounts[projection.kind] += 1; + for (const kind of Object.keys(expectedCounts) as RegistryCorpusKind[]) { + if (observedCounts[kind] !== expectedCounts[kind]) { + throw new Error( + `Registry projection count mismatch for ${kind}: expected ${expectedCounts[kind]}, found ${observedCounts[kind]}.`, + ); + } + } + + const ownerIds = new Set(args.projections.map((projection) => projection.ownerId)); + if (ownerIds.size !== 1) { + throw new Error(`Registry projections must resolve to one owner; found ${ownerIds.size}.`); + } + const ownerId = [...ownerIds][0]; + if (!ownerId) throw new Error("Registry projections did not resolve an owner."); + if (args.expectedOwnerId && ownerId !== args.expectedOwnerId) { + throw new Error(`Registry owner mismatch: expected ${args.expectedOwnerId}, found ${ownerId}.`); + } + + const duplicateSourceIdentities = duplicates( + args.projections.map((projection) => `${projection.kind}:${projection.recordId}`), + ); + const duplicateProjectionIds = duplicates(args.projections.map((projection) => projection.documentId)); + const duplicateDocumentIds = duplicates(args.documents.map((document) => document.id)); + const duplicateGeneratedLabelKeys = duplicates( + args.labels + .filter((label) => label.source === "generated") + .map((label) => `${label.document_id}:${label.label_type}:${label.label}:${label.source}`), + ); + const duplicateProblems = [ + ...duplicateSourceIdentities.map((value) => `source ${value}`), + ...duplicateProjectionIds.map((value) => `projection ${value}`), + ...duplicateDocumentIds.map((value) => `document ${value}`), + ...duplicateGeneratedLabelKeys.map((value) => `generated label ${value}`), + ]; + if (duplicateProblems.length > 0) { + throw new Error(`Duplicate registry governance identities: ${duplicateProblems.slice(0, 10).join(", ")}.`); + } + + const documentById = new Map(args.documents.map((document) => [document.id, document])); + const missingDocumentIds = args.projections + .map((projection) => projection.documentId) + .filter((documentId) => !documentById.has(documentId)); + if (missingDocumentIds.length > 0) { + throw new Error( + `Missing ${missingDocumentIds.length} deterministic registry document projection(s): ${missingDocumentIds.slice(0, 10).join(", ")}.`, + ); + } + + const ownerMismatches = args.projections.filter( + (projection) => documentById.get(projection.documentId)?.owner_id !== projection.ownerId, + ); + if (ownerMismatches.length > 0) { + throw new Error( + `Registry document owner mismatch for ${ownerMismatches.length} projection(s): ${ownerMismatches + .slice(0, 10) + .map((projection) => projection.documentId) + .join(", ")}.`, + ); + } + + const labelsByDocument = new Map(); + for (const label of args.labels) { + labelsByDocument.set(label.document_id, [...(labelsByDocument.get(label.document_id) ?? []), label]); + } + + const documentUpdates: RegistryGovernanceDocumentUpdate[] = []; + const labelsToInsert: TablesInsert<"document_labels">[] = []; + const labelsToUpdate: TablesInsert<"document_labels">[] = []; + const labelIdsToDelete: string[] = []; + + for (const projection of args.projections) { + const document = documentById.get(projection.documentId)!; + const mergedMetadata = { ...metadataRecord(document.metadata), ...projection.requiredMetadata }; + if (stableJson(mergedMetadata) !== stableJson(metadataRecord(document.metadata))) { + documentUpdates.push({ id: document.id, ownerId: projection.ownerId, metadata: mergedMetadata }); + } + + const documentLabels = labelsByDocument.get(projection.documentId) ?? []; + for (const label of documentLabels) { + if (label.source !== "generated") continue; + if (label.label_type === "site") labelIdsToDelete.push(label.id); + if (label.label_type === "document_intent" && label.label !== projection.intentLabel.label) { + labelIdsToDelete.push(label.id); + } + } + + const expectedLabel = documentLabels.find( + (label) => + label.source === "generated" && + label.label_type === "document_intent" && + label.label === projection.intentLabel.label, + ); + if (!expectedLabel) labelsToInsert.push(projection.intentLabel); + else if (!expectedLabelCurrent(expectedLabel, projection.intentLabel)) labelsToUpdate.push(projection.intentLabel); + } + + return { + ownerId, + projections: args.projections, + documentUpdates, + labelsToInsert, + labelsToUpdate, + labelIdsToDelete: [...new Set(labelIdsToDelete)].sort(), + }; +} + +export function assertExpectedRegistryProjectRef(args: { + expectedProjectRef: string; + configuredProjectRef?: string; + supabaseUrl?: string; +}) { + const configuredProjectRef = args.configuredProjectRef?.trim(); + if (configuredProjectRef !== args.expectedProjectRef) { + throw new Error( + `Supabase project ref mismatch: expected ${args.expectedProjectRef}, configured ${configuredProjectRef || "not set"}.`, + ); + } + let urlProjectRef = ""; + try { + urlProjectRef = args.supabaseUrl ? (new URL(args.supabaseUrl).hostname.split(".")[0] ?? "") : ""; + } catch { + throw new Error("NEXT_PUBLIC_SUPABASE_URL is not a valid URL."); + } + if (urlProjectRef !== args.expectedProjectRef) { + throw new Error( + `Supabase URL project mismatch: expected ${args.expectedProjectRef}, observed ${urlProjectRef || "not set"}.`, + ); + } +} + +function chunks(values: T[], size: number) { + const output: T[][] = []; + for (let index = 0; index < values.length; index += size) output.push(values.slice(index, index + size)); + return output; +} + +async function loadAdminClient() { + const { createAdminClient } = await import("@/lib/supabase/admin"); + return createAdminClient(); +} + +type AdminClient = Awaited>; + +async function loadAllRows( + loadPage: (from: number, to: number) => PromiseLike<{ data: Row[] | null; error: { message: string } | null }>, +) { + const rows: Row[] = []; + const pageSize = 1000; + while (true) { + const { data, error } = await loadPage(rows.length, rows.length + pageSize - 1); + if (error) throw new Error(error.message); + const page = data ?? []; + rows.push(...page); + if (page.length < pageSize) return rows; + } +} + +type OwnerCounts = Record; + +async function resolveProductionOwner(supabase: AdminClient, requestedOwnerId?: string) { + const [registryRows, medicationRows, differentialRows] = await Promise.all([ + loadAllRows<{ owner_id: string; kind: string }>((from, to) => + supabase.from("clinical_registry_records").select("owner_id,kind").range(from, to), + ), + loadAllRows<{ owner_id: string }>((from, to) => + supabase.from("medication_records").select("owner_id").range(from, to), + ), + loadAllRows<{ owner_id: string }>((from, to) => + supabase.from("differential_records").select("owner_id").range(from, to), + ), + ]); + const counts = new Map(); + const ownerCounts = (ownerId: string) => { + const current = counts.get(ownerId) ?? { service: 0, form: 0, medication: 0, differential: 0 }; + counts.set(ownerId, current); + return current; + }; + for (const row of registryRows) ownerCounts(row.owner_id)[row.kind === "form" ? "form" : "service"] += 1; + for (const row of medicationRows) ownerCounts(row.owner_id).medication += 1; + for (const row of differentialRows) ownerCounts(row.owner_id).differential += 1; + + const candidates = [...counts.entries()].filter(([, count]) => + (Object.keys(expectedRegistryProjectionCounts) as RegistryCorpusKind[]).every( + (kind) => count[kind] === expectedRegistryProjectionCounts[kind], + ), + ); + if (candidates.length !== 1) { + throw new Error(`Expected exactly one owner with the production registry profile; found ${candidates.length}.`); + } + const ownerId = candidates[0]![0]; + if (requestedOwnerId && ownerId !== requestedOwnerId) { + throw new Error(`Resolved production owner ${ownerId} does not match --owner-id ${requestedOwnerId}.`); + } + return ownerId; +} + +async function loadProjections(supabase: AdminClient, ownerId: string) { + const [registryResult, medicationResult, differentialResult] = await Promise.all([ + supabase.from("clinical_registry_records").select("*").eq("owner_id", ownerId).order("kind").order("title"), + supabase.from("medication_records").select("*").eq("owner_id", ownerId).order("name"), + supabase.from("differential_records").select("*").eq("owner_id", ownerId).order("kind").order("title"), + ]); + if (registryResult.error) throw new Error(`Could not load registry records: ${registryResult.error.message}`); + if (medicationResult.error) throw new Error(`Could not load medication records: ${medicationResult.error.message}`); + if (differentialResult.error) + throw new Error(`Could not load differential records: ${differentialResult.error.message}`); + + const { + clinicalRegistryRowsToCorpusEntries, + differentialRowsToCorpusEntries, + medicationRowsToCorpusEntries, + registryGovernanceProjection, + } = await import("@/lib/registry-corpus"); + return [ + ...clinicalRegistryRowsToCorpusEntries((registryResult.data ?? []) as RegistryRecordRow[]), + ...medicationRowsToCorpusEntries((medicationResult.data ?? []) as MedicationRecordRow[]), + ...differentialRowsToCorpusEntries((differentialResult.data ?? []) as DifferentialRecordRow[]), + ].map(registryGovernanceProjection); +} + +async function loadExistingGovernance(supabase: AdminClient, documentIds: string[]) { + const documents: RegistryGovernanceDocument[] = []; + const labels: RegistryGovernanceLabel[] = []; + for (const documentIdBatch of chunks(documentIds, READ_BATCH_SIZE)) { + const [documentResult, labelResult] = await Promise.all([ + supabase.from("documents").select("id,owner_id,metadata").in("id", documentIdBatch), + supabase + .from("document_labels") + .select("id,document_id,owner_id,label,label_type,source,confidence,metadata") + .in("document_id", documentIdBatch), + ]); + if (documentResult.error) throw new Error(`Registry document preflight failed: ${documentResult.error.message}`); + if (labelResult.error) throw new Error(`Registry label preflight failed: ${labelResult.error.message}`); + documents.push(...((documentResult.data ?? []) as RegistryGovernanceDocument[])); + labels.push(...((labelResult.data ?? []) as RegistryGovernanceLabel[])); + } + return { documents, labels }; +} + +async function runWithConcurrency(items: T[], concurrency: number, worker: (item: T) => Promise) { + let nextIndex = 0; + async function runWorker() { + while (nextIndex < items.length) { + const item = items[nextIndex++]; + if (item) await worker(item); + } + } + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => runWorker())); +} + +async function applyPlan(supabase: AdminClient, plan: RegistryGovernancePlan) { + await runWithConcurrency(plan.documentUpdates, WRITE_CONCURRENCY, async (update) => { + const { data, error } = await supabase + .from("documents") + .update({ metadata: update.metadata }) + .eq("id", update.id) + .eq("owner_id", update.ownerId) + .select("id"); + if (error) throw new Error(`Registry metadata update failed for ${update.id}: ${error.message}`); + if (data?.length !== 1) throw new Error(`Registry metadata update did not match exactly one row for ${update.id}.`); + }); + + for (const labelIdBatch of chunks(plan.labelIdsToDelete, READ_BATCH_SIZE)) { + const { error } = await supabase.from("document_labels").delete().in("id", labelIdBatch); + if (error) throw new Error(`Registry generated-label cleanup failed: ${error.message}`); + } + for (const labelBatch of chunks([...plan.labelsToInsert, ...plan.labelsToUpdate], READ_BATCH_SIZE)) { + const { error } = await supabase + .from("document_labels") + .upsert(labelBatch, { onConflict: "document_id,label_type,label,source" }); + if (error) throw new Error(`Registry intent-label upsert failed: ${error.message}`); + } +} + +function reportForPlan(plan: RegistryGovernancePlan, write: boolean) { + return { + mode: write ? "write" : "dry-run", + owner_id: plan.ownerId, + documents_inspected: plan.projections.length, + documents_updated: plan.documentUpdates.length, + labels_inserted: plan.labelsToInsert.length, + labels_updated: plan.labelsToUpdate.length, + labels_deleted: plan.labelIdsToDelete.length, + chunk_rows_touched: 0, + openai_calls: 0, + }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const expectedProjectRef = args.expectedProjectRef ?? PRODUCTION_PROJECT_REF; + assertExpectedRegistryProjectRef({ + expectedProjectRef, + configuredProjectRef: process.env.SUPABASE_PROJECT_REF, + supabaseUrl: process.env.NEXT_PUBLIC_SUPABASE_URL, + }); + const { requireServerEnv } = await import("@/lib/env"); + requireServerEnv(); + const supabase = await loadAdminClient(); + const ownerId = await resolveProductionOwner(supabase, args.ownerId); + const projections = await loadProjections(supabase, ownerId); + const existing = await loadExistingGovernance( + supabase, + projections.map((projection) => projection.documentId), + ); + const plan = buildRegistryGovernancePlan({ ...existing, projections, expectedOwnerId: ownerId }); + const report = reportForPlan(plan, args.write); + + if (args.write) await applyPlan(supabase, plan); + if (args.json) console.log(JSON.stringify(report, null, 2)); + else { + console.log("[registry:reconcile-governance]"); + for (const [key, value] of Object.entries(report)) console.log(` ${key}: ${value}`); + if (!args.write) { + console.log( + `Dry run only. Re-run with --write --confirm --expected-project-ref ${PRODUCTION_PROJECT_REF} to apply this exact scope.`, + ); + } + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(`[registry:reconcile-governance] ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + }); +} diff --git a/scripts/run-eval-safe.mjs b/scripts/run-eval-safe.mjs index 3f3c9f371..69f3c810e 100644 --- a/scripts/run-eval-safe.mjs +++ b/scripts/run-eval-safe.mjs @@ -7,6 +7,9 @@ import { fileURLToPath } from "node:url"; const projectRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const isWindows = process.platform === "win32"; const [targetScript, ...forwardArgs] = process.argv.slice(2); +const offlineProviderRequested = forwardArgs.some( + (token, index) => token === "--provider-mode" && forwardArgs[index + 1] === "offline", +); if (!targetScript) { console.error("Usage: node scripts/run-eval-safe.mjs