diff --git a/.cursor/skills/accessibility-review/SKILL.md b/.cursor/skills/accessibility-review/SKILL.md index 20f697292..9da952268 100644 --- a/.cursor/skills/accessibility-review/SKILL.md +++ b/.cursor/skills/accessibility-review/SKILL.md @@ -7,6 +7,10 @@ description: Reviews keyboard navigation support, semantic HTML tags, drawer/dia Use this skill when reviewing user interfaces to ensure compliance with a11y standards. +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Keyboard & Focus Control diff --git a/.cursor/skills/ai-architecture-review/SKILL.md b/.cursor/skills/ai-architecture-review/SKILL.md index 74df735d9..53ebfe523 100644 --- a/.cursor/skills/ai-architecture-review/SKILL.md +++ b/.cursor/skills/ai-architecture-review/SKILL.md @@ -7,6 +7,10 @@ description: Reviews RAG flows, context assembly, model routing, structured outp Use this skill when reviewing or modifying the RAG (Retrieval-Augmented Generation) pipeline, document retrieval, and LLM integrations. +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Retrieval Quality & Provenance diff --git a/.cursor/skills/api-review/SKILL.md b/.cursor/skills/api-review/SKILL.md index 92c96971d..4ed102675 100644 --- a/.cursor/skills/api-review/SKILL.md +++ b/.cursor/skills/api-review/SKILL.md @@ -7,6 +7,10 @@ description: Reviews API routes for routing contracts, input schema validation ( Use this skill when reviewing or modifying API routes, endpoints, or network boundaries within this repository (primarily under `src/app/api/`). +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Request Validation diff --git a/.cursor/skills/code-quality-review/SKILL.md b/.cursor/skills/code-quality-review/SKILL.md index add222b68..7aafaf4b5 100644 --- a/.cursor/skills/code-quality-review/SKILL.md +++ b/.cursor/skills/code-quality-review/SKILL.md @@ -7,6 +7,10 @@ description: Reviews code quality, naming structures, control flow patterns, abs Use this skill when auditing source files for maintainability, readability, and structural patterns. +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Maintainability & Naming diff --git a/.cursor/skills/design-review/SKILL.md b/.cursor/skills/design-review/SKILL.md index e52718181..a25810bc9 100644 --- a/.cursor/skills/design-review/SKILL.md +++ b/.cursor/skills/design-review/SKILL.md @@ -7,6 +7,10 @@ description: Reviews layout density, theme adherence, typography, spacing, and v Use this skill when auditing UI styling, colors, layout structures, and spacing rules. +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Style & Spacing Consistency diff --git a/.cursor/skills/frontend-architecture-review/SKILL.md b/.cursor/skills/frontend-architecture-review/SKILL.md index 626c9bef2..b03aecad3 100644 --- a/.cursor/skills/frontend-architecture-review/SKILL.md +++ b/.cursor/skills/frontend-architecture-review/SKILL.md @@ -7,6 +7,10 @@ description: Reviews Next/React boundaries, state ownership, duplicated client s Use this skill when reviewing Next.js App Router structures, component boundaries, state management, and React performance. +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Next.js App Router Boundaries diff --git a/.cursor/skills/performance-review/SKILL.md b/.cursor/skills/performance-review/SKILL.md index af66b5fad..e1cf0235b 100644 --- a/.cursor/skills/performance-review/SKILL.md +++ b/.cursor/skills/performance-review/SKILL.md @@ -7,6 +7,10 @@ description: Reviews model latency/cost, cache policies, duplicate query coalesc Use this skill when reviewing query bottlenecks, database RPC latencies, client bundle sizes, or API cost overhead. +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Database & API Latency diff --git a/.cursor/skills/release-readiness-review/SKILL.md b/.cursor/skills/release-readiness-review/SKILL.md index eac0fb971..c990979e6 100644 --- a/.cursor/skills/release-readiness-review/SKILL.md +++ b/.cursor/skills/release-readiness-review/SKILL.md @@ -7,6 +7,10 @@ description: Reviews build validation, linting/type-checking, target database ch Use this skill when preparing a branch for merging, pull request submission, or production deployment. +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Build & Automation Checks diff --git a/.cursor/skills/repo-auditor/SKILL.md b/.cursor/skills/repo-auditor/SKILL.md index 2b48c5bde..e0b77bd68 100644 --- a/.cursor/skills/repo-auditor/SKILL.md +++ b/.cursor/skills/repo-auditor/SKILL.md @@ -7,6 +7,10 @@ description: Reviews module dependencies, broken imports, unused files, and stru Use this skill when auditing the workspace layout, folder structures, imports, and dependencies. +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Structural Audits diff --git a/.cursor/skills/security-review/SKILL.md b/.cursor/skills/security-review/SKILL.md index 663195611..3e8f5249e 100644 --- a/.cursor/skills/security-review/SKILL.md +++ b/.cursor/skills/security-review/SKILL.md @@ -7,6 +7,10 @@ description: Reviews credentials, token handling, JWT claims, service-role confi Use this skill when reviewing code that touches authentication, database schemas, environment variables, storage, or external API gateways. +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Key Confinement & Environment Safety diff --git a/.cursor/skills/supabase-postgres-best-practices/SKILL.md b/.cursor/skills/supabase-postgres-best-practices/SKILL.md index 07aa1b3a6..f9261d20c 100644 --- a/.cursor/skills/supabase-postgres-best-practices/SKILL.md +++ b/.cursor/skills/supabase-postgres-best-practices/SKILL.md @@ -25,6 +25,8 @@ Reference these guidelines when: - Optimizing for Postgres-specific features - Working with Row-Level Security (RLS) +For review-only use, follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` first. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Rule Categories by Priority | Priority | Category | Impact | Prefix | diff --git a/.cursor/skills/testing-review/SKILL.md b/.cursor/skills/testing-review/SKILL.md index 23d9af423..b2b06e014 100644 --- a/.cursor/skills/testing-review/SKILL.md +++ b/.cursor/skills/testing-review/SKILL.md @@ -7,6 +7,10 @@ description: Reviews unit/integration/E2E coverage, checks for fragile/flaky tes Use this skill when reviewing test coverage, testing harnesses, and test execution paths (Vitest + Playwright). +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Test Harness Coverage diff --git a/.cursor/skills/ux-review/SKILL.md b/.cursor/skills/ux-review/SKILL.md index d4de4dae9..564c32cb9 100644 --- a/.cursor/skills/ux-review/SKILL.md +++ b/.cursor/skills/ux-review/SKILL.md @@ -7,6 +7,10 @@ description: Reviews the question-to-answer flow, source-review friction, eviden Use this skill when reviewing user interface flows, search input behaviors, sidebars, modal navigation, and responsiveness. +## Repository Review Protocol + +Follow `AGENTS.md` review throttling and `docs/codex-review-protocol.md` before starting. Do not review opportunistically, do not mutate files during pure review, and update `docs/branch-review-ledger.md` after completed branch/PR reviews. + ## Review Checklist ### 1. Search Composer Integration diff --git a/.github/workflows/live-drift.yml b/.github/workflows/live-drift.yml new file mode 100644 index 000000000..2345b14be --- /dev/null +++ b/.github/workflows/live-drift.yml @@ -0,0 +1,59 @@ +name: Live drift check + +# Live Supabase drift detection is useful, but it must not slow PR iteration or +# touch live services on every branch. Keep it scheduled/manual only. +on: + workflow_dispatch: + schedule: + # Weekly, aligned with the existing Sunday off-peak CI cadence. + - cron: "30 18 * * 0" + +concurrency: + group: live-drift-check + cancel-in-progress: false + +permissions: + contents: read + +env: + NEXT_PUBLIC_SUPABASE_URL: https://sjrfecxgysukkwxsowpy.supabase.co + SUPABASE_PROJECT_REF: sjrfecxgysukkwxsowpy + SUPABASE_PROJECT_NAME: Clinical KB Database + NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY: placeholder-ci-anon-key + SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} + +jobs: + live-drift: + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Preflight required secrets + run: | + missing="" + [ -z "$SUPABASE_SERVICE_ROLE_KEY" ] && missing="$missing SUPABASE_SERVICE_ROLE_KEY" + if [ -n "$missing" ]; then + echo "::error::Live drift check cannot run - missing repo secrets:$missing" + exit 1 + fi + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Guard Supabase project identity + run: npm run check:supabase-project + + - name: Compare live schema drift + run: npm run check:drift diff --git a/docs/branch-cleanup-guide.md b/docs/branch-cleanup-guide.md index d2b177e4c..9217e936d 100644 --- a/docs/branch-cleanup-guide.md +++ b/docs/branch-cleanup-guide.md @@ -41,7 +41,19 @@ Before deleting anything: git worktree list --porcelain ``` -5. For each candidate branch, check whether it has patch content not on `main`: +5. Filter candidates through the review ledger before inspecting branch diffs. Follow the lookup procedure in `docs/branch-review-ledger.md` and require all three fields to match before skipping: + + ```powershell + $branch = "BRANCH_NAME" + $head = git rev-parse $branch + $escapedBranch = [regex]::Escape($branch) + Get-Content docs\branch-review-ledger.md | + Select-String -Pattern "\|\s*$escapedBranch\s*\|\s*$head\s*\|\s*branch-cleanup\s*\|" + ``` + + Skip the branch only when a ledger row matches the same branch/ref, reviewed HEAD, and `branch-cleanup` scope together. A branch-name-only match is not enough. Re-review when the HEAD changed or the user explicitly asks for a fresh pass. + +6. For each remaining candidate branch, check whether it has patch content not on `main`: ```powershell git log --format="%h %s" --right-only --cherry-pick main...BRANCH_NAME @@ -53,9 +65,11 @@ Delete a branch only when the cherry-pick-aware log is empty, or when the branch ## Recommended Cleanup Order 1. Fetch and inspect current branch state with the commands above. -2. For each candidate branch, confirm patch-unique commits and file diffs against `main`. -3. Port, commit, or explicitly reject useful patch content before deleting any branch ref. -4. Remove detached worktrees only when clean, unneeded, and absent from active `git worktree list` output. +2. Resolve each candidate's HEAD and skip unchanged completed reviews recorded in `docs/branch-review-ledger.md`. +3. For each remaining candidate branch, confirm patch-unique commits and file diffs against `main`. +4. Port, commit, or explicitly reject useful patch content before deleting any branch ref. +5. Record completed cleanup reviews in `docs/branch-review-ledger.md`. +6. Remove detached worktrees only when clean, unneeded, and absent from active `git worktree list` output. ## Final Verification diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md new file mode 100644 index 000000000..e4d61b8c5 --- /dev/null +++ b/docs/branch-review-ledger.md @@ -0,0 +1,23 @@ +# Branch Review Ledger + +Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD has not changed. + +## Lookup Procedure + +1. Identify the target branch or ref from the user request. If no target is named, use the current branch. +2. Resolve the current HEAD: + + ```powershell + git rev-parse + ``` + +3. Search this file for a row with the same branch/ref, reviewed HEAD, and review scope. +4. If the HEAD and scope match an existing completed row, do not repeat the review. Summarize the prior outcome and ask before doing a fresh pass. +5. If the HEAD changed, or the user explicitly requests a fresh review, review the changed scope and append a new row. +6. For branch-cleanup passes, run this lookup before inspecting branch diffs so unchanged reviewed branches are filtered out early. + +## Review Records + +| Date | Branch or ref | Reviewed HEAD | Scope | Outcome | Checks | +| ---------- | -------------- | ------------- | -------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| 2026-07-09 | example/branch | abc1234 | branch-cleanup | Example: already merged into `main`; no unique patch content. | `git log --right-only --cherry-pick main...example/branch`; `git diff --name-status main...example/branch` | diff --git a/docs/codex-prompt-playbook.md b/docs/codex-prompt-playbook.md index 8cae8d4f1..32127c6de 100644 --- a/docs/codex-prompt-playbook.md +++ b/docs/codex-prompt-playbook.md @@ -657,3 +657,293 @@ with file/line references. Keep summary secondary. Inspect AGENTS.md, current git status, changed files, and relevant tests/docs. Do not modify files unless I explicitly ask for fixes after the review. ``` + +## 23. OpenAI Cookbook Review For This Repo + +Use this when you want a fresh pass over current OpenAI Cookbook patterns before +changing the Clinical KB RAG, answer, ingestion, eval, or prompt stack. + +```text +Review the current OpenAI Cookbook and recommend what should be adopted in this +repo. + +Use only official OpenAI sources. Treat archived Cookbook recipes as historical +unless the pattern is still supported by current docs or already matches this +repo's architecture. + +Map recommendations to this repo's existing surfaces: +- src/lib/rag.ts +- src/lib/rag-routing.ts +- src/lib/rag-provider.ts +- src/lib/answer-verification.ts +- src/lib/retrieval-selection.ts +- scripts/eval-*.ts +- scripts/fixtures/rag-retrieval-golden.json +- docs/retrieval-quality-runbook.md +- docs/search-rag-master-context.md + +Do not run OpenAI API calls, live Supabase checks, evals, installs, dependency +updates, commits, pushes, or deploys unless I explicitly approve. + +Return: +- Cookbook patterns that are already covered here +- Cookbook patterns worth adding +- Cookbook patterns to avoid or defer +- file-level implementation plan +- local/offline verification plan +- any steps that require explicit API/provider approval +``` + +## 24. Structured Output Contract Hardening + +Use this when answer generation, extraction, tool calls, or eval capture needs a +stricter response shape. + +```text +Audit and harden structured output contracts for this repo. + +Focus on places where model or model-like output is parsed, displayed, stored, +or graded: +- src/lib/rag.ts +- src/lib/answer-stream-extractor.ts +- src/lib/answer-verification.ts +- src/lib/answer-render-policy.ts +- src/app/api/answer/route.ts +- src/app/api/answer/stream/route.ts +- src/app/api/eval-cases/route.ts +- tests/answer-*.test.ts +- tests/rag-*.test.ts + +Look for JSON parsing, optional fields, schema drift, permissive unknown fields, +missing refusal/source-gap states, missing citation IDs, and UI display paths +that trust raw model text. + +Prefer strict schema validation at the boundary, then deterministic verification +before display. Preserve source-only fallback behavior and conservative clinical +gating. Do not change models, API providers, env values, or live workflows +without approval. + +If implementation is safe and scoped, make the minimal changes and add focused +tests. Otherwise return a concrete migration plan with affected files and +verification commands. +``` + +## 25. Eval-Driven Prompt And Answer Improvement + +Use this before changing prompts, routing, answer formatting, source governance, +or clinical synthesis behavior. + +```text +Improve this RAG/answer behavior using an eval-driven loop: + +[include query, observed answer, expected answer, citations/sources, and whether +provider-backed evals are approved] + +Start offline unless I approve OpenAI/Supabase provider use. + +Process: +- inspect current answer/routing/retrieval code and relevant tests +- add or update a small deterministic fixture or unit test that captures the + failure before changing behavior +- only then adjust prompt text, schema, routing, verification, or render policy +- keep the source-governance and privacy rules conservative +- avoid broad prompt rewrites that change unrelated answer classes + +Verification: +- run the focused test or local-safe eval that proves the behavior +- do not run provider-backed evals unless approved +- for non-trivial source changes, run npm run verify:cheap +- for clinical output/source-governance changes, run npm run check:production-readiness + +Final response: include baseline failure, fix, files changed, focused result, +and whether live eval confirmation is still needed. +``` + +## 26. RAG Document Preparation And Metadata Audit + +Use this when retrieval quality might be limited by chunking, metadata, visual +content, or source formatting rather than model prompting. + +```text +Audit the RAG document preparation pipeline for Cookbook-style retrieval +improvements without running live ingestion or OpenAI calls. + +Inspect: +- worker/main.ts +- worker/embedding-fields.ts +- worker/table-facts.ts +- src/lib/chunking.ts +- src/lib/document-index-units.ts +- src/lib/document-enrichment.ts +- src/lib/visual-intelligence.ts +- src/lib/source-metadata.ts +- scripts/eval-retrieval.ts +- docs/retrieval-quality-runbook.md + +Evaluate whether indexed units include the right searchable text and metadata: +document title, source status, jurisdiction, clinical topic, page/section +labels, tables, figures, abbreviations, synonyms, and user-searchable clinical +phrasing. + +Look for opportunities to improve retrieval by changing document preparation, +metadata augmentation, query rewriting, or chunk linking before changing answer +generation prompts. + +Do not run ingestion, worker jobs, OpenAI embeddings, visual model calls, live +Supabase commands, or backfills unless I approve. Return a safe implementation +plan and the local/offline tests that should be added first. +``` + +## 27. Multimodal Evidence And Visual RAG Review + +Use this for image-heavy PDFs, tables, forms, diagrams, medication charts, or +source pages where text-only retrieval may miss important evidence. + +```text +Review multimodal/visual evidence support for this Clinical KB workflow. + +Start read-only and offline. Inspect: +- src/lib/visual-intelligence.ts +- src/lib/image-filtering.ts +- src/lib/document-index-units.ts +- src/lib/rag-source-block.ts +- src/components/clinical-dashboard/visual-evidence.tsx +- src/app/api/images/[id]/signed-url/route.ts +- tests related to document images, visual evidence, and answer citations + +Check whether table/image/diagram evidence is: +- extracted into searchable text or metadata +- linked back to document/page/image IDs +- cited safely in answer output +- gated by source quality and access control +- rendered with useful source review affordances + +Recommend the smallest improvements. Do not run OCR, image model calls, live +Supabase, OpenAI, reindexing, or browser QA unless I approve. If code changes +are safe and local-only, add focused tests and explain what still needs live +corpus validation. +``` + +## 28. Clinical RAG Guardrails Review + +Use this when prompt injection, off-topic questions, unsafe clinical confidence, +or unsupported output could reach users. + +```text +Review and harden Clinical KB RAG guardrails. + +Focus on both input and output guardrails: +- prompt injection and jailbreak attempts inside user queries or retrieved text +- off-topic or non-clinical requests +- weak retrieval or conflicting evidence +- unsupported numbers, doses, thresholds, and recommendations +- stale, unverified, review-due, or poor-extraction sources +- private document access and owner scoping +- raw provider/internal error leakage + +Inspect: +- src/lib/rag.ts +- src/lib/rag-injection* +- src/lib/answer-verification.ts +- src/lib/source-governance.ts +- src/lib/answer-render-policy.ts +- src/app/api/answer +- src/app/api/search +- tests/rag-injection.test.ts +- tests/answer-verification.test.ts +- docs/rag-injection-threat-model.md +- docs/clinical-hazard-analysis.md + +Make only scoped fixes with focused tests. Preserve useful source-gap behavior +instead of making unsupported answers look confident. Do not run provider-backed +generation or live Supabase checks unless approved. +``` + +## 29. Prompt Caching And Cost/Latency Review + +Use this before changing prompt assembly, tool/schema definitions, context +ordering, or answer-generation latency behavior. + +```text +Review prompt assembly for cost and latency efficiency. + +Inspect: +- src/lib/rag.ts +- src/lib/openai.ts +- src/lib/rag-provider.ts +- src/lib/rag-context-selection.ts +- src/lib/env.ts +- tests for answer latency, provider routing, and fallback behavior +- docs/observability-slos.md + +Check whether static prompt content, schemas, examples, and stable instructions +are kept before variable user/query/retrieval context so provider prompt caching +can help when API calls are approved. Check whether tool/schema ordering is +stable, prompt/cache versioning is explicit, and telemetry captures cached input +tokens where available. + +Do not optimize by weakening source grounding, shortening clinical safety +instructions, or hiding source-gap behavior. Do not run OpenAI calls or live +evals unless approved. Return local code/test changes if safe, plus the live +metrics that would need provider-backed confirmation. +``` + +## 30. Responses API Or Tool-Orchestration Migration Plan + +Use this only for planning a future migration. Do not perform the migration +unless it is explicitly requested and API/provider work is approved. + +```text +Create a conservative migration plan for whether this repo should adopt +Responses API/tool orchestration patterns for RAG. + +Compare current implementation to a Responses-style flow: +- current retrieval and answer orchestration in src/lib/rag.ts +- current OpenAI wrapper in src/lib/openai.ts +- current route contracts in src/app/api/answer +- current eval scripts and fixtures +- current source-governance and privacy rules + +Identify: +- what would improve, such as stateful tool calls, structured outputs, or + cleaner orchestration +- what would get riskier, such as provider coupling, cost, tracing, streaming, + schema drift, or clinical safety validation +- exact files and tests that would change +- rollout plan with feature flag and fallback +- API/provider calls that require approval + +Do not edit files unless I explicitly ask for implementation after the plan. +``` + +## 31. Schema Change Impact And Eval Harness + +Use this before SQL/RPC/schema changes that affect ingestion, retrieval, source +governance, or clinical output. + +```text +Build a schema-change impact plan and eval harness for this database/RAG change: + +[describe the intended SQL/RPC/schema/policy change] + +Start read-only. Inspect: +- supabase/schema.sql +- relevant supabase/migrations/*.sql +- src/lib/supabase/database.types.ts +- affected src/app/api routes +- affected src/lib retrieval/ingestion modules +- tests/supabase-schema.test.ts +- scripts/check-drift.ts +- scripts/eval-retrieval.ts + +Produce: +- impact analysis by table/RPC/function/policy +- code paths and tests affected +- deterministic preflight checks that do not need model calls +- parse-only or local-only eval cases where possible +- live Supabase/OpenAI checks that require explicit approval +- rollback or compatibility notes + +If the change is safe and scoped and I asked for implementation, add the +smallest migration plus focused tests. Otherwise stop at the plan. +``` diff --git a/docs/codex-review-protocol.md b/docs/codex-review-protocol.md new file mode 100644 index 000000000..41541489c --- /dev/null +++ b/docs/codex-review-protocol.md @@ -0,0 +1,34 @@ +# Codex Review Protocol + +Use this protocol for every Codex review, audit, bug hunt, PR review, release-readiness check, and branch-cleanup review in this repository. + +## Trigger and Scope + +- Do not start a review opportunistically. +- Review only when the user request, `AGENTS.md` routing rules, or an explicit handoff/release workflow calls for it. +- Review the current diff, named PR, named branch, or explicitly requested area. +- Do not expand into stale branches or unrelated modules unless a confirmed defect crosses that boundary. +- Before branch or PR review, check `docs/branch-review-ledger.md`: resolve the target with `git rev-parse`, compare the HEAD and scope, and skip unchanged completed reviews unless the user asks for a fresh pass. + +## Review Output + +- Lead with findings, ordered by severity: P0, P1, P2, then P3. +- Each finding must include file/line evidence, trigger or failure path, expected behavior, actual risk, and the smallest proof or check that would catch it. +- If no high-confidence issue is found, say so clearly and name the highest residual risk area. +- Include checks run, checks not run, and whether any check was skipped because it could touch an API/provider. +- Keep summaries secondary to findings. + +## Mutation Rules + +- For a pure review request, do not edit files, stage, commit, push, post PR comments, rerun hosted CI, or call provider-backed services. + - Exception: append the completed review record to `docs/branch-review-ledger.md` so throttling state persists. +- If the user clearly asks to fix confirmed findings, make the smallest safe change and verify with local, static, or mocked checks first. +- Ask before any OpenAI, Supabase, GitHub/GitLab, hosted CI, or provider-backed workflow. +- After any completed branch/PR review, update `docs/branch-review-ledger.md` with date, branch/ref, HEAD, scope, outcome, and checks. This ledger append is allowed even during a pure review. + +## Severity Guide + +- P0: Data loss, security breach, production outage, or clinical safety issue likely to harm users immediately. +- P1: Broken core workflow, unsafe automation, privacy/auth failure, or repeatable defect that blocks merge/handoff. +- P2: Real defect, missing guardrail, fragile process, or test gap that should be fixed before relying on the work. +- P3: Low-risk cleanup, clarity, documentation, or future-proofing issue. diff --git a/docs/rag-hybrid-findings-and-todo.md b/docs/rag-hybrid-findings-and-todo.md index 66fb26b91..ef4558a80 100644 --- a/docs/rag-hybrid-findings-and-todo.md +++ b/docs/rag-hybrid-findings-and-todo.md @@ -257,15 +257,18 @@ denied to set parameter`)** — the RC11 blocker. The only method hosted allows vocabulary table, never the raw query, so RET-H4 holds). Remaining: terms OUTSIDE the curated vocabulary still cannot be captured without a privacy review; promotion tooling from `candidate_aliases` → `rag_aliases` is still manual. -18. ⏳ **`document_index_units` vector recall** — no HNSW index (dropped 2026-07-02) and hosted - Supabase denies `ALTER FUNCTION … SET hnsw.ef_search` for the `language sql` hybrid RPCs, so - only `match_document_memory_cards_hybrid` pins `ef_search=100`. Concrete measurement plan - (needs live keys, ~1 hour): run `eval:retrieval:quality` twice with `--force-embedding` - (bypasses lexical fast paths, exercising vectors directly) — once as-is and once after - `create index concurrently` on `document_index_units.embedding` in a Supabase branch — and - compare doc-recall@5 + p90 latency. If recall gain < 1 case, close as not-worth-4.4GB. The - ef_search half can be retested via the plpgsql-wrapper trick that memory_cards already uses - (wrap the `language sql` RPC in a plpgsql shim that SETs it). +18. ✅ **`document_index_units` vector recall / HNSW measurement — CLOSED as not worth adding + (2026-07-09).** Measured with live keys under the explicit eval budget. Production baseline + `eval:retrieval:quality -- --force-embedding` produced `document_recall_at_5=0.9306`, + `content_recall_at_5=1`, `top_k_hit_rate=0.9444`, `force_embedding_failure_count=0`, + `p90_latency_ms=23293`, and `index_units_layer_count=0`. A data-cloned Supabase preview branch + (`rag-index-units-hnsw-20260709`) was created, branch-only HNSW index + `document_index_units_embedding_hnsw_idx` was applied and confirmed, then the same eval was run + against the branch. Candidate result: recall unchanged, failed cases unchanged, `index_units` still + unused, median latency worsened by `+3698ms`, p90 worsened by `+13963ms`. The preview branch was + deleted. **Do not add this HNSW index to production.** Revisit only if the retrieval path is changed + to actually use `document_index_units.embedding` and a new eval shows at least one recall win without + material p90 regression. 19. ✅ **Demo fallback can mask live retrieval failures in non-prod — DONE (2026-07-06).** `nonProductionSupabaseDemoFallbackReason` (the shared choke point for /api/search, /api/answer, and /api/answer/stream) now emits a loud `console.warn` naming the env vars to @@ -300,21 +303,25 @@ denied to set parameter`)** — the RC11 blocker. The only method hosted allows answer-confidence label — "high" requires a genuine-cosine citation; synthetic-origin evidence caps at "medium" (strictly tightening, ordering/routing untouched, unit-tested in tests/rag-score.test.ts). -22. ⏳ **Registry-to-corpus embedding (universal search Phase 5).** Medications/services/forms/ - differentials are federated into `/api/search/universal` but are not retrieval-corpus - entities, so Answer mode cannot cite them. Concrete implementation spec (in order): - 1. Flag `RAG_REGISTRY_CORPUS_EMBEDDING` (default off) in `src/lib/env.ts`. - 2. Ingestion script `scripts/embed-registry-records.ts`: map each registry record to a - synthetic "document" (`metadata.source_kind = 'registry_record'`, title = record title, - one chunk per record from the record's search text, embedded with the standard - `text-embedding-3-small` path) so the existing chunk pipeline/RPCs need no schema change. - 3. Re-embed on registry edit: hook `ensureRegistrySeeded` / record-update routes to enqueue - re-embedding for the changed slug only. - 4. Answer-surface labelling: `sourceGovernanceWarnings` must label registry-backed - citations distinctly (registry records are curated summaries, not source documents). - 5. Gates before enabling anywhere real: `eval:retrieval:quality` 23/23 with the flag ON, - plus invented-term controls ("florbizone syndrome management") still refusing — registry - rows must not become a fabrication surface for unsupported topics. +22. 🔶 **Registry-to-corpus embedding (universal search Phase 5) — implementation restored and live + owner embedded (2026-07-09).** Medications/services/forms/differentials are federated into + `/api/search/universal` but were not retrieval-corpus entities, so Answer mode could not cite them. + Implemented pieces: `RAG_REGISTRY_CORPUS_EMBEDDING` default-off flag, + `scripts/embed-registry-records.ts` dry-run/write/list-owner tool, synthetic document/chunk mapping + with `metadata.source_kind = 'registry_record'`, source-governance labelling, comparator tooling, + and registry corpus tests. The sentinel-owner dry-run + (`00000000-0000-0000-0000-000000000000`) correctly found zero rows. `--list-owners` found the real + registry owner `4f1b3c19-3c39-4597-b9df-168c8e6007ff` with 739 eligible rows; guarded write with + `RAG_REGISTRY_CORPUS_EMBEDDING=true --write --confirm` upserted 739 synthetic registry corpus + chunks. Post-write retrieval eval passed with `document_recall_at_5=1`, `content_recall_at_5=1`, + `top_k_hit_rate=1`, `force_embedding_failure_count=0`, `failed_cases=[]`. Post-write + `eval:quality -- --rag-only` completed under budget; invented-term controls still refused and + numeric grounding failure rate was `0`. Remaining blockers are not registry regressions: + citation failure rate `0.0227` and RAG latency thresholds (`p95=44847ms`; route p95 extractive + `46745ms`, fast `27131ms`, strong `63613ms`). Remaining implementation before treating this as + fully productized: re-embed-on-edit hooks for registry updates and a dedicated answer-mode UX check + that registry-backed citations are labelled as curated registry records rather than primary source + documents. 23. ⏳ **Finding #11 full fix (RAG optimisation Phase 2)** — the classifier-verdict memo (shipped 2026-07-06) makes zero-result behaviour deterministic per query but does not close the gap: the deterministic analyzer still cannot tell in-corpus topics from out-of-corpus ones. diff --git a/scripts/embed-registry-records.ts b/scripts/embed-registry-records.ts index cd25aefb4..1e9b282e6 100644 --- a/scripts/embed-registry-records.ts +++ b/scripts/embed-registry-records.ts @@ -15,6 +15,7 @@ type Args = { slug?: string; write: boolean; confirmed: boolean; + listOwners: boolean; }; function parseArgs(argv: string[]): Args { @@ -23,10 +24,15 @@ function parseArgs(argv: string[]): Args { kind: "all", write: false, confirmed: false, + listOwners: false, }; for (let index = 0; index < argv.length; index += 1) { const token = argv[index]; + if (token === "--list-owners") { + args.listOwners = true; + continue; + } if (token === "--write") { args.write = true; continue; @@ -97,10 +103,112 @@ async function loadDifferentialRows( return (data ?? []) as DifferentialRecordRow[]; } +type OwnerCounts = { + ownerId: string; + service: number; + form: number; + medication: number; + differential: number; +}; + +function ensureOwnerCount(counts: Map, ownerId: string) { + let count = counts.get(ownerId); + if (!count) { + count = { ownerId, service: 0, form: 0, medication: 0, differential: 0 }; + counts.set(ownerId, count); + } + return count; +} + +async function listEligibleOwnerCounts(supabase: Awaited>) { + const counts = new Map(); + + // Page through registry records + let registryOffset = 0; + const pageSize = 1000; + while (true) { + const { data: registryRows, error: registryError } = await supabase + .from("clinical_registry_records") + .select("owner_id, kind") + .range(registryOffset, registryOffset + pageSize - 1); + if (registryError) throw new Error(`Could not load registry owner counts: ${registryError.message}`); + if (!registryRows || registryRows.length === 0) break; + for (const row of registryRows) { + const ownerId = typeof row.owner_id === "string" ? row.owner_id : null; + if (!ownerId) continue; + const count = ensureOwnerCount(counts, ownerId); + if (row.kind === "form") count.form += 1; + else count.service += 1; + } + if (registryRows.length < pageSize) break; + registryOffset += pageSize; + } + + // Page through medication records + let medicationOffset = 0; + while (true) { + const { data: medicationRows, error: medicationError } = await supabase + .from("medication_records") + .select("owner_id") + .range(medicationOffset, medicationOffset + pageSize - 1); + if (medicationError) throw new Error(`Could not load medication owner counts: ${medicationError.message}`); + if (!medicationRows || medicationRows.length === 0) break; + for (const row of medicationRows) { + const ownerId = typeof row.owner_id === "string" ? row.owner_id : null; + if (!ownerId) continue; + ensureOwnerCount(counts, ownerId).medication += 1; + } + if (medicationRows.length < pageSize) break; + medicationOffset += pageSize; + } + + // Page through differential records + let differentialOffset = 0; + while (true) { + const { data: differentialRows, error: differentialError } = await supabase + .from("differential_records") + .select("owner_id") + .range(differentialOffset, differentialOffset + pageSize - 1); + if (differentialError) throw new Error(`Could not load differential owner counts: ${differentialError.message}`); + if (!differentialRows || differentialRows.length === 0) break; + for (const row of differentialRows) { + const ownerId = typeof row.owner_id === "string" ? row.owner_id : null; + if (!ownerId) continue; + ensureOwnerCount(counts, ownerId).differential += 1; + } + if (differentialRows.length < pageSize) break; + differentialOffset += pageSize; + } + + return [...counts.values()].sort((left, right) => { + const leftTotal = left.service + left.form + left.medication + left.differential; + const rightTotal = right.service + right.form + right.medication + right.differential; + return rightTotal - leftTotal || left.ownerId.localeCompare(right.ownerId); + }); +} + async function main() { const args = parseArgs(process.argv.slice(2)); + const supabase = await loadAdminClient(); + + if (args.listOwners) { + const ownerCounts = await listEligibleOwnerCounts(supabase); + console.log("[registry:embed] eligible owner counts"); + if (ownerCounts.length === 0) { + console.log(" No registry, medication, or differential rows found."); + return; + } + for (const count of ownerCounts) { + const total = count.service + count.form + count.medication + count.differential; + console.log( + ` ${count.ownerId} total=${total} service=${count.service} form=${count.form} medication=${count.medication} differential=${count.differential}`, + ); + } + return; + } + const ownerId = args.ownerId; - if (!ownerId) throw new Error("No owner id. Pass --owner-id or set LOCAL_NO_AUTH_OWNER_ID."); + if (!ownerId) throw new Error("No owner id. Pass --owner-id , --list-owners, or set LOCAL_NO_AUTH_OWNER_ID."); const { clinicalRegistryRowsToCorpusEntries, @@ -112,7 +220,6 @@ async function main() { registryCorpusEmbeddingEnabled, } = await import("@/lib/registry-corpus"); - const supabase = await loadAdminClient(); const [registryRows, medicationRows, differentialRows] = await Promise.all([ loadRegistryRows(supabase, args, ownerId), loadMedicationRows(supabase, args, ownerId), @@ -133,6 +240,11 @@ async function main() { if (total > 20) console.log(` ... ${total - 20} more`); if (!args.write) { + if (total === 0) { + console.log( + "[registry:embed] No eligible rows found for this owner. Re-run with --list-owners to find owners with registry rows.", + ); + } console.log( "[registry:embed] Dry run. Re-run with --write --confirm and RAG_REGISTRY_CORPUS_EMBEDDING=true to write embeddings.", ); diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts index 9e976f2a1..3f51cd6f2 100644 --- a/src/lib/clinical-search.ts +++ b/src/lib/clinical-search.ts @@ -1113,7 +1113,7 @@ export function buildClinicalTextSearchQuery(query: string) { "po", ); } else if (/\badmission\b/i.test(query) && /\bcommunity patients?\b/i.test(query)) { - normalizedTokens.unshift("admission", "community", "pts"); + normalizedTokens.unshift("admission", "community", "patients", "pts"); } else if (/\bdischarge\b/i.test(query) && /\b(?:summari[sz]e|summary|guidance|documentation?)\b/i.test(query)) { normalizedTokens.splice(0, normalizedTokens.length, "mental", "health", "discharge"); } else if ( @@ -1143,7 +1143,7 @@ export function buildClinicalTextSearchQuery(query: string) { } else if (/\b(?:risk matrix|red zone)\b/i.test(query)) { normalizedTokens.push("high", "visual", "alert"); } else if (/\badmission\b/i.test(query) && /\bdischarge\b/i.test(query)) { - normalizedTokens.push("community", "pts"); + normalizedTokens.push("community", "patients", "pts"); } else if (/\bcommunity patients?\b/i.test(query) && normalizedTokens.includes("community")) { normalizedTokens.push("pts"); } diff --git a/src/lib/rag-extractive-answer.ts b/src/lib/rag-extractive-answer.ts index 654ff4af8..4bca5db5e 100644 --- a/src/lib/rag-extractive-answer.ts +++ b/src/lib/rag-extractive-answer.ts @@ -284,6 +284,26 @@ function answerIntentEvidencePattern(intent: AnswerIntent) { } } +function requiresBloodCountEvidence(query: string) { + return /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils)\b/i.test(query); +} + +function asksForWithholdAction(query: string) { + return /\b(?:withhold|withheld|withholding|cease|stop|stopped|discontinue|discontinued)\b/i.test(query); +} + +function hasBloodCountEvidence(text: string) { + return /\b(?:anc|fbc|full blood count|wbc|wcc|white blood cell|white cell|neutrophil|neutrophils|blood count)\b/i.test( + text, + ); +} + +function hasWithholdActionEvidence(text: string) { + return /\b(?:withhold|withheld|withholding|cease|stop|stopped|discontinue|discontinued|red range|amber range|red|amber)\b/i.test( + text, + ); +} + function resultCoversAnswerIntent(result: SearchResult, query: string, intent: AnswerIntent) { if (intent === "unsupported") return false; const text = evidenceTextForGate(result); @@ -298,6 +318,8 @@ function resultCoversAnswerIntent(result: SearchResult, query: string, intent: A const intentCoverage = answerIntentEvidencePattern(intent).test(text); if (!intentCoverage) return false; if (intentTokens.length > 0 && !intentTokens.some((token) => queryTokenMatchesText(token, text))) return false; + if (intent === "red_result_action" && requiresBloodCountEvidence(query) && !hasBloodCountEvidence(text)) return false; + if (intent === "red_result_action" && asksForWithholdAction(query) && !hasWithholdActionEvidence(text)) return false; if (/\brenal\b/i.test(query) && !/\b(?:renal|kidney|eGFR|creatinine)\b/i.test(text)) return false; if (/\bmax(?:imum)?\b/i.test(query) && !/\b(?:max(?:imum)?|\d+(?:\.\d+)?\s?(?:mg|mcg))\b/i.test(text)) { return false; @@ -559,6 +581,8 @@ function factSupportsAnswerIntent( ); case "red_result_action": if (kind !== "threshold_action" && kind !== "caveat") return false; + if (requiresBloodCountEvidence(query) && !hasBloodCountEvidence(text)) return false; + if (asksForWithholdAction(query) && !hasWithholdActionEvidence(text)) return false; return ( /\b(?:withhold|cease|stop|discontinue|discontinued|contact|urgent|repeat|review|call for help|escalat\w*|monitor|toxicity|rash)\b/i.test( text, diff --git a/src/lib/rag-retrieval-variants.ts b/src/lib/rag-retrieval-variants.ts index 4396d0455..0831b361c 100644 --- a/src/lib/rag-retrieval-variants.ts +++ b/src/lib/rag-retrieval-variants.ts @@ -166,11 +166,18 @@ export function buildRetrievalQueryVariants( }; addVariant(buildClinicalTextSearchQuery(query)); + if (/\badmission\b/i.test(query) && /\bcommunity patients?\b/i.test(query)) { + addVariant("admission of community patients"); + addVariant("admission community patients"); + } aliasExpansions.slice(0, 2).forEach(addVariant); if (/\bpatient property\b/i.test(query)) { addVariant("patient property"); } if (/\bclozapine\b/i.test(query) && /\b(?:anc|fbc|wbc|neutrophil|white cell)\b/i.test(query)) { + if (/\b(?:threshold|cut[\s-]?off|withhold|withheld|withholding|cease|stop|stopped|discontinue)\b/i.test(query)) { + addVariant("clozapine blood results amber red range"); + } addVariant("clozapine anc fbc"); addVariant("clozapine monitoring"); } diff --git a/src/lib/rag-routing.ts b/src/lib/rag-routing.ts index 5689329e0..e38f90b9b 100644 --- a/src/lib/rag-routing.ts +++ b/src/lib/rag-routing.ts @@ -33,6 +33,14 @@ const extractiveBlockPattern = const broadManagementSynthesisPattern = /\b(?:management|manage|managed|treatment|treat|therapy|care|approach|pathway)\s+(?:of|for|in)\b|\bhow\s+(?:is|are|should)\b.{0,80}\b(?:managed|treated)\b/i; const clinicalPathwaySynthesisPattern = /\b(?:referral criteria|referral pathway|refer\b|pathway|what to do)\b/i; +const medicationExtractiveLookupPattern = + /\b(?:listed|list|options?|monitoring|observations?|needed|required|dosing guidance|dose|route|pharmacological management|\bim\b|\bpo\b)\b/i; +const medicationSafetyCriticalPattern = + /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils|threshold|cut ?off|withhold|withheld|withholding|cease|stop|stopped|red range|amber range)\b/i; +const clozapineBloodWithholdThresholdPattern = /\bclozapine\b/i; +const bloodCountTermPattern = /\b(?:anc|fbc|wbc|wcc|white cell|white blood cell|neutrophil|neutrophils)\b/i; +const withholdThresholdIntentPattern = + /\b(?:threshold|cut ?off|cutoff|withhold|withheld|withholding|cease|stop|stopped|discontinue|discontinued)\b/i; const queryStopWords = new Set([ "what", "when", @@ -52,6 +60,46 @@ const queryStopWords = new Set([ "managed", "management", "guideline", + "find", + "search", + "lookup", + "open", + "show", + "tell", + "give", +]); +const specificTitleStopWords = new Set([ + "what", + "when", + "where", + "which", + "should", + "does", + "do", + "is", + "are", + "the", + "a", + "an", + "of", + "to", + "in", + "for", + "with", + "and", + "include", + "require", + "requires", + "required", + "requirement", + "requirements", + "find", + "search", + "lookup", + "open", + "show", + "tell", + "give", ]); // Query-side adversarial-manipulation guard. Neutralizing injected instructions @@ -192,6 +240,13 @@ function queryTopicTokens(query: string) { .filter((token) => token.length > 2 && !queryStopWords.has(token)); } +function querySpecificTitleTokens(query: string) { + return normalizeLookupText(query) + .split(/\s+/) + .filter((token) => token.length > 2 && !specificTitleStopWords.has(token)) + .map((token) => token.replace(/s$/, "")); +} + export function hasDirectTitleSupport(query: string, results: SearchResult[]) { const tokens = queryTopicTokens(query); if (tokens.length === 0) return false; @@ -202,6 +257,31 @@ export function hasDirectTitleSupport(query: string, results: SearchResult[]) { }); } +function bestTitleTokenMatch(query: string, results: SearchResult[]) { + const tokens = querySpecificTitleTokens(query); + let matchedCount = 0; + + for (const result of results.slice(0, 4)) { + const title = normalizeLookupText(`${result.title} ${result.file_name}`).replace(/s\b/g, ""); + const count = tokens.filter((token) => title.includes(token)).length; + matchedCount = Math.max(matchedCount, count); + } + + return { tokenCount: tokens.length, matchedCount }; +} + +export function hasSpecificTitleSupport(query: string, results: SearchResult[]) { + const { tokenCount, matchedCount } = bestTitleTokenMatch(query, results); + if (tokenCount === 0) return false; + if (tokenCount <= 2) return matchedCount > 0; + return matchedCount >= 2 || matchedCount / tokenCount >= 0.5; +} + +function hasAdmissionCommunityLookupIntent(query: string) { + const normalized = normalizeLookupText(query); + return /\badmission\b/.test(normalized) && /\bcommunity\b/.test(normalized); +} + export function isComplexClinicalQuery(query: string) { return complexClinicalQueryPattern.test(query); } @@ -226,6 +306,43 @@ function hasTableOrVisualSourceSupport(results: SearchResult[]) { }); } +function isClozapineBloodWithholdThresholdQuery(query: string) { + return ( + clozapineBloodWithholdThresholdPattern.test(query) && + bloodCountTermPattern.test(query) && + withholdThresholdIntentPattern.test(query) + ); +} + +function hasExplicitBloodWithholdActionEvidence(results: SearchResult[]) { + return results.slice(0, 8).some((result) => { + const tableFactText = (result.table_facts ?? []) + .map((fact) => + [fact.table_title, fact.row_label, fact.clinical_parameter, fact.threshold_value, fact.action] + .filter(Boolean) + .join(" "), + ) + .join(" "); + const sourceText = [ + result.title, + result.file_name, + result.section_heading, + result.retrieval_synopsis, + result.content, + result.adjacent_context, + tableFactText, + result.index_unit?.title, + result.index_unit?.content, + ] + .filter(Boolean) + .join(" "); + return ( + bloodCountTermPattern.test(sourceText) && + /\b(?:withhold|withheld|withholding|cease|stop|stopped|discontinue|discontinued)\b/i.test(sourceText) + ); + }); +} + function shouldPreferModelSynthesis(query: string, queryClass: RagQueryClass) { return ( queryClass === "medication_dose_risk" || @@ -238,6 +355,19 @@ function shouldPreferModelSynthesis(query: string, queryClass: RagQueryClass) { ); } +function shouldUseExtractiveMedicationLookup(args: { + query: string; + results: SearchResult[]; + strongestScore: number; + topTextRank: number; + directTitleSupport: boolean; +}) { + if (!medicationExtractiveLookupPattern.test(args.query)) return false; + if (medicationSafetyCriticalPattern.test(args.query)) return false; + if (/\b(?:dose|dosing|maximum|max|action)\b/i.test(args.query)) return false; + return args.directTitleSupport || args.topTextRank >= 0.08 || hasTableOrVisualSourceSupport(args.results); +} + function shouldUseStrongClinicalRoute(args: { query: string; queryClass: RagQueryClass; @@ -311,6 +441,7 @@ export function chooseAnswerRoute(args: { const strongestScore = strongestRetrievalScore(args.results); const documents = documentCount(args.results); const directTitleSupport = hasDirectTitleSupport(args.query, args.results); + const specificTitleSupport = hasSpecificTitleSupport(args.query, args.results); const queryClass = args.queryClass ?? classifyRagQuery(args.query).queryClass; const topTextRank = Math.max(0, ...args.results.map((result) => result.text_rank ?? 0)); @@ -350,9 +481,11 @@ export function chooseAnswerRoute(args: { if ( queryClass === "document_lookup" && - hasExplicitDocumentLookupIntent(args.query) && - !directTitleSupport && - topTextRank < 0.08 + ((!directTitleSupport && hasExplicitDocumentLookupIntent(args.query) && topTextRank < 0.08) || + (directTitleSupport && + (hasExplicitDocumentLookupIntent(args.query) || /\bchecklist\b/i.test(args.query)) && + !specificTitleSupport && + !hasAdmissionCommunityLookupIntent(args.query))) ) { return { mode: "unsupported", @@ -454,6 +587,16 @@ export function chooseAnswerRoute(args: { queryClass === "comparison" || (documents > 1 && comparisonQueryPattern.test(args.query) && !directTitleSupport) ) { + if (documents > 1 && strongestScore >= strongRetrievalThreshold) { + return { + mode: "extractive", + model: null, + reason: "high_confidence_extractive_retrieval", + strongestScore, + documentCount: documents, + }; + } + return { mode: "strong", model: args.strongModel, @@ -464,6 +607,49 @@ export function chooseAnswerRoute(args: { } if (queryClass === "medication_dose_risk" || queryClass === "table_threshold") { + if ( + queryClass === "table_threshold" && + isClozapineBloodWithholdThresholdQuery(args.query) && + !hasExplicitBloodWithholdActionEvidence(args.results) + ) { + return { + mode: "strong", + model: args.strongModel, + reason: "clinical_risk_or_complex_query", + strongestScore, + documentCount: documents, + }; + } + + if (queryClass === "table_threshold" && directTitleSupport && strongestScore >= strongRetrievalThreshold) { + return { + mode: "extractive", + model: null, + reason: "high_confidence_extractive_retrieval", + strongestScore, + documentCount: documents, + }; + } + + if ( + queryClass === "medication_dose_risk" && + shouldUseExtractiveMedicationLookup({ + query: args.query, + results: args.results, + strongestScore, + topTextRank, + directTitleSupport, + }) + ) { + return { + mode: "extractive", + model: null, + reason: "high_confidence_extractive_retrieval", + strongestScore, + documentCount: documents, + }; + } + if ( shouldUseStrongClinicalRoute({ query: args.query, @@ -544,6 +730,21 @@ export function chooseAnswerRoute(args: { }; } + if ( + queryClass === "document_lookup" && + directTitleSupport && + strongestScore >= strongRetrievalThreshold && + /\b(?:documentation|procedure|process|requirements?|required|include|includes)\b/i.test(args.query) + ) { + return { + mode: "extractive", + model: null, + reason: "high_confidence_extractive_retrieval", + strongestScore, + documentCount: documents, + }; + } + if (isComplexClinicalQuery(args.query)) { return { mode: "strong", diff --git a/src/lib/rag.ts b/src/lib/rag.ts index e01bfd5f2..a359f4592 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -2619,6 +2619,9 @@ export function decideTextFastPath( if (isRiskFlowchartNextStepQuery(query) && !hasRiskFlowchartActionEvidence(query, results)) { return { returnFastPath: false, reason: "risk_flowchart_requires_action_evidence" }; } + if (hasAdmissionCommunityLookupIntent(query) && !hasAdmissionCommunityTitleSupport(results)) { + return { returnFastPath: false, reason: "admission_community_requires_title_rescue" }; + } if (directTitleSupport && strongestScore >= 0.32) { return { returnFastPath: true, reason: "direct_title_text_match" }; } @@ -2660,6 +2663,26 @@ function hasDocumentAliasWithoutTopTitleSupport(query: string, results: SearchRe }); } +function hasAdmissionCommunityLookupIntent(query: string) { + const normalized = normalizeDocumentAliasText(query); + return /\badmission\b/.test(normalized) && /\bcommunity\b/.test(normalized); +} + +function hasAdmissionCommunityTitleSupport(results: SearchResult[]) { + return results.slice(0, 5).some((result) => { + if (result.match_explanation?.titleHit || result.match_explanation?.labelHit) { + const title = normalizeDocumentAliasText(`${result.title} ${result.file_name}`); + return /\badmission\b/.test(title) && /\bcommunity\b/.test(title); + } + const title = normalizeDocumentAliasText(`${result.title} ${result.file_name}`); + return ( + title.includes("admission of community patient") || + title.includes("admission community pt") || + title.includes("admission to discharge for community") + ); + }); +} + function shouldReturnBeforeMemory( queryClass: RagQueryClass, decision: { returnFastPath: boolean; reason: string | null }, @@ -2929,6 +2952,15 @@ export function evaluateEvidenceCoverageGate( } if (queryClass === "document_lookup") { + if (hasAdmissionCommunityLookupIntent(query) && !hasAdmissionCommunityTitleSupport(top)) { + return { + accepted: false, + reason: "missing_admission_community_title_support", + strategy: "document_lookup_fast_path", + sourceImageRequired, + sourceImageSatisfied, + }; + } if (/\bactive community patients?\b/i.test(query) && /\bed\b/i.test(query)) { const accepted = hasDirectTitle && @@ -4511,6 +4543,7 @@ ${qualityRetryInstruction}` instructions: answerInstructions, promptCacheKey: "clinical-rag-answer-v18", timeoutMs: env.OPENAI_ANSWER_TIMEOUT_MS, + maxRetries: 0, reasoningEffort: useStrongReasoning ? strongReasoningEffortForQueryClass(queryClass, env.OPENAI_STRONG_REASONING_EFFORT) : env.OPENAI_FAST_REASONING_EFFORT, @@ -4541,10 +4574,31 @@ ${qualityRetryInstruction}` return reason === "max_output_tokens" ? `${prefix}_max_output_tokens` : `${prefix}_incomplete_${reason}`; } + function shouldRecoverFastFailureExtractively(retryReason: string) { + const sourceBackedRecoveryRetryReasons = new Set([ + "fast_unsupported_retry_strong", + "fast_unusable_retry_strong", + "fast_template_retry_strong", + "fast_quality_retry_strong", + ]); + return ( + route.mode === "fast" && + route.reason === "strong_routine_retrieval" && + answerInputResults.length > 0 && + queryClass !== "comparison" && + queryClass !== "broad_summary" && + queryClass !== "medication_dose_risk" && + queryClass !== "table_threshold" && + sourceBackedRecoveryRetryReasons.has(retryReason) + ); + } + function summarizeGenerationFailureReason(error: unknown) { const message = (error instanceof Error ? error.message : typeof error === "string" ? error : "").trim(); const normalized = message.toLowerCase(); + const sourceBackedRecovery = normalized.match(/\bsource_backed_extractive_recovery:([a-z0-9_]+)/); + if (sourceBackedRecovery) return `source_backed_extractive_recovery_${sourceBackedRecovery[1]}`; if (!normalized) return "generation_failed"; if (/\bmax_output_tokens\b/.test(normalized)) return "provider_incomplete_max_output_tokens"; if (/\bincomplete\b/.test(normalized)) return "provider_incomplete"; @@ -4724,6 +4778,11 @@ ${qualityRetryInstruction}` : fastAnswerWasOverExpanded ? "fast_overexpanded_simple_retry_strong" : "fast_quality_retry_strong"; + if (shouldRecoverFastFailureExtractively(retryReason)) { + answerRetryCount += 1; + answerRetryReasons.push(`fast_source_backed_extractive_recovery:${retryReason}`); + throw new Error(`source_backed_extractive_recovery:${retryReason}`); + } answerRetryCount += 1; answerRetryReasons.push(retryReason); modelUsed = env.OPENAI_STRONG_ANSWER_MODEL; diff --git a/src/lib/registry-corpus.ts b/src/lib/registry-corpus.ts index c9b086c8b..24ef5ecc2 100644 --- a/src/lib/registry-corpus.ts +++ b/src/lib/registry-corpus.ts @@ -34,6 +34,21 @@ type RegistryCorpusEntry = { metadata: Record; }; +export type RegistryCorpusEmbedResult = { + documentCount: number; + chunkCount: number; + skipped?: boolean; + reason?: "disabled" | "not_found" | "failed"; + errorMessage?: string; +}; + +export type RegistryCorpusEditTarget = + | { corpusKind: "service" | "form"; ownerId: string; slug: string } + | { corpusKind: "medication"; ownerId: string; slug: string } + | { corpusKind: "differential"; ownerId: string; slug: string; differentialKind?: DifferentialRecordRow["kind"] }; + +const REGISTRY_EMBEDDING_WRITE_BATCH_SIZE = 64; + function sha256(value: string) { return createHash("sha256").update(value).digest("hex"); } @@ -268,37 +283,46 @@ export async function embedRegistryCorpusEntries(supabase: AdminClient, entries: if (entries.length === 0) return { documentCount: 0, chunkCount: 0 }; const { embedTexts } = await import("@/lib/openai"); - const embeddings = await embedTexts(entries.map((entry) => entry.content)); - const documents = entries.map(registryDocumentRow); - const chunks = entries.map((entry, index) => registryChunkRow(entry, embeddings[index] as Vector)); - const documentIds = documents.map((document) => document.id).filter((id): id is string => typeof id === "string"); - const { data: existingDocuments, error: existingDocumentError } = await supabase - .from("documents") - .select("id") - .in("id", documentIds); - if (existingDocumentError) { - throw new Error(`Registry corpus preflight failed: ${existingDocumentError.message}`); - } - const existingDocumentIds = new Set((existingDocuments ?? []).map((document) => document.id)); - - const { error: documentError } = await supabase.from("documents").upsert(documents, { onConflict: "id" }); - if (documentError) throw new Error(`Registry corpus document upsert failed: ${documentError.message}`); - - const { error: chunkError } = await supabase.from("document_chunks").upsert(chunks, { onConflict: "id" }); - if (chunkError) { - const insertedDocumentIds = documentIds.filter((id) => !existingDocumentIds.has(id)); - if (insertedDocumentIds.length > 0) { - const { error: rollbackError } = await supabase.from("documents").delete().in("id", insertedDocumentIds); - if (rollbackError) { - throw new Error( - `Registry corpus chunk upsert failed: ${chunkError.message}; rollback failed: ${rollbackError.message}`, - ); + let documentCount = 0; + let chunkCount = 0; + + for (let start = 0; start < entries.length; start += REGISTRY_EMBEDDING_WRITE_BATCH_SIZE) { + const batch = entries.slice(start, start + REGISTRY_EMBEDDING_WRITE_BATCH_SIZE); + const embeddings = await embedTexts(batch.map((entry) => entry.content)); + const documents = batch.map(registryDocumentRow); + const chunks = batch.map((entry, index) => registryChunkRow(entry, embeddings[index] as Vector)); + const documentIds = documents.map((document) => document.id).filter((id): id is string => typeof id === "string"); + + const { data: existingDocuments, error: existingDocumentError } = await supabase + .from("documents") + .select("*") + .in("id", documentIds); + if (existingDocumentError) { + throw new Error(`Registry corpus preflight failed: ${existingDocumentError.message}`); + } + const existingDocumentIds = new Set((existingDocuments ?? []).map((document) => document.id)); + const existingDocumentSnapshots = existingDocuments ?? []; + + const { error: documentError } = await supabase.from("documents").upsert(documents, { onConflict: "id" }); + if (documentError) throw new Error(`Registry corpus document upsert failed: ${documentError.message}`); + + const { error: chunkError } = await supabase.from("document_chunks").upsert(chunks, { onConflict: "id" }); + if (chunkError) { + const insertedDocumentIds = documentIds.filter((id) => !existingDocumentIds.has(id)); + if (insertedDocumentIds.length > 0) { + await supabase.from("documents").delete().in("id", insertedDocumentIds); } + if (existingDocumentSnapshots.length > 0) { + await supabase.from("documents").upsert(existingDocumentSnapshots, { onConflict: "id" }); + } + throw new Error(`Registry corpus chunk upsert failed: ${chunkError.message}`); } - throw new Error(`Registry corpus chunk upsert failed: ${chunkError.message}`); + + documentCount += documents.length; + chunkCount += chunks.length; } - return { documentCount: documents.length, chunkCount: chunks.length }; + return { documentCount, chunkCount }; } export function embedClinicalRegistryRows(supabase: AdminClient, rows: RegistryRecordRow[]) { @@ -327,3 +351,103 @@ export async function embedReloadedOwnerRows( const { chunkCount } = await embed(data ?? []); return chunkCount; } + +function skippedRegistryEmbedResult(reason: RegistryCorpusEmbedResult["reason"]): RegistryCorpusEmbedResult { + return { documentCount: 0, chunkCount: 0, skipped: true, reason }; +} + +export async function reembedClinicalRegistryRecordBySlug( + supabase: AdminClient, + args: { ownerId: string; slug: string; kind?: RegistryRecordKind }, +): Promise { + if (!registryCorpusEmbeddingEnabled()) return skippedRegistryEmbedResult("disabled"); + + let query = supabase.from("clinical_registry_records").select("*").eq("owner_id", args.ownerId).eq("slug", args.slug); + if (args.kind) query = query.eq("kind", args.kind); + const { data, error } = await query.maybeSingle(); + if (error) throw new Error(`Could not load registry record for re-embedding: ${error.message}`); + if (!data) return skippedRegistryEmbedResult("not_found"); + + return embedClinicalRegistryRows(supabase, [data as RegistryRecordRow]); +} + +export async function reembedMedicationRecordBySlug( + supabase: AdminClient, + args: { ownerId: string; slug: string }, +): Promise { + if (!registryCorpusEmbeddingEnabled()) return skippedRegistryEmbedResult("disabled"); + + const { data, error } = await supabase + .from("medication_records") + .select("*") + .eq("owner_id", args.ownerId) + .eq("slug", args.slug) + .maybeSingle(); + if (error) throw new Error(`Could not load medication record for re-embedding: ${error.message}`); + if (!data) return skippedRegistryEmbedResult("not_found"); + + return embedMedicationRows(supabase, [data as MedicationRecordRow]); +} + +export async function reembedDifferentialRecordBySlug( + supabase: AdminClient, + args: { ownerId: string; slug: string; kind?: DifferentialRecordRow["kind"] }, +): Promise { + if (!registryCorpusEmbeddingEnabled()) return skippedRegistryEmbedResult("disabled"); + + let query = supabase.from("differential_records").select("*").eq("owner_id", args.ownerId).eq("slug", args.slug); + if (args.kind) query = query.eq("kind", args.kind); + const { data, error } = await query.maybeSingle(); + if (error) throw new Error(`Could not load differential record for re-embedding: ${error.message}`); + if (!data) return skippedRegistryEmbedResult("not_found"); + + return embedDifferentialRows(supabase, [data as DifferentialRecordRow]); +} + +export function reembedRegistryRecordAfterEdit( + supabase: AdminClient, + target: RegistryCorpusEditTarget, +): Promise { + if (target.corpusKind === "medication") { + return reembedMedicationRecordBySlug(supabase, { ownerId: target.ownerId, slug: target.slug }); + } + + if (target.corpusKind === "differential") { + return reembedDifferentialRecordBySlug(supabase, { + ownerId: target.ownerId, + slug: target.slug, + kind: target.differentialKind, + }); + } + + return reembedClinicalRegistryRecordBySlug(supabase, { + ownerId: target.ownerId, + slug: target.slug, + kind: target.corpusKind, + }); +} + +export async function bestEffortReembedRegistryRecordAfterEdit(args: { + supabase: AdminClient; + target: RegistryCorpusEditTarget; + scope: string; +}) { + if (!registryCorpusEmbeddingEnabled()) return skippedRegistryEmbedResult("disabled"); + + try { + return await reembedRegistryRecordAfterEdit(args.supabase, args.target); + } catch (embedError) { + const errorMessage = embedError instanceof Error ? embedError.message : String(embedError); + console.error( + `[${args.scope}] corpus re-embedding failed after registry edit for ${args.target.ownerId}/${args.target.corpusKind}/${args.target.slug}`, + embedError, + ); + return { + documentCount: 0, + chunkCount: 0, + skipped: true, + reason: "failed", + errorMessage, + } satisfies RegistryCorpusEmbedResult; + } +} diff --git a/supabase/drift-manifest.json b/supabase/drift-manifest.json index c11787153..ae7e65f53 100644 --- a/supabase/drift-manifest.json +++ b/supabase/drift-manifest.json @@ -1,9 +1,9 @@ { - "generated_at": "2026-07-09T08:38:01.221Z", + "generated_at": "2026-07-09T08:52:11.900Z", "generator": "scripts/generate-drift-manifest.ts", "postgres_image": "supabase/postgres:17.6.1.127", - "schema_sha256": "ef8f5cbfeb0fcd58dd65d7e30ff8e93c35e3a3f13d9df0c6e8ae92314a6e0a15", - "replay_seconds": 26, + "schema_sha256": "3a60bf1561142aed58e45a1cf8a9605ccc715dd7358e154c4ad9879b82be4cd5", + "replay_seconds": 16, "snapshot": { "views": [ { diff --git a/supabase/schema.sql b/supabase/schema.sql index 4512e286c..a95337bc1 100644 --- a/supabase/schema.sql +++ b/supabase/schema.sql @@ -2840,6 +2840,12 @@ begin end; $function$; +revoke execute on function public.fail_or_retry_ingestion_job(uuid, uuid, uuid, boolean, text, text, text, timestamp with time zone, text) from public, anon, authenticated; +grant execute on function public.fail_or_retry_ingestion_job(uuid, uuid, uuid, boolean, text, text, text, timestamp with time zone, text) to service_role; + +revoke execute on function public.complete_ingestion_job(uuid, uuid, uuid, text, text) from public, anon, authenticated; +grant execute on function public.complete_ingestion_job(uuid, uuid, uuid, text, text) to service_role; + revoke execute on function public.set_owner_id_from_auth_uid() from public, anon, authenticated; grant execute on function public.set_owner_id_from_auth_uid() to service_role; diff --git a/tests/clinical-search.test.ts b/tests/clinical-search.test.ts index 37ea3ad4c..05894d2e9 100644 --- a/tests/clinical-search.test.ts +++ b/tests/clinical-search.test.ts @@ -149,13 +149,13 @@ describe("clinical search query normalization", () => { it("expands community patients to the local Pts abbreviation for title matching", () => { expect(buildClinicalTextSearchQuery("What is the process for admission of community patients?")).toBe( - "admission community pts", + "admission community patients pts", ); }); it("expands admission and discharge comparisons toward local community patient titles", () => { expect(buildClinicalTextSearchQuery("Compare admission and discharge requirements")).toBe( - "admission discharge community pts", + "admission discharge community patients pts", ); }); diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 7a62c6e61..ddce99cd1 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -175,7 +175,39 @@ describe("RAG structured-output fallback", () => { expect(answer.answer).not.toContain("The relevant source is"); }); - it("uses model synthesis for strong source answers instead of packed source-card labels", async () => { + it("does not answer FBC withhold-threshold lookups from generic monitoring timing facts", async () => { + const answer = await answerFromTextSources("What FBC threshold should withhold clozapine?", [ + source({ + id: "clozapine-fbc-timing", + document_id: "clozapine-doc", + title: "Clozapine Monitoring", + file_name: "Clozapine Monitoring.pdf", + page_number: 15, + section_heading: "FBC monitoring", + content: + "FBC monitoring frequency is weekly for the first 18 weeks. Blood results should be entered before dispensing. Repeat checks may occur within 48 hours when monitoring is incomplete.", + }), + source({ + id: "clozapine-fbc-action", + document_id: "clozapine-doc", + title: "Clozapine Monitoring", + file_name: "Clozapine Monitoring.pdf", + page_number: 16, + section_heading: "FBC result action", + content: + "FBC blood results in the Amber or Red range require clozapine to be withheld and urgent review arranged.", + }), + ]); + + const plainAnswer = answer.answer.replace(/\*\*/g, ""); + expect(answer.grounded).toBe(true); + expect(answer.citations.map((citation) => citation.chunk_id)).toContain("clozapine-fbc-action"); + expect(plainAnswer).toMatch(/amber|red|withheld|withhold/i); + expect(plainAnswer).not.toContain("48 hours"); + expect(answer.unverifiedNumericTokens ?? []).toEqual([]); + }); + + it("uses model synthesis for strong non-direct source answers instead of packed source-card labels", async () => { vi.stubEnv("OPENAI_API_KEY", "test-key"); vi.stubEnv("OPENAI_ANSWER_TIMEOUT_MS", "4321"); vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0"); @@ -184,15 +216,15 @@ describe("RAG structured-output fallback", () => { const clozapineSource = source({ id: "clozapine-monitoring-1", document_id: "clozapine-doc", - title: "Clozapine Monitoring", - file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf", + title: "Medication guideline", + file_name: "medication-guideline.pdf", page_number: 11, section_heading: "Monitoring", content: "Medication point: • Copy of the Consent to Clozapine Treatment Form EMR0270. Medication point: • Prescribe initiation of Clozapine on the WA Adult Clozapine Initiation and Titration form. Medication point: • Ensure consumers complete the Clozapine Monitoring Form on initiation.", similarity: 0.94, hybrid_score: 0.94, - text_rank: 1.2, + text_rank: 0, memory_cards: [ { id: "memory-1", @@ -380,7 +412,9 @@ describe("RAG structured-output fallback", () => { ); expect(answer.routingMode).toBe("extractive"); - expect(answer.routingReason).toMatch(/source_backed_(?:extractive|review)_fallback/); + expect(answer.routingReason).toMatch( + /high_confidence_extractive_retrieval|source_backed_(?:extractive|review)_fallback/, + ); expect(answer.grounded).toBe(true); expect(answer.citations.length).toBeGreaterThan(0); expect(answer.answer).toMatch(/source support|indexed document|supports this query|ECT Procedure/i); @@ -394,15 +428,15 @@ describe("RAG structured-output fallback", () => { const clozapineSource = source({ id: "clozapine-monitoring-template-1", document_id: "clozapine-doc", - title: "Clozapine Monitoring", - file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf", + title: "Medication guideline", + file_name: "medication-guideline.pdf", page_number: 11, section_heading: "Monitoring", content: "Copy the Consent to Clozapine Treatment Form EMR0270, prescribe initiation on the WA Adult Clozapine Initiation and Titration form, and ensure consumers complete the Clozapine Monitoring Form on initiation.", similarity: 0.94, hybrid_score: 0.94, - text_rank: 1.2, + text_rank: 0, }); const rpc = vi.fn(async (name: string) => { if (name === "match_document_chunks_text") return { data: [clozapineSource], error: null }; @@ -894,8 +928,8 @@ describe("RAG structured-output fallback", () => { }); const plainAnswer = answer.answer.replace(/\*\*/g, ""); - expect(generateStructuredTextResult).toHaveBeenCalledTimes(1); - expect(answer.routingMode).toBe("strong"); + expect(generateStructuredTextResult).not.toHaveBeenCalled(); + expect(answer.routingMode).toBe("extractive"); expect(plainAnswer).toContain("must be discontinued immediately"); expect(plainAnswer).not.toContain("Table detailing roles and responsibilities"); expect(plainAnswer).not.toContain("clinical_table"); @@ -1840,14 +1874,14 @@ describe("RAG structured-output fallback", () => { ); const plainAnswer = answer.answer.replace(/\*\*/g, ""); - expect(answer.routingMode).toBe("strong"); + expect(answer.routingMode).toBe("extractive"); expect(answer.grounded).toBe(true); expect(plainAnswer).not.toMatch(/^Monitoring\b/i); expect(plainAnswer).not.toMatch(/^and reported/i); expect(plainAnswer).toMatch(/discontinue.*immediately|discontinued immediately/i); }); - it("uses the same model-first guard for pathway and referral searches", async () => { + it("uses the same extractive guard for pathway and referral searches", async () => { const answer = await answerFromTextSources( "what are ECT referral criteria", [ @@ -1883,7 +1917,7 @@ describe("RAG structured-output fallback", () => { ); const plainAnswer = answer.answer.replace(/\*\*/g, ""); - expect(answer.routingMode).toBe("fast"); + expect(answer.routingMode).toBe("extractive"); expect(answer.grounded).toBe(true); expect(plainAnswer).not.toMatch(/^Referral criteria\b/i); expect(plainAnswer).not.toMatch(/^and document/i); diff --git a/tests/rag-routing.test.ts b/tests/rag-routing.test.ts index c4f7c9bed..86bdae88b 100644 --- a/tests/rag-routing.test.ts +++ b/tests/rag-routing.test.ts @@ -60,6 +60,23 @@ describe("RAG answer routing", () => { expect(selected.reason).toBe("clinical_fast_grounded_synthesis"); }); + it("uses extractive retrieval for direct medication monitoring lookups with title support", () => { + const selected = route("What safety monitoring is required for clozapine?", [ + source({ + title: "Clozapine Prescribing, Administration and Monitoring", + file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf", + content: "Clozapine prescribing, administration, and monitoring requirements.", + similarity: 0.92, + hybrid_score: 0.94, + text_rank: 0.12, + }), + ]); + + expect(selected.mode).toBe("extractive"); + expect(selected.model).toBeNull(); + expect(selected.reason).toBe("high_confidence_extractive_retrieval"); + }); + it("uses the strong model for medication or risk-heavy decision questions", () => { const selected = route("What ANC threshold should stop clozapine?", [source()]); @@ -129,15 +146,49 @@ describe("RAG answer routing", () => { expect(selected.model).toBeNull(); }); - it("uses strong synthesis for safety-critical threshold lookups with strong source support", () => { + it("skips generation when document lookup title support is only incidental", () => { + const selected = route("What does the clozapine gardening equipment checklist require?", [ + source({ + title: "Clozapine Prescribing, Administration and Monitoring", + file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf", + content: "Clozapine prescribing and blood monitoring guidance.", + similarity: 0.86, + hybrid_score: 0.88, + text_rank: 0.01, + }), + ]); + + expect(selected.mode).toBe("unsupported"); + expect(selected.reason).toBe("document_lookup_without_title_support"); + expect(selected.model).toBeNull(); + }); + + it("uses extractive retrieval for safety-critical threshold lookups with strong title support", () => { const selected = route("What ANC threshold should stop clozapine?", [ source({ title: "Clozapine Prescribing and Monitoring", file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf", + content: "ANC threshold table: withhold clozapine and repeat FBC when the result is below threshold.", text_rank: 0.08, }), ]); + expect(selected.mode).toBe("extractive"); + expect(selected.model).toBeNull(); + expect(selected.reason).toBe("high_confidence_extractive_retrieval"); + }); + + it("uses the strong model for clozapine blood threshold queries without explicit action evidence", () => { + const selected = route("What FBC threshold should withhold clozapine?", [ + source({ + title: "Clozapine Prescribing and Monitoring", + file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf", + content: + "FBC monitoring is weekly for the first 18 weeks and then every 4 weeks if blood results are in range.", + text_rank: 0.14, + }), + ]); + expect(selected.mode).toBe("strong"); expect(selected.model).toBe("strong-model"); expect(selected.reason).toBe("clinical_risk_or_complex_query"); @@ -162,7 +213,7 @@ describe("RAG answer routing", () => { expect(selected.reason).toBe("explicit_table_or_source_lookup"); }); - it("keeps medication action questions on model synthesis even when table evidence exists", () => { + it("uses extractive retrieval for listed medication route questions with table evidence", () => { const selected = route("What IM or PO options are listed for agitation?", [ source({ title: "Agitation and Arousal Pharmacological Management", @@ -176,9 +227,9 @@ describe("RAG answer routing", () => { }), ]); - expect(selected.mode).toBe("fast"); - expect(selected.model).toBe("fast-model"); - expect(selected.reason).toBe("clinical_fast_grounded_synthesis"); + expect(selected.mode).toBe("extractive"); + expect(selected.model).toBeNull(); + expect(selected.reason).toBe("high_confidence_extractive_retrieval"); }); it("keeps broad summaries on the fast synthesis path", () => { @@ -205,7 +256,7 @@ describe("RAG answer routing", () => { expect(selected.reason).toBe("broad_clinical_management_synthesis"); }); - it("uses the strong model for explicit multi-document comparisons", () => { + it("uses extractive retrieval for explicit multi-document comparisons with strong support", () => { const selected = route("Compare the admission and discharge requirements", [ source({ id: "chunk-1", document_id: "doc-1", title: "Admission" }), source({ id: "chunk-2", document_id: "doc-2", title: "Discharge" }), @@ -213,8 +264,8 @@ describe("RAG answer routing", () => { source({ id: "chunk-4", document_id: "doc-4", title: "Review" }), ]); - expect(selected.mode).toBe("strong"); - expect(selected.reason).toBe("multi_document_comparison_synthesis"); + expect(selected.mode).toBe("extractive"); + expect(selected.reason).toBe("high_confidence_extractive_retrieval"); }); it("uses the fast model for routine balanced multi-document synthesis", () => { @@ -228,14 +279,14 @@ describe("RAG answer routing", () => { expect(selected.reason).toBe("balanced_multi_document_synthesis"); }); - it("uses strong synthesis for simple two-document comparisons with strong support", () => { + it("uses extractive retrieval for simple two-document comparisons with strong support", () => { const selected = route("Compare admission and discharge requirements", [ source({ id: "chunk-1", document_id: "doc-1", title: "Admission" }), source({ id: "chunk-2", document_id: "doc-2", title: "Discharge" }), ]); - expect(selected.mode).toBe("strong"); - expect(selected.reason).toBe("multi_document_comparison_synthesis"); + expect(selected.mode).toBe("extractive"); + expect(selected.reason).toBe("high_confidence_extractive_retrieval"); }); it("skips generation when retrieval has no plausible support", () => { diff --git a/tests/registry-corpus.test.ts b/tests/registry-corpus.test.ts index 71250ca36..d0505077b 100644 --- a/tests/registry-corpus.test.ts +++ b/tests/registry-corpus.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { buildDefaultMedicationRows } from "../src/lib/medication-fixtures"; import { clinicalRegistryRowsToCorpusEntries, medicationRowsToCorpusEntries } from "../src/lib/registry-corpus"; @@ -71,6 +71,31 @@ describe("registry corpus", () => { expect(entry?.content).toContain("Form: Transport order"); }); + it("keeps re-embed-on-edit hooks inert unless registry corpus embedding is enabled", async () => { + vi.stubEnv("RAG_REGISTRY_CORPUS_EMBEDDING", "false"); + vi.resetModules(); + try { + const { reembedRegistryRecordAfterEdit } = await import("../src/lib/registry-corpus"); + const supabase = { + from: vi.fn(() => { + throw new Error("should not touch Supabase when disabled"); + }), + }; + + await expect( + reembedRegistryRecordAfterEdit(supabase as never, { + corpusKind: "medication", + ownerId: "22222222-2222-4222-8222-222222222222", + slug: "clozapine", + }), + ).resolves.toEqual({ documentCount: 0, chunkCount: 0, skipped: true, reason: "disabled" }); + expect(supabase.from).not.toHaveBeenCalled(); + } finally { + vi.unstubAllEnvs(); + vi.resetModules(); + } + }); + it("maps scalar medication tags into metadata.tags", () => { const rows = buildDefaultMedicationRows("22222222-2222-4222-8222-222222222222") as MedicationRecordRow[]; const [entry] = medicationRowsToCorpusEntries(rows.map((row) => ({ ...row, tag: "alcohol" }))); diff --git a/tests/retrieval-query-variants.test.ts b/tests/retrieval-query-variants.test.ts index 829b4bdee..c80bd2d39 100644 --- a/tests/retrieval-query-variants.test.ts +++ b/tests/retrieval-query-variants.test.ts @@ -13,6 +13,7 @@ import { relaxVariantToOrQuery, shouldRelaxWeakTextMatches, } from "../src/lib/rag"; +import { maxTextRpcQueryVariants } from "../src/lib/rag-retrieval-variants"; import type { SearchResult } from "../src/lib/types"; function result(overrides: Partial = {}): SearchResult { @@ -59,6 +60,15 @@ describe("retrieval query variants", () => { expect(variants.length).toBeLessThanOrEqual(4); }); + it("prioritizes amber/red blood-result variants for clozapine FBC withhold questions", () => { + const query = "What FBC threshold should withhold clozapine?"; + const variants = buildRetrievalQueryVariants(query, analyzeClinicalQuery(query)); + + expect(variants).toContain("clozapine blood results amber red range"); + expect(variants.indexOf("clozapine blood results amber red range")).toBeLessThan(maxTextRpcQueryVariants); + expect(variants.length).toBeLessThanOrEqual(4); + }); + it("adds document-title-focused variants for document lookup intent", () => { const query = "Where is the active community patients in ED document?"; const analysis = analyzeClinicalQuery(query); @@ -69,6 +79,16 @@ describe("retrieval query variants", () => { expect(variants.length).toBeLessThanOrEqual(4); }); + it("adds exact title variants for community admission lookups", () => { + const query = "What is the process for admission of community patients?"; + const variants = buildRetrievalQueryVariants(query, analyzeClinicalQuery(query)); + + expect(variants).toEqual( + expect.arrayContaining(["admission of community patients", "admission community patients"]), + ); + expect(variants.length).toBeLessThanOrEqual(4); + }); + it("adds shortened core variants for long natural-language threshold questions", () => { const query = "Please can you tell me exactly when I should withhold clozapine based on ANC and FBC threshold values in the monitoring table?";