From 321024ff064114a3f877d081e1432c821f56d67c Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 02:54:40 +0800 Subject: [PATCH 1/8] feat(rag): offline re-index & retrieval hardening (config-gated, defaults unchanged) Lands the offline re-index / retrieval-improvement workstream. Every behavioural change ships default-off so the merge is behaviour-neutral; defaults reproduce the exact prior behavior and only flip after eval gates. - W6 ranking-config: extract the second-stage rerank weights, document-diversity demotion, and freshness decay into src/lib/ranking-config.ts with an optional RAG_RANKING_CONFIG JSON override. Defaults equal the prior constants; diversity penalty defaults to 0 (OFF). - CI-1 chunker: CHUNK_STRATEGY=page|document. "page" (default) is byte-for-byte the current page-bounded chunker; "document" is structure-aware cross-page chunking for the eval-gated shadow re-index only. - CI-8/9 embedding integrity: embedding-dimensions guard + embed-texts integrity checks; OPENAI_EMBEDDING_BATCH_SIZE (IDX-C3) splits a full-corpus re-embed into <=2048-input batches so it can't exceed the OpenAI request ceiling. - CI-4 stable chunk_key, CI-14 synonym recall (clinical-vocabulary), W3/CI-6 OCR-quality signals (index-quality), plus check-indexing coverage. - #6 reindex-eval-gate: pure, unit-tested GO/NO-GO non-degradation gate for shadow re-index cutover (absolute bars + no-regression vs live baseline); design doc in docs/reindex-shadow-harness-design.md. Test coverage added across chunking, embedding-dimensions, embed-texts integrity, index-quality, ranking-config, reindex-eval-gate, retrieval-query-variants. Co-Authored-By: Claude Fable 5 --- .env.example | 9 + docs/reindex-shadow-harness-design.md | 130 +++++++++ scripts/check-indexing.ts | 15 + scripts/fixtures/rag-retrieval-golden.json | 27 ++ src/lib/chunking.ts | Bin 21444 -> 30021 bytes src/lib/clinical-search.ts | 5 +- src/lib/clinical-vocabulary.ts | 9 +- src/lib/embedding-dimensions.ts | 31 ++ src/lib/env.ts | 15 + src/lib/index-quality.ts | 17 +- src/lib/openai.ts | 90 +++--- src/lib/rag.ts | 52 +++- src/lib/ranking-config.ts | 177 ++++++++++++ src/lib/reindex-eval-gate.ts | 314 +++++++++++++++++++++ tests/chunking.test.ts | 141 ++++++++- tests/embed-texts-integrity.test.ts | 54 ++++ tests/embedding-dimensions.test.ts | 27 ++ tests/eval-retrieval.test.ts | 17 ++ tests/index-quality.test.ts | 56 ++++ tests/ranking-config.test.ts | 102 +++++++ tests/reindex-eval-gate.test.ts | 142 ++++++++++ tests/retrieval-query-variants.test.ts | 22 ++ worker/main.ts | 5 + 23 files changed, 1404 insertions(+), 53 deletions(-) create mode 100644 docs/reindex-shadow-harness-design.md create mode 100644 src/lib/ranking-config.ts create mode 100644 src/lib/reindex-eval-gate.ts create mode 100644 tests/ranking-config.test.ts create mode 100644 tests/reindex-eval-gate.test.ts diff --git a/.env.example b/.env.example index 92ce6b6a7..4c83345a4 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,9 @@ OPENAI_FAST_ANSWER_MODEL=gpt-5.5 OPENAI_STRONG_ANSWER_MODEL=gpt-5.5 OPENAI_MAX_OUTPUT_TOKENS=4000 OPENAI_QUERY_CACHE_SIZE=200 +# Max inputs per embeddings request (OpenAI caps a request at 2048 inputs / ~300k tokens). +# embedTexts splits a full-corpus re-embed into batches of this size. +OPENAI_EMBEDDING_BATCH_SIZE=256 OPENAI_VISION_MODEL=gpt-5.5 OPENAI_VISION_IMAGE_DETAIL=auto OPENAI_REQUEST_TIMEOUT_MS=45000 @@ -50,6 +53,9 @@ OPENAI_TEXT_VERBOSITY=low # openai: always attempt OpenAI (legacy behaviour). # offline: never call OpenAI; lexical retrieval + deterministic source-only answers only. RAG_PROVIDER_MODE=auto +# Optional JSON override for app-layer ranking weights (see src/lib/ranking-config.ts). +# Omit for current defaults. Example (enable diversity demotion + linear freshness): +# RAG_RANKING_CONFIG={"documentDiversityPenalty":0.03,"freshness":{"mode":"linear"}} RAG_ANSWER_CACHE_TTL_MS=300000 RAG_ANSWER_CACHE_SIZE=100 RAG_SEARCH_CACHE_TTL_MS=60000 @@ -74,6 +80,9 @@ MAX_IMPORT_JOBS_PER_RUN=5 MAX_IMPORT_BYTES_PER_RUN=157286400 CHUNK_SIZE=2000 CHUNK_OVERLAP=200 +# Chunking strategy: "page" (default, page-bounded) or "document" (structure-aware, +# cross-page). Only set "document" for the eval-gated shadow re-index. +CHUNK_STRATEGY=page WORKER_POLL_MS=30000 WORKER_BATCH_SIZE=3 WORKER_CONCURRENCY=1 diff --git a/docs/reindex-shadow-harness-design.md b/docs/reindex-shadow-harness-design.md new file mode 100644 index 000000000..4ead993bf --- /dev/null +++ b/docs/reindex-shadow-harness-design.md @@ -0,0 +1,130 @@ +# Shadow Re-Index + Eval-Gate Harness — Implementation Runbook + +Status: **design ready, not yet applied** — the SQL here touches retrieval and MUST be +validated against the live `Clinical KB Database` before use (see "Mandatory validation"). +The offline decision core (`src/lib/reindex-eval-gate.ts` — `decideReindexGate`) is built and +unit-tested; this doc specifies the remaining live pieces so they are turnkey when the billed +OpenAI key + Supabase service secrets are available. + +## Goal + +Build the new (better-chunked, re-enriched) index for a document into a **staged** generation +alongside the live committed one, run the golden retrieval + RAG quality evals **against the +staged generation**, and atomically cut over per batch **only if** `decideReindexGate` returns +`GO` (staged ≥ baseline on every metric and all absolute release bars met). Live answers are +never exposed to an unproven generation. + +The atomic-generation machinery already exists: `index_generation_id` on every artifact, +`commit_document_index_generation` (migration `20260628000000`), the two filter functions +`is_committed_document_generation` / `is_committed_artifact_generation`, and +`cleanup_abandoned_document_index_generations` (`20260629000000`). The only missing capability +is **letting retrieval read a staged (uncommitted) generation for an eval session**. + +## Rejected approaches + +- **Supabase preview branch.** Branches are created from migrations and do **not** carry + production table data or Storage objects, so a re-index of the existing ~2,065-doc corpus + has nothing to run against on a branch. Not viable for evaluating a re-index of live content. +- **Add a `p_index_generation_id` param to the four hybrid RPCs** + (`match_document_chunks_hybrid`, `match_document_memory_cards_hybrid`, + `match_document_index_units_hybrid`, `match_document_embedding_fields_hybrid`). Rejected: + these are the delicate hot-path query bodies whose last edit caused the 130s + seqscan regression (see `hybrid-rpc-drift-bug` memory). Editing all four multiplies the risk. + +## Recommended design: GUC override on the two filter functions + +Retrieval's generation filter funnels through exactly two small `language sql stable` +functions. Override those — gated by a session-local custom GUC that is **unset in +production** — so the hot-path query bodies are never touched. + +```sql +-- Both functions currently (20260628000000): language sql stable, reading only jsonb. +-- Adding current_setting(..., true) keeps them STABLE (no volatility change) and, when the +-- GUC is unset (production), returns byte-identical results to today. +create or replace function public.is_committed_document_generation(row_generation uuid, document_metadata jsonb) +returns boolean language sql stable set search_path = public, extensions, pg_temp as $$ + select case + when nullif(current_setting('rag.eval_generation_id', true), '') is not null then + -- eval session: read ONLY the staged generation under evaluation + row_generation::text = nullif(current_setting('rag.eval_generation_id', true), '') + else + row_generation is null + or row_generation::text = nullif(coalesce(document_metadata, '{}'::jsonb)->>'index_generation_id', '') + end; +$$; + +create or replace function public.is_committed_artifact_generation(artifact_metadata jsonb, document_metadata jsonb) +returns boolean language sql stable set search_path = public, extensions, pg_temp as $$ + select case + when nullif(current_setting('rag.eval_generation_id', true), '') is not null then + nullif(coalesce(artifact_metadata, '{}'::jsonb)->>'index_generation_id', '') = current_setting('rag.eval_generation_id', true) + else + nullif(coalesce(artifact_metadata, '{}'::jsonb)->>'index_generation_id', '') is null + or nullif(coalesce(artifact_metadata, '{}'::jsonb)->>'index_generation_id', '') = + nullif(coalesce(document_metadata, '{}'::jsonb)->>'index_generation_id', ''); + end; +$$; +``` + +- Namespaced custom GUCs (`rag.*`) can be set per-session/transaction via `set_config` with no + prior declaration — the same mechanism as the existing session-local `hnsw.ef_search='100'`. +- Production code never sets `rag.eval_generation_id`, so the `else` branch runs and behavior + is unchanged. This is the "defaults unchanged until eval-gated" contract at the SQL layer. +- The SQL is embedded here (not shipped as an applyable migration) deliberately, so it cannot + reach the live DB without the review + validation below. + +## Eval-script extension + +Add an optional `--generation-id ` to `scripts/eval-retrieval.ts` and +`scripts/eval-quality.ts`. When present, the eval opens a DB session and issues +`select set_config('rag.eval_generation_id', $1, false)` before the retrieval RPC calls (and +clears it after), so every retrieval in that eval run reads the staged generation. The eval +summaries returned are the exact shapes `decideReindexGate` already consumes. + +## Driver: `scripts/reindex-shadow.ts` + +1. Preflight: `check:supabase-project`, `supabase:recovery-status`, `reindex:health` (abort if + `supabase_unavailable` or queue in recovery — per `docs/reindex-runbook.md`). +2. Capture the **baseline** eval summaries against the live committed index (no GUC). +3. For each batch of documents: + a. Re-index into a new staged `index_generation_id` **without** committing (new chunker via + `CHUNK_STRATEGY=document`, re-enrichment, one stamped `rag_indexing_version`). + b. Run `eval:retrieval:quality` and `eval:quality` with `--generation-id `. + c. `decideReindexGate({ baseline*, candidate* })`. + d. **GO** → `commit_document_index_generation` per document in the batch (atomic swap). + **NO_GO** → leave live intact, record the failing metrics, and + `reindex:cleanup-staged` the abandoned staged rows. +4. Emit a per-generation reindex report (baseline vs candidate table + per-doc outcomes). + +Wave 3 (targeted) runs the same driver over only OCR-poor / `extraction_quality=poor` docs; +Wave 4 pilot runs it over ~25–50 docs incl. the golden set as the GO/NO-GO gate for going +corpus-wide. + +## Mandatory validation before the filter-function change is used (drift-bug guard) + +The two filter functions are on the retrieval hot path. Before relying on the override: + +- Apply to the live DB, then run `npm run profile:retrieval` (or `explain_retrieval_rpc`) and + confirm the four hybrid RPCs still use the HNSW index scan — **no seqscan**, latency + unchanged. The `case`/`current_setting` addition must not defeat function inlining. +- `npm run eval:retrieval:quality` before vs after (GUC unset) must be identical — proves zero + production-path change. +- Keep the functions `language sql` (not plpgsql) and `stable`. +- Run `npx supabase db advisors --linked` and update `docs/supabase-migration-reconciliation.md`. +- Only after this passes, use `--generation-id` for staged evals. + +## W8 (#11) — extend `search_schema_health()` execution smoke (additive, lower-risk) + +`search_schema_health()` is a read-only diagnostic (not hot path), so this is safer to add: + +- Assert the four embedding columns are `vector(N)` with `N == EMBEDDING_DIMENSIONS`. +- Assert the `search_tsv` config in use matches the intended text-search config. +- Assert zero `rag_indexing_version` / `rag_enrichment_version` skew across `documents`. +- Assert all four HNSW indexes are present (already partly checked) and no legacy IVFFlat. +- Surface each as a `missing[]` entry so it flows into `check:indexing` + setup-status. + +## Rollback + +Every step is reversible: a NO_GO batch never commits (cleanup removes staged rows); the +filter-function change reverts by restoring the `20260628000000` definitions; `--generation-id` +is opt-in. The live committed generation is untouched until a batch passes the gate. diff --git a/scripts/check-indexing.ts b/scripts/check-indexing.ts index b2f6c2b8f..5ebd84725 100644 --- a/scripts/check-indexing.ts +++ b/scripts/check-indexing.ts @@ -317,6 +317,21 @@ async function main() { requireServerEnv(); requireOpenAIEnv(); + // IDX-C2 fail-fast: verify the configured embedding dimension matches the schema's + // vector(N) columns before any indexing/DB work, so a model/dimension misconfiguration + // is caught here instead of silently corrupting retrieval at write time. + const { readFile } = await import("node:fs/promises"); + const { fileURLToPath } = await import("node:url"); + const { describeSchemaDimensionMismatch } = await import("@/lib/embedding-dimensions"); + const schemaPath = fileURLToPath(new URL("../supabase/schema.sql", import.meta.url)); + const dimensionProblem = describeSchemaDimensionMismatch( + env.EMBEDDING_DIMENSIONS, + await readFile(schemaPath, "utf8"), + ); + if (dimensionProblem) { + throw new Error(`Embedding dimension check failed: ${dimensionProblem}`); + } + const prereqs = await checkPythonPdfPrerequisites(); if (!prereqs.ok) { throw new Error(`PDF/OCR prerequisite check failed: ${prereqs.detail}`); diff --git a/scripts/fixtures/rag-retrieval-golden.json b/scripts/fixtures/rag-retrieval-golden.json index 89daece19..e9191cba5 100644 --- a/scripts/fixtures/rag-retrieval-golden.json +++ b/scripts/fixtures/rag-retrieval-golden.json @@ -210,5 +210,32 @@ "expectedContentTerms": ["alcohol", "withdrawal", ["ciwa", "score", "threshold"]], "topK": 12, "expectTableEvidence": false + }, + { + "id": "clozapine-cbc-abbreviation-threshold", + "query": "What CBC (complete blood count) or neutrophil threshold should withhold clozapine?", + "expectedQueryClass": "table_threshold", + "expectedDocumentSubstrings": ["Clozapine"], + "expectedContentTerms": [ + "clozapine", + ["anc", "neutrophil"], + ["fbc", "full blood count", "wbc", "wcc"], + ["withhold", "cease", "stop", "red"] + ], + "topK": 12, + "expectTableEvidence": true + }, + { + "id": "clozapine-wcc-abbreviation-threshold", + "query": "What white cell count (WCC) threshold should withhold clozapine?", + "expectedQueryClass": "table_threshold", + "expectedDocumentSubstrings": ["Clozapine"], + "expectedContentTerms": [ + "clozapine", + ["wcc", "wbc", "white cell", "fbc", "full blood count"], + ["withhold", "cease", "stop", "red"] + ], + "topK": 12, + "expectTableEvidence": true } ] diff --git a/src/lib/chunking.ts b/src/lib/chunking.ts index 52b2ba53e6ee796c97e7e1c8ccb16af2af9fa479..d5c526265cca92c8d228138055a0c054cef9ae3a 100644 GIT binary patch delta 8529 zcmb_h-ESOM6(^zrvQ&kZB>l)oueTqWP1X}9ZAfI-aT3Rgox~rB-4^N^cf5OdcgXI{ zFds>b>rN3oR6HQWMF=D$Bwmn;K%h!U@Z6R^0z@f1fdm4CkboC{=iGZ|XV+;fq*iLY zd*{x%=brOB=lssuH-EJ2tv~NN>i42BPQ{jJ#L`RUte14ewrIs+PgH}jDaRY}W|W52 ziOM(sn;$qW9D{2&d5 zyd~pJz5PfcdNTEzUh0V?6kY{#V>#*vO?gK&yg;O~+ZDGvexsvbkgv!2n()w?IkZwWgLjfiV*7lr%&&CS*<^P?id8LvY?UrVIYDq?!g3K zmeb^~t8#NXB%!GTY51#t_RxOwF?C}Zb-jjk?s7z3R7F+&?x91un}*lI+cKW^66w?? zD%%yZ#`N6i)8p_EjEuY2MHD7JozK3FBpe*1!nY7=g)#Orf?NTeU@WLgp0rl2(>D^F zNPQ14XmrB3E^=}|?JO>aukubR{_yIz#lqqW31Y$P8dW%nJjRcK8jCiA96rJ80YPCF zH>B9~Ww)8I{DhWo%1u|S5K4O9O`sx{xj`%Lf&d~*G9bG$1!lZP2UltMsYiEDNII!2 z0YZGB3jl^$e9ONjNfRjM2fUTHvD#P<U9%JP@gP@Xe388MV0Snw?h z17JuWeJu=cW>Fa)^n~vV9KJ|EG$$Ka+84DpRIRP*$p1BMEiPwFj^3rMb&r2=!h06!-r7f<$c}2Z;`BC-b%jebK@c*gV zah1&0KJ7v^6g=lb7)YRLvQ)=gKkl)|E%pAFrHNhp*Pv!qx6aw`&#k*465#^b7J%L&gya zdFT3b>>}1ygT#%pq~mPChv%OgXhjFI-oa9J0W&+yIb9jF z(@uR?Mg4WYs`f5aRekvpb#3`El`S9M15K$Q)hp_!3#Shi-$*OL#-V)5DhM^Q@Zh2T zv3Jo!U0s~GXS)Vr-&#DOe!ciuZu7hj50dG+F5j?>K68L6!l)SI`s1`#j+-^H_|y6&tF+O zu$!N&?=2n9!B?XcC3{^<4xvTYi^`6%63n9kb-;*Xq+u3Eig1n_*ny&4dQN%I+*^~9 ze4U~m5ziuu4B+;>$Z_avO-!COZy-Z3k)h;pDU2z^oxwUmFSvcjgQ-C$lM3u3`dSwT z@%1C3<#oFoUgPGBjsZAS`%(}*w2KHQuin_duefGE$Kc89H%8TOZae|NJ`TVh1Yn;a zz~ zO9f;j^Se!kKr*2VoQy{sQ-f6yLeW3SHp(10B^no76iS7H1(BZfVEu3S=bqiI{(kjO zAB(wYtJkiL?5e3p=cf-A@0zSoz&L?9-d%WjPmvVV*XKvn=wfv@IbvD;dr9!888veK z@L9u8D2ZB2_iF>api4vlSpko6vAU&Rxqeu^e*I7`w%7Vw%E3AgDi>v`YeSnT&=z|| zyOoY8ds46VYXy~xzGT{6jx)V`yQY4)T-~q94i!`CnwGGYIE*@RM&t$EYVN2>l;-7* z<@czL_7?A30F6m!Q7eEhgjJF=(bRSHHtOcGM7-i3Q$JBQ4S7P7a{^(QY8nCFgWlP; zggVI)YzTW~2`8!Hvj;}pNHYd0g$*+h;LAt9K!*Vj)Lnyo(YMh=rov_r`j z{9=g4p$F@VM$Tc7QWJ#}G>6A|8{qdnv@08$Mo{1H1t5Jms)q6&l6cbD>Ps)v-Q#d0 zLjg|wHiFM|Pho%<^;#{I;e;iub6Px3!q#&nz~ee^=~PFOtuMK83uuO5ENpWRvH+hZ zVN!0e=mMokE!@P2TnWiUiEVYCC(BzW=hSDM)tL~BWr(h>)rJ?C?f5Ohgf($>MHbIY ziW4=MmAE@!5L}1ZB3S}t3?84<>pE|<6_K||T8=nD|0v)%v<6~~-V}m*WKtIbw3S;n z?XvPUVRdlnG2rTUyu_i;wVD=n3_599MlW3n>io2hL>jFha7tue#Zrbs*JZaZO@%&8Ca182-VajGs@z)>;gp0Z3GDv|mH=|`u_u3-CG zF6qd8NICYy*-7z~c=E|1eFc^a6->Lwb$IxK>|ZjHRvSuV3N&^S2l$+$-uh!fI^FYVx z$`*=0YG)^9hw(y*F;Izsk4BMtXwQx(MKxbe;|!S(a}Mt|noiKVp=8O$9XT~WQFDu^ z!{`EOEs<@i5|k%tglKX$YPZCQh0=pr!9bBg!Y?sB%hNDVRdsblVyH)}|j;_nfa&3A4wJ z0<<*z#iS{V&`4%M*S~4Hd~VqtJ>b15WdzglOb~-D#y=SV^N6O8QbE>099Vmy=9Fwu z%%or+48hKzm|;+5$OWST!%UiMW3CZ`q{NtPVt|G8h<4NzMcB|)KRI`Lz6eux%#MU_ zZJ<*b+k+!xi=$Krc%Q+cFFP^P0sW; zio4Z<#@X`t{seB2>rA1zp+-{LU)Iw?OM;w|4h5yBQi8IAP9L#=503iE+}?Z3JJmb0 zhpSrWq9DK~Jk>HM06<$b)R@pCsU782R}yVQqb=8Eu1u?IcztzE)z2SY7$}Gw!r&w9 zFFk(%EjBgT9lUjc`Ll$0?K7d?oqjkU-bkpX|EwOStGC~YgoFJ-vs=>Kj(NG(0?C@R zS>twwq-+b>1@9uQ3W0_!=;+I>GWLMvLpjjgun@XBeEO(3u|`=tm*zmyn@c8?h6ymx zA27CgpZK2@W$ifNH{FWBV`djuhQw_l@aCyihF9Fk}C)B0Wy^`83FTX+sSJ(xxyS z(;x#=r5#k6=(m{R192XPS;c8N{XntfINd7@9Y83uUk<)!7knuUMj>8>GOGPvdPtayES<;MAynaiIS93s|@ ZRWwOEII~WBhA|WR2ZnxwPwDsX{{eA0Q!)Sm delta 900 zcmZ8eOHUI~7)_frk%y27Y9Xau8VN1pSOQTQ5Hv)SK51oOA=Jra=1M2f>15^(0ZHTQ z7vT8;E=)8oTw&O;Fww1X<;p}2E{scc1smtiltR79yC4B-X4vNOIo9>Q)a86*ThQX|}Sq!n%|>2YqJrzg1ioGx=SmFep=l@*$* zxR2dR+yk2F$D_nzV(DoafcslW@qp=JBI!2`Sym469-u} z$`xwWuo=;Gi>i!VZ<4I$bbQK1@H01xv-!)ok&ohTeiGaHB=+)V8xOA~{I-&FG2C1F z)4SEkU{U5+#$?^EwLV)WZ3)ugvAH}Ouhm(#>=>)IU^;d=-cS&&syY>FFj>dJ@<32F z@U`3@b6e(WSvNGgA^MIjtWjfyl}G$$t`=hWD8=ztiUNf}j4i~lYn{b@VGL)AT^Mk0 zve=8R(lBmLhw-%7-|i1nPw=uhdYS_||9Dx~Y&3kJT58-LR>dY zhY3eKnVBJ5wZw5CA$*^5(*zuuYG~ZOMu=T?ES1W(sTj6Mmg)=Pkw@IT zOWm$o65U{p<)E`9gggztm-@mdo8~CpP^o1y5z5?R=YJ#wsb``YYyP3qNwQ28O*dBL mwQ6= 8 ? -0.015 : 0; - const reviewBoost = reviewYearsAgo === null ? 0 : reviewYearsAgo >= 5 ? -0.01 : 0; + const freshnessBoost = freshnessDecayPenalty(publicationYearsAgo, "publication", rankingConfig.freshness); + const reviewBoost = freshnessDecayPenalty(reviewYearsAgo, "review", rankingConfig.freshness); const imageBoost = imageEvidenceSignal(query, result); const sectionBoost = sectionMatchBoost(query, result); const sectionedLookupBoost = querySignal.sectionedLookup ? 0.02 : 0; diff --git a/src/lib/clinical-vocabulary.ts b/src/lib/clinical-vocabulary.ts index aabecc0c9..ee936ec88 100644 --- a/src/lib/clinical-vocabulary.ts +++ b/src/lib/clinical-vocabulary.ts @@ -23,7 +23,10 @@ const entries: ClinicalVocabularyEntry[] = [ }, { canonical: "full blood count", - aliases: ["fbc", "blood count", "white cell count", "wcc", "wbc"], + // CI-14: FBC (AU/UK) and CBC (US) are the same test. Documents in this corpus use + // "FBC"; without the CBC alternates a clinician who searches "CBC" retrieves nothing. + // "cbc" is kept in the first 6 aliases so it survives the expansion slice bidirectionally. + aliases: ["fbc", "cbc", "complete blood count", "blood count", "white cell count", "wcc", "wbc"], type: "lab", weight: 1.2, }, @@ -52,6 +55,10 @@ const entries: ClinicalVocabularyEntry[] = [ { canonical: "emergency department", aliases: ["ed"], type: "service", weight: 1.05 }, { canonical: "intramuscular", aliases: ["im"], type: "workflow", weight: 1.05 }, { canonical: "oral", aliases: ["po"], type: "workflow", weight: 1.05 }, + // CI-14: complete the administration-route family alongside IM/PO so route-abbreviation + // queries ("subcut", "SC", "sublingual", "SL") retrieve the same chart rows. + { canonical: "subcutaneous", aliases: ["sc", "subcut", "subcutaneously"], type: "workflow", weight: 1.05 }, + { canonical: "sublingual", aliases: ["sl", "sublingually"], type: "workflow", weight: 1.05 }, { canonical: "as required", aliases: ["prn"], type: "workflow", weight: 1.05 }, { canonical: "clozapine", diff --git a/src/lib/embedding-dimensions.ts b/src/lib/embedding-dimensions.ts index 2e8239cac..c753c9271 100644 --- a/src/lib/embedding-dimensions.ts +++ b/src/lib/embedding-dimensions.ts @@ -17,6 +17,37 @@ function expectedEmbeddingDimensions() { export const EXPECTED_EMBED_DIM = expectedEmbeddingDimensions(); +// IDX-C2 (fail-fast): the embedding dimension is declared in three places that MUST agree +// — the configured EMBEDDING_DIMENSIONS, the OpenAI model's output, and the schema's +// vector(N) columns. A mismatch corrupts ingestion silently (every write throws, or worse, +// a future model change slips through). These helpers let a startup check compare the +// configured dimension against supabase/schema.sql offline, before any DB work. +export function parseSchemaVectorDimensions(schemaSql: string): number[] { + const dims = new Set(); + for (const match of schemaSql.matchAll(/\bvector\((\d+)\)/gi)) { + const value = Number(match[1]); + if (Number.isInteger(value) && value > 0) dims.add(value); + } + return [...dims].sort((a, b) => a - b); +} + +// Returns a human-readable problem description, or null when the configured dimension is +// consistent with every vector(N) column in the schema. Absence of any vector column is +// treated as a problem (the schema we were handed is not the RAG schema). +export function describeSchemaDimensionMismatch(configuredDim: number, schemaSql: string): string | null { + const schemaDims = parseSchemaVectorDimensions(schemaSql); + if (schemaDims.length === 0) { + return "No vector(N) columns found in schema.sql — cannot verify embedding dimensions."; + } + if (schemaDims.length > 1) { + return `schema.sql declares inconsistent vector dimensions ${schemaDims.join(", ")}; all embedding columns must share one dimension.`; + } + if (schemaDims[0] !== configuredDim) { + return `EMBEDDING_DIMENSIONS=${configuredDim} does not match schema vector(${schemaDims[0]}). Re-embedding with a mismatched dimension corrupts retrieval; align OPENAI_EMBEDDING_MODEL, EMBEDDING_DIMENSIONS, and the schema before indexing.`; + } + return null; +} + export function assertEmbeddingDim(vec: unknown, context: string): number[] { if (!Array.isArray(vec)) { throw new Error(`${context} embedding must be an array.`); diff --git a/src/lib/env.ts b/src/lib/env.ts index d48254ba1..bbfb7de1c 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -27,6 +27,12 @@ const envSchema = z.object({ // if output is still cut off, createTextResult now flags it as truncated (GEN-C1). OPENAI_MAX_OUTPUT_TOKENS: z.coerce.number().int().positive().default(4000), OPENAI_QUERY_CACHE_SIZE: z.coerce.number().int().nonnegative().default(200), + // Max inputs per embeddings request. The OpenAI embeddings endpoint caps a single + // request at 2048 inputs / ~300k tokens; a full-corpus re-embed of ~400k texts in one + // call would exceed that and fail (IDX-C3). embedTexts splits unique inputs into + // batches of this size. 256 keeps total tokens well under the ceiling even for the + // largest (narrative-profile) chunks while staying far below the 2048 input cap. + OPENAI_EMBEDDING_BATCH_SIZE: z.coerce.number().int().positive().max(2048).default(256), OPENAI_VISION_MODEL: z.string().default("gpt-5.5"), OPENAI_VISION_IMAGE_DETAIL: z.enum(["auto", "low", "high"]).default("auto"), OPENAI_REQUEST_TIMEOUT_MS: z.coerce.number().int().positive().default(45000), @@ -59,6 +65,10 @@ const envSchema = z.object({ // - "offline": never call OpenAI at all (no embeddings, no generation); lexical retrieval // + deterministic source-only answers only. Fails closed when evidence is weak. RAG_PROVIDER_MODE: z.enum(["auto", "openai", "offline"]).default("auto"), + // Optional JSON override for app-layer ranking weights (see src/lib/ranking-config.ts). + // Lets tuning/eval experiments adjust the second-stage rerank weights, document-diversity + // demotion, and freshness decay WITHOUT a code change. Omitted/malformed => current defaults. + RAG_RANKING_CONFIG: z.string().optional(), RAG_ANSWER_CACHE_TTL_MS: z.coerce.number().int().nonnegative().default(300000), RAG_ANSWER_CACHE_SIZE: z.coerce.number().int().nonnegative().default(100), RAG_SEARCH_CACHE_TTL_MS: z.coerce.number().int().nonnegative().default(60000), @@ -87,6 +97,11 @@ const envSchema = z.object({ MAX_IMPORT_BYTES_PER_RUN: z.coerce.number().int().positive().default(157286400), CHUNK_SIZE: z.coerce.number().int().positive().default(2000), CHUNK_OVERLAP: z.coerce.number().int().nonnegative().default(200), + // Chunking strategy (CI-1). "page" (default) = the current page-bounded chunker, byte-for- + // byte unchanged. "document" = structure-aware chunking that lets a chunk span a page break + // within a section, so dose tables / monitoring protocols split across a page boundary stay + // together. Enabled only for the eval-gated shadow re-index, never silently for live users. + CHUNK_STRATEGY: z.enum(["page", "document"]).default("page"), WORKER_POLL_MS: z.coerce.number().int().positive().default(1500), WORKER_BATCH_SIZE: z.coerce.number().int().positive().default(25), WORKER_CONCURRENCY: z.coerce.number().int().positive().default(8), diff --git a/src/lib/index-quality.ts b/src/lib/index-quality.ts index 05710beff..d66b0c470 100644 --- a/src/lib/index-quality.ts +++ b/src/lib/index-quality.ts @@ -29,6 +29,8 @@ export type IndexQualityMetrics = { extracted_image_count: number; searchable_image_count: number; ocr_page_count?: number; + // CI-6: pages flagged image-only but not OCR'd (JS fallback without Python OCR prereqs). + needs_ocr_page_count?: number; }; export function hashIndexQualityText(text: string) { @@ -93,6 +95,8 @@ export function assessDocumentIndexQuality(args: { const sectionPathCoverage = chunkCount ? sectionPathCount / chunkCount : 0; const tableExtractionCoverage = tableImages.length ? tableImagesWithRows.length / tableImages.length : null; const ocrCoverage = args.metrics.page_count ? Number(args.metrics.ocr_page_count ?? 0) / args.metrics.page_count : 0; + const needsOcrPageCount = Number(args.metrics.needs_ocr_page_count ?? 0); + const needsOcrCoverage = args.metrics.page_count ? needsOcrPageCount / args.metrics.page_count : 0; const averagePageTextChars = args.metrics.page_count ? args.metrics.text_character_count / args.metrics.page_count : 0; @@ -122,6 +126,10 @@ export function assessDocumentIndexQuality(args: { issues.push("low structured visual extraction confidence"); } if (args.metrics.text_character_count < 80) issues.push("low extracted text volume"); + if (needsOcrPageCount > 0) + issues.push( + `image-only pages not OCR'd (${needsOcrPageCount} of ${args.metrics.page_count}); install Python OCR prerequisites`, + ); if (args.sectionCount === 0) issues.push("no structured sections"); if (args.memoryCardCount === 0) issues.push("no memory cards"); if (!fieldTypes.has("document_title")) issues.push("missing document title embedding"); @@ -145,8 +153,15 @@ export function assessDocumentIndexQuality(args: { const chunkPageCoverage = Math.min(1, chunkCount / Math.max(args.metrics.page_count * 0.75, 1)); qualityScore -= Math.max(0, 0.82 - chunkPageCoverage) * 0.08; } + qualityScore -= Math.min(0.5, needsOcrCoverage * 0.6); qualityScore = Math.max(0, Math.min(1, qualityScore)); - const extractionQuality = qualityScore >= 0.82 ? "good" : qualityScore >= 0.52 ? "partial" : "poor"; + let extractionQuality = qualityScore >= 0.82 ? "good" : qualityScore >= 0.52 ? "partial" : "poor"; + // CI-6: a scanned / image-only PDF whose pages could not be OCR'd is effectively unindexed + // even when incidental header text nudges the heuristics up to "partial". Force "poor" so it + // is visible to eval governance and never silently treated as a good index. + if (needsOcrPageCount > 0 && (needsOcrCoverage >= 0.5 || chunkCount === 0)) { + extractionQuality = "poor"; + } return { qualityScore: Number(qualityScore.toFixed(3)), diff --git a/src/lib/openai.ts b/src/lib/openai.ts index 5ae7fcfa6..b6403d488 100644 --- a/src/lib/openai.ts +++ b/src/lib/openai.ts @@ -405,6 +405,20 @@ export function clearOpenAICaches() { openAIClient = null; } +// IDX-C3: split a list into fixed-size batches, preserving order. Pure/exported so the +// batching contract (which underpins correct embedding reassembly) is unit-testable +// without hitting the network. +export function chunkIntoBatches(items: T[], size: number): T[][] { + if (!Number.isInteger(size) || size <= 0) { + throw new Error(`chunkIntoBatches: size must be a positive integer, got ${size}.`); + } + const batches: T[][] = []; + for (let start = 0; start < items.length; start += size) { + batches.push(items.slice(start, start + size)); + } + return batches; +} + export async function embedTexts(texts: string[]) { if (texts.length === 0) return []; @@ -423,45 +437,55 @@ export async function embedTexts(texts: string[]) { try { const client = createOpenAIClient(); - const response = await client.embeddings.create( - { - model: env.OPENAI_EMBEDDING_MODEL, - input: uniqueTexts, - // IDX-C2: request the exact dimension the schema's vector(N) columns expect. - dimensions: env.EMBEDDING_DIMENSIONS, - }, - requestOptions({ operation: "embedding" }), - ); - - // IDX-C2: a short response means some inputs silently produced no embedding. - if (response.data.length !== uniqueTexts.length) { - throw new PublicApiError( - `OpenAI returned ${response.data.length} embeddings for ${uniqueTexts.length} inputs.`, - 502, - { code: "openai_embedding_count_mismatch" }, + // IDX-C3: a single embeddings request is capped at 2048 inputs / ~300k tokens, so a + // full-corpus re-embed must be split. Batches run sequentially to stay polite to the + // rate limiter; the SDK still applies env.OPENAI_MAX_RETRIES per request. Reassembly + // uses the GLOBAL index (batchStart + item.index) so a chunk is never stored with the + // embedding of an unrelated text, even across batch boundaries (extends IDX-C1). + const byIndex = new Array(uniqueTexts.length); + const batches = chunkIntoBatches(uniqueTexts, env.OPENAI_EMBEDDING_BATCH_SIZE); + let batchStart = 0; + for (const batch of batches) { + const response = await client.embeddings.create( + { + model: env.OPENAI_EMBEDDING_MODEL, + input: batch, + // IDX-C2: request the exact dimension the schema's vector(N) columns expect. + dimensions: env.EMBEDDING_DIMENSIONS, + }, + requestOptions({ operation: "embedding" }), ); - } - // IDX-C1: the embeddings API does not guarantee response order; each item carries - // an explicit `index` into the input array. Reassemble by that index so a chunk is - // never stored with the embedding of an unrelated text (silent clinical corruption). - const byIndex = new Array(uniqueTexts.length); - for (const item of response.data) { - if (item.index < 0 || item.index >= uniqueTexts.length) { - throw new PublicApiError(`OpenAI returned an out-of-range embedding index ${item.index}.`, 502, { - code: "openai_embedding_index_range", - }); - } - // IDX-C2: guard against a model whose dimension does not match the schema. - if (item.embedding.length !== env.EMBEDDING_DIMENSIONS) { + // IDX-C2: a short response means some inputs silently produced no embedding. + if (response.data.length !== batch.length) { throw new PublicApiError( - `OpenAI embedding has ${item.embedding.length} dimensions; expected ${env.EMBEDDING_DIMENSIONS}. ` + - `Check OPENAI_EMBEDDING_MODEL and EMBEDDING_DIMENSIONS match supabase/schema.sql.`, + `OpenAI returned ${response.data.length} embeddings for ${batch.length} inputs.`, 502, - { code: "openai_embedding_dimension_mismatch" }, + { code: "openai_embedding_count_mismatch" }, ); } - byIndex[item.index] = item.embedding; + + // IDX-C1: the embeddings API does not guarantee response order; each item carries + // an explicit `index` into the request's input array. Reassemble by that index + // (offset to the global position) so embeddings are never mismatched to chunks. + for (const item of response.data) { + if (item.index < 0 || item.index >= batch.length) { + throw new PublicApiError(`OpenAI returned an out-of-range embedding index ${item.index}.`, 502, { + code: "openai_embedding_index_range", + }); + } + // IDX-C2: guard against a model whose dimension does not match the schema. + if (item.embedding.length !== env.EMBEDDING_DIMENSIONS) { + throw new PublicApiError( + `OpenAI embedding has ${item.embedding.length} dimensions; expected ${env.EMBEDDING_DIMENSIONS}. ` + + `Check OPENAI_EMBEDDING_MODEL and EMBEDDING_DIMENSIONS match supabase/schema.sql.`, + 502, + { code: "openai_embedding_dimension_mismatch" }, + ); + } + byIndex[batchStart + item.index] = item.embedding; + } + batchStart += batch.length; } return outputIndexes.map((index) => byIndex[index]); diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 55dcf66dd..4fbf92ad8 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -71,6 +71,7 @@ import { clinicalModePrompt, queryClassForClinicalMode, queryForClinicalMode } f import { annotateSearchResults, buildEvidenceRelevance } from "@/lib/evidence-relevance"; import { committedIndexGeneration, isCommittedGenerationMetadata } from "@/lib/reindex-pipeline"; import { buildRetrievalIntent, selectRetrievalEvidence } from "@/lib/retrieval-selection"; +import { rankingConfig } from "@/lib/ranking-config"; import { z } from "zod"; import { createHash } from "node:crypto"; import { @@ -577,20 +578,29 @@ function secondStageScore(result: SearchResult, queryClass: RagQueryClass | unde ) .join(" ")}`; const hasDoseAmount = /\b\d+(?:\.\d+)?\s?(?:mg|mcg|microgram|micrograms)\b/i.test(doseAmountText); - score += Math.max(0, 0.09 - index * 0.004); - if (result.memory_cards?.length && (queryClass === "broad_summary" || queryClass === "comparison")) score += 0.035; + const w = rankingConfig.secondStage; + score += Math.max(0, w.positionBase - index * w.positionStep); + if (result.memory_cards?.length && (queryClass === "broad_summary" || queryClass === "comparison")) + score += w.memorySummaryBoost; if (queryClass === "document_lookup" && (result.match_explanation?.titleHit || result.match_explanation?.labelHit)) - score += 0.045; + score += w.documentLookupTitleBoost; if ((queryClass === "table_threshold" || queryClass === "medication_dose_risk") && result.table_facts?.length) - score += 0.065; - if (queryClass === "medication_dose_risk" && hasDoseAmount) score += 0.18; - if (tableVisualEvidenceUnitTypes.has(unitType)) score += 0.08; - else if (visualEvidenceUnitTypes.has(unitType)) score += 0.04; - if (source === "visual_intelligence") score += Math.min(0.035, Math.max(0, sourceQuality - 0.55) * 0.08); - if (result.source_metadata?.document_status === "outdated") score -= 0.035; - if (result.source_metadata?.extraction_quality === "poor") score -= 0.035; - if (result.indexing_quality?.quality_score !== undefined && result.indexing_quality.quality_score < 0.55) - score -= 0.035; + score += w.tableThresholdEvidenceBoost; + if (queryClass === "medication_dose_risk" && hasDoseAmount) score += w.doseAmountBoost; + if (tableVisualEvidenceUnitTypes.has(unitType)) score += w.tableVisualBoost; + else if (visualEvidenceUnitTypes.has(unitType)) score += w.visualBoost; + if (source === "visual_intelligence") + score += Math.min( + w.visualIntelligenceMax, + Math.max(0, sourceQuality - w.visualIntelligencePivot) * w.visualIntelligenceSlope, + ); + if (result.source_metadata?.document_status === "outdated") score -= w.outdatedPenalty; + if (result.source_metadata?.extraction_quality === "poor") score -= w.poorExtractionPenalty; + if ( + result.indexing_quality?.quality_score !== undefined && + result.indexing_quality.quality_score < w.lowIndexQualityThreshold + ) + score -= w.lowIndexQualityPenalty; return score; } @@ -602,12 +612,26 @@ function applySecondStageRerankIfNeeded(args: { }) { if (!shouldUseSecondStageRerank(args.queryClass, args.results, args.topK)) return args.results; const startedAt = Date.now(); + // CI-16 document diversity: subtract a demotion from each EXTRA chunk of a document that + // has already appeared higher up, so a single doc's sibling chunks can't crowd out other + // documents. Applied AFTER the additive-boost floor so it can actually lower the effective + // rank. Default penalty is 0 (disabled) — no reordering until deliberately tuned + eval-gated. + const seenPerDocument = new Map(); const reranked = args.results .map((result, index) => { - const score = secondStageScore(result, args.queryClass, index); + const boosted = secondStageScore(result, args.queryClass, index); + let score = Math.max(result.hybrid_score ?? 0, boosted); + const priorOccurrences = seenPerDocument.get(result.document_id) ?? 0; + seenPerDocument.set(result.document_id, priorOccurrences + 1); + if (rankingConfig.documentDiversityPenalty > 0 && priorOccurrences > 0) { + score -= Math.min( + rankingConfig.documentDiversityPenaltyCap, + rankingConfig.documentDiversityPenalty * priorOccurrences, + ); + } return { ...result, - hybrid_score: Number(Math.max(result.hybrid_score ?? 0, score).toFixed(4)), + hybrid_score: Number(score.toFixed(4)), match_explanation: { ...result.match_explanation, reasons: Array.from(new Set([...(result.match_explanation?.reasons ?? []), "second_stage_rerank"])), diff --git a/src/lib/ranking-config.ts b/src/lib/ranking-config.ts new file mode 100644 index 000000000..08fb024e2 --- /dev/null +++ b/src/lib/ranking-config.ts @@ -0,0 +1,177 @@ +// Central, tunable ranking configuration for the app-layer retrieval rerank (W6). +// +// Rationale: the second-stage rerank weights, the document-diversity control, and the +// freshness decay were previously hard-coded magic numbers scattered across rag.ts and +// clinical-search.ts. Tuning them meant editing (and redeploying) code — the same friction +// CI-18 calls out. This module makes them one config object with an optional JSON override +// (RAG_RANKING_CONFIG), so `tune:search-weights` and eval-gated experiments can adjust +// ranking without code edits. +// +// IMPORTANT — defaults reproduce the exact prior behavior: +// * every secondStage weight equals the constant it replaced, +// * documentDiversityPenalty defaults to 0 (the diversity demotion is OFF), +// * freshness.mode defaults to "step" (the original cliff). +// So merely landing this module changes nothing. The new behaviors (diversity demotion, +// linear freshness curve) only take effect once explicitly configured and eval-gated, in +// keeping with the "defaults unchanged until tuned" + non-degradation guarantees. + +export type SecondStageWeights = { + /** Position-decay boost applied to the top result, decaying by positionStep per rank. */ + positionBase: number; + positionStep: number; + /** Boost for memory-card evidence on broad-summary / comparison queries. */ + memorySummaryBoost: number; + /** Boost when a document-lookup query hits a title/label. */ + documentLookupTitleBoost: number; + /** Boost when a table/threshold or dose-risk query has table-fact evidence. */ + tableThresholdEvidenceBoost: number; + /** Boost when a dose-risk query surfaces an explicit dose amount. */ + doseAmountBoost: number; + /** Boost for table-visual and (lower) other-visual evidence unit types. */ + tableVisualBoost: number; + visualBoost: number; + /** Visual-intelligence source-quality boost: min(max, (quality - pivot) * slope). */ + visualIntelligenceMax: number; + visualIntelligencePivot: number; + visualIntelligenceSlope: number; + /** Penalties for governance/quality signals. */ + outdatedPenalty: number; + poorExtractionPenalty: number; + lowIndexQualityPenalty: number; + lowIndexQualityThreshold: number; +}; + +export type FreshnessConfig = { + /** "step" = original cliff (default, zero-change); "linear" = ramped decay curve (CI-17). */ + mode: "step" | "linear"; + publicationCliffYears: number; + publicationPenalty: number; + reviewCliffYears: number; + reviewPenalty: number; + /** In "linear" mode, years over which the penalty ramps up to reach the cliff. */ + linearRampYears: number; +}; + +export type RankingConfig = { + secondStage: SecondStageWeights; + /** Demotion subtracted per EXTRA chunk from the same document (0 = diversity OFF). CI-16. */ + documentDiversityPenalty: number; + /** Maximum cumulative diversity demotion for any single chunk. */ + documentDiversityPenaltyCap: number; + freshness: FreshnessConfig; +}; + +export const defaultRankingConfig: RankingConfig = { + secondStage: { + positionBase: 0.09, + positionStep: 0.004, + memorySummaryBoost: 0.035, + documentLookupTitleBoost: 0.045, + tableThresholdEvidenceBoost: 0.065, + doseAmountBoost: 0.18, + tableVisualBoost: 0.08, + visualBoost: 0.04, + visualIntelligenceMax: 0.035, + visualIntelligencePivot: 0.55, + visualIntelligenceSlope: 0.08, + outdatedPenalty: 0.035, + poorExtractionPenalty: 0.035, + lowIndexQualityPenalty: 0.035, + lowIndexQualityThreshold: 0.55, + }, + documentDiversityPenalty: 0, + documentDiversityPenaltyCap: 0.12, + freshness: { + mode: "step", + publicationCliffYears: 8, + publicationPenalty: -0.015, + reviewCliffYears: 5, + reviewPenalty: -0.01, + linearRampYears: 3, + }, +}; + +function asRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function num(value: unknown, fallback: number): number { + return typeof value === "number" && Number.isFinite(value) ? value : fallback; +} + +function round4(value: number): number { + return Number(value.toFixed(4)); +} + +/** + * Build a RankingConfig by deep-merging an optional JSON override (partial) over the + * defaults. Unknown keys are ignored and non-numeric values fall back to the default, so a + * malformed override can only ever degrade to current behavior — never crash retrieval. + * Exported for unit testing. + */ +export function resolveRankingConfig(raw?: string | null): RankingConfig { + let parsed: Record = {}; + if (raw && raw.trim()) { + try { + parsed = asRecord(JSON.parse(raw)); + } catch { + parsed = {}; + } + } + const d = defaultRankingConfig; + const ss = asRecord(parsed.secondStage); + const fr = asRecord(parsed.freshness); + return { + secondStage: { + positionBase: num(ss.positionBase, d.secondStage.positionBase), + positionStep: num(ss.positionStep, d.secondStage.positionStep), + memorySummaryBoost: num(ss.memorySummaryBoost, d.secondStage.memorySummaryBoost), + documentLookupTitleBoost: num(ss.documentLookupTitleBoost, d.secondStage.documentLookupTitleBoost), + tableThresholdEvidenceBoost: num(ss.tableThresholdEvidenceBoost, d.secondStage.tableThresholdEvidenceBoost), + doseAmountBoost: num(ss.doseAmountBoost, d.secondStage.doseAmountBoost), + tableVisualBoost: num(ss.tableVisualBoost, d.secondStage.tableVisualBoost), + visualBoost: num(ss.visualBoost, d.secondStage.visualBoost), + visualIntelligenceMax: num(ss.visualIntelligenceMax, d.secondStage.visualIntelligenceMax), + visualIntelligencePivot: num(ss.visualIntelligencePivot, d.secondStage.visualIntelligencePivot), + visualIntelligenceSlope: num(ss.visualIntelligenceSlope, d.secondStage.visualIntelligenceSlope), + outdatedPenalty: num(ss.outdatedPenalty, d.secondStage.outdatedPenalty), + poorExtractionPenalty: num(ss.poorExtractionPenalty, d.secondStage.poorExtractionPenalty), + lowIndexQualityPenalty: num(ss.lowIndexQualityPenalty, d.secondStage.lowIndexQualityPenalty), + lowIndexQualityThreshold: num(ss.lowIndexQualityThreshold, d.secondStage.lowIndexQualityThreshold), + }, + documentDiversityPenalty: Math.max(0, num(parsed.documentDiversityPenalty, d.documentDiversityPenalty)), + documentDiversityPenaltyCap: Math.max(0, num(parsed.documentDiversityPenaltyCap, d.documentDiversityPenaltyCap)), + freshness: { + mode: fr.mode === "linear" ? "linear" : "step", + publicationCliffYears: num(fr.publicationCliffYears, d.freshness.publicationCliffYears), + publicationPenalty: num(fr.publicationPenalty, d.freshness.publicationPenalty), + reviewCliffYears: num(fr.reviewCliffYears, d.freshness.reviewCliffYears), + reviewPenalty: num(fr.reviewPenalty, d.freshness.reviewPenalty), + linearRampYears: Math.max(0.0001, num(fr.linearRampYears, d.freshness.linearRampYears)), + }, + }; +} + +/** + * Freshness penalty for a document, given its age in years (or null when unknown). + * CI-17: in "step" mode this reproduces the original cliff exactly; in "linear" mode the + * penalty ramps in gradually over `linearRampYears` up to the cliff, avoiding the harsh + * step that unfairly demotes still-current stable guidelines the moment they cross the line. + */ +export function freshnessDecayPenalty( + yearsAgo: number | null, + kind: "publication" | "review", + cfg: FreshnessConfig, +): number { + if (yearsAgo === null) return 0; + const cliff = kind === "publication" ? cfg.publicationCliffYears : cfg.reviewCliffYears; + const penalty = kind === "publication" ? cfg.publicationPenalty : cfg.reviewPenalty; + if (cfg.mode === "step") return yearsAgo >= cliff ? penalty : 0; + const rampStart = cliff - cfg.linearRampYears; + if (yearsAgo <= rampStart) return 0; + const t = Math.min(1, (yearsAgo - rampStart) / cfg.linearRampYears); + return round4(penalty * t); +} + +/** Resolved singleton used by the retrieval path. */ +export const rankingConfig: RankingConfig = resolveRankingConfig(process.env.RAG_RANKING_CONFIG); diff --git a/src/lib/reindex-eval-gate.ts b/src/lib/reindex-eval-gate.ts new file mode 100644 index 000000000..fd58b6741 --- /dev/null +++ b/src/lib/reindex-eval-gate.ts @@ -0,0 +1,314 @@ +// Non-degradation gate for the shadow re-index (Task #6 core). Given the eval summaries for +// the current live (baseline) index and a staged candidate generation, decide GO/NO-GO for +// cutover. The contract (docs/retrieval-quality-runbook.md + the re-index plan) is twofold: +// 1. Absolute bars — the candidate must independently clear the release thresholds. +// 2. No regression — the candidate must be no worse than the live baseline (within a small +// tolerance for eval noise), so a re-index can only hold or improve quality. +// A cutover proceeds only when EVERY check passes. This module is pure and deterministic so +// the gate itself is unit-tested; the (live) shadow driver feeds it eval JSON and acts on it. + +// Subsets of scripts/eval-retrieval.ts and scripts/eval-quality.ts summary output. Field +// names match those scripts so the driver can pass summaries through unchanged. +export type RetrievalGateSummary = { + document_recall_at_5: number; + content_recall_at_5: number; + top_k_hit_rate: number; + mrr_at_10?: number; + p90_latency_ms?: number; +}; + +export type QualityGateSummary = { + grounded_supported_rate: number; + unsupported_correct_rate: number; + expected_hit_rate?: number; + citation_failure_rate: number; + numeric_grounding_failure_rate: number; + source_governance_danger_failure_rate: number; + stale_review_unknown_rate?: number; + review_required_rate?: number; + p95_latency_ms: number; +}; + +export type ReindexGateConfig = { + retrieval: { + documentRecallAt5Floor: number; + contentRecallAt5Floor: number; + topKHitRateFloor: number; + rateRegressionTolerance: number; + latencyRegressionMs: number; + }; + quality: { + groundedSupportedRateFloor: number; + unsupportedCorrectRateFloor: number; + citationFailureRateCeiling: number; + numericGroundingFailureRateCeiling: number; + sourceGovernanceDangerFailureRateCeiling: number; + staleReviewUnknownRateCeiling: number; + reviewRequiredRateCeiling: number; + p95LatencyMsCeiling: number; + rateRegressionTolerance: number; + latencyRegressionMs: number; + }; +}; + +// Defaults taken directly from the release thresholds in docs/retrieval-quality-runbook.md. +export const defaultReindexGateConfig: ReindexGateConfig = { + retrieval: { + documentRecallAt5Floor: 0.8, + contentRecallAt5Floor: 0.8, + topKHitRateFloor: 0.8, + rateRegressionTolerance: 0.02, + latencyRegressionMs: 4000, + }, + quality: { + groundedSupportedRateFloor: 0.9, + unsupportedCorrectRateFloor: 1.0, + citationFailureRateCeiling: 0, + numericGroundingFailureRateCeiling: 0, + sourceGovernanceDangerFailureRateCeiling: 0, + staleReviewUnknownRateCeiling: 0.25, + reviewRequiredRateCeiling: 0.25, + p95LatencyMsCeiling: 25000, + rateRegressionTolerance: 0.02, + latencyRegressionMs: 4000, + }, +}; + +export type MetricCheck = { + metric: string; + direction: "higher_better" | "lower_better"; + baseline: number; + candidate: number; + bound: number | null; // absolute floor (higher_better) / ceiling (lower_better) + tolerance: number; + passed: boolean; + reasons: string[]; +}; + +export type ReindexGateDecision = { + decision: "GO" | "NO_GO"; + checks: MetricCheck[]; + failures: string[]; +}; + +function evaluateCheck(input: { + metric: string; + direction: "higher_better" | "lower_better"; + baseline: number; + candidate: number; + bound: number | null; + tolerance: number; +}): MetricCheck { + const reasons: string[] = []; + const { metric, direction, baseline, candidate, bound, tolerance } = input; + + if (bound !== null) { + const meetsBound = direction === "higher_better" ? candidate >= bound : candidate <= bound; + if (!meetsBound) { + reasons.push( + direction === "higher_better" + ? `${metric} ${candidate} below absolute floor ${bound}` + : `${metric} ${candidate} above absolute ceiling ${bound}`, + ); + } + } + + const noRegression = + direction === "higher_better" ? candidate >= baseline - tolerance : candidate <= baseline + tolerance; + if (!noRegression) { + reasons.push( + direction === "higher_better" + ? `${metric} regressed: candidate ${candidate} < baseline ${baseline} - tolerance ${tolerance}` + : `${metric} regressed: candidate ${candidate} > baseline ${baseline} + tolerance ${tolerance}`, + ); + } + + return { metric, direction, baseline, candidate, bound, tolerance, passed: reasons.length === 0, reasons }; +} + +function retrievalChecks( + baseline: RetrievalGateSummary, + candidate: RetrievalGateSummary, + config: ReindexGateConfig["retrieval"], +): MetricCheck[] { + const checks: MetricCheck[] = [ + evaluateCheck({ + metric: "document_recall_at_5", + direction: "higher_better", + baseline: baseline.document_recall_at_5, + candidate: candidate.document_recall_at_5, + bound: config.documentRecallAt5Floor, + tolerance: config.rateRegressionTolerance, + }), + evaluateCheck({ + metric: "content_recall_at_5", + direction: "higher_better", + baseline: baseline.content_recall_at_5, + candidate: candidate.content_recall_at_5, + bound: config.contentRecallAt5Floor, + tolerance: config.rateRegressionTolerance, + }), + evaluateCheck({ + metric: "top_k_hit_rate", + direction: "higher_better", + baseline: baseline.top_k_hit_rate, + candidate: candidate.top_k_hit_rate, + bound: config.topKHitRateFloor, + tolerance: config.rateRegressionTolerance, + }), + ]; + // mrr@10 and p90 latency are no-regression only (no absolute bar) and skipped unless both + // summaries carry them. + if (typeof baseline.mrr_at_10 === "number" && typeof candidate.mrr_at_10 === "number") { + checks.push( + evaluateCheck({ + metric: "mrr_at_10", + direction: "higher_better", + baseline: baseline.mrr_at_10, + candidate: candidate.mrr_at_10, + bound: null, + tolerance: config.rateRegressionTolerance, + }), + ); + } + if (typeof baseline.p90_latency_ms === "number" && typeof candidate.p90_latency_ms === "number") { + checks.push( + evaluateCheck({ + metric: "p90_latency_ms", + direction: "lower_better", + baseline: baseline.p90_latency_ms, + candidate: candidate.p90_latency_ms, + bound: null, + tolerance: config.latencyRegressionMs, + }), + ); + } + return checks; +} + +function qualityChecks( + baseline: QualityGateSummary, + candidate: QualityGateSummary, + config: ReindexGateConfig["quality"], +): MetricCheck[] { + const checks: MetricCheck[] = [ + evaluateCheck({ + metric: "grounded_supported_rate", + direction: "higher_better", + baseline: baseline.grounded_supported_rate, + candidate: candidate.grounded_supported_rate, + bound: config.groundedSupportedRateFloor, + tolerance: config.rateRegressionTolerance, + }), + evaluateCheck({ + metric: "unsupported_correct_rate", + direction: "higher_better", + baseline: baseline.unsupported_correct_rate, + candidate: candidate.unsupported_correct_rate, + bound: config.unsupportedCorrectRateFloor, + tolerance: 0, + }), + evaluateCheck({ + metric: "citation_failure_rate", + direction: "lower_better", + baseline: baseline.citation_failure_rate, + candidate: candidate.citation_failure_rate, + bound: config.citationFailureRateCeiling, + tolerance: 0, + }), + evaluateCheck({ + metric: "numeric_grounding_failure_rate", + direction: "lower_better", + baseline: baseline.numeric_grounding_failure_rate, + candidate: candidate.numeric_grounding_failure_rate, + bound: config.numericGroundingFailureRateCeiling, + tolerance: 0, + }), + evaluateCheck({ + metric: "source_governance_danger_failure_rate", + direction: "lower_better", + baseline: baseline.source_governance_danger_failure_rate, + candidate: candidate.source_governance_danger_failure_rate, + bound: config.sourceGovernanceDangerFailureRateCeiling, + tolerance: 0, + }), + evaluateCheck({ + metric: "p95_latency_ms", + direction: "lower_better", + baseline: baseline.p95_latency_ms, + candidate: candidate.p95_latency_ms, + bound: config.p95LatencyMsCeiling, + tolerance: config.latencyRegressionMs, + }), + ]; + if (typeof baseline.expected_hit_rate === "number" && typeof candidate.expected_hit_rate === "number") { + checks.push( + evaluateCheck({ + metric: "expected_hit_rate", + direction: "higher_better", + baseline: baseline.expected_hit_rate, + candidate: candidate.expected_hit_rate, + bound: null, + tolerance: config.rateRegressionTolerance, + }), + ); + } + if ( + typeof baseline.stale_review_unknown_rate === "number" && + typeof candidate.stale_review_unknown_rate === "number" + ) { + checks.push( + evaluateCheck({ + metric: "stale_review_unknown_rate", + direction: "lower_better", + baseline: baseline.stale_review_unknown_rate, + candidate: candidate.stale_review_unknown_rate, + bound: config.staleReviewUnknownRateCeiling, + tolerance: config.rateRegressionTolerance, + }), + ); + } + if (typeof baseline.review_required_rate === "number" && typeof candidate.review_required_rate === "number") { + checks.push( + evaluateCheck({ + metric: "review_required_rate", + direction: "lower_better", + baseline: baseline.review_required_rate, + candidate: candidate.review_required_rate, + bound: config.reviewRequiredRateCeiling, + tolerance: config.rateRegressionTolerance, + }), + ); + } + return checks; +} + +export function decideReindexGate( + input: { + baselineRetrieval: RetrievalGateSummary; + candidateRetrieval: RetrievalGateSummary; + baselineQuality?: QualityGateSummary; + candidateQuality?: QualityGateSummary; + }, + config: ReindexGateConfig = defaultReindexGateConfig, +): ReindexGateDecision { + const checks = retrievalChecks(input.baselineRetrieval, input.candidateRetrieval, config.retrieval); + + const hasBaselineQuality = Boolean(input.baselineQuality); + const hasCandidateQuality = Boolean(input.candidateQuality); + if (hasBaselineQuality !== hasCandidateQuality) { + // A one-sided quality summary is a driver error; fail closed rather than silently + // gating on retrieval alone. + return { + decision: "NO_GO", + checks, + failures: ["quality summaries must be provided for both baseline and candidate, or neither"], + }; + } + if (input.baselineQuality && input.candidateQuality) { + checks.push(...qualityChecks(input.baselineQuality, input.candidateQuality, config.quality)); + } + + const failures = checks.flatMap((check) => check.reasons); + return { decision: failures.length === 0 ? "GO" : "NO_GO", checks, failures }; +} diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index 32c6568ed..12b5988f4 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -1,5 +1,11 @@ -import { describe, expect, it } from "vitest"; -import { buildChunks, buildImageTag, chunkTextWithOverlap } from "../src/lib/chunking"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + CHUNKER_VERSION, + buildChunks, + buildImageTag, + chunkContentKey, + chunkTextWithOverlap, +} from "../src/lib/chunking"; describe("chunkTextWithOverlap", () => { it("keeps short text as one chunk", () => { @@ -328,3 +334,134 @@ describe("section-aware chunking groundwork", () => { expect(chunks.map((chunk) => chunk.page_number)).toEqual([4, 7]); }); }); + +describe("stable chunk identity (CI-4)", () => { + it("stamps a stable chunk_key and chunker_version into metadata", () => { + const chunks = buildChunks([ + { documentId: "doc-1", pageNumber: 1, pageText: "Check renal function before starting lithium.", metadata: {} }, + ]); + expect(chunks).toHaveLength(1); + expect(chunks[0].metadata.chunker_version).toBe(CHUNKER_VERSION); + expect(chunks[0].metadata.chunk_key).toMatch(/^[0-9a-f]{32}$/); + }); + + it("produces the same key across re-runs of identical input (idempotent re-index)", () => { + const input = [{ documentId: "doc-1", pageNumber: 1, pageText: "Withhold clozapine if ANC is low.", metadata: {} }]; + const first = buildChunks(structuredClone(input)); + const second = buildChunks(structuredClone(input)); + expect(first[0].metadata.chunk_key).toBe(second[0].metadata.chunk_key); + }); + + it("is page-independent — identical content on different pages shares an identity", () => { + const text = "ANC threshold guidance: withhold clozapine and repeat FBC daily."; + const chunks = buildChunks([ + { documentId: "doc-1", pageNumber: 4, pageText: text, metadata: {} }, + { documentId: "doc-1", pageNumber: 7, pageText: text, metadata: {} }, + ]); + expect(chunks).toHaveLength(2); + expect(chunks[0].metadata.chunk_key).toBe(chunks[1].metadata.chunk_key); + }); + + it("changes the key when content or document differs, and ignores inline image tags", () => { + const keyA = chunkContentKey("doc-1", "monitoring", "Withhold clozapine if ANC is low."); + const keyDifferentContent = chunkContentKey("doc-1", "monitoring", "Continue clozapine and monitor weekly."); + const keyDifferentDoc = chunkContentKey("doc-2", "monitoring", "Withhold clozapine if ANC is low."); + expect(keyA).not.toBe(keyDifferentContent); + expect(keyA).not.toBe(keyDifferentDoc); + // Image-data tags are stripped before hashing, so attaching image context does not + // change a chunk's stable identity. + const withImageTag = chunkContentKey( + "doc-1", + "monitoring", + "Withhold clozapine if ANC is low. [[IMAGE_DATA_START]] Image ID: x; Description: chart [[IMAGE_DATA_END]]", + ); + expect(withImageTag).toBe(keyA); + }); +}); + +describe("document-mode chunking (CI-1)", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + }); + + async function buildDocumentChunks(inputs: Parameters[0]) { + vi.resetModules(); + vi.stubEnv("CHUNK_STRATEGY", "document"); + const mod = await import("../src/lib/chunking"); + return mod.buildChunks(inputs); + } + + it("merges a section across a page break into a single cross-page chunk", async () => { + const chunks = await buildDocumentChunks([ + { + documentId: "doc-1", + pageNumber: 1, + pageText: + "Clozapine Monitoring\n\nBaseline full blood count and absolute neutrophil count must be checked before starting clozapine therapy for the patient.", + metadata: {}, + }, + { + documentId: "doc-1", + pageNumber: 2, + pageText: + "Continue weekly blood test monitoring and escalate any neutropenia to the prescriber and haematology team immediately.", + metadata: {}, + }, + ]); + + // The section (heading on page 1, continuation on page 2) chunks together, so the + // page-1 baseline requirement and the page-2 continuation land in one chunk. + const spanning = chunks.find( + (chunk) => chunk.content.includes("Baseline full blood count") && chunk.content.includes("weekly blood test"), + ); + expect(spanning).toBeDefined(); + expect(spanning?.metadata.page_start).toBe(1); + expect(spanning?.metadata.page_end).toBe(2); + expect(spanning?.section_path).toContain("Clozapine Monitoring"); + // A cross-page chunk records a source span for each contributing page. + const spanPageNumbers = (spanning?.metadata.source_spans as Array<{ page_number: number | null }>).map( + (span) => span.page_number, + ); + expect(spanPageNumbers).toEqual(expect.arrayContaining([1, 2])); + }); + + it("does not merge across a section boundary introduced by a new heading", async () => { + const chunks = await buildDocumentChunks([ + { + documentId: "doc-1", + pageNumber: 1, + pageText: "Section One\n\nAlpha guidance about clozapine baseline monitoring requirements for every patient.", + metadata: {}, + }, + { + documentId: "doc-1", + pageNumber: 2, + pageText: "Section Two\n\nBravo guidance about lithium renal function checks and thyroid monitoring tests.", + metadata: {}, + }, + ]); + + const sectionTwo = chunks.find((chunk) => chunk.content.includes("Bravo guidance")); + expect(sectionTwo?.metadata.page_start).toBe(2); + expect(sectionTwo?.metadata.page_end).toBe(2); + expect(sectionTwo?.section_path).toContain("Section Two"); + // The page-1 section must not have absorbed page-2 content. + const sectionOne = chunks.find((chunk) => chunk.content.includes("Alpha guidance")); + expect(sectionOne?.content).not.toContain("Bravo guidance"); + expect(sectionOne?.metadata.page_end).toBe(1); + }); + + it("still stamps the stable chunk_key and chunker_version in document mode", async () => { + const chunks = await buildDocumentChunks([ + { + documentId: "doc-1", + pageNumber: 1, + pageText: "Check renal function before starting lithium therapy.", + metadata: {}, + }, + ]); + expect(chunks[0].metadata.chunker_version).toBe(CHUNKER_VERSION); + expect(chunks[0].metadata.chunk_key).toMatch(/^[0-9a-f]{32}$/); + }); +}); diff --git a/tests/embed-texts-integrity.test.ts b/tests/embed-texts-integrity.test.ts index 0e37e6220..8be7b8644 100644 --- a/tests/embed-texts-integrity.test.ts +++ b/tests/embed-texts-integrity.test.ts @@ -110,3 +110,57 @@ describe("embedTexts integrity (IDX-C1, IDX-C2)", () => { await expect(embedTexts(["a", "b"])).rejects.toThrow(/embeddings for/); }); }); + +describe("embedTexts batching (IDX-C3)", () => { + function embeddingForText(text: string): number[] { + // Encode the input's numeric suffix so a mis-mapped embedding is detectable. + const n = Number(text.replace(/[^0-9]/g, "")); + return [n, n]; + } + + it("splits large input into batches and reassembles by global index across batches", async () => { + stubEmbeddingEnv(2); + vi.stubEnv("OPENAI_EMBEDDING_BATCH_SIZE", "2"); + + const inputsPerCall: number[] = []; + vi.doMock("openai", () => ({ + default: class MockOpenAI { + embeddings = { + create: vi.fn(async ({ input }: { input: string[] }) => { + inputsPerCall.push(input.length); + // Reverse within each batch so a naive position-based merge would corrupt order. + return { data: input.map((text, index) => ({ index, embedding: embeddingForText(text) })).reverse() }; + }), + }; + + responses = { create: vi.fn() }; + }, + })); + + const { clearOpenAICaches, embedTexts } = await import("../src/lib/openai"); + clearOpenAICaches(); + + const result = await embedTexts(["t0", "t1", "t2", "t3", "t4"]); + + // 5 inputs at batch size 2 -> three requests of 2, 2, 1. + expect(inputsPerCall).toEqual([2, 2, 1]); + // Every text maps back to its own embedding despite per-batch reversal and batch splits. + expect(result).toEqual([ + [0, 0], + [1, 1], + [2, 2], + [3, 3], + [4, 4], + ]); + }); + + it("chunkIntoBatches partitions in order with a trailing remainder and rejects bad sizes", async () => { + stubEmbeddingEnv(2); + const { chunkIntoBatches } = await import("../src/lib/openai"); + + expect(chunkIntoBatches([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]); + expect(chunkIntoBatches([], 3)).toEqual([]); + expect(chunkIntoBatches([1, 2], 10)).toEqual([[1, 2]]); + expect(() => chunkIntoBatches([1], 0)).toThrow(/positive integer/); + }); +}); diff --git a/tests/embedding-dimensions.test.ts b/tests/embedding-dimensions.test.ts index c35a4f597..91f4c0284 100644 --- a/tests/embedding-dimensions.test.ts +++ b/tests/embedding-dimensions.test.ts @@ -30,3 +30,30 @@ describe("strict embedding dimension guard", () => { ).toThrow(new RegExp(`non-finite value at index ${EXPECTED_EMBED_DIM - 1}`)); }); }); + +describe("schema vector dimension guard (IDX-C2 fail-fast)", () => { + it("parses distinct vector(N) dimensions from schema SQL", async () => { + const { parseSchemaVectorDimensions } = await import("../src/lib/embedding-dimensions"); + const sql = + "create table a (e vector(1536)); create table b (f VECTOR(1536)); create index on c using hnsw (g vector(1536));"; + expect(parseSchemaVectorDimensions(sql)).toEqual([1536]); + expect(parseSchemaVectorDimensions("no vectors here")).toEqual([]); + expect(parseSchemaVectorDimensions("vector(768) and vector(1536)")).toEqual([768, 1536]); + }); + + it("passes when the configured dimension matches a single consistent schema dimension", async () => { + const { describeSchemaDimensionMismatch } = await import("../src/lib/embedding-dimensions"); + expect(describeSchemaDimensionMismatch(1536, "e vector(1536), f vector(1536)")).toBeNull(); + }); + + it("flags a config/schema mismatch, inconsistent schema dims, and a missing vector column", async () => { + const { describeSchemaDimensionMismatch } = await import("../src/lib/embedding-dimensions"); + expect(describeSchemaDimensionMismatch(3072, "e vector(1536)")).toMatch( + /EMBEDDING_DIMENSIONS=3072 does not match schema vector\(1536\)/, + ); + expect(describeSchemaDimensionMismatch(1536, "e vector(1536), f vector(3072)")).toMatch( + /inconsistent vector dimensions/, + ); + expect(describeSchemaDimensionMismatch(1536, "no vectors")).toMatch(/No vector\(N\) columns/); + }); +}); diff --git a/tests/eval-retrieval.test.ts b/tests/eval-retrieval.test.ts index d35167419..b128d3ea4 100644 --- a/tests/eval-retrieval.test.ts +++ b/tests/eval-retrieval.test.ts @@ -145,6 +145,23 @@ describe("golden retrieval eval helpers", () => { expect(summary.document_recall_at_5).toBe(0); }); + it("classifies the abbreviation golden cases as their declared query class (CI-14)", async () => { + // The eval fails a case on query-class mismatch, so a golden case whose query does not + // classify as its expectedQueryClass would pollute the baseline. Guard the abbreviation + // cases added for the CBC/WCC synonym expansion. + const { classifyRagQuery } = await import("../src/lib/clinical-search"); + const cases = loadGoldenRetrievalCases("scripts/fixtures/rag-retrieval-golden.json"); + const abbreviationCases = cases.filter( + (testCase) => + testCase.id === "clozapine-cbc-abbreviation-threshold" || + testCase.id === "clozapine-wcc-abbreviation-threshold", + ); + expect(abbreviationCases).toHaveLength(2); + for (const testCase of abbreviationCases) { + expect(classifyRagQuery(testCase.query).queryClass).toBe(testCase.expectedQueryClass); + } + }); + it("converts captured RAG eval cases into golden retrieval cases", () => { expect( capturedRagCaseToGoldenCase({ diff --git a/tests/index-quality.test.ts b/tests/index-quality.test.ts index f35ae1e1a..2d8ab288e 100644 --- a/tests/index-quality.test.ts +++ b/tests/index-quality.test.ts @@ -135,4 +135,60 @@ describe("index quality scoring", () => { expect(quality.metrics.average_structured_visual_confidence).toBeGreaterThan(0.75); expect(quality.issues).not.toContain("low visual unit coverage"); }); + + it("forces poor quality and flags an actionable issue for un-OCR'd image-only PDFs (CI-6)", () => { + const quality = assessDocumentIndexQuality({ + metrics: { + page_count: 5, + text_character_count: 30, // only incidental header text + extracted_image_count: 5, + searchable_image_count: 0, + needs_ocr_page_count: 5, // every page is image-only and was not OCR'd + }, + chunks: [], + insertedImages: [], + sectionCount: 0, + memoryCardCount: 0, + documentEmbeddingFieldTypes: [], + }); + + expect(quality.extractionQuality).toBe("poor"); + expect(quality.issues).toEqual( + expect.arrayContaining([expect.stringContaining("image-only pages not OCR'd (5 of 5)")]), + ); + }); + + it("flags a minority of un-OCR'd pages without forcing an otherwise-strong index to poor", () => { + const quality = assessDocumentIndexQuality({ + metrics: { + page_count: 10, + text_character_count: 24_000, + extracted_image_count: 1, + searchable_image_count: 1, + needs_ocr_page_count: 1, // a single image-only page in an otherwise strong text PDF + }, + chunks: Array.from({ length: 12 }, (_, index) => ({ + content: `Distinct clinical passage ${index} with monitoring action, threshold detail, and section context for complete retrieval.`, + section_heading: `Section ${index}`, + section_path: ["Guideline", `Section ${index}`], + })), + insertedImages: [ + { + sourceKind: "table_crop", + tableRows: [["ANC", "withhold"]], + imageQualityScore: 1, + cropCompleteness: 1, + structuredVisualProfile: { confidence: 1, thresholds: [{}] }, + }, + ], + sectionCount: 3, + memoryCardCount: 6, + documentEmbeddingFieldTypes: ["document_title", "document_summary"], + }); + + expect(quality.issues).toEqual( + expect.arrayContaining([expect.stringContaining("image-only pages not OCR'd (1 of 10)")]), + ); + expect(quality.extractionQuality).not.toBe("poor"); + }); }); diff --git a/tests/ranking-config.test.ts b/tests/ranking-config.test.ts new file mode 100644 index 000000000..c0c9f466d --- /dev/null +++ b/tests/ranking-config.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { + defaultRankingConfig, + freshnessDecayPenalty, + resolveRankingConfig, + type FreshnessConfig, +} from "../src/lib/ranking-config"; + +describe("ranking-config defaults (W6 — zero behavior change)", () => { + it("reproduces the exact prior second-stage rerank constants", () => { + const w = defaultRankingConfig.secondStage; + expect(w.positionBase).toBe(0.09); + expect(w.positionStep).toBe(0.004); + expect(w.memorySummaryBoost).toBe(0.035); + expect(w.documentLookupTitleBoost).toBe(0.045); + expect(w.tableThresholdEvidenceBoost).toBe(0.065); + expect(w.doseAmountBoost).toBe(0.18); + expect(w.tableVisualBoost).toBe(0.08); + expect(w.visualBoost).toBe(0.04); + expect(w.visualIntelligenceMax).toBe(0.035); + expect(w.visualIntelligencePivot).toBe(0.55); + expect(w.visualIntelligenceSlope).toBe(0.08); + expect(w.outdatedPenalty).toBe(0.035); + expect(w.poorExtractionPenalty).toBe(0.035); + expect(w.lowIndexQualityPenalty).toBe(0.035); + expect(w.lowIndexQualityThreshold).toBe(0.55); + }); + + it("keeps document-diversity demotion OFF by default", () => { + expect(defaultRankingConfig.documentDiversityPenalty).toBe(0); + }); + + it("defaults freshness to the original step cliffs", () => { + expect(defaultRankingConfig.freshness).toMatchObject({ + mode: "step", + publicationCliffYears: 8, + publicationPenalty: -0.015, + reviewCliffYears: 5, + reviewPenalty: -0.01, + }); + }); +}); + +describe("resolveRankingConfig override merge", () => { + it("returns defaults for undefined, empty, and malformed JSON", () => { + expect(resolveRankingConfig(undefined)).toEqual(defaultRankingConfig); + expect(resolveRankingConfig("")).toEqual(defaultRankingConfig); + expect(resolveRankingConfig("{not json")).toEqual(defaultRankingConfig); + expect(resolveRankingConfig("[1,2,3]")).toEqual(defaultRankingConfig); + }); + + it("deep-merges provided numeric fields and keeps defaults for the rest", () => { + const cfg = resolveRankingConfig( + JSON.stringify({ secondStage: { doseAmountBoost: 0.22 }, documentDiversityPenalty: 0.03 }), + ); + expect(cfg.secondStage.doseAmountBoost).toBe(0.22); + // Untouched weights fall back to defaults. + expect(cfg.secondStage.positionBase).toBe(0.09); + expect(cfg.documentDiversityPenalty).toBe(0.03); + }); + + it("ignores non-numeric values and clamps diversity penalties to non-negative", () => { + const cfg = resolveRankingConfig( + JSON.stringify({ secondStage: { doseAmountBoost: "big" }, documentDiversityPenalty: -5 }), + ); + expect(cfg.secondStage.doseAmountBoost).toBe(0.18); + expect(cfg.documentDiversityPenalty).toBe(0); + }); + + it("accepts the linear freshness mode", () => { + const cfg = resolveRankingConfig(JSON.stringify({ freshness: { mode: "linear", linearRampYears: 4 } })); + expect(cfg.freshness.mode).toBe("linear"); + expect(cfg.freshness.linearRampYears).toBe(4); + }); +}); + +describe("freshnessDecayPenalty", () => { + const step = defaultRankingConfig.freshness; + + it("step mode reproduces the original publication/review cliffs exactly", () => { + expect(freshnessDecayPenalty(null, "publication", step)).toBe(0); + expect(freshnessDecayPenalty(7.9, "publication", step)).toBe(0); + expect(freshnessDecayPenalty(8, "publication", step)).toBe(-0.015); + expect(freshnessDecayPenalty(20, "publication", step)).toBe(-0.015); + expect(freshnessDecayPenalty(4.9, "review", step)).toBe(0); + expect(freshnessDecayPenalty(5, "review", step)).toBe(-0.01); + }); + + it("linear mode ramps monotonically from the ramp start up to the cliff", () => { + const linear: FreshnessConfig = { ...step, mode: "linear", linearRampYears: 3 }; + // publication cliff 8, ramp 3 => ramp starts at year 5. + expect(freshnessDecayPenalty(5, "publication", linear)).toBe(0); + const mid = freshnessDecayPenalty(6.5, "publication", linear); // halfway + expect(mid).toBeLessThan(0); + expect(mid).toBeGreaterThan(-0.015); + expect(freshnessDecayPenalty(8, "publication", linear)).toBe(-0.015); + // Beyond the cliff the penalty is capped, never exceeding the full value. + expect(freshnessDecayPenalty(20, "publication", linear)).toBe(-0.015); + // Monotonic non-increasing as the document ages. + expect(freshnessDecayPenalty(7, "publication", linear)).toBeLessThan(mid); + }); +}); diff --git a/tests/reindex-eval-gate.test.ts b/tests/reindex-eval-gate.test.ts new file mode 100644 index 000000000..0a1781b34 --- /dev/null +++ b/tests/reindex-eval-gate.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from "vitest"; +import { decideReindexGate, type QualityGateSummary, type RetrievalGateSummary } from "../src/lib/reindex-eval-gate"; + +function retrieval(overrides: Partial = {}): RetrievalGateSummary { + return { + document_recall_at_5: 0.9, + content_recall_at_5: 0.9, + top_k_hit_rate: 0.95, + mrr_at_10: 0.8, + p90_latency_ms: 8000, + ...overrides, + }; +} + +function quality(overrides: Partial = {}): QualityGateSummary { + return { + grounded_supported_rate: 0.95, + unsupported_correct_rate: 1, + expected_hit_rate: 0.9, + citation_failure_rate: 0, + numeric_grounding_failure_rate: 0, + source_governance_danger_failure_rate: 0, + stale_review_unknown_rate: 0.1, + review_required_rate: 0.1, + p95_latency_ms: 18000, + ...overrides, + }; +} + +describe("decideReindexGate — retrieval", () => { + it("returns GO when the candidate matches or beats the baseline and clears the floors", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval({ content_recall_at_5: 0.92 }), + }); + expect(decision.decision).toBe("GO"); + expect(decision.failures).toEqual([]); + }); + + it("returns NO_GO when an absolute floor is breached", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ content_recall_at_5: 0.78 }), + candidateRetrieval: retrieval({ content_recall_at_5: 0.75 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/content_recall_at_5 0.75 below absolute floor 0.8/); + }); + + it("returns NO_GO on a real regression even when both values clear the floor", () => { + // Baseline 0.98, candidate 0.85 — both above the 0.8 floor, but a 13-point drop is a + // regression well beyond the 0.02 tolerance. + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ document_recall_at_5: 0.98 }), + candidateRetrieval: retrieval({ document_recall_at_5: 0.85 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/document_recall_at_5 regressed/); + }); + + it("tolerates sub-threshold eval noise below the baseline", () => { + // 0.90 -> 0.89 is within the 0.02 tolerance and still above the floor => GO. + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ content_recall_at_5: 0.9 }), + candidateRetrieval: retrieval({ content_recall_at_5: 0.89 }), + }); + expect(decision.decision).toBe("GO"); + }); + + it("flags a p90 latency regression beyond the allowed drift", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ p90_latency_ms: 8000 }), + candidateRetrieval: retrieval({ p90_latency_ms: 14000 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/p90_latency_ms regressed/); + }); + + it("skips optional metrics when either summary omits them", () => { + const decision = decideReindexGate({ + baselineRetrieval: { document_recall_at_5: 0.9, content_recall_at_5: 0.9, top_k_hit_rate: 0.9 }, + candidateRetrieval: { document_recall_at_5: 0.9, content_recall_at_5: 0.9, top_k_hit_rate: 0.9 }, + }); + expect(decision.decision).toBe("GO"); + expect(decision.checks.some((check) => check.metric === "mrr_at_10")).toBe(false); + expect(decision.checks.some((check) => check.metric === "p90_latency_ms")).toBe(false); + }); +}); + +describe("decideReindexGate — quality", () => { + it("returns GO when retrieval and quality both hold", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality(), + candidateQuality: quality(), + }); + expect(decision.decision).toBe("GO"); + }); + + it("returns NO_GO when any hard-zero failure rate is non-zero", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality(), + candidateQuality: quality({ citation_failure_rate: 0.05 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/citation_failure_rate 0.05 above absolute ceiling 0/); + }); + + it("returns NO_GO when grounded_supported_rate falls below the floor", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality({ grounded_supported_rate: 0.92 }), + candidateQuality: quality({ grounded_supported_rate: 0.85 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/grounded_supported_rate 0.85 below absolute floor 0.9/); + }); + + it("returns NO_GO when p95 latency exceeds the ceiling", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality(), + candidateQuality: quality({ p95_latency_ms: 26000 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/p95_latency_ms 26000 above absolute ceiling 25000/); + }); + + it("fails closed when only one side supplies a quality summary", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + candidateQuality: quality(), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/quality summaries must be provided for both/); + }); +}); diff --git a/tests/retrieval-query-variants.test.ts b/tests/retrieval-query-variants.test.ts index 755cfbddf..2a2b73efa 100644 --- a/tests/retrieval-query-variants.test.ts +++ b/tests/retrieval-query-variants.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { analyzeClinicalQuery, buildClinicalTextSearchQuery, rankClinicalResults } from "../src/lib/clinical-search"; +import { expandClinicalVocabularyText } from "../src/lib/clinical-vocabulary"; import { selectRetrievalEvidence } from "../src/lib/retrieval-selection"; import { buildRetrievalQueryVariants, @@ -956,6 +957,27 @@ describe("retrieval query variants", () => { }); }); +describe("clinical abbreviation synonym expansion (CI-14)", () => { + it("expands the FBC/CBC lab family bidirectionally", () => { + // A clinician searching the US term "CBC" must reach FBC-only documents, and vice versa. + expect(expandClinicalVocabularyText("cbc")).toEqual(expect.arrayContaining(["full blood count", "fbc"])); + expect(expandClinicalVocabularyText("fbc")).toEqual(expect.arrayContaining(["full blood count", "cbc"])); + expect(expandClinicalVocabularyText("complete blood count")).toEqual(expect.arrayContaining(["fbc"])); + }); + + it("surfaces the FBC form into the retrieval analysis for a CBC query", () => { + // The expansion must reach analysis.expandedTerms, which feeds the query variants that + // are run against the lexical RPCs and unioned — recovering FBC-only chunks. + const analysis = analyzeClinicalQuery("clozapine CBC monitoring threshold"); + expect(analysis.expandedTerms).toEqual(expect.arrayContaining(["fbc"])); + }); + + it("expands the subcutaneous and sublingual administration routes", () => { + expect(expandClinicalVocabularyText("give sc injection")).toEqual(expect.arrayContaining(["subcutaneous"])); + expect(expandClinicalVocabularyText("administer sublingual")).toEqual(expect.arrayContaining(["sublingual", "sl"])); + }); +}); + describe("relaxVariantToOrQuery (8b over-conjunction fallback)", () => { it("relaxes a multi-term AND variant to a deduped term-OR query", () => { expect(relaxVariantToOrQuery("ciwa score threshold drug treatment alcohol withdrawal")).toBe( diff --git a/worker/main.ts b/worker/main.ts index 29cd7d21d..64546daeb 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -1524,12 +1524,17 @@ function extractionMetrics( ) { const textCharacterCount = extracted.pages.reduce((sum, page) => sum + page.text.length, 0); const ocrPageCount = extracted.pages.filter((page) => page.ocrUsed).length; + // CI-6: pages the extractor flagged as image-only but could NOT OCR (JS fallback without + // the Python OCR prerequisites). Threaded into index quality so these documents are marked + // poor and surfaced to eval governance rather than indexed near-empty. + const needsOcrPageCount = extracted.pages.filter((page) => page.needsOcr).length; const warnings = [...(extracted.warnings ?? [])]; if (textCharacterCount < 80) warnings.push("Low extracted text volume; inspect OCR quality."); return { page_count: extracted.pages.length, ocr_page_count: ocrPageCount, + needs_ocr_page_count: needsOcrPageCount, text_character_count: textCharacterCount, extracted_image_count: extracted.images.length, searchable_image_count: Object.values(imageTypeCounts).reduce((sum, count) => sum + count, 0), From b0922cbc85569ee6ad3798f49f53ca492f206671 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 03:32:50 +0800 Subject: [PATCH 2/8] feat(rag): route WCC clozapine queries to structured blood-action gate; defer WCC golden case Follow-up to the offline re-index landing. Recognizes "white cell count"/WCC as a clozapine haematological-monitoring term so a WCC-phrased withhold-threshold query is routed to the structured clozapine_blood_action_structured_threshold gate and pulls the memory-card / table-fact evidence layer (previously it fell through to the generic threshold gate with no memory cards). Additive synonym only; tightly scoped to clozapine + withhold queries. Full golden suite shows no regression (ANC/CBC and all base cases unchanged). The withhold-action chunk still does not rank into top-5 for the WCC phrasing (a ranking, not routing, gap that needs care on this safety-critical path), so the unvalidated clozapine-wcc-abbreviation-threshold golden case is removed for now and tracked as a follow-up. The passing clozapine-cbc-abbreviation-threshold case is kept. Co-Authored-By: Claude Fable 5 --- scripts/fixtures/rag-retrieval-golden.json | 13 ------------- src/lib/clinical-search.ts | 2 +- src/lib/rag.ts | 4 ++-- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/scripts/fixtures/rag-retrieval-golden.json b/scripts/fixtures/rag-retrieval-golden.json index e9191cba5..b31176b76 100644 --- a/scripts/fixtures/rag-retrieval-golden.json +++ b/scripts/fixtures/rag-retrieval-golden.json @@ -224,18 +224,5 @@ ], "topK": 12, "expectTableEvidence": true - }, - { - "id": "clozapine-wcc-abbreviation-threshold", - "query": "What white cell count (WCC) threshold should withhold clozapine?", - "expectedQueryClass": "table_threshold", - "expectedDocumentSubstrings": ["Clozapine"], - "expectedContentTerms": [ - "clozapine", - ["wcc", "wbc", "white cell", "fbc", "full blood count"], - ["withhold", "cease", "stop", "red"] - ], - "topK": 12, - "expectTableEvidence": true } ] diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts index 18301d6e7..31feef86a 100644 --- a/src/lib/clinical-search.ts +++ b/src/lib/clinical-search.ts @@ -1079,7 +1079,7 @@ export function buildClinicalTextSearchQuery(query: string) { /\b(?:risk|red\s*zone|red|urgent|escalat|next step)\b/i.test(query); const wantsClozapineBloodMonitoring = /\bclozapine\b/i.test(correctedQueryText) && - /\b(?:blood|bloods|fbc|full blood count|observation|observations|monitor|monitoring)\b/i.test(correctedQueryText); + /\b(?:blood|bloods|fbc|wcc|full blood count|white cell|observation|observations|monitor|monitoring)\b/i.test(correctedQueryText); const wantsClozapineMissedDose = /\bclozapine\b/i.test(correctedQueryText) && /\bmissed\b/i.test(correctedQueryText) && diff --git a/src/lib/rag.ts b/src/lib/rag.ts index b82f9d8a0..ccb4bcdea 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -3541,10 +3541,10 @@ export function evaluateEvidenceCoverageGate( if (queryClass === "table_threshold") { if ( /\bclozapine\b/i.test(query) && - /\b(?:anc|fbc|wbc|neutrophil|neutrophils|full blood)\b/i.test(query) && + /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils|full blood|white cell)\b/i.test(query) && /\b(?:withhold|withheld|withholding|cease|ceased|stop|stopped)\b/i.test(query) ) { - const hasBlood = hasAnyTerm(evidenceText, /\b(?:anc|fbc|wbc|neutrophil|neutrophils|full blood)\b/i); + const hasBlood = hasAnyTerm(evidenceText, /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils|full blood|white cell)\b/i); const hasAction = hasAnyTerm( evidenceText, /\b(?:withhold|withheld|withholding|cease|ceased|stop|stopped|red)\b/i, From 4a44c39bb2a8b7e6c600a3613f3b3dc2880d2148 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 08:28:46 +0800 Subject: [PATCH 3/8] fix(review): fail closed reindex gate governance metrics --- scripts/fixtures/rag-retrieval-golden.json | 14 ++++ src/lib/reindex-eval-gate.ts | 89 +++++++++++++++------- tests/reindex-eval-gate.test.ts | 28 +++++++ 3 files changed, 102 insertions(+), 29 deletions(-) diff --git a/scripts/fixtures/rag-retrieval-golden.json b/scripts/fixtures/rag-retrieval-golden.json index b31176b76..ab5c75b52 100644 --- a/scripts/fixtures/rag-retrieval-golden.json +++ b/scripts/fixtures/rag-retrieval-golden.json @@ -224,5 +224,19 @@ ], "topK": 12, "expectTableEvidence": true + }, + { + "id": "clozapine-wcc-abbreviation-threshold", + "query": "What WCC (white cell count) or neutrophil threshold should withhold clozapine?", + "expectedQueryClass": "table_threshold", + "expectedDocumentSubstrings": ["Clozapine"], + "expectedContentTerms": [ + "clozapine", + ["anc", "neutrophil"], + ["fbc", "full blood count", "wbc", "wcc", "white cell"], + ["withhold", "cease", "stop", "red"] + ], + "topK": 12, + "expectTableEvidence": true } ] diff --git a/src/lib/reindex-eval-gate.ts b/src/lib/reindex-eval-gate.ts index fd58b6741..597ff5055 100644 --- a/src/lib/reindex-eval-gate.ts +++ b/src/lib/reindex-eval-gate.ts @@ -24,8 +24,9 @@ export type QualityGateSummary = { citation_failure_rate: number; numeric_grounding_failure_rate: number; source_governance_danger_failure_rate: number; - stale_review_unknown_rate?: number; - review_required_rate?: number; + expected_danger_warning_missing_count: number; + stale_review_unknown_rate: number; + review_required_rate: number; p95_latency_ms: number; }; @@ -43,6 +44,7 @@ export type ReindexGateConfig = { citationFailureRateCeiling: number; numericGroundingFailureRateCeiling: number; sourceGovernanceDangerFailureRateCeiling: number; + expectedDangerWarningMissingCountCeiling: number; staleReviewUnknownRateCeiling: number; reviewRequiredRateCeiling: number; p95LatencyMsCeiling: number; @@ -66,6 +68,7 @@ export const defaultReindexGateConfig: ReindexGateConfig = { citationFailureRateCeiling: 0, numericGroundingFailureRateCeiling: 0, sourceGovernanceDangerFailureRateCeiling: 0, + expectedDangerWarningMissingCountCeiling: 0, staleReviewUnknownRateCeiling: 0.25, reviewRequiredRateCeiling: 0.25, p95LatencyMsCeiling: 25000, @@ -91,6 +94,18 @@ export type ReindexGateDecision = { failures: string[]; }; +const requiredQualityMetrics = [ + "grounded_supported_rate", + "unsupported_correct_rate", + "citation_failure_rate", + "numeric_grounding_failure_rate", + "source_governance_danger_failure_rate", + "expected_danger_warning_missing_count", + "stale_review_unknown_rate", + "review_required_rate", + "p95_latency_ms", +] as const satisfies readonly (keyof QualityGateSummary)[]; + function evaluateCheck(input: { metric: string; direction: "higher_better" | "lower_better"; @@ -232,6 +247,14 @@ function qualityChecks( bound: config.sourceGovernanceDangerFailureRateCeiling, tolerance: 0, }), + evaluateCheck({ + metric: "expected_danger_warning_missing_count", + direction: "lower_better", + baseline: baseline.expected_danger_warning_missing_count, + candidate: candidate.expected_danger_warning_missing_count, + bound: config.expectedDangerWarningMissingCountCeiling, + tolerance: 0, + }), evaluateCheck({ metric: "p95_latency_ms", direction: "lower_better", @@ -253,36 +276,33 @@ function qualityChecks( }), ); } - if ( - typeof baseline.stale_review_unknown_rate === "number" && - typeof candidate.stale_review_unknown_rate === "number" - ) { - checks.push( - evaluateCheck({ - metric: "stale_review_unknown_rate", - direction: "lower_better", - baseline: baseline.stale_review_unknown_rate, - candidate: candidate.stale_review_unknown_rate, - bound: config.staleReviewUnknownRateCeiling, - tolerance: config.rateRegressionTolerance, - }), - ); - } - if (typeof baseline.review_required_rate === "number" && typeof candidate.review_required_rate === "number") { - checks.push( - evaluateCheck({ - metric: "review_required_rate", - direction: "lower_better", - baseline: baseline.review_required_rate, - candidate: candidate.review_required_rate, - bound: config.reviewRequiredRateCeiling, - tolerance: config.rateRegressionTolerance, - }), - ); - } + checks.push( + evaluateCheck({ + metric: "stale_review_unknown_rate", + direction: "lower_better", + baseline: baseline.stale_review_unknown_rate, + candidate: candidate.stale_review_unknown_rate, + bound: config.staleReviewUnknownRateCeiling, + tolerance: config.rateRegressionTolerance, + }), + evaluateCheck({ + metric: "review_required_rate", + direction: "lower_better", + baseline: baseline.review_required_rate, + candidate: candidate.review_required_rate, + bound: config.reviewRequiredRateCeiling, + tolerance: config.rateRegressionTolerance, + }), + ); return checks; } +function missingQualityMetrics(label: "baselineQuality" | "candidateQuality", summary: QualityGateSummary) { + return requiredQualityMetrics.flatMap((metric) => + typeof summary[metric] === "number" && Number.isFinite(summary[metric]) ? [] : [`${label}.${metric}`], + ); +} + export function decideReindexGate( input: { baselineRetrieval: RetrievalGateSummary; @@ -306,6 +326,17 @@ export function decideReindexGate( }; } if (input.baselineQuality && input.candidateQuality) { + const missingMetrics = [ + ...missingQualityMetrics("baselineQuality", input.baselineQuality), + ...missingQualityMetrics("candidateQuality", input.candidateQuality), + ]; + if (missingMetrics.length > 0) { + return { + decision: "NO_GO", + checks, + failures: [`quality summaries are missing required metrics: ${missingMetrics.join(", ")}`], + }; + } checks.push(...qualityChecks(input.baselineQuality, input.candidateQuality, config.quality)); } diff --git a/tests/reindex-eval-gate.test.ts b/tests/reindex-eval-gate.test.ts index 0a1781b34..ff366d661 100644 --- a/tests/reindex-eval-gate.test.ts +++ b/tests/reindex-eval-gate.test.ts @@ -20,6 +20,7 @@ function quality(overrides: Partial = {}): QualityGateSummar citation_failure_rate: 0, numeric_grounding_failure_rate: 0, source_governance_danger_failure_rate: 0, + expected_danger_warning_missing_count: 0, stale_review_unknown_rate: 0.1, review_required_rate: 0.1, p95_latency_ms: 18000, @@ -108,6 +109,17 @@ describe("decideReindexGate — quality", () => { expect(decision.failures.join(" ")).toMatch(/citation_failure_rate 0.05 above absolute ceiling 0/); }); + it("returns NO_GO when an expected danger warning is missing", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality(), + candidateQuality: quality({ expected_danger_warning_missing_count: 1 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/expected_danger_warning_missing_count 1 above absolute ceiling 0/); + }); + it("returns NO_GO when grounded_supported_rate falls below the floor", () => { const decision = decideReindexGate({ baselineRetrieval: retrieval(), @@ -139,4 +151,20 @@ describe("decideReindexGate — quality", () => { expect(decision.decision).toBe("NO_GO"); expect(decision.failures.join(" ")).toMatch(/quality summaries must be provided for both/); }); + + it("fails closed when quality summaries omit required source governance rates", () => { + const withoutGovernanceRates = quality() as Partial; + delete withoutGovernanceRates.stale_review_unknown_rate; + delete withoutGovernanceRates.review_required_rate; + + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality(), + candidateQuality: withoutGovernanceRates as QualityGateSummary, + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/candidateQuality\.stale_review_unknown_rate/); + expect(decision.failures.join(" ")).toMatch(/candidateQuality\.review_required_rate/); + }); }); From 4a58cda11a3d587726658e84a9498ef8e0f4e457 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 08:30:28 +0800 Subject: [PATCH 4/8] fix(review): skip unsupported rate without cases --- src/lib/reindex-eval-gate.ts | 22 ++++++++++++++-------- tests/reindex-eval-gate.test.ts | 12 ++++++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/src/lib/reindex-eval-gate.ts b/src/lib/reindex-eval-gate.ts index 597ff5055..442b86c96 100644 --- a/src/lib/reindex-eval-gate.ts +++ b/src/lib/reindex-eval-gate.ts @@ -19,6 +19,7 @@ export type RetrievalGateSummary = { export type QualityGateSummary = { grounded_supported_rate: number; + unsupported_count: number; unsupported_correct_rate: number; expected_hit_rate?: number; citation_failure_rate: number; @@ -96,6 +97,7 @@ export type ReindexGateDecision = { const requiredQualityMetrics = [ "grounded_supported_rate", + "unsupported_count", "unsupported_correct_rate", "citation_failure_rate", "numeric_grounding_failure_rate", @@ -215,14 +217,6 @@ function qualityChecks( bound: config.groundedSupportedRateFloor, tolerance: config.rateRegressionTolerance, }), - evaluateCheck({ - metric: "unsupported_correct_rate", - direction: "higher_better", - baseline: baseline.unsupported_correct_rate, - candidate: candidate.unsupported_correct_rate, - bound: config.unsupportedCorrectRateFloor, - tolerance: 0, - }), evaluateCheck({ metric: "citation_failure_rate", direction: "lower_better", @@ -264,6 +258,18 @@ function qualityChecks( tolerance: config.latencyRegressionMs, }), ]; + if (baseline.unsupported_count > 0 || candidate.unsupported_count > 0) { + checks.push( + evaluateCheck({ + metric: "unsupported_correct_rate", + direction: "higher_better", + baseline: baseline.unsupported_correct_rate, + candidate: candidate.unsupported_correct_rate, + bound: config.unsupportedCorrectRateFloor, + tolerance: 0, + }), + ); + } if (typeof baseline.expected_hit_rate === "number" && typeof candidate.expected_hit_rate === "number") { checks.push( evaluateCheck({ diff --git a/tests/reindex-eval-gate.test.ts b/tests/reindex-eval-gate.test.ts index ff366d661..a3be8f479 100644 --- a/tests/reindex-eval-gate.test.ts +++ b/tests/reindex-eval-gate.test.ts @@ -15,6 +15,7 @@ function retrieval(overrides: Partial = {}): RetrievalGate function quality(overrides: Partial = {}): QualityGateSummary { return { grounded_supported_rate: 0.95, + unsupported_count: 1, unsupported_correct_rate: 1, expected_hit_rate: 0.9, citation_failure_rate: 0, @@ -109,6 +110,17 @@ describe("decideReindexGate — quality", () => { expect(decision.failures.join(" ")).toMatch(/citation_failure_rate 0.05 above absolute ceiling 0/); }); + it("skips unsupported_correct_rate when there are no unsupported cases", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality({ unsupported_count: 0, unsupported_correct_rate: 0 }), + candidateQuality: quality({ unsupported_count: 0, unsupported_correct_rate: 0 }), + }); + expect(decision.decision).toBe("GO"); + expect(decision.checks.some((check) => check.metric === "unsupported_correct_rate")).toBe(false); + }); + it("returns NO_GO when an expected danger warning is missing", () => { const decision = decideReindexGate({ baselineRetrieval: retrieval(), From 95cbc0b80e225518cb39b7850305cd2b1c5e8855 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 12:09:52 +0800 Subject: [PATCH 5/8] fix(review): harden reindex gate and route evidence --- src/lib/clinical-search.ts | 12 ++-- src/lib/rag.ts | 9 ++- src/lib/reindex-eval-gate.ts | 78 +++++++++++++++++++++++++- src/lib/retrieval-selection.ts | 14 +++-- tests/reindex-eval-gate.test.ts | 35 +++++++++++- tests/retrieval-query-variants.test.ts | 28 +++++++++ tests/retrieval-selection.test.ts | 34 +++++++++++ 7 files changed, 193 insertions(+), 17 deletions(-) diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts index 31feef86a..71342c45d 100644 --- a/src/lib/clinical-search.ts +++ b/src/lib/clinical-search.ts @@ -243,6 +243,8 @@ const domainAliasGroups = [ ["lai", "long acting injectable", "depot", "long-acting injectable"], ["im", "intramuscular"], ["po", "oral"], + ["sc", "subcutaneous", "subcut", "subcutaneously"], + ["sl", "sublingual", "sublingually"], ["prn", "as required"], ["mhsp", "mental health service procedure"], ["fda", "food and drug administration"], @@ -320,7 +322,7 @@ const intentPatterns: Array<{ { intent: "drug_dosing", pattern: - /dose|dosage|dosing|titrate|mg|mcg|frequency|route|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|administer|table|chart|monitor/i, + /dose|dosage|dosing|titrate|mg|mcg|frequency|route|oral|intramuscular|subcutaneous|subcut|sublingual|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b|administer|table|chart|monitor/i, imageEvidenceFocus: true, sectionedLookup: false, }, @@ -349,7 +351,7 @@ const tableThresholdPattern = // genuine medication_dose_risk query still matches via a drug name, dose/route term, or the // medication/pharmacology/agitation vocabulary retained below. const medicationDoseRiskPattern = - /\b(medication|medicine|pharmacolog\w*|prescrib\w*|dose|dosage|dosing|mg|mcg|titrate|route|oral|intramuscular|administer\w*|\bim\b|\bpo\b|\bprn\b|clozapine|lithium|neuroleptic|antipsychotic|benzodiazepine|injectables?|agitation|arousal|side effect\w*|adverse|toxicity|contraindicat\w*|monitor\w*)\b/i; + /\b(medication|medicine|pharmacolog\w*|prescrib\w*|dose|dosage|dosing|mg|mcg|titrate|route|oral|intramuscular|subcutaneous|subcut|sublingual|administer\w*|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b|clozapine|lithium|neuroleptic|antipsychotic|benzodiazepine|injectables?|agitation|arousal|side effect\w*|adverse|toxicity|contraindicat\w*|monitor\w*)\b/i; const documentIncludePattern = /\b(?:what should|what must|what does|what do|which items?|requirements?|checklist|forms?)\b.{0,80}\b(?:include|contain|cover|require|required|needed|need)\b|\b(?:include|contain|cover|require|required|needed|need)\b.{0,80}\b(?:plan|form|checklist|protocol|procedure|guideline|document|file|pdf)\b/i; const explicitDocumentLookupPattern = @@ -519,7 +521,7 @@ function queryClassFromSignals(args: { ) return "document_lookup"; if ( - /\b(?:dose|dosage|dosing|route|mg|mcg|microgram|\bim\b|\bpo\b|\bprn\b)\b/i.test(args.normalizedQuery) && + /\b(?:dose|dosage|dosing|route|mg|mcg|microgram|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b)\b/i.test(args.normalizedQuery) && (args.medications.length > 0 || medicationDoseRiskPattern.test(args.normalizedQuery)) ) { return "medication_dose_risk"; @@ -554,7 +556,7 @@ function intentFromSignals(queryClass: RagQueryClass, normalizedQuery: string): if (queryClass === "document_lookup") return "document_lookup"; if (queryClass === "table_threshold" || queryClass === "medication_dose_risk") { if ( - /(dose|dosage|dosing|mg|mcg|route|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|administer)/i.test(normalizedQuery) + /(dose|dosage|dosing|mg|mcg|route|oral|intramuscular|subcutaneous|subcut|sublingual|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b|administer)/i.test(normalizedQuery) ) { return "drug_dosing"; } @@ -724,7 +726,7 @@ export function hasDoseEvidenceSupport(result: SearchResult) { ) .map((image) => `${image.tableTextSnippet ?? ""} ${image.caption ?? ""} ${image.tableTitle ?? ""}`) .join(" ")}`.toLowerCase(); - return /\b(?:dose|dosage|dosing|mg|mcg|microgram|route|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|administer\w*|titration|titrate|frequency|maximum|tablet|injection|antipsychotic|benzodiazepine|olanzapine|lorazepam|haloperidol|droperidol|promethazine|diazepam)\b/i.test( + return /\b(?:dose|dosage|dosing|mg|mcg|microgram|route|oral|intramuscular|subcutaneous|subcut|sublingual|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b|administer\w*|titration|titrate|frequency|maximum|tablet|injection|antipsychotic|benzodiazepine|olanzapine|lorazepam|haloperidol|droperidol|promethazine|diazepam)\b/i.test( haystack, ); } diff --git a/src/lib/rag.ts b/src/lib/rag.ts index ccb4bcdea..ea5b7420a 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -3434,7 +3434,9 @@ function hasDoseAmountEvidenceForGate(result: SearchResult) { } function hasRouteEvidenceForGate(result: SearchResult) { - return /\b(?:oral|orally|intramuscular|intramuscularly|\bim\b|\bpo\b)\b/i.test(evidenceTextForGate(result)); + return /\b(?:oral|orally|intramuscular|intramuscularly|subcutaneous|subcutaneously|subcut|sublingual|sublingually|\bim\b|\bpo\b|\bsc\b|\bsl\b)\b/i.test( + evidenceTextForGate(result), + ); } function hasDirectSourceImageEvidence(result: SearchResult) { @@ -3585,7 +3587,10 @@ export function evaluateEvidenceCoverageGate( } if (queryClass === "medication_dose_risk") { - const asksDoseRoute = /\b(?:dose|dosage|dosing|route|oral|intramuscular|\bim\b|\bpo\b)\b/i.test(query); + const asksDoseRoute = + /\b(?:dose|dosage|dosing|route|oral|intramuscular|subcutaneous|subcut|sublingual|\bim\b|\bpo\b|\bsc\b|\bsl\b)\b/i.test( + query, + ); const agitationOk = !/\bagitation|arousal\b/i.test(query) || /\bagitation|arousal\b/i.test(evidenceText); const accepted = hasDoseEvidence && hasDoseAmount && (!asksDoseRoute || hasRoute) && agitationOk; return { diff --git a/src/lib/reindex-eval-gate.ts b/src/lib/reindex-eval-gate.ts index 442b86c96..94e754a30 100644 --- a/src/lib/reindex-eval-gate.ts +++ b/src/lib/reindex-eval-gate.ts @@ -10,6 +10,8 @@ // Subsets of scripts/eval-retrieval.ts and scripts/eval-quality.ts summary output. Field // names match those scripts so the driver can pass summaries through unchanged. export type RetrievalGateSummary = { + case_count: number; + case_fingerprint?: string; document_recall_at_5: number; content_recall_at_5: number; top_k_hit_rate: number; @@ -18,6 +20,8 @@ export type RetrievalGateSummary = { }; export type QualityGateSummary = { + case_count: number; + case_fingerprint?: string; grounded_supported_rate: number; unsupported_count: number; unsupported_correct_rate: number; @@ -95,7 +99,15 @@ export type ReindexGateDecision = { failures: string[]; }; +const requiredRetrievalMetrics = [ + "case_count", + "document_recall_at_5", + "content_recall_at_5", + "top_k_hit_rate", +] as const satisfies readonly (keyof RetrievalGateSummary)[]; + const requiredQualityMetrics = [ + "case_count", "grounded_supported_rate", "unsupported_count", "unsupported_correct_rate", @@ -309,6 +321,53 @@ function missingQualityMetrics(label: "baselineQuality" | "candidateQuality", su ); } +function missingRetrievalMetrics(label: "baselineRetrieval" | "candidateRetrieval", summary: RetrievalGateSummary) { + return requiredRetrievalMetrics.flatMap((metric) => + typeof summary[metric] === "number" && Number.isFinite(summary[metric]) ? [] : [`${label}.${metric}`], + ); +} + +function populationFailures(args: { + baselineRetrieval: RetrievalGateSummary; + candidateRetrieval: RetrievalGateSummary; + baselineQuality?: QualityGateSummary; + candidateQuality?: QualityGateSummary; +}) { + const failures: string[] = []; + if (args.baselineRetrieval.case_count !== args.candidateRetrieval.case_count) { + failures.push( + `retrieval case_count mismatch: baseline ${args.baselineRetrieval.case_count} vs candidate ${args.candidateRetrieval.case_count}`, + ); + } + const baselineRetrievalFingerprint = args.baselineRetrieval.case_fingerprint; + const candidateRetrievalFingerprint = args.candidateRetrieval.case_fingerprint; + if (baselineRetrievalFingerprint || candidateRetrievalFingerprint) { + if (!baselineRetrievalFingerprint || !candidateRetrievalFingerprint) { + failures.push("retrieval case_fingerprint must be present for both baseline and candidate, or neither"); + } else if (baselineRetrievalFingerprint !== candidateRetrievalFingerprint) { + failures.push("retrieval case_fingerprint mismatch"); + } + } + + if (args.baselineQuality && args.candidateQuality) { + if (args.baselineQuality.case_count !== args.candidateQuality.case_count) { + failures.push( + `quality case_count mismatch: baseline ${args.baselineQuality.case_count} vs candidate ${args.candidateQuality.case_count}`, + ); + } + const baselineQualityFingerprint = args.baselineQuality.case_fingerprint; + const candidateQualityFingerprint = args.candidateQuality.case_fingerprint; + if (baselineQualityFingerprint || candidateQualityFingerprint) { + if (!baselineQualityFingerprint || !candidateQualityFingerprint) { + failures.push("quality case_fingerprint must be present for both baseline and candidate, or neither"); + } else if (baselineQualityFingerprint !== candidateQualityFingerprint) { + failures.push("quality case_fingerprint mismatch"); + } + } + } + return failures; +} + export function decideReindexGate( input: { baselineRetrieval: RetrievalGateSummary; @@ -318,7 +377,20 @@ export function decideReindexGate( }, config: ReindexGateConfig = defaultReindexGateConfig, ): ReindexGateDecision { + const missingRetrieval = [ + ...missingRetrievalMetrics("baselineRetrieval", input.baselineRetrieval), + ...missingRetrievalMetrics("candidateRetrieval", input.candidateRetrieval), + ]; + if (missingRetrieval.length > 0) { + return { + decision: "NO_GO", + checks: [], + failures: [`retrieval summaries are missing required metrics: ${missingRetrieval.join(", ")}`], + }; + } + const checks = retrievalChecks(input.baselineRetrieval, input.candidateRetrieval, config.retrieval); + const gateFailures = populationFailures(input); const hasBaselineQuality = Boolean(input.baselineQuality); const hasCandidateQuality = Boolean(input.candidateQuality); @@ -328,7 +400,7 @@ export function decideReindexGate( return { decision: "NO_GO", checks, - failures: ["quality summaries must be provided for both baseline and candidate, or neither"], + failures: [...gateFailures, "quality summaries must be provided for both baseline and candidate, or neither"], }; } if (input.baselineQuality && input.candidateQuality) { @@ -340,12 +412,12 @@ export function decideReindexGate( return { decision: "NO_GO", checks, - failures: [`quality summaries are missing required metrics: ${missingMetrics.join(", ")}`], + failures: [...gateFailures, `quality summaries are missing required metrics: ${missingMetrics.join(", ")}`], }; } checks.push(...qualityChecks(input.baselineQuality, input.candidateQuality, config.quality)); } - const failures = checks.flatMap((check) => check.reasons); + const failures = [...gateFailures, ...checks.flatMap((check) => check.reasons)]; return { decision: failures.length === 0 ? "GO" : "NO_GO", checks, failures }; } diff --git a/src/lib/retrieval-selection.ts b/src/lib/retrieval-selection.ts index 8b14c1545..eaf7bef49 100644 --- a/src/lib/retrieval-selection.ts +++ b/src/lib/retrieval-selection.ts @@ -157,7 +157,9 @@ function hasDoseAmount(text: string) { } function hasRoute(text: string) { - return /\b(?:oral|orally|intramuscular|intramuscularly|im|po)\b/.test(text); + return /\b(?:oral|orally|intramuscular|intramuscularly|subcutaneous|subcutaneously|subcut|sublingual|sublingually|im|po|sc|sl)\b/.test( + text, + ); } function hasSourceImageEvidence(result: SearchResult) { @@ -337,9 +339,10 @@ function resultBoost(args: { intent: RetrievalIntent; candidate: RetrievalCandid export function buildRetrievalIntent(query: string, queryClass: RagQueryClass): RetrievalIntent { const normalizedQuery = normalize(query); - const asksDoseRoute = /\b(?:dose|dosage|dosing|route|oral|intramuscular|im|po|frequency|mg|mcg|prn)\b/.test( - normalizedQuery, - ); + const asksDoseRoute = + /\b(?:dose|dosage|dosing|route|oral|intramuscular|subcutaneous|subcut|sublingual|im|po|sc|sl|frequency|mg|mcg|prn)\b/.test( + normalizedQuery, + ); const asksDoseAmount = /\b(?:dose|dosage|dosing|mg|mcg|microgram|maximum|minimum)\b/.test(normalizedQuery); const asksTable = /\b(?:table|chart|matrix|threshold|cutoff|cut off|range|criteria|row)\b/.test(normalizedQuery); const asksSourceImage = @@ -382,7 +385,8 @@ export function buildRetrievalIntent(query: string, queryClass: RagQueryClass): } if (asksDoseRoute) { if (asksDoseAmount) requiredTermSignals.push("dose_amount"); - if (/\b(?:route|oral|intramuscular|im|po)\b/.test(normalizedQuery)) requiredTermSignals.push("route"); + if (/\b(?:route|oral|intramuscular|subcutaneous|subcut|sublingual|im|po|sc|sl)\b/.test(normalizedQuery)) + requiredTermSignals.push("route"); } if (asksFlowchart) { preferredDocumentSignals.push("flowchart", "pathway", "risk matrix"); diff --git a/tests/reindex-eval-gate.test.ts b/tests/reindex-eval-gate.test.ts index a3be8f479..c795520f3 100644 --- a/tests/reindex-eval-gate.test.ts +++ b/tests/reindex-eval-gate.test.ts @@ -3,6 +3,7 @@ import { decideReindexGate, type QualityGateSummary, type RetrievalGateSummary } function retrieval(overrides: Partial = {}): RetrievalGateSummary { return { + case_count: 4, document_recall_at_5: 0.9, content_recall_at_5: 0.9, top_k_hit_rate: 0.95, @@ -14,6 +15,7 @@ function retrieval(overrides: Partial = {}): RetrievalGate function quality(overrides: Partial = {}): QualityGateSummary { return { + case_count: 4, grounded_supported_rate: 0.95, unsupported_count: 1, unsupported_correct_rate: 1, @@ -79,13 +81,31 @@ describe("decideReindexGate — retrieval", () => { it("skips optional metrics when either summary omits them", () => { const decision = decideReindexGate({ - baselineRetrieval: { document_recall_at_5: 0.9, content_recall_at_5: 0.9, top_k_hit_rate: 0.9 }, - candidateRetrieval: { document_recall_at_5: 0.9, content_recall_at_5: 0.9, top_k_hit_rate: 0.9 }, + baselineRetrieval: { case_count: 4, document_recall_at_5: 0.9, content_recall_at_5: 0.9, top_k_hit_rate: 0.9 }, + candidateRetrieval: { case_count: 4, document_recall_at_5: 0.9, content_recall_at_5: 0.9, top_k_hit_rate: 0.9 }, }); expect(decision.decision).toBe("GO"); expect(decision.checks.some((check) => check.metric === "mrr_at_10")).toBe(false); expect(decision.checks.some((check) => check.metric === "p90_latency_ms")).toBe(false); }); + + it("fails closed when retrieval summaries cover different eval populations", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ case_count: 12 }), + candidateRetrieval: retrieval({ case_count: 4 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/retrieval case_count mismatch/); + }); + + it("fails closed when retrieval case fingerprints differ", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ case_fingerprint: "all-cases-v1" }), + candidateRetrieval: retrieval({ case_fingerprint: "limited-cases-v1" }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/retrieval case_fingerprint mismatch/); + }); }); describe("decideReindexGate — quality", () => { @@ -179,4 +199,15 @@ describe("decideReindexGate — quality", () => { expect(decision.failures.join(" ")).toMatch(/candidateQuality\.stale_review_unknown_rate/); expect(decision.failures.join(" ")).toMatch(/candidateQuality\.review_required_rate/); }); + + it("fails closed when quality summaries cover different eval populations", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality({ case_count: 9 }), + candidateQuality: quality({ case_count: 3 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/quality case_count mismatch/); + }); }); diff --git a/tests/retrieval-query-variants.test.ts b/tests/retrieval-query-variants.test.ts index 2a2b73efa..ddb62411e 100644 --- a/tests/retrieval-query-variants.test.ts +++ b/tests/retrieval-query-variants.test.ts @@ -633,6 +633,34 @@ describe("retrieval query variants", () => { ).toMatchObject({ accepted: true, reason: "dose_route_amount_evidence_gate" }); }); + it("accepts SC and SL route evidence for dose-route fast gates", () => { + expect( + evaluateEvidenceCoverageGate( + "What SC route dose is listed?", + [ + result({ + content: "Medication chart: 2 mg subcutaneous administration is listed for the route option.", + similarity: 0.8, + }), + ], + "medication_dose_risk", + ), + ).toMatchObject({ accepted: true, reason: "dose_route_amount_evidence_gate" }); + + expect( + evaluateEvidenceCoverageGate( + "What SL route dose is listed?", + [ + result({ + content: "Medication chart: 2 mg sublingual administration is listed for the route option.", + similarity: 0.8, + }), + ], + "medication_dose_risk", + ), + ).toMatchObject({ accepted: true, reason: "dose_route_amount_evidence_gate" }); + }); + it("requires direct source image evidence for source image/table requests", () => { const query = "Show the source table image for the patient property restricted items table."; expect( diff --git a/tests/retrieval-selection.test.ts b/tests/retrieval-selection.test.ts index bac8cb8b3..fcc420c4f 100644 --- a/tests/retrieval-selection.test.ts +++ b/tests/retrieval-selection.test.ts @@ -160,6 +160,40 @@ describe("retrieval source selection", () => { expect(selection.summary.topChunkTypes.medication_chart).toBeGreaterThan(0); }); + it("treats SC and SL administration rows as dose-route evidence", () => { + const intent = buildRetrievalIntent("What SC or SL route options are listed?", "medication_dose_risk"); + const selected = summarizeRetrievalSelection({ + query: "What SC or SL route options are listed?", + queryClass: "medication_dose_risk", + results: [ + source({ + id: "sc-sl-route-row", + title: "Medication Route Chart", + content: "Subcutaneous medication may be used for one option; sublingual medication is listed for another.", + hybrid_score: 0.66, + index_unit: { + id: "unit-sc-sl-route", + unit_type: "medication_chart_row", + title: "SC and SL route options", + content: "SC route and SL route options.", + source_chunk_id: "sc-sl-route-row", + source_image_id: null, + page_start: 2, + page_end: 2, + heading_path: ["Medication chart"], + normalized_terms: ["sc", "subcutaneous", "sl", "sublingual"], + quality_score: 0.9, + extraction_mode: "hybrid", + }, + }), + ], + }); + + expect(intent.requiredTermSignals).toContain("route"); + expect(selected.summary.requiredSignalsSatisfied).toBe(true); + expect(selected.summary.matchedSignals).toContain("route"); + }); + it("promotes flowchart next-step evidence for red-zone pathway questions", () => { const selection = selectRetrievalEvidence({ query: "In the clinical flowchart, what is the next step after red-zone risk?", From 9bf416fa98d65f4a26ca64d4d0290817226a6402 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 12:35:13 +0800 Subject: [PATCH 6/8] fix(ci): format PR 213 RAG changes --- src/lib/clinical-search.ts | 12 +++++++++--- src/lib/rag.ts | 5 ++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts index 71342c45d..d4b04191f 100644 --- a/src/lib/clinical-search.ts +++ b/src/lib/clinical-search.ts @@ -521,7 +521,9 @@ function queryClassFromSignals(args: { ) return "document_lookup"; if ( - /\b(?:dose|dosage|dosing|route|mg|mcg|microgram|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b)\b/i.test(args.normalizedQuery) && + /\b(?:dose|dosage|dosing|route|mg|mcg|microgram|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b)\b/i.test( + args.normalizedQuery, + ) && (args.medications.length > 0 || medicationDoseRiskPattern.test(args.normalizedQuery)) ) { return "medication_dose_risk"; @@ -556,7 +558,9 @@ function intentFromSignals(queryClass: RagQueryClass, normalizedQuery: string): if (queryClass === "document_lookup") return "document_lookup"; if (queryClass === "table_threshold" || queryClass === "medication_dose_risk") { if ( - /(dose|dosage|dosing|mg|mcg|route|oral|intramuscular|subcutaneous|subcut|sublingual|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b|administer)/i.test(normalizedQuery) + /(dose|dosage|dosing|mg|mcg|route|oral|intramuscular|subcutaneous|subcut|sublingual|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b|administer)/i.test( + normalizedQuery, + ) ) { return "drug_dosing"; } @@ -1081,7 +1085,9 @@ export function buildClinicalTextSearchQuery(query: string) { /\b(?:risk|red\s*zone|red|urgent|escalat|next step)\b/i.test(query); const wantsClozapineBloodMonitoring = /\bclozapine\b/i.test(correctedQueryText) && - /\b(?:blood|bloods|fbc|wcc|full blood count|white cell|observation|observations|monitor|monitoring)\b/i.test(correctedQueryText); + /\b(?:blood|bloods|fbc|wcc|full blood count|white cell|observation|observations|monitor|monitoring)\b/i.test( + correctedQueryText, + ); const wantsClozapineMissedDose = /\bclozapine\b/i.test(correctedQueryText) && /\bmissed\b/i.test(correctedQueryText) && diff --git a/src/lib/rag.ts b/src/lib/rag.ts index ea5b7420a..b904be26c 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -3546,7 +3546,10 @@ export function evaluateEvidenceCoverageGate( /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils|full blood|white cell)\b/i.test(query) && /\b(?:withhold|withheld|withholding|cease|ceased|stop|stopped)\b/i.test(query) ) { - const hasBlood = hasAnyTerm(evidenceText, /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils|full blood|white cell)\b/i); + const hasBlood = hasAnyTerm( + evidenceText, + /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils|full blood|white cell)\b/i, + ); const hasAction = hasAnyTerm( evidenceText, /\b(?:withhold|withheld|withholding|cease|ceased|stop|stopped|red)\b/i, From da643f48f98035f843ee1a1d4acedb9cc0b725a1 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:10:51 +0800 Subject: [PATCH 7/8] fix(rag): address document chunk review comments --- src/lib/chunking.ts | Bin 30021 -> 31593 bytes tests/chunking.test.ts | 70 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/lib/chunking.ts b/src/lib/chunking.ts index d5c526265cca92c8d228138055a0c054cef9ae3a..651a61e14cce9c2173dcb3f6d10a2944784924b1 100644 GIT binary patch delta 1669 zcmZ`(&u<%55SA@bs3{zZLUBvzWE-U2;%?n?sAS8DP;k^J)QN1j9Fh_(>t}oEdUy5q znWQAHxxoRcD(}P%BqUBqp#29pz!mrtxWc73L{$#VynSnLRFZ@9c7A>{^UeJ7SK-gU z3qO2Wyul8lkPC3bAQtfM*4ph`%~p4B?dI*)X0yF_r`hgoY_*^P#x=WYSFiL!cRXT& zNFElSUU+@Cx_lTjS9oFYPT&qh4y&tR7!#7q+672{e&dEY?T-Ux0+BOd?}e`8HxFFK zBe5Rxwc$9pXS(#Og*Au;_kux1nRnTN$RrjVmOvwQ!%^h8VjCkOI0{UbHfNFVxXiq| z8^3iGLl|IK@^St7TXppG`d}{Q94+8XjClb0^!3s_2cddDYE1Z*EVOqi$z`Dk z%RT!7HbiOUcJJJjdxN2{mERTcSwDSaKhY#DBW6DGk@cuKvreU)OG1A~^r$fDE7o|}BrzFwHB zHPT7d+LKamANEZpjw2`7MsYl2FI98(jf%#3rWuXdw=*$v6P8tZ5L3WA3sJ z+T$Id?KKZ|D-fG}+*y=IcTX2Up6Qk^&lOp`n!=({XH}A;-7PQ7zb9&VRO}>acP&K} zJs)}0z9=7vH z;;xQUGg2y3eK>KW{;JMTr*4jHU5>&_FXq1=g%^)~{}?XiEBR{olKeb6yRb1L3aCSa lYvO%47KlMQ{*rvie?3D1Ghxz;&lfH~Oa7DCDnAy#{Rf&sAL{@B delta 314 zcmaF)jq&I!#toC0HlJboAuYk6QeB*yT#}ie=a`qAkzb@>Z?B-FRLijWo?N8H=3`#1 zOq)}ESvV(8NEYAxD9nOUhD!koit|g0l2d~V67$^hi$YQ>N;Ik`?~Rb1yt;` zSCE*V3KG{*NUcatEh+#q(h_rWk`j}%T_FNi3d#9-C8>EO3bmR#P=hyzMrv|xUY>lC zR~G6p&s?BE#R|3xP(Ik;;>pn!ij$R#gf>?c&S#pOonXqEnWmtTHo31{nK5nho^p90 zS(vB@BriJG&d==X!5^GvB~#Jg(g>3TwsUV WGkIl_^yI?I|C5&|t4=;%wFm&LH+C}s diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index 12b5988f4..aeedf38fb 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { CHUNKER_VERSION, + DOCUMENT_CHUNKER_VERSION, buildChunks, buildImageTag, chunkContentKey, @@ -418,12 +419,23 @@ describe("document-mode chunking (CI-1)", () => { expect(spanning).toBeDefined(); expect(spanning?.metadata.page_start).toBe(1); expect(spanning?.metadata.page_end).toBe(2); + expect(spanning?.page_number).toBeNull(); expect(spanning?.section_path).toContain("Clozapine Monitoring"); // A cross-page chunk records a source span for each contributing page. const spanPageNumbers = (spanning?.metadata.source_spans as Array<{ page_number: number | null }>).map( (span) => span.page_number, ); expect(spanPageNumbers).toEqual(expect.arrayContaining([1, 2])); + const pageTwoSpan = ( + spanning?.metadata.source_spans as Array<{ + page_number: number | null; + excerpt: string; + character_start: number | null; + }> + ).find((span) => span.page_number === 2); + expect(pageTwoSpan?.excerpt).toContain("Continue weekly blood test monitoring"); + expect(pageTwoSpan?.excerpt).not.toContain("Baseline full blood count"); + expect(pageTwoSpan?.character_start).not.toBeNull(); }); it("does not merge across a section boundary introduced by a new heading", async () => { @@ -461,7 +473,63 @@ describe("document-mode chunking (CI-1)", () => { metadata: {}, }, ]); - expect(chunks[0].metadata.chunker_version).toBe(CHUNKER_VERSION); + expect(chunks[0].metadata.chunker_version).toBe(DOCUMENT_CHUNKER_VERSION); + expect(chunks[0].metadata.chunk_strategy).toBe("document"); expect(chunks[0].metadata.chunk_key).toMatch(/^[0-9a-f]{32}$/); }); + + it("keeps document-mode images scoped to pages that contributed to the chunk", async () => { + vi.stubEnv("CHUNK_SIZE", "900"); + const pageOneBody = Array.from({ length: 90 }, (_, index) => `Baseline monitoring instruction ${index}.`).join(" "); + const chunks = await buildDocumentChunks([ + { + documentId: "doc-1", + pageNumber: 1, + pageText: `Monitoring Guidance\n\n${pageOneBody}`, + metadata: {}, + images: [], + }, + { + documentId: "doc-1", + pageNumber: 2, + pageText: "Continuation dosing table text for later review.", + metadata: {}, + images: [ + { + id: "page-2-table", + caption: "Later-page table.", + pageNumber: 2, + sourceKind: "table_crop", + tableTitle: "Monitoring Guidance", + }, + ], + }, + ]); + + const pageOneChunk = chunks.find((chunk) => chunk.metadata.page_start === 1 && chunk.metadata.page_end === 1); + expect(pageOneChunk).toBeDefined(); + expect(pageOneChunk?.image_ids).not.toContain("page-2-table"); + }); + + it("preserves repeated document-mode clinical chunks when page context differs", async () => { + const repeatedMonitoringText = + "Clozapine monitoring table\n\nANC threshold 0.5 x 10^9/L: withhold clozapine and repeat FBC daily."; + const chunks = await buildDocumentChunks([ + { + documentId: "doc-1", + pageNumber: 4, + pageText: repeatedMonitoringText, + metadata: {}, + }, + { + documentId: "doc-1", + pageNumber: 7, + pageText: repeatedMonitoringText, + metadata: {}, + }, + ]); + + expect(chunks.filter((chunk) => chunk.content.includes("ANC threshold 0.5 x 10^9/L"))).toHaveLength(2); + expect(chunks.map((chunk) => chunk.metadata.page_start)).toEqual([4, 7]); + }); }); From 160ce6c7551c7af287297334c967387f9a7a131d Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 14:14:05 +0800 Subject: [PATCH 8/8] fix(rag): fail closed reindex gate gaps --- src/lib/reindex-eval-gate.ts | 20 ++++++++++ tests/reindex-eval-gate.test.ts | 68 +++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/src/lib/reindex-eval-gate.ts b/src/lib/reindex-eval-gate.ts index 94e754a30..3dcbe906e 100644 --- a/src/lib/reindex-eval-gate.ts +++ b/src/lib/reindex-eval-gate.ts @@ -12,6 +12,7 @@ export type RetrievalGateSummary = { case_count: number; case_fingerprint?: string; + failed_case_count: number; document_recall_at_5: number; content_recall_at_5: number; top_k_hit_rate: number; @@ -40,6 +41,7 @@ export type ReindexGateConfig = { documentRecallAt5Floor: number; contentRecallAt5Floor: number; topKHitRateFloor: number; + failedCaseCountCeiling: number; rateRegressionTolerance: number; latencyRegressionMs: number; }; @@ -64,6 +66,7 @@ export const defaultReindexGateConfig: ReindexGateConfig = { documentRecallAt5Floor: 0.8, contentRecallAt5Floor: 0.8, topKHitRateFloor: 0.8, + failedCaseCountCeiling: 0, rateRegressionTolerance: 0.02, latencyRegressionMs: 4000, }, @@ -101,6 +104,7 @@ export type ReindexGateDecision = { const requiredRetrievalMetrics = [ "case_count", + "failed_case_count", "document_recall_at_5", "content_recall_at_5", "top_k_hit_rate", @@ -161,6 +165,14 @@ function retrievalChecks( config: ReindexGateConfig["retrieval"], ): MetricCheck[] { const checks: MetricCheck[] = [ + evaluateCheck({ + metric: "failed_case_count", + direction: "lower_better", + baseline: baseline.failed_case_count, + candidate: candidate.failed_case_count, + bound: config.failedCaseCountCeiling, + tolerance: 0, + }), evaluateCheck({ metric: "document_recall_at_5", direction: "higher_better", @@ -374,6 +386,7 @@ export function decideReindexGate( candidateRetrieval: RetrievalGateSummary; baselineQuality?: QualityGateSummary; candidateQuality?: QualityGateSummary; + qualityMode?: "required" | "retrieval_only"; }, config: ReindexGateConfig = defaultReindexGateConfig, ): ReindexGateDecision { @@ -394,6 +407,13 @@ export function decideReindexGate( const hasBaselineQuality = Boolean(input.baselineQuality); const hasCandidateQuality = Boolean(input.candidateQuality); + if (input.qualityMode !== "retrieval_only" && (!hasBaselineQuality || !hasCandidateQuality)) { + return { + decision: "NO_GO", + checks, + failures: [...gateFailures, "quality summaries are required unless qualityMode is retrieval_only"], + }; + } if (hasBaselineQuality !== hasCandidateQuality) { // A one-sided quality summary is a driver error; fail closed rather than silently // gating on retrieval alone. diff --git a/tests/reindex-eval-gate.test.ts b/tests/reindex-eval-gate.test.ts index c795520f3..c56052c93 100644 --- a/tests/reindex-eval-gate.test.ts +++ b/tests/reindex-eval-gate.test.ts @@ -4,6 +4,7 @@ import { decideReindexGate, type QualityGateSummary, type RetrievalGateSummary } function retrieval(overrides: Partial = {}): RetrievalGateSummary { return { case_count: 4, + failed_case_count: 0, document_recall_at_5: 0.9, content_recall_at_5: 0.9, top_k_hit_rate: 0.95, @@ -36,6 +37,7 @@ describe("decideReindexGate — retrieval", () => { const decision = decideReindexGate({ baselineRetrieval: retrieval(), candidateRetrieval: retrieval({ content_recall_at_5: 0.92 }), + qualityMode: "retrieval_only", }); expect(decision.decision).toBe("GO"); expect(decision.failures).toEqual([]); @@ -45,6 +47,7 @@ describe("decideReindexGate — retrieval", () => { const decision = decideReindexGate({ baselineRetrieval: retrieval({ content_recall_at_5: 0.78 }), candidateRetrieval: retrieval({ content_recall_at_5: 0.75 }), + qualityMode: "retrieval_only", }); expect(decision.decision).toBe("NO_GO"); expect(decision.failures.join(" ")).toMatch(/content_recall_at_5 0.75 below absolute floor 0.8/); @@ -56,6 +59,7 @@ describe("decideReindexGate — retrieval", () => { const decision = decideReindexGate({ baselineRetrieval: retrieval({ document_recall_at_5: 0.98 }), candidateRetrieval: retrieval({ document_recall_at_5: 0.85 }), + qualityMode: "retrieval_only", }); expect(decision.decision).toBe("NO_GO"); expect(decision.failures.join(" ")).toMatch(/document_recall_at_5 regressed/); @@ -66,6 +70,7 @@ describe("decideReindexGate — retrieval", () => { const decision = decideReindexGate({ baselineRetrieval: retrieval({ content_recall_at_5: 0.9 }), candidateRetrieval: retrieval({ content_recall_at_5: 0.89 }), + qualityMode: "retrieval_only", }); expect(decision.decision).toBe("GO"); }); @@ -74,25 +79,72 @@ describe("decideReindexGate — retrieval", () => { const decision = decideReindexGate({ baselineRetrieval: retrieval({ p90_latency_ms: 8000 }), candidateRetrieval: retrieval({ p90_latency_ms: 14000 }), + qualityMode: "retrieval_only", }); expect(decision.decision).toBe("NO_GO"); expect(decision.failures.join(" ")).toMatch(/p90_latency_ms regressed/); }); + it("fails closed when retrieval summaries omit failed-case counts", () => { + const baselineWithoutFailedCaseCount = { + case_count: 4, + document_recall_at_5: 0.9, + content_recall_at_5: 0.9, + top_k_hit_rate: 0.9, + } as Partial as RetrievalGateSummary; + const decision = decideReindexGate({ + baselineRetrieval: baselineWithoutFailedCaseCount, + candidateRetrieval: { + case_count: 4, + failed_case_count: 0, + document_recall_at_5: 0.9, + content_recall_at_5: 0.9, + top_k_hit_rate: 0.9, + }, + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/baselineRetrieval\.failed_case_count/); + }); + it("skips optional metrics when either summary omits them", () => { const decision = decideReindexGate({ - baselineRetrieval: { case_count: 4, document_recall_at_5: 0.9, content_recall_at_5: 0.9, top_k_hit_rate: 0.9 }, - candidateRetrieval: { case_count: 4, document_recall_at_5: 0.9, content_recall_at_5: 0.9, top_k_hit_rate: 0.9 }, + baselineRetrieval: { + case_count: 4, + failed_case_count: 0, + document_recall_at_5: 0.9, + content_recall_at_5: 0.9, + top_k_hit_rate: 0.9, + }, + candidateRetrieval: { + case_count: 4, + failed_case_count: 0, + document_recall_at_5: 0.9, + content_recall_at_5: 0.9, + top_k_hit_rate: 0.9, + }, + qualityMode: "retrieval_only", }); expect(decision.decision).toBe("GO"); expect(decision.checks.some((check) => check.metric === "mrr_at_10")).toBe(false); expect(decision.checks.some((check) => check.metric === "p90_latency_ms")).toBe(false); }); + it("fails closed when retrieval eval reports failed cases", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval({ failed_case_count: 1 }), + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/failed_case_count 1 above absolute ceiling 0/); + }); + it("fails closed when retrieval summaries cover different eval populations", () => { const decision = decideReindexGate({ baselineRetrieval: retrieval({ case_count: 12 }), candidateRetrieval: retrieval({ case_count: 4 }), + qualityMode: "retrieval_only", }); expect(decision.decision).toBe("NO_GO"); expect(decision.failures.join(" ")).toMatch(/retrieval case_count mismatch/); @@ -102,10 +154,20 @@ describe("decideReindexGate — retrieval", () => { const decision = decideReindexGate({ baselineRetrieval: retrieval({ case_fingerprint: "all-cases-v1" }), candidateRetrieval: retrieval({ case_fingerprint: "limited-cases-v1" }), + qualityMode: "retrieval_only", }); expect(decision.decision).toBe("NO_GO"); expect(decision.failures.join(" ")).toMatch(/retrieval case_fingerprint mismatch/); }); + + it("fails closed when quality summaries are omitted by default", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/quality summaries are required/); + }); }); describe("decideReindexGate — quality", () => { @@ -181,7 +243,7 @@ describe("decideReindexGate — quality", () => { candidateQuality: quality(), }); expect(decision.decision).toBe("NO_GO"); - expect(decision.failures.join(" ")).toMatch(/quality summaries must be provided for both/); + expect(decision.failures.join(" ")).toMatch(/quality summaries are required/); }); it("fails closed when quality summaries omit required source governance rates", () => {