diff --git a/.env.example b/.env.example index 92ce6b6a7..4c83345a4 100644 --- a/.env.example +++ b/.env.example @@ -29,6 +29,9 @@ OPENAI_FAST_ANSWER_MODEL=gpt-5.5 OPENAI_STRONG_ANSWER_MODEL=gpt-5.5 OPENAI_MAX_OUTPUT_TOKENS=4000 OPENAI_QUERY_CACHE_SIZE=200 +# Max inputs per embeddings request (OpenAI caps a request at 2048 inputs / ~300k tokens). +# embedTexts splits a full-corpus re-embed into batches of this size. +OPENAI_EMBEDDING_BATCH_SIZE=256 OPENAI_VISION_MODEL=gpt-5.5 OPENAI_VISION_IMAGE_DETAIL=auto OPENAI_REQUEST_TIMEOUT_MS=45000 @@ -50,6 +53,9 @@ OPENAI_TEXT_VERBOSITY=low # openai: always attempt OpenAI (legacy behaviour). # offline: never call OpenAI; lexical retrieval + deterministic source-only answers only. RAG_PROVIDER_MODE=auto +# Optional JSON override for app-layer ranking weights (see src/lib/ranking-config.ts). +# Omit for current defaults. Example (enable diversity demotion + linear freshness): +# RAG_RANKING_CONFIG={"documentDiversityPenalty":0.03,"freshness":{"mode":"linear"}} RAG_ANSWER_CACHE_TTL_MS=300000 RAG_ANSWER_CACHE_SIZE=100 RAG_SEARCH_CACHE_TTL_MS=60000 @@ -74,6 +80,9 @@ MAX_IMPORT_JOBS_PER_RUN=5 MAX_IMPORT_BYTES_PER_RUN=157286400 CHUNK_SIZE=2000 CHUNK_OVERLAP=200 +# Chunking strategy: "page" (default, page-bounded) or "document" (structure-aware, +# cross-page). Only set "document" for the eval-gated shadow re-index. +CHUNK_STRATEGY=page WORKER_POLL_MS=30000 WORKER_BATCH_SIZE=3 WORKER_CONCURRENCY=1 diff --git a/docs/reindex-shadow-harness-design.md b/docs/reindex-shadow-harness-design.md new file mode 100644 index 000000000..4ead993bf --- /dev/null +++ b/docs/reindex-shadow-harness-design.md @@ -0,0 +1,130 @@ +# Shadow Re-Index + Eval-Gate Harness — Implementation Runbook + +Status: **design ready, not yet applied** — the SQL here touches retrieval and MUST be +validated against the live `Clinical KB Database` before use (see "Mandatory validation"). +The offline decision core (`src/lib/reindex-eval-gate.ts` — `decideReindexGate`) is built and +unit-tested; this doc specifies the remaining live pieces so they are turnkey when the billed +OpenAI key + Supabase service secrets are available. + +## Goal + +Build the new (better-chunked, re-enriched) index for a document into a **staged** generation +alongside the live committed one, run the golden retrieval + RAG quality evals **against the +staged generation**, and atomically cut over per batch **only if** `decideReindexGate` returns +`GO` (staged ≥ baseline on every metric and all absolute release bars met). Live answers are +never exposed to an unproven generation. + +The atomic-generation machinery already exists: `index_generation_id` on every artifact, +`commit_document_index_generation` (migration `20260628000000`), the two filter functions +`is_committed_document_generation` / `is_committed_artifact_generation`, and +`cleanup_abandoned_document_index_generations` (`20260629000000`). The only missing capability +is **letting retrieval read a staged (uncommitted) generation for an eval session**. + +## Rejected approaches + +- **Supabase preview branch.** Branches are created from migrations and do **not** carry + production table data or Storage objects, so a re-index of the existing ~2,065-doc corpus + has nothing to run against on a branch. Not viable for evaluating a re-index of live content. +- **Add a `p_index_generation_id` param to the four hybrid RPCs** + (`match_document_chunks_hybrid`, `match_document_memory_cards_hybrid`, + `match_document_index_units_hybrid`, `match_document_embedding_fields_hybrid`). Rejected: + these are the delicate hot-path query bodies whose last edit caused the 130s + seqscan regression (see `hybrid-rpc-drift-bug` memory). Editing all four multiplies the risk. + +## Recommended design: GUC override on the two filter functions + +Retrieval's generation filter funnels through exactly two small `language sql stable` +functions. Override those — gated by a session-local custom GUC that is **unset in +production** — so the hot-path query bodies are never touched. + +```sql +-- Both functions currently (20260628000000): language sql stable, reading only jsonb. +-- Adding current_setting(..., true) keeps them STABLE (no volatility change) and, when the +-- GUC is unset (production), returns byte-identical results to today. +create or replace function public.is_committed_document_generation(row_generation uuid, document_metadata jsonb) +returns boolean language sql stable set search_path = public, extensions, pg_temp as $$ + select case + when nullif(current_setting('rag.eval_generation_id', true), '') is not null then + -- eval session: read ONLY the staged generation under evaluation + row_generation::text = nullif(current_setting('rag.eval_generation_id', true), '') + else + row_generation is null + or row_generation::text = nullif(coalesce(document_metadata, '{}'::jsonb)->>'index_generation_id', '') + end; +$$; + +create or replace function public.is_committed_artifact_generation(artifact_metadata jsonb, document_metadata jsonb) +returns boolean language sql stable set search_path = public, extensions, pg_temp as $$ + select case + when nullif(current_setting('rag.eval_generation_id', true), '') is not null then + nullif(coalesce(artifact_metadata, '{}'::jsonb)->>'index_generation_id', '') = current_setting('rag.eval_generation_id', true) + else + nullif(coalesce(artifact_metadata, '{}'::jsonb)->>'index_generation_id', '') is null + or nullif(coalesce(artifact_metadata, '{}'::jsonb)->>'index_generation_id', '') = + nullif(coalesce(document_metadata, '{}'::jsonb)->>'index_generation_id', ''); + end; +$$; +``` + +- Namespaced custom GUCs (`rag.*`) can be set per-session/transaction via `set_config` with no + prior declaration — the same mechanism as the existing session-local `hnsw.ef_search='100'`. +- Production code never sets `rag.eval_generation_id`, so the `else` branch runs and behavior + is unchanged. This is the "defaults unchanged until eval-gated" contract at the SQL layer. +- The SQL is embedded here (not shipped as an applyable migration) deliberately, so it cannot + reach the live DB without the review + validation below. + +## Eval-script extension + +Add an optional `--generation-id ` to `scripts/eval-retrieval.ts` and +`scripts/eval-quality.ts`. When present, the eval opens a DB session and issues +`select set_config('rag.eval_generation_id', $1, false)` before the retrieval RPC calls (and +clears it after), so every retrieval in that eval run reads the staged generation. The eval +summaries returned are the exact shapes `decideReindexGate` already consumes. + +## Driver: `scripts/reindex-shadow.ts` + +1. Preflight: `check:supabase-project`, `supabase:recovery-status`, `reindex:health` (abort if + `supabase_unavailable` or queue in recovery — per `docs/reindex-runbook.md`). +2. Capture the **baseline** eval summaries against the live committed index (no GUC). +3. For each batch of documents: + a. Re-index into a new staged `index_generation_id` **without** committing (new chunker via + `CHUNK_STRATEGY=document`, re-enrichment, one stamped `rag_indexing_version`). + b. Run `eval:retrieval:quality` and `eval:quality` with `--generation-id `. + c. `decideReindexGate({ baseline*, candidate* })`. + d. **GO** → `commit_document_index_generation` per document in the batch (atomic swap). + **NO_GO** → leave live intact, record the failing metrics, and + `reindex:cleanup-staged` the abandoned staged rows. +4. Emit a per-generation reindex report (baseline vs candidate table + per-doc outcomes). + +Wave 3 (targeted) runs the same driver over only OCR-poor / `extraction_quality=poor` docs; +Wave 4 pilot runs it over ~25–50 docs incl. the golden set as the GO/NO-GO gate for going +corpus-wide. + +## Mandatory validation before the filter-function change is used (drift-bug guard) + +The two filter functions are on the retrieval hot path. Before relying on the override: + +- Apply to the live DB, then run `npm run profile:retrieval` (or `explain_retrieval_rpc`) and + confirm the four hybrid RPCs still use the HNSW index scan — **no seqscan**, latency + unchanged. The `case`/`current_setting` addition must not defeat function inlining. +- `npm run eval:retrieval:quality` before vs after (GUC unset) must be identical — proves zero + production-path change. +- Keep the functions `language sql` (not plpgsql) and `stable`. +- Run `npx supabase db advisors --linked` and update `docs/supabase-migration-reconciliation.md`. +- Only after this passes, use `--generation-id` for staged evals. + +## W8 (#11) — extend `search_schema_health()` execution smoke (additive, lower-risk) + +`search_schema_health()` is a read-only diagnostic (not hot path), so this is safer to add: + +- Assert the four embedding columns are `vector(N)` with `N == EMBEDDING_DIMENSIONS`. +- Assert the `search_tsv` config in use matches the intended text-search config. +- Assert zero `rag_indexing_version` / `rag_enrichment_version` skew across `documents`. +- Assert all four HNSW indexes are present (already partly checked) and no legacy IVFFlat. +- Surface each as a `missing[]` entry so it flows into `check:indexing` + setup-status. + +## Rollback + +Every step is reversible: a NO_GO batch never commits (cleanup removes staged rows); the +filter-function change reverts by restoring the `20260628000000` definitions; `--generation-id` +is opt-in. The live committed generation is untouched until a batch passes the gate. diff --git a/scripts/check-indexing.ts b/scripts/check-indexing.ts index b2f6c2b8f..5ebd84725 100644 --- a/scripts/check-indexing.ts +++ b/scripts/check-indexing.ts @@ -317,6 +317,21 @@ async function main() { requireServerEnv(); requireOpenAIEnv(); + // IDX-C2 fail-fast: verify the configured embedding dimension matches the schema's + // vector(N) columns before any indexing/DB work, so a model/dimension misconfiguration + // is caught here instead of silently corrupting retrieval at write time. + const { readFile } = await import("node:fs/promises"); + const { fileURLToPath } = await import("node:url"); + const { describeSchemaDimensionMismatch } = await import("@/lib/embedding-dimensions"); + const schemaPath = fileURLToPath(new URL("../supabase/schema.sql", import.meta.url)); + const dimensionProblem = describeSchemaDimensionMismatch( + env.EMBEDDING_DIMENSIONS, + await readFile(schemaPath, "utf8"), + ); + if (dimensionProblem) { + throw new Error(`Embedding dimension check failed: ${dimensionProblem}`); + } + const prereqs = await checkPythonPdfPrerequisites(); if (!prereqs.ok) { throw new Error(`PDF/OCR prerequisite check failed: ${prereqs.detail}`); diff --git a/scripts/fixtures/rag-retrieval-golden.json b/scripts/fixtures/rag-retrieval-golden.json index 89daece19..ab5c75b52 100644 --- a/scripts/fixtures/rag-retrieval-golden.json +++ b/scripts/fixtures/rag-retrieval-golden.json @@ -210,5 +210,33 @@ "expectedContentTerms": ["alcohol", "withdrawal", ["ciwa", "score", "threshold"]], "topK": 12, "expectTableEvidence": false + }, + { + "id": "clozapine-cbc-abbreviation-threshold", + "query": "What CBC (complete blood count) or neutrophil threshold should withhold clozapine?", + "expectedQueryClass": "table_threshold", + "expectedDocumentSubstrings": ["Clozapine"], + "expectedContentTerms": [ + "clozapine", + ["anc", "neutrophil"], + ["fbc", "full blood count", "wbc", "wcc"], + ["withhold", "cease", "stop", "red"] + ], + "topK": 12, + "expectTableEvidence": true + }, + { + "id": "clozapine-wcc-abbreviation-threshold", + "query": "What WCC (white cell count) or neutrophil threshold should withhold clozapine?", + "expectedQueryClass": "table_threshold", + "expectedDocumentSubstrings": ["Clozapine"], + "expectedContentTerms": [ + "clozapine", + ["anc", "neutrophil"], + ["fbc", "full blood count", "wbc", "wcc", "white cell"], + ["withhold", "cease", "stop", "red"] + ], + "topK": 12, + "expectTableEvidence": true } ] diff --git a/src/lib/chunking.ts b/src/lib/chunking.ts index 52b2ba53e..651a61e14 100644 --- a/src/lib/chunking.ts +++ b/src/lib/chunking.ts @@ -1,8 +1,16 @@ +import { createHash } from "node:crypto"; import { env } from "@/lib/env"; import { sourceSpanForText } from "@/lib/source-spans"; import { normalizeExtractedGlyphs, stripClassificationBanner } from "@/lib/source-text-sanitizer"; import type { ChunkInput, DocumentChunk } from "@/lib/types"; +// Identity of the chunking strategy + params. Stamped into every chunk's metadata so a +// re-index can tell which chunker produced a row, and so bumping the strategy deliberately +// invalidates stale chunks. Bump this when chunk boundaries change (e.g. when cross-page +// document-mode chunking is enabled by default). +export const CHUNKER_VERSION = "1.0.0-page"; +export const DOCUMENT_CHUNKER_VERSION = "1.0.0-document"; + const sentenceBoundary = /(?<=[.!?])\s+/; const paragraphBoundary = /\n{2,}/; const metadataNoisePatterns: RegExp[] = [ @@ -169,6 +177,26 @@ function dedupeChunkFingerprint(text: string) { return normalizeLookupText(text).replace(/\s+/g, " ").trim(); } +const imageDataTagPattern = + /\[\[IMAGE_DATA_START\]\][\s\S]*?\[\[IMAGE_DATA_END\]\]|\[\[IMAGE_DATA_OMITTED\]\][\s\S]*?\[\[\/IMAGE_DATA_OMITTED\]\]/g; + +function normalizeChunkKeyContent(content: string) { + return content.replace(imageDataTagPattern, " ").replace(/\s+/g, " ").trim().toLowerCase(); +} + +// CI-4: a stable, position-independent identity for a chunk. Keyed on the document, the +// section anchor, and the normalized text — NOT on chunk_index or page — so re-indexing the +// same source yields the same key. This makes re-index idempotent and lets cached citations +// and eval anchors survive re-pagination and chunk reordering. Genuinely-identical content +// within one section shares a key, which is the intended "same content = same identity" +// semantics (a duplicate key is informational metadata, not a DB uniqueness constraint). +export function chunkContentKey(documentId: string, sectionAnchor: string | null, content: string) { + return createHash("sha256") + .update(`${documentId}\0${sectionAnchor ?? ""}\0${normalizeChunkKeyContent(content)}`) + .digest("hex") + .slice(0, 32); +} + function clampChunkSize(value: number, minimum: number, maximum: number) { return Math.min(maximum, Math.max(minimum, Math.round(value))); } @@ -446,7 +474,139 @@ function buildPageImageContext(pageImages: NonNullable) { return selectedImages.join("\n"); } -export function buildChunks(inputs: ChunkInput[]) { +type ChunkImages = NonNullable; +type ChunkProfile = ReturnType; +type SpanPage = { pageNumber: number | null; pageText: string }; + +function pageLocalExcerptForChunk(chunkExcerpt: string, pageText: string) { + const compactPage = pageText.replace(/\s+/g, " ").trim(); + if (!compactPage) return chunkExcerpt; + + const chunkWords = new Set( + normalizeLookupText(chunkExcerpt) + .split(/\s+/) + .filter((word) => word.length >= 3), + ); + if (chunkWords.size === 0) return compactPage; + + const candidates = compactPage + .split(/(?<=[.!?])\s+|\n+/) + .map((part) => part.trim()) + .filter(Boolean) + .map((part) => { + const words = normalizeLookupText(part).split(/\s+/).filter(Boolean); + const hits = words.filter((word) => chunkWords.has(word)).length; + return { part, hits }; + }) + .filter((entry) => entry.hits > 0) + .sort((left, right) => right.hits - left.hits); + + const selected = candidates + .slice(0, 3) + .map((entry) => entry.part) + .join(" "); + return selected || compactPage; +} + +// Shared chunk emission used by BOTH the page and document strategies so they produce an +// identical DocumentChunk shape. Page mode passes a single page (pageStart===pageEnd, one +// span page); document mode passes a page range and one span page per contributing page. +function emitChunk(args: { + chunks: DocumentChunk[]; + content: string; + documentId: string; + pageNumber: number | null; + pageStart: number | null; + pageEnd: number | null; + sectionPath: string[]; + images: ChunkImages; + spanPages: SpanPage[]; + baseMetadata: Record; + pageLookupText: string; + chunkProfile: ChunkProfile; + pageChunkIndex: number; + chunkerVersion: string; +}) { + const { content, sectionPath } = args; + const contentLookup = normalizeLookupText(content); + const heading = detectHeading(content); + const sectionContext = sectionPath.includes(heading ?? "") ? sectionPath : [...sectionPath]; + const sectionAnchor = sectionAnchorId(heading); + const level = headingLevel(heading, sectionContext); + const parentHeading = sectionContext.length > 1 ? sectionContext[sectionContext.length - 2] : null; + const referencedImageIds = args.images + .filter((image) => { + const label = normalizeLookupText(image.tableLabel ?? ""); + const title = normalizeLookupText(image.tableTitle ?? ""); + const caption = normalizeLookupText(image.caption); + const imageText = [label, title, caption].filter(Boolean).flatMap((value) => value.split(/\s+/).filter(Boolean)); + const imageLookup = imageText.join(" "); + const headerBoost = heading && imageText.some((token) => normalizeLookupText(heading).includes(token)) ? 1 : 0; + const direct = imageMatchScore(caption, contentLookup) >= 1 || imageMatchScore(imageLookup, contentLookup) >= 2; + const pathHit = + sectionContext.some((candidate) => + normalizeLookupText(candidate) + .split(/\s+/) + .some((token) => imageLookup.includes(token)), + ) && image.sourceKind !== "embedded"; + return direct || pathHit || (image.sourceKind === "table_crop" && headerBoost > 0) || headerBoost >= 1; + }) + .map((image) => image.id); + + const excerpt = content.replace(/\[\[IMAGE_DATA_START\]\][\s\S]*?\[\[IMAGE_DATA_END\]\]/g, "").trim(); + args.chunks.push({ + document_id: args.documentId, + page_number: args.pageNumber, + chunk_index: args.chunks.length, + section_heading: heading, + section_path: sectionContext, + heading_level: level, + parent_heading: parentHeading, + anchor_id: sectionAnchor, + content, + retrieval_synopsis: buildRetrievalSynopsis({ + content, + heading, + sectionContext, + pageNumber: args.pageNumber, + referencedImageCount: referencedImageIds.length, + }), + token_estimate: estimateTokens(content), + image_ids: referencedImageIds, + metadata: { + ...args.baseMetadata, + chunk_key: chunkContentKey(args.documentId, sectionAnchor, content), + chunker_version: args.chunkerVersion, + chunk_strategy: args.chunkerVersion === DOCUMENT_CHUNKER_VERSION ? "document" : "page", + page_chunk_index: args.pageChunkIndex, + chunk_profile: args.chunkProfile.profile, + adaptive_chunk_size: args.chunkProfile.chunkSize, + adaptive_chunk_overlap: args.chunkProfile.overlap, + page_start: args.pageStart, + page_end: args.pageEnd, + source_spans: args.spanPages.map((span) => { + const pageExcerpt = args.spanPages.length > 1 ? pageLocalExcerptForChunk(excerpt, span.pageText) : excerpt; + return sourceSpanForText({ + pageNumber: span.pageNumber, + pageText: span.pageText, + excerpt: pageExcerpt, + fallbackExcerpt: content, + }); + }), + heading_lookup: args.pageLookupText, + subsection_path: sectionContext, + section_anchor: sectionAnchor, + section_path: sectionContext, + heading_level: level, + parent_heading: parentHeading, + anchor_id: sectionAnchor, + }, + }); +} + +// Page-bounded chunking (default). Behavior is intentionally unchanged from the original +// buildChunks; the per-chunk emission is delegated to emitChunk. +function buildPageModeChunks(inputs: ChunkInput[]) { const chunks: DocumentChunk[] = []; const chunkFingerprint = new Map(); const repeatedBoilerplateLines = buildRepeatedBoilerplateLines(inputs); @@ -469,90 +629,161 @@ export function buildChunks(inputs: ChunkInput[]) { const pageChunks = chunkTextWithOverlap(pageText, chunkProfile.chunkSize, chunkProfile.overlap); pageChunks.forEach((content, pageChunkIndex) => { - const contentLookup = normalizeLookupText(content); - const heading = detectHeading(content); - const sectionContext = sectionPath.includes(heading ?? "") ? sectionPath : [...sectionPath]; - const sectionAnchor = sectionAnchorId(heading); - const level = headingLevel(heading, sectionContext); - const parentHeading = sectionContext.length > 1 ? sectionContext[sectionContext.length - 2] : null; - const referencedImageIds = pageImages - .filter((image) => { - const label = normalizeLookupText(image.tableLabel ?? ""); - const title = normalizeLookupText(image.tableTitle ?? ""); - const caption = normalizeLookupText(image.caption); - const imageText = [label, title, caption] - .filter(Boolean) - .flatMap((value) => value.split(/\s+/).filter(Boolean)); - const imageLookup = imageText.join(" "); - const headerBoost = - heading && imageText.some((token) => normalizeLookupText(heading).includes(token)) ? 1 : 0; - const direct = - imageMatchScore(caption, contentLookup) >= 1 || imageMatchScore(imageLookup, contentLookup) >= 2; - const pathHit = - sectionContext.some((candidate) => - normalizeLookupText(candidate) - .split(/\s+/) - .some((token) => imageLookup.includes(token)), - ) && image.sourceKind !== "embedded"; - return direct || pathHit || (image.sourceKind === "table_crop" && headerBoost > 0) || headerBoost >= 1; - }) - .map((image) => image.id); - const fingerprint = dedupeChunkFingerprint(content); const pageScopedFingerprint = fingerprint ? `${input.pageNumber ?? "unknown"}:${fingerprint}` : ""; - if (pageScopedFingerprint && chunkFingerprint.has(pageScopedFingerprint)) { - return; - } + if (pageScopedFingerprint && chunkFingerprint.has(pageScopedFingerprint)) return; + if (pageScopedFingerprint) chunkFingerprint.set(pageScopedFingerprint, chunks.length); + emitChunk({ + chunks, + content, + documentId: input.documentId, + pageNumber: input.pageNumber, + pageStart: input.pageNumber, + pageEnd: input.pageNumber, + sectionPath, + images: pageImages, + spanPages: [{ pageNumber: input.pageNumber, pageText: normalizedPageText }], + baseMetadata: input.metadata ?? {}, + pageLookupText, + chunkProfile, + pageChunkIndex, + chunkerVersion: CHUNKER_VERSION, + }); + }); + } - if (pageScopedFingerprint) { - chunkFingerprint.set(pageScopedFingerprint, chunks.length); - } - chunks.push({ - document_id: input.documentId, - page_number: input.pageNumber, - chunk_index: chunks.length, - section_heading: heading, - section_path: sectionContext, - heading_level: level, - parent_heading: parentHeading, - anchor_id: sectionAnchor, + return chunks; +} + +type DocumentPage = { + pageNumber: number | null; + normalizedText: string; + cleanedText: string; + images: ChunkImages; + metadata: Record; + sectionHeadings: string[]; + wordSet: Set; +}; + +// CI-1: attribute a cross-page chunk to the page(s) that actually contributed its words, +// by word-set overlap. Offset recovery is unreliable here (removePageNoise, whitespace +// collapse and paragraph rejoin mean chunk text is not a verbatim substring of any page), so +// word overlap is the robust signal. This only affects page_start/page_end/source_spans +// metadata — never chunk content or the retrieval text. +function attributeChunkToPages(contentWords: Set, pages: DocumentPage[]): DocumentPage[] { + if (pages.length <= 1) return pages; + const scored = pages.map((page) => { + let hits = 0; + for (const word of contentWords) if (page.wordSet.has(word)) hits += 1; + return { page, hits, fraction: contentWords.size ? hits / contentWords.size : 0 }; + }); + const best = Math.max(...scored.map((entry) => entry.fraction)); + if (best <= 0) return [pages[0]]; + const threshold = Math.min(0.15, best * 0.5); + const contributing = scored + .filter((entry) => entry.hits >= 3 && entry.fraction >= threshold) + .map((entry) => entry.page); + const result = contributing.length + ? contributing + : [scored.reduce((top, entry) => (entry.fraction > top.fraction ? entry : top)).page]; + return result.slice().sort((left, right) => (left.pageNumber ?? 0) - (right.pageNumber ?? 0)); +} + +// Structure-aware, cross-page chunking (CI-1). Documents are segmented into sections; each +// section's pages are chunked together so a chunk can span a page break within the section. +// A new section starts whenever a page introduces its own headings; heading-less continuation +// pages append to the current section (this is where cross-page merging happens). +function buildDocumentModeChunks(inputs: ChunkInput[]) { + const chunks: DocumentChunk[] = []; + if (inputs.length === 0) return chunks; + const documentId = inputs[0].documentId; + const repeatedBoilerplateLines = buildRepeatedBoilerplateLines(inputs); + // Document-scoped dedupe: unlike page mode (which keeps identical content on different + // pages), cross-page duplicates within a document are repeated boilerplate worth collapsing. + const chunkFingerprint = new Map(); + + const pages: DocumentPage[] = inputs.map((input) => { + const normalizedText = normalizeExtractedGlyphs(input.pageText); + const cleanedText = removePageNoise(normalizedText, repeatedBoilerplateLines); + return { + pageNumber: input.pageNumber, + normalizedText, + cleanedText, + images: input.images ?? [], + metadata: input.metadata ?? {}, + sectionHeadings: extractSectionHeadings(cleanedText), + wordSet: new Set(normalizeLookupText(cleanedText).split(/\s+/).filter(Boolean)), + }; + }); + + type Section = { sectionPath: string[]; pages: DocumentPage[] }; + const sections: Section[] = []; + let activeSectionPath: string[] = []; + for (const page of pages) { + if (page.sectionHeadings.length > 0) { + activeSectionPath = page.sectionHeadings; + sections.push({ sectionPath: activeSectionPath, pages: [page] }); + } else if (sections.length === 0) { + sections.push({ sectionPath: activeSectionPath, pages: [page] }); + } else { + sections[sections.length - 1].pages.push(page); + } + } + + for (const section of sections) { + const combinedText = section.pages + .map((page) => [page.cleanedText, buildPageImageContext(page.images)].filter(Boolean).join("\n\n")) + .filter(Boolean) + .join("\n\n"); + const combinedCleanText = section.pages + .map((page) => page.cleanedText) + .filter(Boolean) + .join("\n\n"); + const chunkProfile = adaptiveChunkProfile(combinedCleanText, section.sectionPath); + const pageLookupText = normalizeLookupText(section.pages.map((page) => page.normalizedText).join(" ")); + const sectionChunks = chunkTextWithOverlap(combinedText, chunkProfile.chunkSize, chunkProfile.overlap); + + sectionChunks.forEach((content, pageChunkIndex) => { + const contentWords = new Set( + normalizeLookupText(content.replace(imageDataTagPattern, " ")).split(/\s+/).filter(Boolean), + ); + const contributing = attributeChunkToPages(contentWords, section.pages); + const pageNumbers = contributing + .map((page) => page.pageNumber) + .filter((value): value is number => typeof value === "number"); + const pageStart = pageNumbers.length ? Math.min(...pageNumbers) : (section.pages[0]?.pageNumber ?? null); + const pageEnd = pageNumbers.length ? Math.max(...pageNumbers) : pageStart; + const representativePage = pageStart === pageEnd ? pageStart : null; + const fingerprint = dedupeChunkFingerprint(content); + const contextualFingerprint = fingerprint + ? [section.sectionPath.join(" > "), pageStart ?? "unknown", pageEnd ?? "unknown", fingerprint].join(":") + : ""; + if (contextualFingerprint && chunkFingerprint.has(contextualFingerprint)) return; + if (contextualFingerprint) chunkFingerprint.set(contextualFingerprint, chunks.length); + const contributingImages = contributing.flatMap((page) => page.images); + + emitChunk({ + chunks, content, - retrieval_synopsis: buildRetrievalSynopsis({ - content, - heading, - sectionContext, - pageNumber: input.pageNumber, - referencedImageCount: referencedImageIds.length, - }), - token_estimate: estimateTokens(content), - image_ids: referencedImageIds, - metadata: { - ...(input.metadata ?? {}), - page_chunk_index: pageChunkIndex, - chunk_profile: chunkProfile.profile, - adaptive_chunk_size: chunkProfile.chunkSize, - adaptive_chunk_overlap: chunkProfile.overlap, - page_start: input.pageNumber, - page_end: input.pageNumber, - source_spans: [ - sourceSpanForText({ - pageNumber: input.pageNumber, - pageText: normalizedPageText, - excerpt: content.replace(/\[\[IMAGE_DATA_START\]\][\s\S]*?\[\[IMAGE_DATA_END\]\]/g, "").trim(), - fallbackExcerpt: content, - }), - ], - heading_lookup: pageLookupText, - subsection_path: sectionContext, - section_anchor: sectionAnchor, - section_path: sectionContext, - heading_level: level, - parent_heading: parentHeading, - anchor_id: sectionAnchor, - }, + documentId, + pageNumber: representativePage, + pageStart, + pageEnd, + sectionPath: section.sectionPath, + images: contributingImages, + spanPages: contributing.map((page) => ({ pageNumber: page.pageNumber, pageText: page.normalizedText })), + baseMetadata: contributing[0]?.metadata ?? section.pages[0]?.metadata ?? {}, + pageLookupText, + chunkProfile, + pageChunkIndex, + chunkerVersion: DOCUMENT_CHUNKER_VERSION, }); }); } return chunks; } + +export function buildChunks(inputs: ChunkInput[]) { + return env.CHUNK_STRATEGY === "document" ? buildDocumentModeChunks(inputs) : buildPageModeChunks(inputs); +} diff --git a/src/lib/clinical-search.ts b/src/lib/clinical-search.ts index daa908e68..a6a0bd5a8 100644 --- a/src/lib/clinical-search.ts +++ b/src/lib/clinical-search.ts @@ -1,5 +1,6 @@ import { isClinicalImageEvidence } from "@/lib/image-filtering"; import { expandClinicalVocabularyText } from "@/lib/clinical-vocabulary"; +import { freshnessDecayPenalty, rankingConfig } from "@/lib/ranking-config"; import type { ClinicalQueryAnalysis, ClinicalQueryIntent, @@ -242,6 +243,8 @@ const domainAliasGroups = [ ["lai", "long acting injectable", "depot", "long-acting injectable"], ["im", "intramuscular"], ["po", "oral"], + ["sc", "subcutaneous", "subcut", "subcutaneously"], + ["sl", "sublingual", "sublingually"], ["prn", "as required"], ["mhsp", "mental health service procedure"], ["fda", "food and drug administration"], @@ -319,7 +322,7 @@ const intentPatterns: Array<{ { intent: "drug_dosing", pattern: - /dose|dosage|dosing|titrate|mg|mcg|frequency|route|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|administer|table|chart|monitor/i, + /dose|dosage|dosing|titrate|mg|mcg|frequency|route|oral|intramuscular|subcutaneous|subcut|sublingual|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b|administer|table|chart|monitor/i, imageEvidenceFocus: true, sectionedLookup: false, }, @@ -348,7 +351,7 @@ const tableThresholdPattern = // genuine medication_dose_risk query still matches via a drug name, dose/route term, or the // medication/pharmacology/agitation vocabulary retained below. const medicationDoseRiskPattern = - /\b(medication|medicine|pharmacolog\w*|prescrib\w*|dose|dosage|dosing|mg|mcg|titrate|route|oral|intramuscular|administer\w*|\bim\b|\bpo\b|\bprn\b|clozapine|lithium|neuroleptic|antipsychotic|benzodiazepine|injectables?|agitation|arousal|side effect\w*|adverse|toxicity|contraindicat\w*|monitor\w*)\b/i; + /\b(medication|medicine|pharmacolog\w*|prescrib\w*|dose|dosage|dosing|mg|mcg|titrate|route|oral|intramuscular|subcutaneous|subcut|sublingual|administer\w*|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b|clozapine|lithium|neuroleptic|antipsychotic|benzodiazepine|injectables?|agitation|arousal|side effect\w*|adverse|toxicity|contraindicat\w*|monitor\w*)\b/i; const documentIncludePattern = /\b(?:what should|what must|what does|what do|which items?|requirements?|checklist|forms?)\b.{0,80}\b(?:include|contain|cover|require|required|needed|need)\b|\b(?:include|contain|cover|require|required|needed|need)\b.{0,80}\b(?:plan|form|checklist|protocol|procedure|guideline|document|file|pdf)\b/i; const explicitDocumentLookupPattern = @@ -518,7 +521,9 @@ function queryClassFromSignals(args: { ) return "document_lookup"; if ( - /\b(?:dose|dosage|dosing|route|mg|mcg|microgram|\bim\b|\bpo\b|\bprn\b)\b/i.test(args.normalizedQuery) && + /\b(?:dose|dosage|dosing|route|mg|mcg|microgram|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b)\b/i.test( + args.normalizedQuery, + ) && (args.medications.length > 0 || medicationDoseRiskPattern.test(args.normalizedQuery)) ) { return "medication_dose_risk"; @@ -553,7 +558,9 @@ function intentFromSignals(queryClass: RagQueryClass, normalizedQuery: string): if (queryClass === "document_lookup") return "document_lookup"; if (queryClass === "table_threshold" || queryClass === "medication_dose_risk") { if ( - /(dose|dosage|dosing|mg|mcg|route|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|administer)/i.test(normalizedQuery) + /(dose|dosage|dosing|mg|mcg|route|oral|intramuscular|subcutaneous|subcut|sublingual|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b|administer)/i.test( + normalizedQuery, + ) ) { return "drug_dosing"; } @@ -723,7 +730,7 @@ export function hasDoseEvidenceSupport(result: SearchResult) { ) .map((image) => `${image.tableTextSnippet ?? ""} ${image.caption ?? ""} ${image.tableTitle ?? ""}`) .join(" ")}`.toLowerCase(); - return /\b(?:dose|dosage|dosing|mg|mcg|microgram|route|oral|intramuscular|\bim\b|\bpo\b|\bprn\b|administer\w*|titration|titrate|frequency|maximum|tablet|injection|antipsychotic|benzodiazepine|olanzapine|lorazepam|haloperidol|droperidol|promethazine|diazepam)\b/i.test( + return /\b(?:dose|dosage|dosing|mg|mcg|microgram|route|oral|intramuscular|subcutaneous|subcut|sublingual|\bim\b|\bpo\b|\bsc\b|\bsl\b|\bprn\b|administer\w*|titration|titrate|frequency|maximum|tablet|injection|antipsychotic|benzodiazepine|olanzapine|lorazepam|haloperidol|droperidol|promethazine|diazepam)\b/i.test( haystack, ); } @@ -1078,7 +1085,9 @@ export function buildClinicalTextSearchQuery(query: string) { /\b(?:risk|red\s*zone|red|urgent|escalat|next step)\b/i.test(query); const wantsClozapineBloodMonitoring = /\bclozapine\b/i.test(correctedQueryText) && - /\b(?:blood|bloods|fbc|full blood count|observation|observations|monitor|monitoring)\b/i.test(correctedQueryText); + /\b(?:blood|bloods|fbc|wcc|full blood count|white cell|observation|observations|monitor|monitoring)\b/i.test( + correctedQueryText, + ); const wantsClozapineMissedDose = /\bclozapine\b/i.test(correctedQueryText) && /\bmissed\b/i.test(correctedQueryText) && @@ -1217,8 +1226,8 @@ export function clinicalRankExplanation(query: string, result: SearchResult): Se const statusBoost = status === "outdated" ? -0.04 : 0; const publicationYearsAgo = parseDateAsYearsAgo(result.source_metadata?.publication_date); const reviewYearsAgo = parseDateAsYearsAgo(result.source_metadata?.review_date); - const freshnessBoost = publicationYearsAgo === null ? 0 : publicationYearsAgo >= 8 ? -0.015 : 0; - const reviewBoost = reviewYearsAgo === null ? 0 : reviewYearsAgo >= 5 ? -0.01 : 0; + const freshnessBoost = freshnessDecayPenalty(publicationYearsAgo, "publication", rankingConfig.freshness); + const reviewBoost = freshnessDecayPenalty(reviewYearsAgo, "review", rankingConfig.freshness); const imageBoost = imageEvidenceSignal(query, result); const sectionBoost = sectionMatchBoost(query, result); const sectionedLookupBoost = querySignal.sectionedLookup ? 0.02 : 0; diff --git a/src/lib/clinical-vocabulary.ts b/src/lib/clinical-vocabulary.ts index aabecc0c9..ee936ec88 100644 --- a/src/lib/clinical-vocabulary.ts +++ b/src/lib/clinical-vocabulary.ts @@ -23,7 +23,10 @@ const entries: ClinicalVocabularyEntry[] = [ }, { canonical: "full blood count", - aliases: ["fbc", "blood count", "white cell count", "wcc", "wbc"], + // CI-14: FBC (AU/UK) and CBC (US) are the same test. Documents in this corpus use + // "FBC"; without the CBC alternates a clinician who searches "CBC" retrieves nothing. + // "cbc" is kept in the first 6 aliases so it survives the expansion slice bidirectionally. + aliases: ["fbc", "cbc", "complete blood count", "blood count", "white cell count", "wcc", "wbc"], type: "lab", weight: 1.2, }, @@ -52,6 +55,10 @@ const entries: ClinicalVocabularyEntry[] = [ { canonical: "emergency department", aliases: ["ed"], type: "service", weight: 1.05 }, { canonical: "intramuscular", aliases: ["im"], type: "workflow", weight: 1.05 }, { canonical: "oral", aliases: ["po"], type: "workflow", weight: 1.05 }, + // CI-14: complete the administration-route family alongside IM/PO so route-abbreviation + // queries ("subcut", "SC", "sublingual", "SL") retrieve the same chart rows. + { canonical: "subcutaneous", aliases: ["sc", "subcut", "subcutaneously"], type: "workflow", weight: 1.05 }, + { canonical: "sublingual", aliases: ["sl", "sublingually"], type: "workflow", weight: 1.05 }, { canonical: "as required", aliases: ["prn"], type: "workflow", weight: 1.05 }, { canonical: "clozapine", diff --git a/src/lib/embedding-dimensions.ts b/src/lib/embedding-dimensions.ts index 2e8239cac..c753c9271 100644 --- a/src/lib/embedding-dimensions.ts +++ b/src/lib/embedding-dimensions.ts @@ -17,6 +17,37 @@ function expectedEmbeddingDimensions() { export const EXPECTED_EMBED_DIM = expectedEmbeddingDimensions(); +// IDX-C2 (fail-fast): the embedding dimension is declared in three places that MUST agree +// — the configured EMBEDDING_DIMENSIONS, the OpenAI model's output, and the schema's +// vector(N) columns. A mismatch corrupts ingestion silently (every write throws, or worse, +// a future model change slips through). These helpers let a startup check compare the +// configured dimension against supabase/schema.sql offline, before any DB work. +export function parseSchemaVectorDimensions(schemaSql: string): number[] { + const dims = new Set(); + for (const match of schemaSql.matchAll(/\bvector\((\d+)\)/gi)) { + const value = Number(match[1]); + if (Number.isInteger(value) && value > 0) dims.add(value); + } + return [...dims].sort((a, b) => a - b); +} + +// Returns a human-readable problem description, or null when the configured dimension is +// consistent with every vector(N) column in the schema. Absence of any vector column is +// treated as a problem (the schema we were handed is not the RAG schema). +export function describeSchemaDimensionMismatch(configuredDim: number, schemaSql: string): string | null { + const schemaDims = parseSchemaVectorDimensions(schemaSql); + if (schemaDims.length === 0) { + return "No vector(N) columns found in schema.sql — cannot verify embedding dimensions."; + } + if (schemaDims.length > 1) { + return `schema.sql declares inconsistent vector dimensions ${schemaDims.join(", ")}; all embedding columns must share one dimension.`; + } + if (schemaDims[0] !== configuredDim) { + return `EMBEDDING_DIMENSIONS=${configuredDim} does not match schema vector(${schemaDims[0]}). Re-embedding with a mismatched dimension corrupts retrieval; align OPENAI_EMBEDDING_MODEL, EMBEDDING_DIMENSIONS, and the schema before indexing.`; + } + return null; +} + export function assertEmbeddingDim(vec: unknown, context: string): number[] { if (!Array.isArray(vec)) { throw new Error(`${context} embedding must be an array.`); diff --git a/src/lib/env.ts b/src/lib/env.ts index d48254ba1..bbfb7de1c 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -27,6 +27,12 @@ const envSchema = z.object({ // if output is still cut off, createTextResult now flags it as truncated (GEN-C1). OPENAI_MAX_OUTPUT_TOKENS: z.coerce.number().int().positive().default(4000), OPENAI_QUERY_CACHE_SIZE: z.coerce.number().int().nonnegative().default(200), + // Max inputs per embeddings request. The OpenAI embeddings endpoint caps a single + // request at 2048 inputs / ~300k tokens; a full-corpus re-embed of ~400k texts in one + // call would exceed that and fail (IDX-C3). embedTexts splits unique inputs into + // batches of this size. 256 keeps total tokens well under the ceiling even for the + // largest (narrative-profile) chunks while staying far below the 2048 input cap. + OPENAI_EMBEDDING_BATCH_SIZE: z.coerce.number().int().positive().max(2048).default(256), OPENAI_VISION_MODEL: z.string().default("gpt-5.5"), OPENAI_VISION_IMAGE_DETAIL: z.enum(["auto", "low", "high"]).default("auto"), OPENAI_REQUEST_TIMEOUT_MS: z.coerce.number().int().positive().default(45000), @@ -59,6 +65,10 @@ const envSchema = z.object({ // - "offline": never call OpenAI at all (no embeddings, no generation); lexical retrieval // + deterministic source-only answers only. Fails closed when evidence is weak. RAG_PROVIDER_MODE: z.enum(["auto", "openai", "offline"]).default("auto"), + // Optional JSON override for app-layer ranking weights (see src/lib/ranking-config.ts). + // Lets tuning/eval experiments adjust the second-stage rerank weights, document-diversity + // demotion, and freshness decay WITHOUT a code change. Omitted/malformed => current defaults. + RAG_RANKING_CONFIG: z.string().optional(), RAG_ANSWER_CACHE_TTL_MS: z.coerce.number().int().nonnegative().default(300000), RAG_ANSWER_CACHE_SIZE: z.coerce.number().int().nonnegative().default(100), RAG_SEARCH_CACHE_TTL_MS: z.coerce.number().int().nonnegative().default(60000), @@ -87,6 +97,11 @@ const envSchema = z.object({ MAX_IMPORT_BYTES_PER_RUN: z.coerce.number().int().positive().default(157286400), CHUNK_SIZE: z.coerce.number().int().positive().default(2000), CHUNK_OVERLAP: z.coerce.number().int().nonnegative().default(200), + // Chunking strategy (CI-1). "page" (default) = the current page-bounded chunker, byte-for- + // byte unchanged. "document" = structure-aware chunking that lets a chunk span a page break + // within a section, so dose tables / monitoring protocols split across a page boundary stay + // together. Enabled only for the eval-gated shadow re-index, never silently for live users. + CHUNK_STRATEGY: z.enum(["page", "document"]).default("page"), WORKER_POLL_MS: z.coerce.number().int().positive().default(1500), WORKER_BATCH_SIZE: z.coerce.number().int().positive().default(25), WORKER_CONCURRENCY: z.coerce.number().int().positive().default(8), diff --git a/src/lib/index-quality.ts b/src/lib/index-quality.ts index 05710beff..d66b0c470 100644 --- a/src/lib/index-quality.ts +++ b/src/lib/index-quality.ts @@ -29,6 +29,8 @@ export type IndexQualityMetrics = { extracted_image_count: number; searchable_image_count: number; ocr_page_count?: number; + // CI-6: pages flagged image-only but not OCR'd (JS fallback without Python OCR prereqs). + needs_ocr_page_count?: number; }; export function hashIndexQualityText(text: string) { @@ -93,6 +95,8 @@ export function assessDocumentIndexQuality(args: { const sectionPathCoverage = chunkCount ? sectionPathCount / chunkCount : 0; const tableExtractionCoverage = tableImages.length ? tableImagesWithRows.length / tableImages.length : null; const ocrCoverage = args.metrics.page_count ? Number(args.metrics.ocr_page_count ?? 0) / args.metrics.page_count : 0; + const needsOcrPageCount = Number(args.metrics.needs_ocr_page_count ?? 0); + const needsOcrCoverage = args.metrics.page_count ? needsOcrPageCount / args.metrics.page_count : 0; const averagePageTextChars = args.metrics.page_count ? args.metrics.text_character_count / args.metrics.page_count : 0; @@ -122,6 +126,10 @@ export function assessDocumentIndexQuality(args: { issues.push("low structured visual extraction confidence"); } if (args.metrics.text_character_count < 80) issues.push("low extracted text volume"); + if (needsOcrPageCount > 0) + issues.push( + `image-only pages not OCR'd (${needsOcrPageCount} of ${args.metrics.page_count}); install Python OCR prerequisites`, + ); if (args.sectionCount === 0) issues.push("no structured sections"); if (args.memoryCardCount === 0) issues.push("no memory cards"); if (!fieldTypes.has("document_title")) issues.push("missing document title embedding"); @@ -145,8 +153,15 @@ export function assessDocumentIndexQuality(args: { const chunkPageCoverage = Math.min(1, chunkCount / Math.max(args.metrics.page_count * 0.75, 1)); qualityScore -= Math.max(0, 0.82 - chunkPageCoverage) * 0.08; } + qualityScore -= Math.min(0.5, needsOcrCoverage * 0.6); qualityScore = Math.max(0, Math.min(1, qualityScore)); - const extractionQuality = qualityScore >= 0.82 ? "good" : qualityScore >= 0.52 ? "partial" : "poor"; + let extractionQuality = qualityScore >= 0.82 ? "good" : qualityScore >= 0.52 ? "partial" : "poor"; + // CI-6: a scanned / image-only PDF whose pages could not be OCR'd is effectively unindexed + // even when incidental header text nudges the heuristics up to "partial". Force "poor" so it + // is visible to eval governance and never silently treated as a good index. + if (needsOcrPageCount > 0 && (needsOcrCoverage >= 0.5 || chunkCount === 0)) { + extractionQuality = "poor"; + } return { qualityScore: Number(qualityScore.toFixed(3)), diff --git a/src/lib/openai.ts b/src/lib/openai.ts index be0438686..31c99b42e 100644 --- a/src/lib/openai.ts +++ b/src/lib/openai.ts @@ -406,6 +406,20 @@ export function clearOpenAICaches() { openAIClient = null; } +// IDX-C3: split a list into fixed-size batches, preserving order. Pure/exported so the +// batching contract (which underpins correct embedding reassembly) is unit-testable +// without hitting the network. +export function chunkIntoBatches(items: T[], size: number): T[][] { + if (!Number.isInteger(size) || size <= 0) { + throw new Error(`chunkIntoBatches: size must be a positive integer, got ${size}.`); + } + const batches: T[][] = []; + for (let start = 0; start < items.length; start += size) { + batches.push(items.slice(start, start + size)); + } + return batches; +} + export async function embedTexts(texts: string[]) { if (texts.length === 0) return []; @@ -424,45 +438,55 @@ export async function embedTexts(texts: string[]) { try { const client = createOpenAIClient(); - const response = await client.embeddings.create( - { - model: env.OPENAI_EMBEDDING_MODEL, - input: uniqueTexts, - // IDX-C2: request the exact dimension the schema's vector(N) columns expect. - dimensions: env.EMBEDDING_DIMENSIONS, - }, - requestOptions({ operation: "embedding" }), - ); - - // IDX-C2: a short response means some inputs silently produced no embedding. - if (response.data.length !== uniqueTexts.length) { - throw new PublicApiError( - `OpenAI returned ${response.data.length} embeddings for ${uniqueTexts.length} inputs.`, - 502, - { code: "openai_embedding_count_mismatch" }, + // IDX-C3: a single embeddings request is capped at 2048 inputs / ~300k tokens, so a + // full-corpus re-embed must be split. Batches run sequentially to stay polite to the + // rate limiter; the SDK still applies env.OPENAI_MAX_RETRIES per request. Reassembly + // uses the GLOBAL index (batchStart + item.index) so a chunk is never stored with the + // embedding of an unrelated text, even across batch boundaries (extends IDX-C1). + const byIndex = new Array(uniqueTexts.length); + const batches = chunkIntoBatches(uniqueTexts, env.OPENAI_EMBEDDING_BATCH_SIZE); + let batchStart = 0; + for (const batch of batches) { + const response = await client.embeddings.create( + { + model: env.OPENAI_EMBEDDING_MODEL, + input: batch, + // IDX-C2: request the exact dimension the schema's vector(N) columns expect. + dimensions: env.EMBEDDING_DIMENSIONS, + }, + requestOptions({ operation: "embedding" }), ); - } - // IDX-C1: the embeddings API does not guarantee response order; each item carries - // an explicit `index` into the input array. Reassemble by that index so a chunk is - // never stored with the embedding of an unrelated text (silent clinical corruption). - const byIndex = new Array(uniqueTexts.length); - for (const item of response.data) { - if (item.index < 0 || item.index >= uniqueTexts.length) { - throw new PublicApiError(`OpenAI returned an out-of-range embedding index ${item.index}.`, 502, { - code: "openai_embedding_index_range", - }); - } - // IDX-C2: guard against a model whose dimension does not match the schema. - if (item.embedding.length !== env.EMBEDDING_DIMENSIONS) { + // IDX-C2: a short response means some inputs silently produced no embedding. + if (response.data.length !== batch.length) { throw new PublicApiError( - `OpenAI embedding has ${item.embedding.length} dimensions; expected ${env.EMBEDDING_DIMENSIONS}. ` + - `Check OPENAI_EMBEDDING_MODEL and EMBEDDING_DIMENSIONS match supabase/schema.sql.`, + `OpenAI returned ${response.data.length} embeddings for ${batch.length} inputs.`, 502, - { code: "openai_embedding_dimension_mismatch" }, + { code: "openai_embedding_count_mismatch" }, ); } - byIndex[item.index] = item.embedding; + + // IDX-C1: the embeddings API does not guarantee response order; each item carries + // an explicit `index` into the request's input array. Reassemble by that index + // (offset to the global position) so embeddings are never mismatched to chunks. + for (const item of response.data) { + if (item.index < 0 || item.index >= batch.length) { + throw new PublicApiError(`OpenAI returned an out-of-range embedding index ${item.index}.`, 502, { + code: "openai_embedding_index_range", + }); + } + // IDX-C2: guard against a model whose dimension does not match the schema. + if (item.embedding.length !== env.EMBEDDING_DIMENSIONS) { + throw new PublicApiError( + `OpenAI embedding has ${item.embedding.length} dimensions; expected ${env.EMBEDDING_DIMENSIONS}. ` + + `Check OPENAI_EMBEDDING_MODEL and EMBEDDING_DIMENSIONS match supabase/schema.sql.`, + 502, + { code: "openai_embedding_dimension_mismatch" }, + ); + } + byIndex[batchStart + item.index] = item.embedding; + } + batchStart += batch.length; } return outputIndexes.map((index) => byIndex[index]); diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 5fba4ab77..6764accd2 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -73,6 +73,7 @@ import { clinicalModePrompt, queryClassForClinicalMode, queryForClinicalMode } f import { annotateSearchResults, buildEvidenceRelevance } from "@/lib/evidence-relevance"; import { committedIndexGeneration, isCommittedGenerationMetadata } from "@/lib/reindex-pipeline"; import { buildRetrievalIntent, selectRetrievalEvidence } from "@/lib/retrieval-selection"; +import { rankingConfig } from "@/lib/ranking-config"; import { z } from "zod"; import { createHash } from "node:crypto"; import { @@ -579,20 +580,29 @@ function secondStageScore(result: SearchResult, queryClass: RagQueryClass | unde ) .join(" ")}`; const hasDoseAmount = /\b\d+(?:\.\d+)?\s?(?:mg|mcg|microgram|micrograms)\b/i.test(doseAmountText); - score += Math.max(0, 0.09 - index * 0.004); - if (result.memory_cards?.length && (queryClass === "broad_summary" || queryClass === "comparison")) score += 0.035; + const w = rankingConfig.secondStage; + score += Math.max(0, w.positionBase - index * w.positionStep); + if (result.memory_cards?.length && (queryClass === "broad_summary" || queryClass === "comparison")) + score += w.memorySummaryBoost; if (queryClass === "document_lookup" && (result.match_explanation?.titleHit || result.match_explanation?.labelHit)) - score += 0.045; + score += w.documentLookupTitleBoost; if ((queryClass === "table_threshold" || queryClass === "medication_dose_risk") && result.table_facts?.length) - score += 0.065; - if (queryClass === "medication_dose_risk" && hasDoseAmount) score += 0.18; - if (tableVisualEvidenceUnitTypes.has(unitType)) score += 0.08; - else if (visualEvidenceUnitTypes.has(unitType)) score += 0.04; - if (source === "visual_intelligence") score += Math.min(0.035, Math.max(0, sourceQuality - 0.55) * 0.08); - if (result.source_metadata?.document_status === "outdated") score -= 0.035; - if (result.source_metadata?.extraction_quality === "poor") score -= 0.035; - if (result.indexing_quality?.quality_score !== undefined && result.indexing_quality.quality_score < 0.55) - score -= 0.035; + score += w.tableThresholdEvidenceBoost; + if (queryClass === "medication_dose_risk" && hasDoseAmount) score += w.doseAmountBoost; + if (tableVisualEvidenceUnitTypes.has(unitType)) score += w.tableVisualBoost; + else if (visualEvidenceUnitTypes.has(unitType)) score += w.visualBoost; + if (source === "visual_intelligence") + score += Math.min( + w.visualIntelligenceMax, + Math.max(0, sourceQuality - w.visualIntelligencePivot) * w.visualIntelligenceSlope, + ); + if (result.source_metadata?.document_status === "outdated") score -= w.outdatedPenalty; + if (result.source_metadata?.extraction_quality === "poor") score -= w.poorExtractionPenalty; + if ( + result.indexing_quality?.quality_score !== undefined && + result.indexing_quality.quality_score < w.lowIndexQualityThreshold + ) + score -= w.lowIndexQualityPenalty; return score; } @@ -604,12 +614,26 @@ function applySecondStageRerankIfNeeded(args: { }) { if (!shouldUseSecondStageRerank(args.queryClass, args.results, args.topK)) return args.results; const startedAt = Date.now(); + // CI-16 document diversity: subtract a demotion from each EXTRA chunk of a document that + // has already appeared higher up, so a single doc's sibling chunks can't crowd out other + // documents. Applied AFTER the additive-boost floor so it can actually lower the effective + // rank. Default penalty is 0 (disabled) — no reordering until deliberately tuned + eval-gated. + const seenPerDocument = new Map(); const reranked = args.results .map((result, index) => { - const score = secondStageScore(result, args.queryClass, index); + const boosted = secondStageScore(result, args.queryClass, index); + let score = Math.max(result.hybrid_score ?? 0, boosted); + const priorOccurrences = seenPerDocument.get(result.document_id) ?? 0; + seenPerDocument.set(result.document_id, priorOccurrences + 1); + if (rankingConfig.documentDiversityPenalty > 0 && priorOccurrences > 0) { + score -= Math.min( + rankingConfig.documentDiversityPenaltyCap, + rankingConfig.documentDiversityPenalty * priorOccurrences, + ); + } return { ...result, - hybrid_score: Number(Math.max(result.hybrid_score ?? 0, score).toFixed(4)), + hybrid_score: Number(score.toFixed(4)), match_explanation: { ...result.match_explanation, reasons: Array.from(new Set([...(result.match_explanation?.reasons ?? []), "second_stage_rerank"])), @@ -3414,7 +3438,9 @@ function hasDoseAmountEvidenceForGate(result: SearchResult) { } function hasRouteEvidenceForGate(result: SearchResult) { - return /\b(?:oral|orally|intramuscular|intramuscularly|\bim\b|\bpo\b)\b/i.test(evidenceTextForGate(result)); + return /\b(?:oral|orally|intramuscular|intramuscularly|subcutaneous|subcutaneously|subcut|sublingual|sublingually|\bim\b|\bpo\b|\bsc\b|\bsl\b)\b/i.test( + evidenceTextForGate(result), + ); } function hasDirectSourceImageEvidence(result: SearchResult) { @@ -3521,10 +3547,13 @@ export function evaluateEvidenceCoverageGate( if (queryClass === "table_threshold") { if ( /\bclozapine\b/i.test(query) && - /\b(?:anc|fbc|wbc|neutrophil|neutrophils|full blood)\b/i.test(query) && + /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils|full blood|white cell)\b/i.test(query) && /\b(?:withhold|withheld|withholding|cease|ceased|stop|stopped)\b/i.test(query) ) { - const hasBlood = hasAnyTerm(evidenceText, /\b(?:anc|fbc|wbc|neutrophil|neutrophils|full blood)\b/i); + const hasBlood = hasAnyTerm( + evidenceText, + /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils|full blood|white cell)\b/i, + ); const hasAction = hasAnyTerm( evidenceText, /\b(?:withhold|withheld|withholding|cease|ceased|stop|stopped|red)\b/i, @@ -3565,7 +3594,10 @@ export function evaluateEvidenceCoverageGate( } if (queryClass === "medication_dose_risk") { - const asksDoseRoute = /\b(?:dose|dosage|dosing|route|oral|intramuscular|\bim\b|\bpo\b)\b/i.test(query); + const asksDoseRoute = + /\b(?:dose|dosage|dosing|route|oral|intramuscular|subcutaneous|subcut|sublingual|\bim\b|\bpo\b|\bsc\b|\bsl\b)\b/i.test( + query, + ); const agitationOk = !/\bagitation|arousal\b/i.test(query) || /\bagitation|arousal\b/i.test(evidenceText); const accepted = hasDoseEvidence && hasDoseAmount && (!asksDoseRoute || hasRoute) && agitationOk; return { diff --git a/src/lib/ranking-config.ts b/src/lib/ranking-config.ts new file mode 100644 index 000000000..08fb024e2 --- /dev/null +++ b/src/lib/ranking-config.ts @@ -0,0 +1,177 @@ +// Central, tunable ranking configuration for the app-layer retrieval rerank (W6). +// +// Rationale: the second-stage rerank weights, the document-diversity control, and the +// freshness decay were previously hard-coded magic numbers scattered across rag.ts and +// clinical-search.ts. Tuning them meant editing (and redeploying) code — the same friction +// CI-18 calls out. This module makes them one config object with an optional JSON override +// (RAG_RANKING_CONFIG), so `tune:search-weights` and eval-gated experiments can adjust +// ranking without code edits. +// +// IMPORTANT — defaults reproduce the exact prior behavior: +// * every secondStage weight equals the constant it replaced, +// * documentDiversityPenalty defaults to 0 (the diversity demotion is OFF), +// * freshness.mode defaults to "step" (the original cliff). +// So merely landing this module changes nothing. The new behaviors (diversity demotion, +// linear freshness curve) only take effect once explicitly configured and eval-gated, in +// keeping with the "defaults unchanged until tuned" + non-degradation guarantees. + +export type SecondStageWeights = { + /** Position-decay boost applied to the top result, decaying by positionStep per rank. */ + positionBase: number; + positionStep: number; + /** Boost for memory-card evidence on broad-summary / comparison queries. */ + memorySummaryBoost: number; + /** Boost when a document-lookup query hits a title/label. */ + documentLookupTitleBoost: number; + /** Boost when a table/threshold or dose-risk query has table-fact evidence. */ + tableThresholdEvidenceBoost: number; + /** Boost when a dose-risk query surfaces an explicit dose amount. */ + doseAmountBoost: number; + /** Boost for table-visual and (lower) other-visual evidence unit types. */ + tableVisualBoost: number; + visualBoost: number; + /** Visual-intelligence source-quality boost: min(max, (quality - pivot) * slope). */ + visualIntelligenceMax: number; + visualIntelligencePivot: number; + visualIntelligenceSlope: number; + /** Penalties for governance/quality signals. */ + outdatedPenalty: number; + poorExtractionPenalty: number; + lowIndexQualityPenalty: number; + lowIndexQualityThreshold: number; +}; + +export type FreshnessConfig = { + /** "step" = original cliff (default, zero-change); "linear" = ramped decay curve (CI-17). */ + mode: "step" | "linear"; + publicationCliffYears: number; + publicationPenalty: number; + reviewCliffYears: number; + reviewPenalty: number; + /** In "linear" mode, years over which the penalty ramps up to reach the cliff. */ + linearRampYears: number; +}; + +export type RankingConfig = { + secondStage: SecondStageWeights; + /** Demotion subtracted per EXTRA chunk from the same document (0 = diversity OFF). CI-16. */ + documentDiversityPenalty: number; + /** Maximum cumulative diversity demotion for any single chunk. */ + documentDiversityPenaltyCap: number; + freshness: FreshnessConfig; +}; + +export const defaultRankingConfig: RankingConfig = { + secondStage: { + positionBase: 0.09, + positionStep: 0.004, + memorySummaryBoost: 0.035, + documentLookupTitleBoost: 0.045, + tableThresholdEvidenceBoost: 0.065, + doseAmountBoost: 0.18, + tableVisualBoost: 0.08, + visualBoost: 0.04, + visualIntelligenceMax: 0.035, + visualIntelligencePivot: 0.55, + visualIntelligenceSlope: 0.08, + outdatedPenalty: 0.035, + poorExtractionPenalty: 0.035, + lowIndexQualityPenalty: 0.035, + lowIndexQualityThreshold: 0.55, + }, + documentDiversityPenalty: 0, + documentDiversityPenaltyCap: 0.12, + freshness: { + mode: "step", + publicationCliffYears: 8, + publicationPenalty: -0.015, + reviewCliffYears: 5, + reviewPenalty: -0.01, + linearRampYears: 3, + }, +}; + +function asRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function num(value: unknown, fallback: number): number { + return typeof value === "number" && Number.isFinite(value) ? value : fallback; +} + +function round4(value: number): number { + return Number(value.toFixed(4)); +} + +/** + * Build a RankingConfig by deep-merging an optional JSON override (partial) over the + * defaults. Unknown keys are ignored and non-numeric values fall back to the default, so a + * malformed override can only ever degrade to current behavior — never crash retrieval. + * Exported for unit testing. + */ +export function resolveRankingConfig(raw?: string | null): RankingConfig { + let parsed: Record = {}; + if (raw && raw.trim()) { + try { + parsed = asRecord(JSON.parse(raw)); + } catch { + parsed = {}; + } + } + const d = defaultRankingConfig; + const ss = asRecord(parsed.secondStage); + const fr = asRecord(parsed.freshness); + return { + secondStage: { + positionBase: num(ss.positionBase, d.secondStage.positionBase), + positionStep: num(ss.positionStep, d.secondStage.positionStep), + memorySummaryBoost: num(ss.memorySummaryBoost, d.secondStage.memorySummaryBoost), + documentLookupTitleBoost: num(ss.documentLookupTitleBoost, d.secondStage.documentLookupTitleBoost), + tableThresholdEvidenceBoost: num(ss.tableThresholdEvidenceBoost, d.secondStage.tableThresholdEvidenceBoost), + doseAmountBoost: num(ss.doseAmountBoost, d.secondStage.doseAmountBoost), + tableVisualBoost: num(ss.tableVisualBoost, d.secondStage.tableVisualBoost), + visualBoost: num(ss.visualBoost, d.secondStage.visualBoost), + visualIntelligenceMax: num(ss.visualIntelligenceMax, d.secondStage.visualIntelligenceMax), + visualIntelligencePivot: num(ss.visualIntelligencePivot, d.secondStage.visualIntelligencePivot), + visualIntelligenceSlope: num(ss.visualIntelligenceSlope, d.secondStage.visualIntelligenceSlope), + outdatedPenalty: num(ss.outdatedPenalty, d.secondStage.outdatedPenalty), + poorExtractionPenalty: num(ss.poorExtractionPenalty, d.secondStage.poorExtractionPenalty), + lowIndexQualityPenalty: num(ss.lowIndexQualityPenalty, d.secondStage.lowIndexQualityPenalty), + lowIndexQualityThreshold: num(ss.lowIndexQualityThreshold, d.secondStage.lowIndexQualityThreshold), + }, + documentDiversityPenalty: Math.max(0, num(parsed.documentDiversityPenalty, d.documentDiversityPenalty)), + documentDiversityPenaltyCap: Math.max(0, num(parsed.documentDiversityPenaltyCap, d.documentDiversityPenaltyCap)), + freshness: { + mode: fr.mode === "linear" ? "linear" : "step", + publicationCliffYears: num(fr.publicationCliffYears, d.freshness.publicationCliffYears), + publicationPenalty: num(fr.publicationPenalty, d.freshness.publicationPenalty), + reviewCliffYears: num(fr.reviewCliffYears, d.freshness.reviewCliffYears), + reviewPenalty: num(fr.reviewPenalty, d.freshness.reviewPenalty), + linearRampYears: Math.max(0.0001, num(fr.linearRampYears, d.freshness.linearRampYears)), + }, + }; +} + +/** + * Freshness penalty for a document, given its age in years (or null when unknown). + * CI-17: in "step" mode this reproduces the original cliff exactly; in "linear" mode the + * penalty ramps in gradually over `linearRampYears` up to the cliff, avoiding the harsh + * step that unfairly demotes still-current stable guidelines the moment they cross the line. + */ +export function freshnessDecayPenalty( + yearsAgo: number | null, + kind: "publication" | "review", + cfg: FreshnessConfig, +): number { + if (yearsAgo === null) return 0; + const cliff = kind === "publication" ? cfg.publicationCliffYears : cfg.reviewCliffYears; + const penalty = kind === "publication" ? cfg.publicationPenalty : cfg.reviewPenalty; + if (cfg.mode === "step") return yearsAgo >= cliff ? penalty : 0; + const rampStart = cliff - cfg.linearRampYears; + if (yearsAgo <= rampStart) return 0; + const t = Math.min(1, (yearsAgo - rampStart) / cfg.linearRampYears); + return round4(penalty * t); +} + +/** Resolved singleton used by the retrieval path. */ +export const rankingConfig: RankingConfig = resolveRankingConfig(process.env.RAG_RANKING_CONFIG); diff --git a/src/lib/reindex-eval-gate.ts b/src/lib/reindex-eval-gate.ts new file mode 100644 index 000000000..3dcbe906e --- /dev/null +++ b/src/lib/reindex-eval-gate.ts @@ -0,0 +1,443 @@ +// Non-degradation gate for the shadow re-index (Task #6 core). Given the eval summaries for +// the current live (baseline) index and a staged candidate generation, decide GO/NO-GO for +// cutover. The contract (docs/retrieval-quality-runbook.md + the re-index plan) is twofold: +// 1. Absolute bars — the candidate must independently clear the release thresholds. +// 2. No regression — the candidate must be no worse than the live baseline (within a small +// tolerance for eval noise), so a re-index can only hold or improve quality. +// A cutover proceeds only when EVERY check passes. This module is pure and deterministic so +// the gate itself is unit-tested; the (live) shadow driver feeds it eval JSON and acts on it. + +// Subsets of scripts/eval-retrieval.ts and scripts/eval-quality.ts summary output. Field +// names match those scripts so the driver can pass summaries through unchanged. +export type RetrievalGateSummary = { + case_count: number; + case_fingerprint?: string; + failed_case_count: number; + document_recall_at_5: number; + content_recall_at_5: number; + top_k_hit_rate: number; + mrr_at_10?: number; + p90_latency_ms?: number; +}; + +export type QualityGateSummary = { + case_count: number; + case_fingerprint?: string; + grounded_supported_rate: number; + unsupported_count: number; + unsupported_correct_rate: number; + expected_hit_rate?: number; + citation_failure_rate: number; + numeric_grounding_failure_rate: number; + source_governance_danger_failure_rate: number; + expected_danger_warning_missing_count: number; + stale_review_unknown_rate: number; + review_required_rate: number; + p95_latency_ms: number; +}; + +export type ReindexGateConfig = { + retrieval: { + documentRecallAt5Floor: number; + contentRecallAt5Floor: number; + topKHitRateFloor: number; + failedCaseCountCeiling: number; + rateRegressionTolerance: number; + latencyRegressionMs: number; + }; + quality: { + groundedSupportedRateFloor: number; + unsupportedCorrectRateFloor: number; + citationFailureRateCeiling: number; + numericGroundingFailureRateCeiling: number; + sourceGovernanceDangerFailureRateCeiling: number; + expectedDangerWarningMissingCountCeiling: number; + staleReviewUnknownRateCeiling: number; + reviewRequiredRateCeiling: number; + p95LatencyMsCeiling: number; + rateRegressionTolerance: number; + latencyRegressionMs: number; + }; +}; + +// Defaults taken directly from the release thresholds in docs/retrieval-quality-runbook.md. +export const defaultReindexGateConfig: ReindexGateConfig = { + retrieval: { + documentRecallAt5Floor: 0.8, + contentRecallAt5Floor: 0.8, + topKHitRateFloor: 0.8, + failedCaseCountCeiling: 0, + rateRegressionTolerance: 0.02, + latencyRegressionMs: 4000, + }, + quality: { + groundedSupportedRateFloor: 0.9, + unsupportedCorrectRateFloor: 1.0, + citationFailureRateCeiling: 0, + numericGroundingFailureRateCeiling: 0, + sourceGovernanceDangerFailureRateCeiling: 0, + expectedDangerWarningMissingCountCeiling: 0, + staleReviewUnknownRateCeiling: 0.25, + reviewRequiredRateCeiling: 0.25, + p95LatencyMsCeiling: 25000, + rateRegressionTolerance: 0.02, + latencyRegressionMs: 4000, + }, +}; + +export type MetricCheck = { + metric: string; + direction: "higher_better" | "lower_better"; + baseline: number; + candidate: number; + bound: number | null; // absolute floor (higher_better) / ceiling (lower_better) + tolerance: number; + passed: boolean; + reasons: string[]; +}; + +export type ReindexGateDecision = { + decision: "GO" | "NO_GO"; + checks: MetricCheck[]; + failures: string[]; +}; + +const requiredRetrievalMetrics = [ + "case_count", + "failed_case_count", + "document_recall_at_5", + "content_recall_at_5", + "top_k_hit_rate", +] as const satisfies readonly (keyof RetrievalGateSummary)[]; + +const requiredQualityMetrics = [ + "case_count", + "grounded_supported_rate", + "unsupported_count", + "unsupported_correct_rate", + "citation_failure_rate", + "numeric_grounding_failure_rate", + "source_governance_danger_failure_rate", + "expected_danger_warning_missing_count", + "stale_review_unknown_rate", + "review_required_rate", + "p95_latency_ms", +] as const satisfies readonly (keyof QualityGateSummary)[]; + +function evaluateCheck(input: { + metric: string; + direction: "higher_better" | "lower_better"; + baseline: number; + candidate: number; + bound: number | null; + tolerance: number; +}): MetricCheck { + const reasons: string[] = []; + const { metric, direction, baseline, candidate, bound, tolerance } = input; + + if (bound !== null) { + const meetsBound = direction === "higher_better" ? candidate >= bound : candidate <= bound; + if (!meetsBound) { + reasons.push( + direction === "higher_better" + ? `${metric} ${candidate} below absolute floor ${bound}` + : `${metric} ${candidate} above absolute ceiling ${bound}`, + ); + } + } + + const noRegression = + direction === "higher_better" ? candidate >= baseline - tolerance : candidate <= baseline + tolerance; + if (!noRegression) { + reasons.push( + direction === "higher_better" + ? `${metric} regressed: candidate ${candidate} < baseline ${baseline} - tolerance ${tolerance}` + : `${metric} regressed: candidate ${candidate} > baseline ${baseline} + tolerance ${tolerance}`, + ); + } + + return { metric, direction, baseline, candidate, bound, tolerance, passed: reasons.length === 0, reasons }; +} + +function retrievalChecks( + baseline: RetrievalGateSummary, + candidate: RetrievalGateSummary, + config: ReindexGateConfig["retrieval"], +): MetricCheck[] { + const checks: MetricCheck[] = [ + evaluateCheck({ + metric: "failed_case_count", + direction: "lower_better", + baseline: baseline.failed_case_count, + candidate: candidate.failed_case_count, + bound: config.failedCaseCountCeiling, + tolerance: 0, + }), + evaluateCheck({ + metric: "document_recall_at_5", + direction: "higher_better", + baseline: baseline.document_recall_at_5, + candidate: candidate.document_recall_at_5, + bound: config.documentRecallAt5Floor, + tolerance: config.rateRegressionTolerance, + }), + evaluateCheck({ + metric: "content_recall_at_5", + direction: "higher_better", + baseline: baseline.content_recall_at_5, + candidate: candidate.content_recall_at_5, + bound: config.contentRecallAt5Floor, + tolerance: config.rateRegressionTolerance, + }), + evaluateCheck({ + metric: "top_k_hit_rate", + direction: "higher_better", + baseline: baseline.top_k_hit_rate, + candidate: candidate.top_k_hit_rate, + bound: config.topKHitRateFloor, + tolerance: config.rateRegressionTolerance, + }), + ]; + // mrr@10 and p90 latency are no-regression only (no absolute bar) and skipped unless both + // summaries carry them. + if (typeof baseline.mrr_at_10 === "number" && typeof candidate.mrr_at_10 === "number") { + checks.push( + evaluateCheck({ + metric: "mrr_at_10", + direction: "higher_better", + baseline: baseline.mrr_at_10, + candidate: candidate.mrr_at_10, + bound: null, + tolerance: config.rateRegressionTolerance, + }), + ); + } + if (typeof baseline.p90_latency_ms === "number" && typeof candidate.p90_latency_ms === "number") { + checks.push( + evaluateCheck({ + metric: "p90_latency_ms", + direction: "lower_better", + baseline: baseline.p90_latency_ms, + candidate: candidate.p90_latency_ms, + bound: null, + tolerance: config.latencyRegressionMs, + }), + ); + } + return checks; +} + +function qualityChecks( + baseline: QualityGateSummary, + candidate: QualityGateSummary, + config: ReindexGateConfig["quality"], +): MetricCheck[] { + const checks: MetricCheck[] = [ + evaluateCheck({ + metric: "grounded_supported_rate", + direction: "higher_better", + baseline: baseline.grounded_supported_rate, + candidate: candidate.grounded_supported_rate, + bound: config.groundedSupportedRateFloor, + tolerance: config.rateRegressionTolerance, + }), + evaluateCheck({ + metric: "citation_failure_rate", + direction: "lower_better", + baseline: baseline.citation_failure_rate, + candidate: candidate.citation_failure_rate, + bound: config.citationFailureRateCeiling, + tolerance: 0, + }), + evaluateCheck({ + metric: "numeric_grounding_failure_rate", + direction: "lower_better", + baseline: baseline.numeric_grounding_failure_rate, + candidate: candidate.numeric_grounding_failure_rate, + bound: config.numericGroundingFailureRateCeiling, + tolerance: 0, + }), + evaluateCheck({ + metric: "source_governance_danger_failure_rate", + direction: "lower_better", + baseline: baseline.source_governance_danger_failure_rate, + candidate: candidate.source_governance_danger_failure_rate, + bound: config.sourceGovernanceDangerFailureRateCeiling, + tolerance: 0, + }), + evaluateCheck({ + metric: "expected_danger_warning_missing_count", + direction: "lower_better", + baseline: baseline.expected_danger_warning_missing_count, + candidate: candidate.expected_danger_warning_missing_count, + bound: config.expectedDangerWarningMissingCountCeiling, + tolerance: 0, + }), + evaluateCheck({ + metric: "p95_latency_ms", + direction: "lower_better", + baseline: baseline.p95_latency_ms, + candidate: candidate.p95_latency_ms, + bound: config.p95LatencyMsCeiling, + tolerance: config.latencyRegressionMs, + }), + ]; + if (baseline.unsupported_count > 0 || candidate.unsupported_count > 0) { + checks.push( + evaluateCheck({ + metric: "unsupported_correct_rate", + direction: "higher_better", + baseline: baseline.unsupported_correct_rate, + candidate: candidate.unsupported_correct_rate, + bound: config.unsupportedCorrectRateFloor, + tolerance: 0, + }), + ); + } + if (typeof baseline.expected_hit_rate === "number" && typeof candidate.expected_hit_rate === "number") { + checks.push( + evaluateCheck({ + metric: "expected_hit_rate", + direction: "higher_better", + baseline: baseline.expected_hit_rate, + candidate: candidate.expected_hit_rate, + bound: null, + tolerance: config.rateRegressionTolerance, + }), + ); + } + checks.push( + evaluateCheck({ + metric: "stale_review_unknown_rate", + direction: "lower_better", + baseline: baseline.stale_review_unknown_rate, + candidate: candidate.stale_review_unknown_rate, + bound: config.staleReviewUnknownRateCeiling, + tolerance: config.rateRegressionTolerance, + }), + evaluateCheck({ + metric: "review_required_rate", + direction: "lower_better", + baseline: baseline.review_required_rate, + candidate: candidate.review_required_rate, + bound: config.reviewRequiredRateCeiling, + tolerance: config.rateRegressionTolerance, + }), + ); + return checks; +} + +function missingQualityMetrics(label: "baselineQuality" | "candidateQuality", summary: QualityGateSummary) { + return requiredQualityMetrics.flatMap((metric) => + typeof summary[metric] === "number" && Number.isFinite(summary[metric]) ? [] : [`${label}.${metric}`], + ); +} + +function missingRetrievalMetrics(label: "baselineRetrieval" | "candidateRetrieval", summary: RetrievalGateSummary) { + return requiredRetrievalMetrics.flatMap((metric) => + typeof summary[metric] === "number" && Number.isFinite(summary[metric]) ? [] : [`${label}.${metric}`], + ); +} + +function populationFailures(args: { + baselineRetrieval: RetrievalGateSummary; + candidateRetrieval: RetrievalGateSummary; + baselineQuality?: QualityGateSummary; + candidateQuality?: QualityGateSummary; +}) { + const failures: string[] = []; + if (args.baselineRetrieval.case_count !== args.candidateRetrieval.case_count) { + failures.push( + `retrieval case_count mismatch: baseline ${args.baselineRetrieval.case_count} vs candidate ${args.candidateRetrieval.case_count}`, + ); + } + const baselineRetrievalFingerprint = args.baselineRetrieval.case_fingerprint; + const candidateRetrievalFingerprint = args.candidateRetrieval.case_fingerprint; + if (baselineRetrievalFingerprint || candidateRetrievalFingerprint) { + if (!baselineRetrievalFingerprint || !candidateRetrievalFingerprint) { + failures.push("retrieval case_fingerprint must be present for both baseline and candidate, or neither"); + } else if (baselineRetrievalFingerprint !== candidateRetrievalFingerprint) { + failures.push("retrieval case_fingerprint mismatch"); + } + } + + if (args.baselineQuality && args.candidateQuality) { + if (args.baselineQuality.case_count !== args.candidateQuality.case_count) { + failures.push( + `quality case_count mismatch: baseline ${args.baselineQuality.case_count} vs candidate ${args.candidateQuality.case_count}`, + ); + } + const baselineQualityFingerprint = args.baselineQuality.case_fingerprint; + const candidateQualityFingerprint = args.candidateQuality.case_fingerprint; + if (baselineQualityFingerprint || candidateQualityFingerprint) { + if (!baselineQualityFingerprint || !candidateQualityFingerprint) { + failures.push("quality case_fingerprint must be present for both baseline and candidate, or neither"); + } else if (baselineQualityFingerprint !== candidateQualityFingerprint) { + failures.push("quality case_fingerprint mismatch"); + } + } + } + return failures; +} + +export function decideReindexGate( + input: { + baselineRetrieval: RetrievalGateSummary; + candidateRetrieval: RetrievalGateSummary; + baselineQuality?: QualityGateSummary; + candidateQuality?: QualityGateSummary; + qualityMode?: "required" | "retrieval_only"; + }, + config: ReindexGateConfig = defaultReindexGateConfig, +): ReindexGateDecision { + const missingRetrieval = [ + ...missingRetrievalMetrics("baselineRetrieval", input.baselineRetrieval), + ...missingRetrievalMetrics("candidateRetrieval", input.candidateRetrieval), + ]; + if (missingRetrieval.length > 0) { + return { + decision: "NO_GO", + checks: [], + failures: [`retrieval summaries are missing required metrics: ${missingRetrieval.join(", ")}`], + }; + } + + const checks = retrievalChecks(input.baselineRetrieval, input.candidateRetrieval, config.retrieval); + const gateFailures = populationFailures(input); + + const hasBaselineQuality = Boolean(input.baselineQuality); + const hasCandidateQuality = Boolean(input.candidateQuality); + if (input.qualityMode !== "retrieval_only" && (!hasBaselineQuality || !hasCandidateQuality)) { + return { + decision: "NO_GO", + checks, + failures: [...gateFailures, "quality summaries are required unless qualityMode is retrieval_only"], + }; + } + if (hasBaselineQuality !== hasCandidateQuality) { + // A one-sided quality summary is a driver error; fail closed rather than silently + // gating on retrieval alone. + return { + decision: "NO_GO", + checks, + failures: [...gateFailures, "quality summaries must be provided for both baseline and candidate, or neither"], + }; + } + if (input.baselineQuality && input.candidateQuality) { + const missingMetrics = [ + ...missingQualityMetrics("baselineQuality", input.baselineQuality), + ...missingQualityMetrics("candidateQuality", input.candidateQuality), + ]; + if (missingMetrics.length > 0) { + return { + decision: "NO_GO", + checks, + failures: [...gateFailures, `quality summaries are missing required metrics: ${missingMetrics.join(", ")}`], + }; + } + checks.push(...qualityChecks(input.baselineQuality, input.candidateQuality, config.quality)); + } + + const failures = [...gateFailures, ...checks.flatMap((check) => check.reasons)]; + return { decision: failures.length === 0 ? "GO" : "NO_GO", checks, failures }; +} diff --git a/src/lib/retrieval-selection.ts b/src/lib/retrieval-selection.ts index 8b14c1545..eaf7bef49 100644 --- a/src/lib/retrieval-selection.ts +++ b/src/lib/retrieval-selection.ts @@ -157,7 +157,9 @@ function hasDoseAmount(text: string) { } function hasRoute(text: string) { - return /\b(?:oral|orally|intramuscular|intramuscularly|im|po)\b/.test(text); + return /\b(?:oral|orally|intramuscular|intramuscularly|subcutaneous|subcutaneously|subcut|sublingual|sublingually|im|po|sc|sl)\b/.test( + text, + ); } function hasSourceImageEvidence(result: SearchResult) { @@ -337,9 +339,10 @@ function resultBoost(args: { intent: RetrievalIntent; candidate: RetrievalCandid export function buildRetrievalIntent(query: string, queryClass: RagQueryClass): RetrievalIntent { const normalizedQuery = normalize(query); - const asksDoseRoute = /\b(?:dose|dosage|dosing|route|oral|intramuscular|im|po|frequency|mg|mcg|prn)\b/.test( - normalizedQuery, - ); + const asksDoseRoute = + /\b(?:dose|dosage|dosing|route|oral|intramuscular|subcutaneous|subcut|sublingual|im|po|sc|sl|frequency|mg|mcg|prn)\b/.test( + normalizedQuery, + ); const asksDoseAmount = /\b(?:dose|dosage|dosing|mg|mcg|microgram|maximum|minimum)\b/.test(normalizedQuery); const asksTable = /\b(?:table|chart|matrix|threshold|cutoff|cut off|range|criteria|row)\b/.test(normalizedQuery); const asksSourceImage = @@ -382,7 +385,8 @@ export function buildRetrievalIntent(query: string, queryClass: RagQueryClass): } if (asksDoseRoute) { if (asksDoseAmount) requiredTermSignals.push("dose_amount"); - if (/\b(?:route|oral|intramuscular|im|po)\b/.test(normalizedQuery)) requiredTermSignals.push("route"); + if (/\b(?:route|oral|intramuscular|subcutaneous|subcut|sublingual|im|po|sc|sl)\b/.test(normalizedQuery)) + requiredTermSignals.push("route"); } if (asksFlowchart) { preferredDocumentSignals.push("flowchart", "pathway", "risk matrix"); diff --git a/tests/chunking.test.ts b/tests/chunking.test.ts index 32c6568ed..aeedf38fb 100644 --- a/tests/chunking.test.ts +++ b/tests/chunking.test.ts @@ -1,5 +1,12 @@ -import { describe, expect, it } from "vitest"; -import { buildChunks, buildImageTag, chunkTextWithOverlap } from "../src/lib/chunking"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + CHUNKER_VERSION, + DOCUMENT_CHUNKER_VERSION, + buildChunks, + buildImageTag, + chunkContentKey, + chunkTextWithOverlap, +} from "../src/lib/chunking"; describe("chunkTextWithOverlap", () => { it("keeps short text as one chunk", () => { @@ -328,3 +335,201 @@ describe("section-aware chunking groundwork", () => { expect(chunks.map((chunk) => chunk.page_number)).toEqual([4, 7]); }); }); + +describe("stable chunk identity (CI-4)", () => { + it("stamps a stable chunk_key and chunker_version into metadata", () => { + const chunks = buildChunks([ + { documentId: "doc-1", pageNumber: 1, pageText: "Check renal function before starting lithium.", metadata: {} }, + ]); + expect(chunks).toHaveLength(1); + expect(chunks[0].metadata.chunker_version).toBe(CHUNKER_VERSION); + expect(chunks[0].metadata.chunk_key).toMatch(/^[0-9a-f]{32}$/); + }); + + it("produces the same key across re-runs of identical input (idempotent re-index)", () => { + const input = [{ documentId: "doc-1", pageNumber: 1, pageText: "Withhold clozapine if ANC is low.", metadata: {} }]; + const first = buildChunks(structuredClone(input)); + const second = buildChunks(structuredClone(input)); + expect(first[0].metadata.chunk_key).toBe(second[0].metadata.chunk_key); + }); + + it("is page-independent — identical content on different pages shares an identity", () => { + const text = "ANC threshold guidance: withhold clozapine and repeat FBC daily."; + const chunks = buildChunks([ + { documentId: "doc-1", pageNumber: 4, pageText: text, metadata: {} }, + { documentId: "doc-1", pageNumber: 7, pageText: text, metadata: {} }, + ]); + expect(chunks).toHaveLength(2); + expect(chunks[0].metadata.chunk_key).toBe(chunks[1].metadata.chunk_key); + }); + + it("changes the key when content or document differs, and ignores inline image tags", () => { + const keyA = chunkContentKey("doc-1", "monitoring", "Withhold clozapine if ANC is low."); + const keyDifferentContent = chunkContentKey("doc-1", "monitoring", "Continue clozapine and monitor weekly."); + const keyDifferentDoc = chunkContentKey("doc-2", "monitoring", "Withhold clozapine if ANC is low."); + expect(keyA).not.toBe(keyDifferentContent); + expect(keyA).not.toBe(keyDifferentDoc); + // Image-data tags are stripped before hashing, so attaching image context does not + // change a chunk's stable identity. + const withImageTag = chunkContentKey( + "doc-1", + "monitoring", + "Withhold clozapine if ANC is low. [[IMAGE_DATA_START]] Image ID: x; Description: chart [[IMAGE_DATA_END]]", + ); + expect(withImageTag).toBe(keyA); + }); +}); + +describe("document-mode chunking (CI-1)", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + }); + + async function buildDocumentChunks(inputs: Parameters[0]) { + vi.resetModules(); + vi.stubEnv("CHUNK_STRATEGY", "document"); + const mod = await import("../src/lib/chunking"); + return mod.buildChunks(inputs); + } + + it("merges a section across a page break into a single cross-page chunk", async () => { + const chunks = await buildDocumentChunks([ + { + documentId: "doc-1", + pageNumber: 1, + pageText: + "Clozapine Monitoring\n\nBaseline full blood count and absolute neutrophil count must be checked before starting clozapine therapy for the patient.", + metadata: {}, + }, + { + documentId: "doc-1", + pageNumber: 2, + pageText: + "Continue weekly blood test monitoring and escalate any neutropenia to the prescriber and haematology team immediately.", + metadata: {}, + }, + ]); + + // The section (heading on page 1, continuation on page 2) chunks together, so the + // page-1 baseline requirement and the page-2 continuation land in one chunk. + const spanning = chunks.find( + (chunk) => chunk.content.includes("Baseline full blood count") && chunk.content.includes("weekly blood test"), + ); + expect(spanning).toBeDefined(); + expect(spanning?.metadata.page_start).toBe(1); + expect(spanning?.metadata.page_end).toBe(2); + expect(spanning?.page_number).toBeNull(); + expect(spanning?.section_path).toContain("Clozapine Monitoring"); + // A cross-page chunk records a source span for each contributing page. + const spanPageNumbers = (spanning?.metadata.source_spans as Array<{ page_number: number | null }>).map( + (span) => span.page_number, + ); + expect(spanPageNumbers).toEqual(expect.arrayContaining([1, 2])); + const pageTwoSpan = ( + spanning?.metadata.source_spans as Array<{ + page_number: number | null; + excerpt: string; + character_start: number | null; + }> + ).find((span) => span.page_number === 2); + expect(pageTwoSpan?.excerpt).toContain("Continue weekly blood test monitoring"); + expect(pageTwoSpan?.excerpt).not.toContain("Baseline full blood count"); + expect(pageTwoSpan?.character_start).not.toBeNull(); + }); + + it("does not merge across a section boundary introduced by a new heading", async () => { + const chunks = await buildDocumentChunks([ + { + documentId: "doc-1", + pageNumber: 1, + pageText: "Section One\n\nAlpha guidance about clozapine baseline monitoring requirements for every patient.", + metadata: {}, + }, + { + documentId: "doc-1", + pageNumber: 2, + pageText: "Section Two\n\nBravo guidance about lithium renal function checks and thyroid monitoring tests.", + metadata: {}, + }, + ]); + + const sectionTwo = chunks.find((chunk) => chunk.content.includes("Bravo guidance")); + expect(sectionTwo?.metadata.page_start).toBe(2); + expect(sectionTwo?.metadata.page_end).toBe(2); + expect(sectionTwo?.section_path).toContain("Section Two"); + // The page-1 section must not have absorbed page-2 content. + const sectionOne = chunks.find((chunk) => chunk.content.includes("Alpha guidance")); + expect(sectionOne?.content).not.toContain("Bravo guidance"); + expect(sectionOne?.metadata.page_end).toBe(1); + }); + + it("still stamps the stable chunk_key and chunker_version in document mode", async () => { + const chunks = await buildDocumentChunks([ + { + documentId: "doc-1", + pageNumber: 1, + pageText: "Check renal function before starting lithium therapy.", + metadata: {}, + }, + ]); + expect(chunks[0].metadata.chunker_version).toBe(DOCUMENT_CHUNKER_VERSION); + expect(chunks[0].metadata.chunk_strategy).toBe("document"); + expect(chunks[0].metadata.chunk_key).toMatch(/^[0-9a-f]{32}$/); + }); + + it("keeps document-mode images scoped to pages that contributed to the chunk", async () => { + vi.stubEnv("CHUNK_SIZE", "900"); + const pageOneBody = Array.from({ length: 90 }, (_, index) => `Baseline monitoring instruction ${index}.`).join(" "); + const chunks = await buildDocumentChunks([ + { + documentId: "doc-1", + pageNumber: 1, + pageText: `Monitoring Guidance\n\n${pageOneBody}`, + metadata: {}, + images: [], + }, + { + documentId: "doc-1", + pageNumber: 2, + pageText: "Continuation dosing table text for later review.", + metadata: {}, + images: [ + { + id: "page-2-table", + caption: "Later-page table.", + pageNumber: 2, + sourceKind: "table_crop", + tableTitle: "Monitoring Guidance", + }, + ], + }, + ]); + + const pageOneChunk = chunks.find((chunk) => chunk.metadata.page_start === 1 && chunk.metadata.page_end === 1); + expect(pageOneChunk).toBeDefined(); + expect(pageOneChunk?.image_ids).not.toContain("page-2-table"); + }); + + it("preserves repeated document-mode clinical chunks when page context differs", async () => { + const repeatedMonitoringText = + "Clozapine monitoring table\n\nANC threshold 0.5 x 10^9/L: withhold clozapine and repeat FBC daily."; + const chunks = await buildDocumentChunks([ + { + documentId: "doc-1", + pageNumber: 4, + pageText: repeatedMonitoringText, + metadata: {}, + }, + { + documentId: "doc-1", + pageNumber: 7, + pageText: repeatedMonitoringText, + metadata: {}, + }, + ]); + + expect(chunks.filter((chunk) => chunk.content.includes("ANC threshold 0.5 x 10^9/L"))).toHaveLength(2); + expect(chunks.map((chunk) => chunk.metadata.page_start)).toEqual([4, 7]); + }); +}); diff --git a/tests/embed-texts-integrity.test.ts b/tests/embed-texts-integrity.test.ts index 0e37e6220..8be7b8644 100644 --- a/tests/embed-texts-integrity.test.ts +++ b/tests/embed-texts-integrity.test.ts @@ -110,3 +110,57 @@ describe("embedTexts integrity (IDX-C1, IDX-C2)", () => { await expect(embedTexts(["a", "b"])).rejects.toThrow(/embeddings for/); }); }); + +describe("embedTexts batching (IDX-C3)", () => { + function embeddingForText(text: string): number[] { + // Encode the input's numeric suffix so a mis-mapped embedding is detectable. + const n = Number(text.replace(/[^0-9]/g, "")); + return [n, n]; + } + + it("splits large input into batches and reassembles by global index across batches", async () => { + stubEmbeddingEnv(2); + vi.stubEnv("OPENAI_EMBEDDING_BATCH_SIZE", "2"); + + const inputsPerCall: number[] = []; + vi.doMock("openai", () => ({ + default: class MockOpenAI { + embeddings = { + create: vi.fn(async ({ input }: { input: string[] }) => { + inputsPerCall.push(input.length); + // Reverse within each batch so a naive position-based merge would corrupt order. + return { data: input.map((text, index) => ({ index, embedding: embeddingForText(text) })).reverse() }; + }), + }; + + responses = { create: vi.fn() }; + }, + })); + + const { clearOpenAICaches, embedTexts } = await import("../src/lib/openai"); + clearOpenAICaches(); + + const result = await embedTexts(["t0", "t1", "t2", "t3", "t4"]); + + // 5 inputs at batch size 2 -> three requests of 2, 2, 1. + expect(inputsPerCall).toEqual([2, 2, 1]); + // Every text maps back to its own embedding despite per-batch reversal and batch splits. + expect(result).toEqual([ + [0, 0], + [1, 1], + [2, 2], + [3, 3], + [4, 4], + ]); + }); + + it("chunkIntoBatches partitions in order with a trailing remainder and rejects bad sizes", async () => { + stubEmbeddingEnv(2); + const { chunkIntoBatches } = await import("../src/lib/openai"); + + expect(chunkIntoBatches([1, 2, 3, 4, 5], 2)).toEqual([[1, 2], [3, 4], [5]]); + expect(chunkIntoBatches([], 3)).toEqual([]); + expect(chunkIntoBatches([1, 2], 10)).toEqual([[1, 2]]); + expect(() => chunkIntoBatches([1], 0)).toThrow(/positive integer/); + }); +}); diff --git a/tests/embedding-dimensions.test.ts b/tests/embedding-dimensions.test.ts index c35a4f597..91f4c0284 100644 --- a/tests/embedding-dimensions.test.ts +++ b/tests/embedding-dimensions.test.ts @@ -30,3 +30,30 @@ describe("strict embedding dimension guard", () => { ).toThrow(new RegExp(`non-finite value at index ${EXPECTED_EMBED_DIM - 1}`)); }); }); + +describe("schema vector dimension guard (IDX-C2 fail-fast)", () => { + it("parses distinct vector(N) dimensions from schema SQL", async () => { + const { parseSchemaVectorDimensions } = await import("../src/lib/embedding-dimensions"); + const sql = + "create table a (e vector(1536)); create table b (f VECTOR(1536)); create index on c using hnsw (g vector(1536));"; + expect(parseSchemaVectorDimensions(sql)).toEqual([1536]); + expect(parseSchemaVectorDimensions("no vectors here")).toEqual([]); + expect(parseSchemaVectorDimensions("vector(768) and vector(1536)")).toEqual([768, 1536]); + }); + + it("passes when the configured dimension matches a single consistent schema dimension", async () => { + const { describeSchemaDimensionMismatch } = await import("../src/lib/embedding-dimensions"); + expect(describeSchemaDimensionMismatch(1536, "e vector(1536), f vector(1536)")).toBeNull(); + }); + + it("flags a config/schema mismatch, inconsistent schema dims, and a missing vector column", async () => { + const { describeSchemaDimensionMismatch } = await import("../src/lib/embedding-dimensions"); + expect(describeSchemaDimensionMismatch(3072, "e vector(1536)")).toMatch( + /EMBEDDING_DIMENSIONS=3072 does not match schema vector\(1536\)/, + ); + expect(describeSchemaDimensionMismatch(1536, "e vector(1536), f vector(3072)")).toMatch( + /inconsistent vector dimensions/, + ); + expect(describeSchemaDimensionMismatch(1536, "no vectors")).toMatch(/No vector\(N\) columns/); + }); +}); diff --git a/tests/eval-retrieval.test.ts b/tests/eval-retrieval.test.ts index d35167419..b128d3ea4 100644 --- a/tests/eval-retrieval.test.ts +++ b/tests/eval-retrieval.test.ts @@ -145,6 +145,23 @@ describe("golden retrieval eval helpers", () => { expect(summary.document_recall_at_5).toBe(0); }); + it("classifies the abbreviation golden cases as their declared query class (CI-14)", async () => { + // The eval fails a case on query-class mismatch, so a golden case whose query does not + // classify as its expectedQueryClass would pollute the baseline. Guard the abbreviation + // cases added for the CBC/WCC synonym expansion. + const { classifyRagQuery } = await import("../src/lib/clinical-search"); + const cases = loadGoldenRetrievalCases("scripts/fixtures/rag-retrieval-golden.json"); + const abbreviationCases = cases.filter( + (testCase) => + testCase.id === "clozapine-cbc-abbreviation-threshold" || + testCase.id === "clozapine-wcc-abbreviation-threshold", + ); + expect(abbreviationCases).toHaveLength(2); + for (const testCase of abbreviationCases) { + expect(classifyRagQuery(testCase.query).queryClass).toBe(testCase.expectedQueryClass); + } + }); + it("converts captured RAG eval cases into golden retrieval cases", () => { expect( capturedRagCaseToGoldenCase({ diff --git a/tests/index-quality.test.ts b/tests/index-quality.test.ts index f35ae1e1a..2d8ab288e 100644 --- a/tests/index-quality.test.ts +++ b/tests/index-quality.test.ts @@ -135,4 +135,60 @@ describe("index quality scoring", () => { expect(quality.metrics.average_structured_visual_confidence).toBeGreaterThan(0.75); expect(quality.issues).not.toContain("low visual unit coverage"); }); + + it("forces poor quality and flags an actionable issue for un-OCR'd image-only PDFs (CI-6)", () => { + const quality = assessDocumentIndexQuality({ + metrics: { + page_count: 5, + text_character_count: 30, // only incidental header text + extracted_image_count: 5, + searchable_image_count: 0, + needs_ocr_page_count: 5, // every page is image-only and was not OCR'd + }, + chunks: [], + insertedImages: [], + sectionCount: 0, + memoryCardCount: 0, + documentEmbeddingFieldTypes: [], + }); + + expect(quality.extractionQuality).toBe("poor"); + expect(quality.issues).toEqual( + expect.arrayContaining([expect.stringContaining("image-only pages not OCR'd (5 of 5)")]), + ); + }); + + it("flags a minority of un-OCR'd pages without forcing an otherwise-strong index to poor", () => { + const quality = assessDocumentIndexQuality({ + metrics: { + page_count: 10, + text_character_count: 24_000, + extracted_image_count: 1, + searchable_image_count: 1, + needs_ocr_page_count: 1, // a single image-only page in an otherwise strong text PDF + }, + chunks: Array.from({ length: 12 }, (_, index) => ({ + content: `Distinct clinical passage ${index} with monitoring action, threshold detail, and section context for complete retrieval.`, + section_heading: `Section ${index}`, + section_path: ["Guideline", `Section ${index}`], + })), + insertedImages: [ + { + sourceKind: "table_crop", + tableRows: [["ANC", "withhold"]], + imageQualityScore: 1, + cropCompleteness: 1, + structuredVisualProfile: { confidence: 1, thresholds: [{}] }, + }, + ], + sectionCount: 3, + memoryCardCount: 6, + documentEmbeddingFieldTypes: ["document_title", "document_summary"], + }); + + expect(quality.issues).toEqual( + expect.arrayContaining([expect.stringContaining("image-only pages not OCR'd (1 of 10)")]), + ); + expect(quality.extractionQuality).not.toBe("poor"); + }); }); diff --git a/tests/ranking-config.test.ts b/tests/ranking-config.test.ts new file mode 100644 index 000000000..c0c9f466d --- /dev/null +++ b/tests/ranking-config.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "vitest"; +import { + defaultRankingConfig, + freshnessDecayPenalty, + resolveRankingConfig, + type FreshnessConfig, +} from "../src/lib/ranking-config"; + +describe("ranking-config defaults (W6 — zero behavior change)", () => { + it("reproduces the exact prior second-stage rerank constants", () => { + const w = defaultRankingConfig.secondStage; + expect(w.positionBase).toBe(0.09); + expect(w.positionStep).toBe(0.004); + expect(w.memorySummaryBoost).toBe(0.035); + expect(w.documentLookupTitleBoost).toBe(0.045); + expect(w.tableThresholdEvidenceBoost).toBe(0.065); + expect(w.doseAmountBoost).toBe(0.18); + expect(w.tableVisualBoost).toBe(0.08); + expect(w.visualBoost).toBe(0.04); + expect(w.visualIntelligenceMax).toBe(0.035); + expect(w.visualIntelligencePivot).toBe(0.55); + expect(w.visualIntelligenceSlope).toBe(0.08); + expect(w.outdatedPenalty).toBe(0.035); + expect(w.poorExtractionPenalty).toBe(0.035); + expect(w.lowIndexQualityPenalty).toBe(0.035); + expect(w.lowIndexQualityThreshold).toBe(0.55); + }); + + it("keeps document-diversity demotion OFF by default", () => { + expect(defaultRankingConfig.documentDiversityPenalty).toBe(0); + }); + + it("defaults freshness to the original step cliffs", () => { + expect(defaultRankingConfig.freshness).toMatchObject({ + mode: "step", + publicationCliffYears: 8, + publicationPenalty: -0.015, + reviewCliffYears: 5, + reviewPenalty: -0.01, + }); + }); +}); + +describe("resolveRankingConfig override merge", () => { + it("returns defaults for undefined, empty, and malformed JSON", () => { + expect(resolveRankingConfig(undefined)).toEqual(defaultRankingConfig); + expect(resolveRankingConfig("")).toEqual(defaultRankingConfig); + expect(resolveRankingConfig("{not json")).toEqual(defaultRankingConfig); + expect(resolveRankingConfig("[1,2,3]")).toEqual(defaultRankingConfig); + }); + + it("deep-merges provided numeric fields and keeps defaults for the rest", () => { + const cfg = resolveRankingConfig( + JSON.stringify({ secondStage: { doseAmountBoost: 0.22 }, documentDiversityPenalty: 0.03 }), + ); + expect(cfg.secondStage.doseAmountBoost).toBe(0.22); + // Untouched weights fall back to defaults. + expect(cfg.secondStage.positionBase).toBe(0.09); + expect(cfg.documentDiversityPenalty).toBe(0.03); + }); + + it("ignores non-numeric values and clamps diversity penalties to non-negative", () => { + const cfg = resolveRankingConfig( + JSON.stringify({ secondStage: { doseAmountBoost: "big" }, documentDiversityPenalty: -5 }), + ); + expect(cfg.secondStage.doseAmountBoost).toBe(0.18); + expect(cfg.documentDiversityPenalty).toBe(0); + }); + + it("accepts the linear freshness mode", () => { + const cfg = resolveRankingConfig(JSON.stringify({ freshness: { mode: "linear", linearRampYears: 4 } })); + expect(cfg.freshness.mode).toBe("linear"); + expect(cfg.freshness.linearRampYears).toBe(4); + }); +}); + +describe("freshnessDecayPenalty", () => { + const step = defaultRankingConfig.freshness; + + it("step mode reproduces the original publication/review cliffs exactly", () => { + expect(freshnessDecayPenalty(null, "publication", step)).toBe(0); + expect(freshnessDecayPenalty(7.9, "publication", step)).toBe(0); + expect(freshnessDecayPenalty(8, "publication", step)).toBe(-0.015); + expect(freshnessDecayPenalty(20, "publication", step)).toBe(-0.015); + expect(freshnessDecayPenalty(4.9, "review", step)).toBe(0); + expect(freshnessDecayPenalty(5, "review", step)).toBe(-0.01); + }); + + it("linear mode ramps monotonically from the ramp start up to the cliff", () => { + const linear: FreshnessConfig = { ...step, mode: "linear", linearRampYears: 3 }; + // publication cliff 8, ramp 3 => ramp starts at year 5. + expect(freshnessDecayPenalty(5, "publication", linear)).toBe(0); + const mid = freshnessDecayPenalty(6.5, "publication", linear); // halfway + expect(mid).toBeLessThan(0); + expect(mid).toBeGreaterThan(-0.015); + expect(freshnessDecayPenalty(8, "publication", linear)).toBe(-0.015); + // Beyond the cliff the penalty is capped, never exceeding the full value. + expect(freshnessDecayPenalty(20, "publication", linear)).toBe(-0.015); + // Monotonic non-increasing as the document ages. + expect(freshnessDecayPenalty(7, "publication", linear)).toBeLessThan(mid); + }); +}); diff --git a/tests/reindex-eval-gate.test.ts b/tests/reindex-eval-gate.test.ts new file mode 100644 index 000000000..c56052c93 --- /dev/null +++ b/tests/reindex-eval-gate.test.ts @@ -0,0 +1,275 @@ +import { describe, expect, it } from "vitest"; +import { decideReindexGate, type QualityGateSummary, type RetrievalGateSummary } from "../src/lib/reindex-eval-gate"; + +function retrieval(overrides: Partial = {}): RetrievalGateSummary { + return { + case_count: 4, + failed_case_count: 0, + document_recall_at_5: 0.9, + content_recall_at_5: 0.9, + top_k_hit_rate: 0.95, + mrr_at_10: 0.8, + p90_latency_ms: 8000, + ...overrides, + }; +} + +function quality(overrides: Partial = {}): QualityGateSummary { + return { + case_count: 4, + grounded_supported_rate: 0.95, + unsupported_count: 1, + unsupported_correct_rate: 1, + expected_hit_rate: 0.9, + citation_failure_rate: 0, + numeric_grounding_failure_rate: 0, + source_governance_danger_failure_rate: 0, + expected_danger_warning_missing_count: 0, + stale_review_unknown_rate: 0.1, + review_required_rate: 0.1, + p95_latency_ms: 18000, + ...overrides, + }; +} + +describe("decideReindexGate — retrieval", () => { + it("returns GO when the candidate matches or beats the baseline and clears the floors", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval({ content_recall_at_5: 0.92 }), + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("GO"); + expect(decision.failures).toEqual([]); + }); + + it("returns NO_GO when an absolute floor is breached", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ content_recall_at_5: 0.78 }), + candidateRetrieval: retrieval({ content_recall_at_5: 0.75 }), + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/content_recall_at_5 0.75 below absolute floor 0.8/); + }); + + it("returns NO_GO on a real regression even when both values clear the floor", () => { + // Baseline 0.98, candidate 0.85 — both above the 0.8 floor, but a 13-point drop is a + // regression well beyond the 0.02 tolerance. + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ document_recall_at_5: 0.98 }), + candidateRetrieval: retrieval({ document_recall_at_5: 0.85 }), + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/document_recall_at_5 regressed/); + }); + + it("tolerates sub-threshold eval noise below the baseline", () => { + // 0.90 -> 0.89 is within the 0.02 tolerance and still above the floor => GO. + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ content_recall_at_5: 0.9 }), + candidateRetrieval: retrieval({ content_recall_at_5: 0.89 }), + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("GO"); + }); + + it("flags a p90 latency regression beyond the allowed drift", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ p90_latency_ms: 8000 }), + candidateRetrieval: retrieval({ p90_latency_ms: 14000 }), + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/p90_latency_ms regressed/); + }); + + it("fails closed when retrieval summaries omit failed-case counts", () => { + const baselineWithoutFailedCaseCount = { + case_count: 4, + document_recall_at_5: 0.9, + content_recall_at_5: 0.9, + top_k_hit_rate: 0.9, + } as Partial as RetrievalGateSummary; + const decision = decideReindexGate({ + baselineRetrieval: baselineWithoutFailedCaseCount, + candidateRetrieval: { + case_count: 4, + failed_case_count: 0, + document_recall_at_5: 0.9, + content_recall_at_5: 0.9, + top_k_hit_rate: 0.9, + }, + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/baselineRetrieval\.failed_case_count/); + }); + + it("skips optional metrics when either summary omits them", () => { + const decision = decideReindexGate({ + baselineRetrieval: { + case_count: 4, + failed_case_count: 0, + document_recall_at_5: 0.9, + content_recall_at_5: 0.9, + top_k_hit_rate: 0.9, + }, + candidateRetrieval: { + case_count: 4, + failed_case_count: 0, + document_recall_at_5: 0.9, + content_recall_at_5: 0.9, + top_k_hit_rate: 0.9, + }, + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("GO"); + expect(decision.checks.some((check) => check.metric === "mrr_at_10")).toBe(false); + expect(decision.checks.some((check) => check.metric === "p90_latency_ms")).toBe(false); + }); + + it("fails closed when retrieval eval reports failed cases", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval({ failed_case_count: 1 }), + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/failed_case_count 1 above absolute ceiling 0/); + }); + + it("fails closed when retrieval summaries cover different eval populations", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ case_count: 12 }), + candidateRetrieval: retrieval({ case_count: 4 }), + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/retrieval case_count mismatch/); + }); + + it("fails closed when retrieval case fingerprints differ", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval({ case_fingerprint: "all-cases-v1" }), + candidateRetrieval: retrieval({ case_fingerprint: "limited-cases-v1" }), + qualityMode: "retrieval_only", + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/retrieval case_fingerprint mismatch/); + }); + + it("fails closed when quality summaries are omitted by default", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/quality summaries are required/); + }); +}); + +describe("decideReindexGate — quality", () => { + it("returns GO when retrieval and quality both hold", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality(), + candidateQuality: quality(), + }); + expect(decision.decision).toBe("GO"); + }); + + it("returns NO_GO when any hard-zero failure rate is non-zero", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality(), + candidateQuality: quality({ citation_failure_rate: 0.05 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/citation_failure_rate 0.05 above absolute ceiling 0/); + }); + + it("skips unsupported_correct_rate when there are no unsupported cases", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality({ unsupported_count: 0, unsupported_correct_rate: 0 }), + candidateQuality: quality({ unsupported_count: 0, unsupported_correct_rate: 0 }), + }); + expect(decision.decision).toBe("GO"); + expect(decision.checks.some((check) => check.metric === "unsupported_correct_rate")).toBe(false); + }); + + it("returns NO_GO when an expected danger warning is missing", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality(), + candidateQuality: quality({ expected_danger_warning_missing_count: 1 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/expected_danger_warning_missing_count 1 above absolute ceiling 0/); + }); + + it("returns NO_GO when grounded_supported_rate falls below the floor", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality({ grounded_supported_rate: 0.92 }), + candidateQuality: quality({ grounded_supported_rate: 0.85 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/grounded_supported_rate 0.85 below absolute floor 0.9/); + }); + + it("returns NO_GO when p95 latency exceeds the ceiling", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality(), + candidateQuality: quality({ p95_latency_ms: 26000 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/p95_latency_ms 26000 above absolute ceiling 25000/); + }); + + it("fails closed when only one side supplies a quality summary", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + candidateQuality: quality(), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/quality summaries are required/); + }); + + it("fails closed when quality summaries omit required source governance rates", () => { + const withoutGovernanceRates = quality() as Partial; + delete withoutGovernanceRates.stale_review_unknown_rate; + delete withoutGovernanceRates.review_required_rate; + + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality(), + candidateQuality: withoutGovernanceRates as QualityGateSummary, + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/candidateQuality\.stale_review_unknown_rate/); + expect(decision.failures.join(" ")).toMatch(/candidateQuality\.review_required_rate/); + }); + + it("fails closed when quality summaries cover different eval populations", () => { + const decision = decideReindexGate({ + baselineRetrieval: retrieval(), + candidateRetrieval: retrieval(), + baselineQuality: quality({ case_count: 9 }), + candidateQuality: quality({ case_count: 3 }), + }); + expect(decision.decision).toBe("NO_GO"); + expect(decision.failures.join(" ")).toMatch(/quality case_count mismatch/); + }); +}); diff --git a/tests/retrieval-query-variants.test.ts b/tests/retrieval-query-variants.test.ts index 755cfbddf..ddb62411e 100644 --- a/tests/retrieval-query-variants.test.ts +++ b/tests/retrieval-query-variants.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { analyzeClinicalQuery, buildClinicalTextSearchQuery, rankClinicalResults } from "../src/lib/clinical-search"; +import { expandClinicalVocabularyText } from "../src/lib/clinical-vocabulary"; import { selectRetrievalEvidence } from "../src/lib/retrieval-selection"; import { buildRetrievalQueryVariants, @@ -632,6 +633,34 @@ describe("retrieval query variants", () => { ).toMatchObject({ accepted: true, reason: "dose_route_amount_evidence_gate" }); }); + it("accepts SC and SL route evidence for dose-route fast gates", () => { + expect( + evaluateEvidenceCoverageGate( + "What SC route dose is listed?", + [ + result({ + content: "Medication chart: 2 mg subcutaneous administration is listed for the route option.", + similarity: 0.8, + }), + ], + "medication_dose_risk", + ), + ).toMatchObject({ accepted: true, reason: "dose_route_amount_evidence_gate" }); + + expect( + evaluateEvidenceCoverageGate( + "What SL route dose is listed?", + [ + result({ + content: "Medication chart: 2 mg sublingual administration is listed for the route option.", + similarity: 0.8, + }), + ], + "medication_dose_risk", + ), + ).toMatchObject({ accepted: true, reason: "dose_route_amount_evidence_gate" }); + }); + it("requires direct source image evidence for source image/table requests", () => { const query = "Show the source table image for the patient property restricted items table."; expect( @@ -956,6 +985,27 @@ describe("retrieval query variants", () => { }); }); +describe("clinical abbreviation synonym expansion (CI-14)", () => { + it("expands the FBC/CBC lab family bidirectionally", () => { + // A clinician searching the US term "CBC" must reach FBC-only documents, and vice versa. + expect(expandClinicalVocabularyText("cbc")).toEqual(expect.arrayContaining(["full blood count", "fbc"])); + expect(expandClinicalVocabularyText("fbc")).toEqual(expect.arrayContaining(["full blood count", "cbc"])); + expect(expandClinicalVocabularyText("complete blood count")).toEqual(expect.arrayContaining(["fbc"])); + }); + + it("surfaces the FBC form into the retrieval analysis for a CBC query", () => { + // The expansion must reach analysis.expandedTerms, which feeds the query variants that + // are run against the lexical RPCs and unioned — recovering FBC-only chunks. + const analysis = analyzeClinicalQuery("clozapine CBC monitoring threshold"); + expect(analysis.expandedTerms).toEqual(expect.arrayContaining(["fbc"])); + }); + + it("expands the subcutaneous and sublingual administration routes", () => { + expect(expandClinicalVocabularyText("give sc injection")).toEqual(expect.arrayContaining(["subcutaneous"])); + expect(expandClinicalVocabularyText("administer sublingual")).toEqual(expect.arrayContaining(["sublingual", "sl"])); + }); +}); + describe("relaxVariantToOrQuery (8b over-conjunction fallback)", () => { it("relaxes a multi-term AND variant to a deduped term-OR query", () => { expect(relaxVariantToOrQuery("ciwa score threshold drug treatment alcohol withdrawal")).toBe( diff --git a/tests/retrieval-selection.test.ts b/tests/retrieval-selection.test.ts index bac8cb8b3..fcc420c4f 100644 --- a/tests/retrieval-selection.test.ts +++ b/tests/retrieval-selection.test.ts @@ -160,6 +160,40 @@ describe("retrieval source selection", () => { expect(selection.summary.topChunkTypes.medication_chart).toBeGreaterThan(0); }); + it("treats SC and SL administration rows as dose-route evidence", () => { + const intent = buildRetrievalIntent("What SC or SL route options are listed?", "medication_dose_risk"); + const selected = summarizeRetrievalSelection({ + query: "What SC or SL route options are listed?", + queryClass: "medication_dose_risk", + results: [ + source({ + id: "sc-sl-route-row", + title: "Medication Route Chart", + content: "Subcutaneous medication may be used for one option; sublingual medication is listed for another.", + hybrid_score: 0.66, + index_unit: { + id: "unit-sc-sl-route", + unit_type: "medication_chart_row", + title: "SC and SL route options", + content: "SC route and SL route options.", + source_chunk_id: "sc-sl-route-row", + source_image_id: null, + page_start: 2, + page_end: 2, + heading_path: ["Medication chart"], + normalized_terms: ["sc", "subcutaneous", "sl", "sublingual"], + quality_score: 0.9, + extraction_mode: "hybrid", + }, + }), + ], + }); + + expect(intent.requiredTermSignals).toContain("route"); + expect(selected.summary.requiredSignalsSatisfied).toBe(true); + expect(selected.summary.matchedSignals).toContain("route"); + }); + it("promotes flowchart next-step evidence for red-zone pathway questions", () => { const selection = selectRetrievalEvidence({ query: "In the clinical flowchart, what is the next step after red-zone risk?", diff --git a/worker/main.ts b/worker/main.ts index 29cd7d21d..64546daeb 100644 --- a/worker/main.ts +++ b/worker/main.ts @@ -1524,12 +1524,17 @@ function extractionMetrics( ) { const textCharacterCount = extracted.pages.reduce((sum, page) => sum + page.text.length, 0); const ocrPageCount = extracted.pages.filter((page) => page.ocrUsed).length; + // CI-6: pages the extractor flagged as image-only but could NOT OCR (JS fallback without + // the Python OCR prerequisites). Threaded into index quality so these documents are marked + // poor and surfaced to eval governance rather than indexed near-empty. + const needsOcrPageCount = extracted.pages.filter((page) => page.needsOcr).length; const warnings = [...(extracted.warnings ?? [])]; if (textCharacterCount < 80) warnings.push("Low extracted text volume; inspect OCR quality."); return { page_count: extracted.pages.length, ocr_page_count: ocrPageCount, + needs_ocr_page_count: needsOcrPageCount, text_character_count: textCharacterCount, extracted_image_count: extracted.images.length, searchable_image_count: Object.values(imageTypeCounts).reduce((sum, count) => sum + count, 0),