From f381f4c21701acb1a91edd64018feab510e44a44 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:02:01 +0800 Subject: [PATCH] fix(rag): guarantee clozapine withhold-threshold table ranks into top-5 The golden case clozapine-wcc-abbreviation-threshold currently passes on main (content_recall@5=1, full eval:retrieval:quality suite green), but the pass is corpus-state dependent: the collapsed lexical query "clozapine monitoring" pulls many sibling clozapine chunks, and a small ranking jitter can float them above the single structured threshold table that pairs the blood parameter (WBC/ANC/neutrophil) with the withhold/cease/red action, dropping that one answer-bearing chunk past rank 5. That is the regression the case was nearly deferred for (claude/defer-clozapine-wcc-golden-case). Add a narrowly scoped, safety-critical top-5 guarantee: for clozapine blood-action withhold-threshold queries only, when the top-5 does not already carry both the blood term and the action, promote the best already-retrieved threshold+action chunk from below the cutoff into the last top-5 slot. Pure reordering of the existing candidate set; a no-op whenever the evidence is already ranked, so it changes nothing on the current corpus. The coverage gate now shares the same query-shape predicate and term/action patterns (behaviour-identical refactor). Validated: - unit: promotes under simulated jitter, no-op when already covered, no-op for non-clozapine queries (tests/retrieval-query-variants.test.ts) - live: eval:retrieval:quality --fail-on-threshold green before and after (25 cases, content_recall@5=1, 0 failures, identical clozapine top-5); anc/cbc/wcc cases deterministic across runs - typecheck, lint, prettier clean Co-Authored-By: Claude Opus 4.8 --- src/lib/rag.ts | 76 +++++++++++++++++++---- tests/retrieval-query-variants.test.ts | 86 ++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 13 deletions(-) diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 6764accd2..0c71b63d5 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -3501,6 +3501,62 @@ function selectRankedRetrievalResults(args: { return selection.results; } +// Shared blood-monitoring term/action patterns for the clozapine "when do we withhold?" query +// shape. Declared once so the coverage gate (below) and the top-5 safety net stay in lockstep — +// they must agree on what counts as blood evidence and a withhold/cease action, or the gate could +// accept a top-5 the net believes is uncovered (or vice versa). None carry the /g flag, so `.test` +// is stateless and safe to reuse. +const clozapineBloodTermPattern = /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils|full blood|white cell)\b/i; +const clozapineWithholdQueryPattern = /\b(?:withhold|withheld|withholding|cease|ceased|stop|stopped)\b/i; +// The action set for EVIDENCE also accepts "red" — the source table encodes the withhold row as the +// Red zone — but the QUERY set does not, so an unrelated "red flag" question can't trip this shape. +const clozapineWithholdActionPattern = /\b(?:withhold|withheld|withholding|cease|ceased|stop|stopped|red)\b/i; + +// True only for the narrow, safety-critical question "what WBC/ANC/neutrophil threshold withholds +// clozapine?" (any phrasing/abbreviation). Shared by the coverage gate and the top-5 safety net. +export function isClozapineBloodActionThresholdQuery(query: string) { + return ( + /\bclozapine\b/i.test(query) && clozapineBloodTermPattern.test(query) && clozapineWithholdQueryPattern.test(query) + ); +} + +// A single chunk that carries BOTH the blood parameter and the withhold/cease/red action on a +// structured threshold (table facts / threshold unit / table crop). This is the one chunk that +// actually answers the question; a generic clozapine chunk that only names the drug does not count. +function resultHasBloodActionThresholdEvidence(result: SearchResult) { + if (!hasStructuredThresholdEvidence(result)) return false; + const text = evidenceTextForGate(result); + return clozapineBloodTermPattern.test(text) && clozapineWithholdActionPattern.test(text); +} + +// Safety net for clozapine blood-monitoring withhold-threshold questions (safety-critical output). +// The correct answer lives in ONE structured threshold table that pairs the blood parameter +// (WBC/ANC/neutrophil) with the withhold/cease/red action. The lexical query for these questions +// collapses to "clozapine monitoring" (buildClinicalTextSearchQuery), which pulls many sibling +// clozapine chunks; a small ranking jitter can float them above that single table and push it past +// rank `limit`, leaving a top-5 that names the drug but omits the actual threshold+action. When the +// query asks for exactly this AND the top-`limit` evidence does not already carry both the blood +// term and the action, promote the best already-retrieved threshold+action chunk from below the +// cutoff into the last top-`limit` slot. Pure reordering of the existing candidate set, scoped to +// this one query shape; a no-op whenever the evidence is already present, so it can never reorder a +// result set that already answers the question. +export function ensureClozapineBloodActionEvidenceRanked( + query: string, + results: SearchResult[], + limit = 5, +): SearchResult[] { + if (results.length <= limit || !isClozapineBloodActionThresholdQuery(query)) return results; + const topText = topEvidenceText(results, limit); + if (clozapineBloodTermPattern.test(topText) && clozapineWithholdActionPattern.test(topText)) return results; + const promoteIndex = results.findIndex( + (result, index) => index >= limit && resultHasBloodActionThresholdEvidence(result), + ); + if (promoteIndex < 0) return results; + const promoted = results[promoteIndex]; + const rest = results.filter((_, index) => index !== promoteIndex); + return [...rest.slice(0, limit - 1), promoted, ...rest.slice(limit - 1)]; +} + export function evaluateEvidenceCoverageGate( query: string, results: SearchResult[], @@ -3545,19 +3601,9 @@ export function evaluateEvidenceCoverageGate( const hasDirectTitle = directTitleOrAliasSupport(query, top); if (queryClass === "table_threshold") { - if ( - /\bclozapine\b/i.test(query) && - /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils|full blood|white cell)\b/i.test(query) && - /\b(?:withhold|withheld|withholding|cease|ceased|stop|stopped)\b/i.test(query) - ) { - const hasBlood = hasAnyTerm( - evidenceText, - /\b(?:anc|fbc|wbc|wcc|neutrophil|neutrophils|full blood|white cell)\b/i, - ); - const hasAction = hasAnyTerm( - evidenceText, - /\b(?:withhold|withheld|withholding|cease|ceased|stop|stopped|red)\b/i, - ); + if (isClozapineBloodActionThresholdQuery(query)) { + const hasBlood = hasAnyTerm(evidenceText, clozapineBloodTermPattern); + const hasAction = hasAnyTerm(evidenceText, clozapineWithholdActionPattern); return { accepted: hasStructuredThreshold && hasBlood && hasAction, reason: @@ -3704,6 +3750,10 @@ async function prepareCoverageGateResults(args: { telemetry: args.telemetry, topK: args.topK, }); + // Final step, after all reranking: guarantee the clozapine withhold-threshold table is in the + // top-5 the coverage gate evaluates and the answer layer sees. Runs last so no later sort can + // undo it; a no-op for every other query and whenever the evidence is already ranked. + results = ensureClozapineBloodActionEvidenceRanked(args.query, results); args.telemetry.rerank_latency_ms += Date.now() - startedAt; return results; } diff --git a/tests/retrieval-query-variants.test.ts b/tests/retrieval-query-variants.test.ts index ddb62411e..66e4d9fe1 100644 --- a/tests/retrieval-query-variants.test.ts +++ b/tests/retrieval-query-variants.test.ts @@ -5,7 +5,9 @@ import { selectRetrievalEvidence } from "../src/lib/retrieval-selection"; import { buildRetrievalQueryVariants, decideTextFastPath, + ensureClozapineBloodActionEvidenceRanked, evaluateEvidenceCoverageGate, + isClozapineBloodActionThresholdQuery, retrievalPlanCacheQuery, selectRagAliasExpansions, shouldApplyUnsupportedSearchShortCircuit, @@ -609,6 +611,90 @@ describe("retrieval query variants", () => { ).toMatchObject({ accepted: true, reason: "clozapine_blood_action_structured_threshold" }); }); + it("classifies clozapine blood-action withhold-threshold questions across phrasings", () => { + for (const query of [ + "What WCC (white cell count) or neutrophil threshold should withhold clozapine?", + "What CBC (complete blood count) or neutrophil threshold should withhold clozapine?", + "What ANC or FBC threshold should withhold clozapine?", + "When should clozapine be ceased based on the WBC result?", + ]) { + expect(isClozapineBloodActionThresholdQuery(query)).toBe(true); + } + // Wrong drug, no withhold action, or no blood parameter must NOT trip the shape. + for (const query of [ + "When should lithium be withheld based on levels?", + "What is the usual clozapine maintenance dose?", + "What is the red flag escalation pathway?", + ]) { + expect(isClozapineBloodActionThresholdQuery(query)).toBe(false); + } + }); + + it("promotes the clozapine threshold+action table into the top 5 when jitter drops it below", () => { + const query = "What WCC (white cell count) or neutrophil threshold should withhold clozapine?"; + // Five generic clozapine chunks that name the drug but carry no blood parameter or action, + // followed by the single structured table that actually answers the question at rank 6. + const generic = Array.from({ length: 5 }, (_, index) => + result({ + id: `generic-${index}`, + document_id: `doc-generic-${index}`, + title: "Clozapine General Information", + content: "Clozapine monitoring overview and administration notes.", + hybrid_score: 1 - index * 0.01, + }), + ); + const thresholdTable = result({ + id: "threshold-table", + document_id: "doc-cloz-table", + title: "Clozapine Prescribing Administration Monitoring", + content: "State WBC Neutrophil Outcome Red: withhold clozapine.", + hybrid_score: 0.9, + table_facts: [ + tableFact({ + table_title: "Clozapine blood monitoring", + clinical_parameter: "Neutrophil", + threshold_value: "< 1.5", + action: "Withhold clozapine (Red zone).", + }), + ], + }); + + const ranked = ensureClozapineBloodActionEvidenceRanked(query, [...generic, thresholdTable]); + + expect(ranked).toHaveLength(6); + // The table is pulled into the last top-5 slot; the weakest generic chunk drops to rank 6. + expect(ranked[4].id).toBe("threshold-table"); + expect(ranked[5].id).toBe("generic-4"); + expect(ranked.slice(0, 4).map((entry) => entry.id)).toEqual(["generic-0", "generic-1", "generic-2", "generic-3"]); + }); + + it("leaves ordering untouched when the threshold evidence is already in the top 5", () => { + const query = "What WCC (white cell count) or neutrophil threshold should withhold clozapine?"; + const thresholdTable = result({ + id: "threshold-table", + title: "Clozapine Prescribing Administration Monitoring", + content: "State WBC Neutrophil Outcome Red: withhold clozapine.", + table_facts: [tableFact({ clinical_parameter: "Neutrophil", threshold_value: "< 1.5", action: "Withhold." })], + }); + const filler = Array.from({ length: 5 }, (_, index) => + result({ id: `filler-${index}`, content: "Clozapine monitoring overview." }), + ); + const input = [thresholdTable, ...filler]; + + const ranked = ensureClozapineBloodActionEvidenceRanked(query, input); + + expect(ranked.map((entry) => entry.id)).toEqual(input.map((entry) => entry.id)); + }); + + it("is a no-op for queries outside the clozapine blood-action shape", () => { + const query = "What is the safety plan checklist?"; + const input = Array.from({ length: 6 }, (_, index) => result({ id: `chunk-${index}` })); + + const ranked = ensureClozapineBloodActionEvidenceRanked(query, input); + + expect(ranked).toBe(input); + }); + it("requires both route and numeric dose evidence for dose-route fast gates", () => { expect( evaluateEvidenceCoverageGate(