From 8f39ab4fe2f4b96a89fae9a13f93940ed5b040f1 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:52:15 +0800 Subject: [PATCH 1/2] fix(rag): recover source-backed clinical answers Use validated deterministic recovery for routine document-content queries and scope clinical-value checks to claim-supporting citations. Focused RAG safety tests and the offline RAG contract pass. --- src/lib/answer-verification.ts | 57 +++++++++++----- src/lib/rag-extractive-answer.ts | 45 ++++++------- src/lib/rag.ts | 76 ++++++++++++++++++++- tests/rag-answer-fallback.test.ts | 108 ++++++++++++++++++++++++++++++ tests/rag-trust.test.ts | 91 +++++++++++++++++++++++++ 5 files changed, 335 insertions(+), 42 deletions(-) diff --git a/src/lib/answer-verification.ts b/src/lib/answer-verification.ts index 385d06325..1813e1690 100644 --- a/src/lib/answer-verification.ts +++ b/src/lib/answer-verification.ts @@ -403,6 +403,7 @@ export function verifyAnswerNumbers( answerText: string, citations: Array>, results: SearchResult[], + options: { collectiveCitationSupport?: boolean } = {}, ): NumericVerification { const answerAtoms = extractClinicalValueAtoms(answerText); const answerTokens = Array.from( @@ -428,7 +429,12 @@ export function verifyAnswerNumbers( // rather than allowing an unrelated citation to verify it by union membership. const sourceAtomSets = citedResults.map((result) => sourceClinicalValueAtomSet([result])); const unverifiedTokens = answerAtoms - .filter((atom) => !sourceAtomSets.every((atoms) => atoms.has(clinicalValueAtomKey(atom)))) + .filter((atom) => { + const key = clinicalValueAtomKey(atom); + return options.collectiveCitationSupport + ? !sourceAtomSets.some((atoms) => atoms.has(key)) + : !sourceAtomSets.every((atoms) => atoms.has(key)); + }) .map(clinicalValueAtomDisplay); return { @@ -490,22 +496,39 @@ export function applyNumericVerification(answer: RagAnswer, verificationSources? const sources = verificationSources ?? answer.sources ?? []; const unverified = new Set(); - // B4: the model is instructed to put dose details in structured - // answerSections (kind medication_dose), so a top-level-only scan never sees - // section-body doses. Verify the top-level answer AND every section body. - // Each section is scoped to its own citation_chunk_ids when present, so a - // dose is only credited against the chunks that section actually cites; - // sections with no citations fall back to the answer-level citations. - const answerVerification = verifyAnswerNumbers(answer.answer, answer.citations, sources); - for (const token of answerVerification.unverifiedTokens) unverified.add(token); - - for (const section of answer.answerSections ?? []) { - const sectionCitations = - section.citation_chunk_ids.length > 0 - ? section.citation_chunk_ids.map((chunk_id) => ({ chunk_id })) - : answer.citations; - const sectionVerification = verifyAnswerNumbers(section.body, sectionCitations, sources); - for (const token of sectionVerification.unverifiedTokens) unverified.add(token); + const claimScopedValues = (answer.supportedClaims ?? []).filter( + (claim) => extractClinicalValueAtoms(claim.text).length > 0, + ); + if (claimScopedValues.length > 0) { + // Claim assessment has already checked entity, polarity, clinical dimension, + // and exact value co-location. Verify each value only against the chunks that + // directly support its claim. Requiring a value to appear in every top-level + // citation incorrectly rejects valid multi-source answers (for example, a + // step number supported by one cited agitation-management passage). + for (const claim of claimScopedValues) { + const verification = verifyAnswerNumbers( + claim.text, + claim.supportingChunkIds.map((chunk_id) => ({ chunk_id })), + sources, + { collectiveCitationSupport: answer.responseMode === "comparison_matrix" }, + ); + for (const token of verification.unverifiedTokens) unverified.add(token); + } + } else { + // B4: before claim provenance is available, verify top-level prose and every + // section body against their explicit citation scopes. This preserves the + // fail-closed parse/fallback paths as well as direct callers of this helper. + const answerVerification = verifyAnswerNumbers(answer.answer, answer.citations, sources); + for (const token of answerVerification.unverifiedTokens) unverified.add(token); + + for (const section of answer.answerSections ?? []) { + const sectionCitations = + section.citation_chunk_ids.length > 0 + ? section.citation_chunk_ids.map((chunk_id) => ({ chunk_id })) + : answer.citations; + const sectionVerification = verifyAnswerNumbers(section.body, sectionCitations, sources); + for (const token of sectionVerification.unverifiedTokens) unverified.add(token); + } } if (unverified.size === 0) return answer; diff --git a/src/lib/rag-extractive-answer.ts b/src/lib/rag-extractive-answer.ts index a5839dee8..7fbb79975 100644 --- a/src/lib/rag-extractive-answer.ts +++ b/src/lib/rag-extractive-answer.ts @@ -288,12 +288,16 @@ export function classifyAnswerIntent(query: string, queryClass: RagQueryClass): if (hasResultActionSignal) return "red_result_action"; if (/\b(?:dose|dosing|dosage|max(?:imum)?|mg|mcg|renal|eGFR|creatinine)\b/i.test(query)) return "dose"; if (/\b(?:pathway|refer|referral|criteria|ect|electroconvulsive)\b/.test(normalized)) return "pathway_referral"; + // Retrieval classification and answer intent are different concerns. A + // document_lookup route can still ask for the document's clinical content + // (for example, "What should a safety plan include?"). Treat it as a source + // lookup only when the wording explicitly asks to find/open/select a source; + // otherwise the extractive path must select responsive clinical facts rather + // than reference-list lines that merely mention a guideline or procedure. if ( - queryClass === "document_lookup" || /\b(?:find|show|open|which)\b.*\b(?:document|guideline|procedure|policy|protocol|form|source|file)\b/.test( normalized, - ) || - /\b(?:documentation|forms?|documents?|sources?|guidelines?|procedure|policy|protocol)\b/.test(normalized) + ) ) { return "document_lookup"; } @@ -343,7 +347,7 @@ function answerIntentEvidencePattern(intent: AnswerIntent) { case "document_lookup": return /\b(?:document|guideline|procedure|policy|protocol|form|source|file|support|supports|covers|contains)\b/i; default: - return /\b(?:assess|arrange|check|continue|review|treat|manage|monitor|refer|dose|risk|therapy|diagnos\w*)\b/i; + return /\b(?:assess|arrange|check|collaborat\w*|complete|conduct|continue|develop|diagnos\w*|document|dose|ensure|identify|include|incorporate|involve|link|manage|monitor|provide|record|refer|revise|review\w*|risk|share|therapy|treat|update)\b/i; } } @@ -736,7 +740,7 @@ function factSupportsAnswerIntent( if (/^what\s+is\b/i.test(query)) { return /\b(?:is|are|means|defined|characteri[sz]ed|involves|refers\s+to)\b/i.test(text); } - return /\b(?:assess|arrange|check|continue|review|treat|manage|monitor|refer|dose|risk|therapy|diagnos\w*)\b/i.test( + return /\b(?:assess|arrange|check|collaborat\w*|complete|conduct|continue|develop|diagnos\w*|document|dose|ensure|identify|include|incorporate|involve|link|manage|monitor|provide|record|refer|revise|review\w*|risk|share|therapy|treat|update)\b/i.test( text, ); } @@ -964,7 +968,10 @@ function sectionForFactKind(kind: ExtractedClinicalFactKind): Pick => Boolean(section)); - return applyNumericVerification( - { - ...answer, - answer: boldHighYieldClinicalText(cleanedAnswer, query), - answerSections, - }, - verificationSources, - ); + return { + ...answer, + answer: boldHighYieldClinicalText(cleanedAnswer, query), + answerSections, + }; } diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 9db00d193..63cfec7af 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -893,6 +893,61 @@ function applyConfidenceGate( }; } +/** + * Allow a score-blocked routine document-content query to use the deterministic + * answer only when that answer independently passes the final safety gates. + * + * This is deliberately narrower than the normal extractive router: it cannot + * recover medication, threshold, comparison, broad-summary, complex, or weakly + * related queries. The retrieval diagnostic remains blocked so the UI still + * presents the recovered answer with low-trust guidance. + */ +function hasValidatedRoutineExtractiveRecovery(args: { + query: string; + queryClass: RagQueryClass; + results: SearchResult[]; + route: { mode: "unsupported" | "extractive" | "fast" | "strong"; reason: string }; + sourceBacked: boolean; +}) { + if ( + args.queryClass !== "document_lookup" || + args.route.mode !== "fast" || + args.route.reason !== "strong_routine_retrieval" || + !args.sourceBacked + ) { + return false; + } + + const candidate = finalizeRagAnswerQuality( + buildExtractiveAnswer({ + query: args.query, + queryClass: args.queryClass, + results: args.results, + quoteCards: [], + documentBreakdown: [], + evidenceSummary: undefined, + sourceCoverage: undefined, + conflictsOrGaps: [], + visualEvidence: [], + bestSource: null, + smartPanel: undefined, + relatedDocuments: [], + routeReason: `${args.route.reason}; validated_routine_extractive_recovery`, + timings: undefined, + }), + args.query, + args.queryClass, + ); + + return ( + candidate.grounded && + candidate.confidence !== "unsupported" && + candidate.citations.length > 0 && + candidate.responseMode !== "evidence_gap" && + !/final_quality_gate:/.test(candidate.routingReason ?? "") + ); +} + /** Clamp confidence. */ function clampConfidence( proposed: RagAnswer["confidence"] | undefined, @@ -4147,13 +4202,30 @@ async function answerQuestionWithScopeUncoalesced( selectedDocuments: explicitlySelectedComparisonDocuments, }) : null; + const validatedRoutineExtractiveRecovery = hasValidatedRoutineExtractiveRecovery({ + query: args.query, + queryClass, + results: answerInputResults, + route: routeFromRouting, + sourceBacked: relevance.isSourceBacked, + }); + const routeBeforeConfidenceGate = validatedRoutineExtractiveRecovery + ? { + ...routeFromRouting, + mode: "extractive" as const, + model: null, + reason: `${routeFromRouting.reason}; validated_routine_extractive_recovery`, + } + : routeFromRouting; const initialRetrievalDiagnostics = buildRetrievalDiagnostics({ queryClass, query: answerFocusQuery, results: answerInputResults, - answerMode: routeFromRouting.mode, + answerMode: routeBeforeConfidenceGate.mode, }); - const gatedRoute = applyConfidenceGate(routeFromRouting, queryClass, initialRetrievalDiagnostics); + const gatedRoute = validatedRoutineExtractiveRecovery + ? { route: routeBeforeConfidenceGate } + : applyConfidenceGate(routeBeforeConfidenceGate, queryClass, initialRetrievalDiagnostics); // In source-only mode (offline, or auto with no usable key) we never call the model. Route to // the deterministic extractive path when evidence is usable, but preserve the confidence gate's // "unsupported" decision so weak evidence still fails closed to a source-gap answer rather than diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 80c395b2a..1caba60b0 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -437,6 +437,42 @@ describe("RAG structured-output fallback", () => { expect(answer.answer).toMatch(/IM medication|oral medication|agitation/i); }); + it("keeps source-backed agitation step numbers grounded across multiple citations", async () => { + const answer = await answerFromTextSources( + "What should be considered for agitation and arousal pharmacological management?", + [ + source({ + id: "agitation-step-table", + document_id: "agitation-doc", + title: "Agitation and Arousal Pharmacological Management", + file_name: "MHSP.AgitationArousalPharmaMgt.pdf", + section_heading: "Stepwise management", + content: + "Step 1: agitation and arousal pharmacological management should assess severity and use oral medication when the patient is willing. Step 2: monitor the agitation response and consider intramuscular medication when oral medication is refused.", + }), + source({ + id: "agitation-clinical-review", + document_id: "agitation-doc", + title: "Agitation and Arousal Pharmacological Management", + file_name: "MHSP.AgitationArousalPharmaMgt.pdf", + section_heading: "Clinical review", + content: + "Review physical causes, medicine-related risks, observations, and the least restrictive management option.", + similarity: 0.94, + hybrid_score: 0.94, + text_rank: 1, + }), + ], + ); + + expect(answer.routingMode).toBe("extractive"); + expect(answer.grounded).toBe(true); + expect(answer.citations.length).toBeGreaterThanOrEqual(2); + expect(answer.answer).toMatch(/step [12]/i); + expect(answer.unverifiedNumericTokens ?? []).toEqual([]); + expect(answer.faithfulnessWarning).toBeUndefined(); + }); + it("returns a grounded document-support fallback for procedure queries when no clean fact can be synthesized", async () => { const answer = await answerFromTextSources( "What is the process for ECT procedure?", @@ -1655,6 +1691,78 @@ describe("RAG structured-output fallback", () => { expect(generateStructuredTextResult).not.toHaveBeenCalled(); }); + it("recovers a directly supported routine document-content answer without model generation", async () => { + vi.stubEnv("OPENAI_API_KEY", "test-key"); + vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0"); + vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0"); + + const sources = [ + source({ + id: "safety-plan-reference", + document_id: "safety-plan-doc", + title: "Patient Safety Planning Guideline", + file_name: "patient-safety-planning.pdf", + section_heading: "Related procedures", + content: + "Related procedures and guidelines. Women's and Perinatal Mental Health Referral and Management Guideline.", + similarity: 0.48, + hybrid_score: 0.48, + text_rank: 0.12, + }), + source({ + id: "safety-plan-requirements", + document_id: "safety-plan-doc", + title: "Patient Safety Planning Guideline", + file_name: "patient-safety-planning.pdf", + section_heading: "Safety planning for identified risks", + content: + "The Consumer Safety Plan must be developed in collaboration with the consumer, involve carers and family where appropriate, identify actions for a crisis and who is responsible, and be reviewed when clinical status changes.", + similarity: 0.48, + hybrid_score: 0.48, + text_rank: 0.11, + }), + ]; + const rpc = vi.fn(async (name: string) => { + if (retrievalRpcBaseName(name) === "match_document_chunks_text") return { data: sources, error: null }; + if (retrievalRpcBaseName(name) === "get_related_document_metadata") return { data: [], error: null }; + return { data: [], error: null }; + }); + const generateStructuredTextResult = vi.fn(); + + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ + rpc, + from: vi.fn(() => new EmptyQuery()), + }), + })); + vi.doMock("@/lib/openai", () => ({ + embedTextWithTelemetry: vi.fn(async () => ({ embedding: [0.1, 0.2, 0.3], cacheHit: false })), + generateStructuredTextResult, + })); + + const { answerQuestionWithScope } = await import("../src/lib/rag"); + const answer = await answerQuestionWithScope({ + query: "What should a patient safety plan include?", + ownerId: undefined, + logQuery: false, + skipCache: true, + }); + + expect(answer.routingMode).toBe("extractive"); + expect(answer.routingReason).toContain("validated_routine_extractive_recovery"); + expect(answer.retrievalDiagnostics).toMatchObject({ + gateStatus: "blocked", + routeMode: "extractive", + topScore: 0.48, + }); + expect(answer.grounded).toBe(true); + expect(answer.confidence).not.toBe("unsupported"); + expect(answer.answer).toContain("developed in collaboration with the consumer"); + expect(answer.answer).not.toContain("Perinatal Mental Health Referral"); + expect(answer.citations.some((citation) => citation.chunk_id === "safety-plan-requirements")).toBe(true); + expect(generateStructuredTextResult).not.toHaveBeenCalled(); + }); + it("does not gate-block a moderate score clustered across several distinct documents", async () => { // Regression: a topic with rich coverage (e.g. clozapine) returns many relevant // documents whose scores cluster tightly at a moderate value. A tiny spread there diff --git a/tests/rag-trust.test.ts b/tests/rag-trust.test.ts index 226401a93..2dc7d5bad 100644 --- a/tests/rag-trust.test.ts +++ b/tests/rag-trust.test.ts @@ -5,6 +5,7 @@ import { classifyAnswerIntent, parseAnswerJson, } from "../src/lib/rag"; +import { buildExtractiveAnswer, finalizeRagAnswerQuality } from "../src/lib/rag-extractive-answer"; import type { SearchResult } from "../src/lib/types"; function source(overrides: Partial = {}): SearchResult { @@ -51,6 +52,96 @@ describe("RAG trust validation", () => { "red_result_action", ); expect(classifyAnswerIntent("What are ECT referral criteria?", "document_lookup")).toBe("pathway_referral"); + expect(classifyAnswerIntent("What should a patient safety plan include?", "document_lookup")).toBe("general"); + expect(classifyAnswerIntent("What does the metabolic screening document require?", "document_lookup")).toBe( + "general", + ); + }); + + it("selects responsive requirements instead of related-policy references for document-routed content questions", () => { + const results = [ + source({ + id: "related-policy", + section_heading: "Safety planning", + content: + "Related procedures and guidelines. Women's and Perinatal Mental Health Referral and Management Guideline.", + }), + source({ + id: "direct-requirements", + section_heading: "Safety planning for identified risks", + content: + "The Consumer Safety Plan must be developed in collaboration with the consumer, involve carers and family where appropriate, identify actions for a crisis and who is responsible, and be reviewed when clinical status changes.", + }), + ]; + const candidate = buildExtractiveAnswer({ + query: "What should a patient safety plan include?", + queryClass: "document_lookup", + results, + quoteCards: [], + documentBreakdown: [], + evidenceSummary: undefined, + sourceCoverage: undefined, + conflictsOrGaps: [], + visualEvidence: [], + bestSource: null, + smartPanel: undefined, + relatedDocuments: [], + routeReason: "test_source_backed_recovery", + timings: undefined, + }); + const finalized = finalizeRagAnswerQuality( + candidate, + "What should a patient safety plan include?", + "document_lookup", + ); + + expect(finalized.grounded).toBe(true); + expect(finalized.answer).toContain("developed in collaboration with the consumer"); + expect(finalized.answer).not.toContain("Perinatal Mental Health Referral"); + expect(finalized.citations.some((citation) => citation.chunk_id === "direct-requirements")).toBe(true); + }); + + it("verifies a clinical value against its claim-supporting citation in a multi-source answer", () => { + const doseSource = source({ + id: "clozapine-dose", + document_id: "clozapine-doc", + title: "Clozapine prescribing guideline", + content: "Clozapine treatment starts at 12.5 mg daily.", + }); + const monitoringSource = source({ + id: "clozapine-monitoring", + document_id: "clozapine-doc", + title: "Clozapine prescribing guideline", + content: "Clozapine treatment requires regular blood monitoring and clinical review.", + }); + const answer = finalizeRagAnswerQuality( + { + answer: "Clozapine treatment starts at 12.5 mg daily.", + grounded: true, + confidence: "medium", + citations: [ + { ...doseSource, chunk_id: doseSource.id, provenance: "deterministic_support" }, + { ...monitoringSource, chunk_id: monitoringSource.id, provenance: "deterministic_support" }, + ], + sources: [doseSource, monitoringSource], + routingMode: "extractive", + routingReason: "test_claim_scoped_numeric_verification", + queryClass: "medication_dose_risk", + answerSections: [], + }, + "What is the clozapine starting dose?", + "medication_dose_risk", + ); + + expect(answer.grounded).toBe(true); + expect(answer.unverifiedNumericTokens ?? []).toEqual([]); + expect(answer.supportedClaims).toContainEqual( + expect.objectContaining({ + text: "clozapine treatment starts at 12.5 mg daily.", + supportStatus: "direct", + supportingChunkIds: ["clozapine-dose"], + }), + ); }); // B5: on model-JSON parse failure the fallback must fail closed — it must NOT From 711e7583a6a2ee0973598dc32a37199feeff89d3 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:20:42 +0800 Subject: [PATCH 2/2] fix(rag): limit routine recovery to blocked retrieval Preserve model synthesis for gate-passed routine document queries while retaining validated extractive recovery for blocked low-signal cases. Focused tests, ESLint, and Prettier pass. --- src/lib/rag.ts | 22 +++++++++-------- tests/rag-answer-fallback.test.ts | 41 +++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 63cfec7af..dc0984013 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -4202,13 +4202,21 @@ async function answerQuestionWithScopeUncoalesced( selectedDocuments: explicitlySelectedComparisonDocuments, }) : null; - const validatedRoutineExtractiveRecovery = hasValidatedRoutineExtractiveRecovery({ - query: args.query, + const initialRetrievalDiagnostics = buildRetrievalDiagnostics({ queryClass, + query: answerFocusQuery, results: answerInputResults, - route: routeFromRouting, - sourceBacked: relevance.isSourceBacked, + answerMode: routeFromRouting.mode, }); + const validatedRoutineExtractiveRecovery = + initialRetrievalDiagnostics.gateStatus === "blocked" && + hasValidatedRoutineExtractiveRecovery({ + query: args.query, + queryClass, + results: answerInputResults, + route: routeFromRouting, + sourceBacked: relevance.isSourceBacked, + }); const routeBeforeConfidenceGate = validatedRoutineExtractiveRecovery ? { ...routeFromRouting, @@ -4217,12 +4225,6 @@ async function answerQuestionWithScopeUncoalesced( reason: `${routeFromRouting.reason}; validated_routine_extractive_recovery`, } : routeFromRouting; - const initialRetrievalDiagnostics = buildRetrievalDiagnostics({ - queryClass, - query: answerFocusQuery, - results: answerInputResults, - answerMode: routeBeforeConfidenceGate.mode, - }); const gatedRoute = validatedRoutineExtractiveRecovery ? { route: routeBeforeConfidenceGate } : applyConfidenceGate(routeBeforeConfidenceGate, queryClass, initialRetrievalDiagnostics); diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 1caba60b0..55796d6fa 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1763,6 +1763,47 @@ describe("RAG structured-output fallback", () => { expect(generateStructuredTextResult).not.toHaveBeenCalled(); }); + it("keeps gate-passed routine document-content queries on model synthesis", async () => { + const answer = await answerFromTextSources( + "How is patient safety planning handled?", + [ + source({ + id: "safety-plan-requirements", + document_id: "safety-plan-doc", + title: "Patient Safety Planning Guideline", + file_name: "patient-safety-planning.pdf", + section_heading: "Safety planning for identified risks", + content: + "Patient safety planning must be developed collaboratively with the consumer and reviewed when clinical status changes.", + similarity: 0.9, + hybrid_score: 0.9, + text_rank: 0.4, + }), + ], + { + answer: + "Patient safety planning is handled collaboratively with the consumer and reviewed when clinical status changes.", + grounded: true, + confidence: "medium", + answerSections: [], + citations: [{ chunk_id: "safety-plan-requirements" }], + quoteCards: [], + conflictsOrGaps: [], + }, + ); + + expect(answer.retrievalDiagnostics).toMatchObject({ + gateStatus: "passed", + routeMode: "fast", + topScore: 0.9, + }); + expect(answer.routingMode).toBe("fast"); + expect(answer.routingReason).toBe("strong_routine_retrieval"); + expect(answer.modelUsed).not.toBeNull(); + expect(answer.openAIRequestIds).toEqual(["req_answer_from_text_sources"]); + expect(answer.grounded).toBe(true); + }); + it("does not gate-block a moderate score clustered across several distinct documents", async () => { // Regression: a topic with rich coverage (e.g. clozapine) returns many relevant // documents whose scores cluster tightly at a moderate value. A tiny spread there