diff --git a/src/lib/answer-verification.ts b/src/lib/answer-verification.ts index 385d06325..e17c54da3 100644 --- a/src/lib/answer-verification.ts +++ b/src/lib/answer-verification.ts @@ -423,12 +423,20 @@ export function verifyAnswerNumbers( return { answerTokens, unverifiedTokens, hasUnverifiedNumbers: unverifiedTokens.length > 0 }; } - // Top-level citations do not identify which sentence each citation supports. - // Require each semantic clinical value to be present in every cited chunk, - // rather than allowing an unrelated citation to verify it by union membership. - const sourceAtomSets = citedResults.map((result) => sourceClinicalValueAtomSet([result])); + // A clinical value is verified when it appears verbatim in at least one chunk the + // answer actually cites (union over cited chunks). Multi-source answers legitimately + // draw different figures from different cited chunks — e.g. a dose-review interval + // from one chunk and a maximum daily dose from another — so requiring every atom to + // appear in EVERY cited chunk (intersection) flagged nearly all stitched multi-chunk + // answers as unverified and demoted correct grounded answers to "unsupported". + // Fabricated or mis-transcribed figures still fail: they appear in NO cited chunk. + // Cross-entity misattribution (drug A's sentence carrying drug B's cited dose) is + // out of reach for bare atom membership either way; that risk is owned by the + // claim-support verification layer (rag-claim-support.ts), and sections with their + // own citation_chunk_ids are still verified only against their scoped chunks. + const sourceAtoms = sourceClinicalValueAtomSet(citedResults); const unverifiedTokens = answerAtoms - .filter((atom) => !sourceAtomSets.every((atoms) => atoms.has(clinicalValueAtomKey(atom)))) + .filter((atom) => !sourceAtoms.has(clinicalValueAtomKey(atom))) .map(clinicalValueAtomDisplay); return { diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 9db00d193..b3d19b5a6 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -827,7 +827,18 @@ function buildRetrievalDiagnostics(args: { answerMode: "unsupported" | "extractive" | "fast" | "strong"; fallbackReason?: string | null; }) { - const resultScores = args.results.map(scoreValue); + // Lexical-only retrieval rows carry a truthful score contract since migration + // 20260713062107_restore_text_fallback_lexical_score: similarity is 0 (no vector + // ran) and hybrid_score is deliberately capped at 0.48 so a keyword hit can never + // masquerade as a moderate/strong cosine match downstream. The honest lexical + // signal lives in lexical_score (0.4..0.99). This gate must therefore read + // max(scoreValue, lexical_score) — reading the capped hybrid_score alone makes + // topScore < 0.5 unconditional for every text-fast-path answer, refusing + // well-supported documentation lookups whose expected document is at rank 1. + // Ranking/selection ordering still uses scoreValue and is unchanged. + const resultScores = args.results.map((result) => + Math.max(scoreValue(result), Math.min(1, result.lexical_score ?? 0)), + ); const sortedScores = [...resultScores].sort((a, b) => b - a); const topScore = sortedScores[0] ?? 0; const secondScore = sortedScores[1] ?? 0; diff --git a/tests/answer-verification.test.ts b/tests/answer-verification.test.ts index dc68ba4d0..e364337be 100644 --- a/tests/answer-verification.test.ts +++ b/tests/answer-verification.test.ts @@ -245,15 +245,37 @@ describe("answer-verification (GEN-C2 / GEN-H2)", () => { expect(verification.unverifiedTokens).toContain("12.5mg"); }); - it("does not let an unrelated answer citation verify a clinical number", () => { + // Verification is union-over-cited-chunks: a figure passes when at least one chunk + // the answer cites contains it verbatim. The earlier intersection semantics (require + // the figure in EVERY cited chunk) flagged nearly all legitimate multi-source answers + // — a stitched answer citing a dose table and a review-interval chunk can never have + // each figure in both — and demoted correct grounded answers to "unsupported" + // (4 golden-eval regressions, canary #459). Cross-entity misattribution (drug A's + // sentence carrying drug B's cited dose) is not detectable by bare atom membership + // under either semantics; that risk belongs to the claim-support layer + // (rag-claim-support.ts) and to per-section citation scoping. + it("verifies figures drawn from different cited chunks (union over citations)", () => { + const doseChunk = source({ id: "chunk-a", content: "Olanzapine maximum 20 mg in 24 hours." }); + const reviewChunk = source({ id: "chunk-b", content: "Review all oral doses after 60 minutes." }); + const verification = verifyAnswerNumbers( + "Olanzapine maximum 20 mg in 24 hours; review oral doses after 60 minutes.", + [{ chunk_id: "chunk-a" }, { chunk_id: "chunk-b" }], + [doseChunk, reviewChunk], + ); + expect(verification.hasUnverifiedNumbers).toBe(false); + expect(verification.unverifiedTokens).toEqual([]); + }); + + it("still flags a figure that appears in no cited chunk even with multiple citations", () => { const drugA = source({ id: "chunk-a", content: "Drug A requires renal monitoring." }); const drugB = source({ id: "chunk-b", content: "Drug B is given at 30 mg daily." }); const verification = verifyAnswerNumbers( - "Give drug A at 30 mg daily.", + "Give drug A at 45 mg daily.", [{ chunk_id: "chunk-a" }, { chunk_id: "chunk-b" }], [drugA, drugB], ); - expect(verification.unverifiedTokens).toContain("30mg"); + expect(verification.hasUnverifiedNumbers).toBe(true); + expect(verification.unverifiedTokens).toContain("45mg"); }); // B1: substring matching previously let a wrong dose verify against a longer diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 80c395b2a..8b3af26a4 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1712,6 +1712,127 @@ describe("RAG structured-output fallback", () => { expect(generateStructuredTextResult).toHaveBeenCalled(); }); + it("does not gate-block strong lexical-only evidence under the truthful score contract", async () => { + // Regression (canary #459 family A): migration 20260713062107 made lexical-only + // retrieval honest — similarity 0, hybrid_score hard-capped at 0.48, with the real + // signal in lexical_score (0.4..0.99). Reading hybrid_score alone made + // topScore < 0.5 unconditional for every text-fast-path answer, so well-supported + // documentation lookups ("What does the metabolic screening document require?", + // expected document at rank 1) were refused as confidence_gate_blocked. The gate + // must read the lexical evidence. + vi.stubEnv("OPENAI_API_KEY", "test-key"); + vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0"); + vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0"); + + const sources = [1, 2, 3].map((n) => + source({ + id: `lexical-${n}`, + document_id: n <= 2 ? "metabolic-doc" : `doc-${n}`, + title: "Metabolic Screening", + file_name: "metabolic-screening.pdf", + content: "Metabolic screening requires baseline observations and scheduled physical health monitoring.", + similarity: 0, + hybrid_score: 0.48, + text_rank: 1.2, + lexical_score: 0.99, + }), + ); + const rpc = vi.fn(async (name: string) => { + if (retrievalRpcBaseName(name) === "match_document_chunks_text") return { data: sources, error: null }; + if (retrievalRpcBaseName(name) === "get_related_document_metadata") return { data: [], error: null }; + return { data: [], error: null }; + }); + const generateStructuredTextResult = vi.fn(); + const embedTextWithTelemetry = vi.fn(async () => ({ embedding: [0.1, 0.2, 0.3], cacheHit: false })); + + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ + rpc, + from: vi.fn(() => new EmptyQuery()), + }), + })); + vi.doMock("@/lib/openai", () => ({ + embedTextWithTelemetry, + generateStructuredTextResult, + })); + + const { answerQuestionWithScope } = await import("../src/lib/rag"); + + const answer = await answerQuestionWithScope({ + query: "What does the metabolic screening document require?", + ownerId: undefined, + logQuery: false, + skipCache: true, + }); + + expect(answer.retrievalDiagnostics).toMatchObject({ gateStatus: "passed" }); + expect(answer.routingReason).not.toContain("confidence_gate_blocked"); + }); + + it("still gate-blocks weak lexical-only evidence", async () => { + // Control for the lexical-aware gate: a marginal keyword hit (lexical_score at its + // 0.4 floor) must stay below the 0.5 evidence bar and refuse, preserving + // unsupported_correct behaviour for junk/near-miss lexical rows. + vi.stubEnv("OPENAI_API_KEY", "test-key"); + vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0"); + vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0"); + + const sources = [ + source({ + id: "weak-lexical-1", + document_id: "doc-1", + title: "Unrelated Process", + file_name: "unrelated.pdf", + content: "A passing mention of screening in an unrelated administrative document.", + similarity: 0, + hybrid_score: 0.19, + text_rank: 0.02, + lexical_score: 0.41, + }), + source({ + id: "weak-lexical-2", + document_id: "doc-2", + title: "General Overview", + file_name: "overview.pdf", + content: "General notes without specific screening guidance.", + similarity: 0, + hybrid_score: 0.18, + text_rank: 0.01, + lexical_score: 0.4, + }), + ]; + const rpc = vi.fn(async (name: string) => { + if (retrievalRpcBaseName(name) === "match_document_chunks_text") return { data: sources, error: null }; + if (retrievalRpcBaseName(name) === "get_related_document_metadata") return { data: [], error: null }; + return { data: [], error: null }; + }); + const generateStructuredTextResult = vi.fn(); + const embedTextWithTelemetry = vi.fn(async () => ({ embedding: [0.1, 0.2, 0.3], cacheHit: false })); + + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ + rpc, + from: vi.fn(() => new EmptyQuery()), + }), + })); + vi.doMock("@/lib/openai", () => ({ + embedTextWithTelemetry, + generateStructuredTextResult, + })); + + const { answerQuestionWithScope } = await import("../src/lib/rag"); + + const answer = await answerQuestionWithScope({ + query: "How is discharge planning handled?", + ownerId: undefined, + logQuery: false, + skipCache: true, + }); + + expect(answer.retrievalDiagnostics).toMatchObject({ gateStatus: "blocked" }); + expect(answer.routingMode).toBe("unsupported"); + }); + it("returns document names for source-support questions instead of clinical advice", async () => { vi.stubEnv("OPENAI_API_KEY", "test-key"); vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0");