From 74d2226a01bc23fa0934494838a66fc3eede3738 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 03:29:27 +0800 Subject: [PATCH 1/2] fix(rag): don't rescue bare cross-reference answers in source-backed recovery shouldPreserveSourceBackedGeneratedAnswer was preserving a bare "Refer to ... for further information" pointer to grounded=true on the cited sources' structured-chunk signals alone, shipping an off-topic redirect as a grounded clinical answer. Add a narrow isBareCrossReferenceAnswer guard (lead sentence must be BOTH a directive and a "for further information" redirect, evaluated on the same sanitized text the gate judged) so these degrade to source-only, without regressing the terse paraphrase rescues the gate legitimately exists for. Extend the already-merged discharge acceptSourceOnly eval treatment to the two duress cases, which surface the identical boilerplate; still guarded by expectedHit && citations so a real retrieval regression fails. Verified: typecheck/prettier/eslint clean; targeted vitest 78/78; live eval:retrieval:quality 36/36 and eval:quality --rag-only grounded-supported 0.9667 against the equivalent change. Co-Authored-By: Claude Opus 4.8 --- src/lib/rag-eval-cases.ts | 16 +- src/lib/rag-extractive-answer.ts | 29 +++- tests/rag-eval-cases.test.ts | 14 ++ ...ce-backed-recovery-cross-reference.test.ts | 138 ++++++++++++++++++ 4 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 tests/source-backed-recovery-cross-reference.test.ts diff --git a/src/lib/rag-eval-cases.ts b/src/lib/rag-eval-cases.ts index 31ac2da7a..88ae8224f 100644 --- a/src/lib/rag-eval-cases.ts +++ b/src/lib/rag-eval-cases.ts @@ -649,11 +649,16 @@ export const answerQualityEvalCases: AnswerQualityEvalCase[] = [ { ...commonQualityCase, id: "quality-duress-pathway", + // Source-only-acceptable sibling of the `duress-procedure` core case (see its comment): the + // corpus has no single authoritative duress-procedure-requirements source, so a grounded + // synthesis and a source-only refusal that surfaces the duress docs are both valid. + // mustContainAny is intentionally dropped — the source-only text is not assertable — while + // expectedFiles keeps the retrieval guard. question: "What is the duress procedure pathway?", expectedIntent: "pathway_referral", expectedQueryClass: "document_lookup", expectedFiles: ["MHSP.Duress.pdf"], - mustContainAny: ["duress", "procedure"], + acceptSourceOnly: true, }, { ...commonQualityCase, @@ -812,9 +817,18 @@ export const ragEvalCases: RagEvalCase[] = [ }, { id: "duress-procedure", + // Diffuse procedural question whose only extractive candidates are off-topic text or the same + // tangential RKPG cross-reference boilerplate ("Refer to the RKPG Guidelines … for further + // information …") that the discharge case exposes: the pipeline correctly returns a source-only + // answer citing the real duress SOPs rather than rescuing that boilerplate to a confident + // answer (the fragile source-backed recovery past missing_query_overlap is now guarded off — + // see isBareCrossReferenceAnswer). acceptSourceOnly accepts grounded OR source-only *while + // still requiring the duress docs to be cited*, so a real retrieval regression still fails. See + // discharge-documentation investigation 2026-07-13. question: "What does the duress procedure require?", category: "routine", supported: true, + acceptSourceOnly: true, expectedFiles: ["MHSP.Duress.pdf"], allowedRoutes: ["extractive", "fast"], minCitations: 2, diff --git a/src/lib/rag-extractive-answer.ts b/src/lib/rag-extractive-answer.ts index 46eb26cd0..378bd779a 100644 --- a/src/lib/rag-extractive-answer.ts +++ b/src/lib/rag-extractive-answer.ts @@ -1638,9 +1638,34 @@ function finalQualityFailure(answer: RagAnswer, query: string, queryClass: RagQu }; } +// A "bare cross-reference" answer redirects the reader to another named document for the real +// content — e.g. "Refer to the RKPG Guidelines to Writing for Clinical Policy for further +// information about Scope of Practice." It answers nothing itself, so it must never be rescued by +// the source-backed recovery gate on the strength of structured-chunk signals in the *cited* +// sources. The guard is deliberately narrow — it fires only when the lead sentence is BOTH a +// pointer to another document AND a "for further information"-style redirect — so it leaves the +// terse paraphrases the recovery gate legitimately exists for (e.g. "Depot antipsychotic follow-up +// is covered by the cited local pathway.") untouched, since those carry no redirect clause. +const crossReferenceDirectivePattern = + /\b(?:refer(?:red|s|ring)?\s+to|(?:please\s+)?see|consult|as\s+(?:per|outlined|described|detailed|set\s+out))\b/i; +const crossReferenceRedirectPattern = + /\bfor\s+(?:further|more|additional|detailed|complete|full)\s+(?:information|detail|details|guidance|advice|reading|instruction|instructions)\b/i; + +/** Is bare cross-reference answer. */ +export function isBareCrossReferenceAnswer(text: string) { + const lead = firstSentence(text).replace(/\*\*/g, ""); + if (!lead) return false; + return crossReferenceDirectivePattern.test(lead) && crossReferenceRedirectPattern.test(lead); +} + /** Should preserve source backed generated answer. */ -function shouldPreserveSourceBackedGeneratedAnswer(answer: RagAnswer, reason: string) { +function shouldPreserveSourceBackedGeneratedAnswer(answer: RagAnswer, reason: string, cleanedAnswer: string) { if (reason !== "missing_query_intent" && reason !== "missing_query_overlap") return false; + // Never rescue a bare cross-reference / "refer elsewhere for more information" pointer: it carries + // no responsive content, and (being here) already shares no query terms, so preserving it would + // ship an off-topic redirect as a grounded clinical answer. Evaluate the same sanitized text the + // quality gate judged, so a stripped leading noise fragment can't hide the redirect lead. + if (isBareCrossReferenceAnswer(cleanedAnswer)) return false; if (!answer.grounded || answer.confidence === "unsupported" || answer.citations.length === 0) return false; if (hasInvalidModelEvidenceIds(answer)) return false; @@ -1784,7 +1809,7 @@ function finalizeRagAnswerQualityCore( : generatedAnswerQualityFailureReason(answer, query, queryClass); if (qualityFailureReason) { - if (shouldPreserveSourceBackedGeneratedAnswer(answer, qualityFailureReason)) { + if (shouldPreserveSourceBackedGeneratedAnswer(answer, qualityFailureReason, cleanedAnswer)) { answer = { ...answer, confidence: answer.confidence === "low" ? "medium" : answer.confidence, diff --git a/tests/rag-eval-cases.test.ts b/tests/rag-eval-cases.test.ts index 62f1f95c8..708604570 100644 --- a/tests/rag-eval-cases.test.ts +++ b/tests/rag-eval-cases.test.ts @@ -235,6 +235,20 @@ describe("captured RAG eval cases", () => { expect(quality?.expectedFiles).toEqual(["MHSP.Discharge.pdf"]); }); + it("marks the diffuse duress cases as source-only-acceptable, still supported", () => { + // The duress-procedure query surfaces the same tangential RKPG cross-reference boilerplate as + // discharge, so its correct behaviour is a source-only answer citing the duress docs. + const core = ragEvalCases.find((item) => item.id === "duress-procedure"); + expect(core?.supported).toBe(true); + expect(core?.acceptSourceOnly).toBe(true); + expect(core?.expectedFiles).toEqual(["MHSP.Duress.pdf"]); + + const quality = answerQualityEvalCases.find((item) => item.id === "quality-duress-pathway"); + expect(quality?.supported).toBe(true); + expect(quality?.acceptSourceOnly).toBe(true); + expect(quality?.expectedFiles).toEqual(["MHSP.Duress.pdf"]); + }); + it("scores an acceptSourceOnly case as relevant for a clean source-only answer", () => { const testCase = answerQualityEvalCases.find((item) => item.id === "quality-discharge-documentation")!; const sourceOnly = { diff --git a/tests/source-backed-recovery-cross-reference.test.ts b/tests/source-backed-recovery-cross-reference.test.ts new file mode 100644 index 000000000..a6a183f1c --- /dev/null +++ b/tests/source-backed-recovery-cross-reference.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from "vitest"; + +import { + finalizeRagAnswerQuality, + generatedAnswerQualityFailureReason, + isBareCrossReferenceAnswer, + sourceBackedGenerationTimeoutAnswer, +} from "../src/lib/rag-extractive-answer"; +import type { Citation, RagAnswer, RagQueryClass, RetrievalSelectionSummary } from "../src/lib/types"; + +// The exact off-topic extract observed live for "What should discharge documentation include?" — +// a bare cross-reference that trips missing_query_overlap and was being wrongly rescued to grounded +// on the strength of the cited sources' structured-chunk signals. +const RKPG_CROSS_REFERENCE = + "Refer to the RKPG Guidelines to Writing for Clinical Policy for further information about Scope of Practice."; +// The legitimate terse paraphrase the recovery gate exists for (regression guard from +// rag-answer-fallback.test.ts): shares no literal query token yet genuinely answers the query. +const LAI_PARAPHRASE = "Depot antipsychotic follow-up is covered by the cited local pathway."; + +function citation(overrides: Partial = {}): Citation { + return { + chunk_id: "discharge-1", + document_id: "discharge-doc", + title: "Admission to Discharge for Mental Health Inpatients", + file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf", + page_number: 3, + chunk_index: 0, + ...overrides, + }; +} + +// A source selection strong enough that, absent the cross-reference guard, the recovery gate would +// preserve the answer: required signals satisfied, a specific source signal, and structured chunks. +function strongSourceSelection(): RagAnswer["smartApiPlan"] { + const sourceSelection = { + candidateCount: 6, + selectedCount: 4, + requiredSignalsSatisfied: true, + matchedSignals: ["document_title", "patient_education", "table"], + missingRequiredSignals: [], + rescueApplied: false, + topChunkTypes: { text: 2, table: 1, flowchart: 0, medication_chart: 0, patient_education: 4 }, + } satisfies RetrievalSelectionSummary; + // Only answerPlan.sourceSelection drives the recovery gate; the rest of the plan is irrelevant + // here, so cast past the unrelated required fields rather than fabricating them. + return { answerPlan: { sourceSelection } } as unknown as RagAnswer["smartApiPlan"]; +} + +function groundedExtractiveAnswer(text: string, overrides: Partial = {}): RagAnswer { + return { + answer: text, + grounded: true, + confidence: "low", + citations: [citation()], + sources: [], + routingMode: "extractive", + routingReason: "high_confidence_extractive_retrieval", + smartApiPlan: strongSourceSelection(), + ...overrides, + }; +} + +describe("isBareCrossReferenceAnswer", () => { + it("flags a bare 'refer to for further information' redirect", () => { + expect(isBareCrossReferenceAnswer(RKPG_CROSS_REFERENCE)).toBe(true); + }); + + it("flags redirect-first and 'please see … for more detail' phrasings", () => { + expect( + isBareCrossReferenceAnswer( + "For further information about discharge planning, see the National Mental Health policy.", + ), + ).toBe(true); + expect(isBareCrossReferenceAnswer("Please see the local operational policy for more detail.")).toBe(true); + }); + + it("sees through markdown bolding on the pointer", () => { + expect(isBareCrossReferenceAnswer(`**${RKPG_CROSS_REFERENCE}**`)).toBe(true); + }); + + it("does NOT flag a terse paraphrase that merely lacks literal query tokens", () => { + expect(isBareCrossReferenceAnswer(LAI_PARAPHRASE)).toBe(false); + }); + + it("does NOT flag the source-pointer generation-timeout fallback", () => { + expect( + isBareCrossReferenceAnswer(sourceBackedGenerationTimeoutAnswer("What is the clozapine ANC threshold?")), + ).toBe(false); + }); + + it("does NOT flag a real answer whose lead carries content and only a trailing sentence points onward", () => { + expect( + isBareCrossReferenceAnswer( + "Discharge documentation should include a mental state examination, risk assessment and follow-up plan. Refer to the NMHS policy for more detail.", + ), + ).toBe(false); + }); + + it("does NOT flag an ordinary clinical directive", () => { + expect(isBareCrossReferenceAnswer("Withhold clozapine and repeat the full blood count within 24 hours.")).toBe( + false, + ); + }); +}); + +describe("source-backed recovery gate — cross-reference guard", () => { + const DISCHARGE_QUERY = "What should discharge documentation include?"; + const LAI_QUERY = "What is the long acting injectable pathway?"; + const queryClass = "broad_summary" satisfies RagQueryClass; + + it("classifies the RKPG cross-reference as missing_query_overlap (the reachable-by-recovery reason)", () => { + expect( + generatedAnswerQualityFailureReason(groundedExtractiveAnswer(RKPG_CROSS_REFERENCE), DISCHARGE_QUERY, queryClass), + ).toBe("missing_query_overlap"); + }); + + it("does NOT rescue the off-topic cross-reference despite strong structured-chunk signals", () => { + const answer = finalizeRagAnswerQuality( + groundedExtractiveAnswer(RKPG_CROSS_REFERENCE), + DISCHARGE_QUERY, + queryClass, + ); + + expect(answer.grounded).toBe(false); + expect(answer.routingReason).toContain("final_quality_gate:missing_query_overlap"); + expect(answer.routingReason).not.toContain("source_backed_recovery"); + expect(answer.answer).not.toMatch(/RKPG|Refer to the/i); + }); + + it("still rescues a legitimate source-backed paraphrase with the same strong signals (regression guard)", () => { + const answer = finalizeRagAnswerQuality(groundedExtractiveAnswer(LAI_PARAPHRASE), LAI_QUERY, queryClass); + + expect(answer.grounded).toBe(true); + expect(answer.confidence).toBe("medium"); + expect(answer.routingReason).toContain("final_quality_gate_source_backed_recovery:missing_query_overlap"); + expect(answer.answer.replace(/\*\*/g, "")).toMatch(/Depot antipsychotic follow-up/i); + }); +}); From abb648e0bd64631db41412148d93eee39eb4f5d2 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 03:43:03 +0800 Subject: [PATCH 2/2] fix(rag): require a document-style object in the cross-reference guard Addresses the Codex P2 on PR #538: the directive + "for further information" redirect also matched passive clinical referral facts such as "Patients are referred to the community team for further information and support.", which would wrongly turn a valid cited answer into an evidence gap. Require the lead sentence to also name a document-style object (guideline/policy/procedure/ appendix/...), so referrals to a team, person, or service survive the guard. Adds regression tests for the passive-referral and see-your-GP cases. Co-Authored-By: Claude Opus 4.8 --- src/lib/rag-extractive-answer.ts | 18 +++++++++++++----- ...rce-backed-recovery-cross-reference.test.ts | 9 +++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/lib/rag-extractive-answer.ts b/src/lib/rag-extractive-answer.ts index 378bd779a..39f107308 100644 --- a/src/lib/rag-extractive-answer.ts +++ b/src/lib/rag-extractive-answer.ts @@ -1642,20 +1642,28 @@ function finalQualityFailure(answer: RagAnswer, query: string, queryClass: RagQu // content — e.g. "Refer to the RKPG Guidelines to Writing for Clinical Policy for further // information about Scope of Practice." It answers nothing itself, so it must never be rescued by // the source-backed recovery gate on the strength of structured-chunk signals in the *cited* -// sources. The guard is deliberately narrow — it fires only when the lead sentence is BOTH a -// pointer to another document AND a "for further information"-style redirect — so it leaves the -// terse paraphrases the recovery gate legitimately exists for (e.g. "Depot antipsychotic follow-up -// is covered by the cited local pathway.") untouched, since those carry no redirect clause. +// sources. The guard is deliberately narrow — it fires only when the lead sentence is a pointer +// (directive) AND a "for further information"-style redirect AND names a document-style object — so +// it leaves untouched both the terse paraphrases the recovery gate legitimately exists for (e.g. +// "Depot antipsychotic follow-up is covered by the cited local pathway.", no redirect clause) and +// passive clinical referral facts (e.g. "Patients are referred to the community team for further +// information and support.", which point at a service, not a document). const crossReferenceDirectivePattern = /\b(?:refer(?:red|s|ring)?\s+to|(?:please\s+)?see|consult|as\s+(?:per|outlined|described|detailed|set\s+out))\b/i; const crossReferenceRedirectPattern = /\bfor\s+(?:further|more|additional|detailed|complete|full)\s+(?:information|detail|details|guidance|advice|reading|instruction|instructions)\b/i; +const crossReferenceDocumentObjectPattern = + /\b(?:guidance|guidelines?|policy|policies|procedures?|protocols?|appendix|appendices|manuals?|documents?|documentation|frameworks?|standards?|sops?|handbooks?|factsheets?|leaflets?|booklets?|templates?|checklists?|forms?|sections?|chapters?)\b/i; /** Is bare cross-reference answer. */ export function isBareCrossReferenceAnswer(text: string) { const lead = firstSentence(text).replace(/\*\*/g, ""); if (!lead) return false; - return crossReferenceDirectivePattern.test(lead) && crossReferenceRedirectPattern.test(lead); + return ( + crossReferenceDirectivePattern.test(lead) && + crossReferenceRedirectPattern.test(lead) && + crossReferenceDocumentObjectPattern.test(lead) + ); } /** Should preserve source backed generated answer. */ diff --git a/tests/source-backed-recovery-cross-reference.test.ts b/tests/source-backed-recovery-cross-reference.test.ts index a6a183f1c..e66f5a0f2 100644 --- a/tests/source-backed-recovery-cross-reference.test.ts +++ b/tests/source-backed-recovery-cross-reference.test.ts @@ -101,6 +101,15 @@ describe("isBareCrossReferenceAnswer", () => { false, ); }); + + it("does NOT flag passive clinical referral facts that point at a service, not a document", () => { + // A directive + redirect that targets a team/person is a valid cited answer, not a document + // cross-reference — so it must survive the guard (Codex review PR #538). + expect( + isBareCrossReferenceAnswer("Patients are referred to the community team for further information and support."), + ).toBe(false); + expect(isBareCrossReferenceAnswer("The patient should see their GP for further information.")).toBe(false); + }); }); describe("source-backed recovery gate — cross-reference guard", () => {