From bb4da7d9fe3665922475800e963f6036ab04ef1a Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 02:18:48 +0800 Subject: [PATCH] fix(rag): exempt deterministic document-list/table answers from prose sanitizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document-support-list and table/visual source-reference answers are built deterministically from source metadata by buildDocumentSupportListAnswer / buildTableOrVisualSourceLookupAnswer — not model prose. But finalizeRagAnswerQualityCore ran them through sanitizeAnswerText, a clinical- *prose* extractor, which stripped their document names: facility-code suffixes like "(NOCC)(AKG)" and semicolon-separated entries read as non-prose and were dropped, turning a valid answer ("I found 5 indexed documents…: National Outcomes and Casemix Collection (NOCC)(AKG); …") into garble the quality gate then correctly refused. So `direct-document-lookup-nocc` failed with 5 citations but grounded=false. Add a `preformatted` flag on RagAnswer, set by those two builders and propagated by buildExtractiveAnswer; finalizeRagAnswerQualityCore returns preformatted grounded answers untouched (they are well-formed by construction and carry no free-text clinical claims, so the prose sanitizer/gate is neither needed nor safe for them). The shared sanitizer and gate are unchanged for model prose. Scope: this fixes the document-list case only. `agitation-arousal-table-lookup` is class table_threshold, which routes to fact synthesis (not a structured builder) and produces genuinely incoherent table-cell output the gate correctly refuses — a separate synthesis-quality issue, intentionally out of scope. Validated (full golden RAG eval, live): NOCC now grounded (5 citations), grounded_supported_rate 0.933 → 0.967. No regression from this change — the one unsupported flip in the run (unsupported-close-title-noise) is pre-existing LLM variance: it is preformatted:None (untouched by this fix) and refuses 3/3 on re-run. New unit test asserts facility-code suffixes survive; rag-answer-fallback 26/26; typecheck/lint/prettier clean. Co-Authored-By: Claude Fable 5 --- src/lib/rag.ts | 11 ++++ src/lib/types.ts | 5 ++ tests/rag-answer-fallback.test.ts | 88 +++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+) diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 55dcf66dd..b86f09a98 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -4668,6 +4668,7 @@ function buildTableOrVisualSourceLookupAnswer(args: { query: string; results: Se return { answer, citationChunkIds: [source.id], + preformatted: true, answerSections: [ { heading: "Source match", @@ -4700,6 +4701,7 @@ function buildDocumentSupportListAnswer(args: { query: string; results: SearchRe : `I found ${names.length} indexed documents that support this query: ${names.slice(0, -1).join("; ")}; and ${names.at(-1)}.`; return { answer, + preformatted: true, citationChunkIds: Array.from( new Set( documents.flatMap((document) => @@ -4812,6 +4814,7 @@ function buildExtractiveAnswer(args: { sources: args.results, modelUsed: null, routingMode: "extractive", + preformatted: hasExtractedAnswer && Boolean((naturalAnswer as { preformatted?: boolean }).preformatted), routingReason: args.routeReason, queryClass: args.queryClass, latencyTimings: args.timings, @@ -5237,6 +5240,14 @@ function finalizeRagAnswerQuality(answer: RagAnswer, query: string, queryClass: } function finalizeRagAnswerQualityCore(answer: RagAnswer, query: string, queryClass: RagQueryClass): RagAnswer { + // Deterministic, template-built answers (document-support lists, table/visual source + // references) are well-formed by construction and carry no free-text clinical claims. + // The clinical-prose sanitizer/quality gate below is designed for model prose and would + // strip their document names (facility codes like "(NOCC)(AKG)" read as non-prose), + // turning a valid answer into garble that then fails the gate. Return them untouched. + if (answer.preformatted && answer.grounded) { + return answer; + } const cleanedAnswer = sanitizeAnswerText(answer.answer); const gapLikeAnswer = /could not find enough clean|no relevant clinical source|no current source|cannot provide a clinical answer|cannot provide a source-backed clinical answer|nearby indexed passages|not strong enough to support a reliable answer|no specific\b.*\bcan be confirmed|do not contain indexed guidance|do not contain (?:specific\s+)?information|do not provide specific|no\b.*\bguidance\b.*\bincluded|defer to other sources/i.test( diff --git a/src/lib/types.ts b/src/lib/types.ts index 6370ac5af..3e481ac6e 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -860,6 +860,11 @@ export type RagAnswer = { queryClass?: RagQueryClass; queryAnalysis?: ClinicalQueryAnalysis; responseMode?: AnswerResponseMode; + // True for deterministic, template-built answers (document-support lists, table/visual + // source references) assembled from source metadata rather than model prose. These are + // well-formed by construction, so the clinical-prose sanitizer/quality gate — which would + // strip their document names (facility codes read as non-prose) — must be skipped. + preformatted?: boolean; latencyTimings?: { search_cache_hit?: boolean; shared_cache_hit?: boolean; diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 45a4c7e09..7a62c6e61 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1588,6 +1588,94 @@ describe("RAG structured-output fallback", () => { expect(answer.answer).not.toMatch(/level checks|renal review|baseline tests/i); }); + it("preserves parenthetical facility codes in the document-support list answer", async () => { + // Regression: the document-list answer is deterministic and pre-formatted, but the + // clinical-prose sanitizer used to strip facility-code suffixes like "(NOCC) (AKG)" + // (read as non-prose), mangling a valid answer into garble the quality gate then + // refused. The `preformatted` flag now exempts it from that sanitizer. + vi.stubEnv("OPENAI_API_KEY", "test-key"); + vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0"); + vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0"); + + const noccDocument = { + id: "nocc-doc", + title: "National Outcomes and Casemix Collection (NOCC) (AKG)", + file_name: "NOCC.pdf", + metadata: { + source_title: "National Outcomes and Casemix Collection", + publisher: "Local service", + jurisdiction: "Australia/WA", + document_status: "current", + clinical_validation_status: "approved", + extraction_quality: "good", + }, + text_rank: 0.31, + }; + const noccChunk = { + id: "nocc-doc-chunk-1", + document_id: "nocc-doc", + page_number: 1, + chunk_index: 0, + section_heading: "Outcome measures", + section_path: ["Outcome measures"], + content: "National Outcomes and Casemix Collection guidance for outcome measures completion.", + retrieval_synopsis: "NOCC source summary.", + image_ids: [], + text_rank: 0.42, + }; + const rpc = vi.fn(async (name: string) => { + if (name === "match_documents_for_query") return { data: [noccDocument], error: null }; + if (name === "match_document_lookup_chunks_text") return { data: [noccChunk], error: null }; + if (name === "match_document_chunks_hybrid") { + return { + data: [ + source({ + id: "nocc-doc-chunk-1", + document_id: "nocc-doc", + title: "National Outcomes and Casemix Collection (NOCC) (AKG)", + file_name: "NOCC.pdf", + section_heading: "Outcome measures", + content: "National Outcomes and Casemix Collection guidance for outcome measures completion.", + similarity: 0.91, + hybrid_score: 0.93, + text_rank: 0.42, + }), + ], + error: null, + }; + } + if (name === "get_related_document_metadata") return { data: [], error: null }; + return { data: [], error: null }; + }); + + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ + rpc, + from: vi.fn(() => new EmptyQuery()), + }), + })); + vi.doMock("@/lib/openai", () => ({ + embedTextWithTelemetry: vi.fn(async () => ({ embedding: [0.1, 0.2, 0.3], cacheHit: false })), + generateStructuredTextResult: vi.fn(), + })); + + const { answerQuestionWithScope } = await import("../src/lib/rag"); + + const answer = await answerQuestionWithScope({ + query: "What documents support outcome measures completion?", + ownerId: undefined, + logQuery: false, + skipCache: true, + }); + + expect(answer.routingMode).toBe("extractive"); + expect(answer.grounded).toBe(true); + expect(answer.preformatted).toBe(true); + // The facility-code suffix survives intact (previously mangled to a dangling "("). + expect(answer.answer).toContain("(NOCC) (AKG)"); + expect(answer.answer).not.toMatch(/\(\s*(?:;|$| Outcome)/); + }); + it("does not promote lithium dosage headings or continuation fragments as the primary answer", async () => { vi.stubEnv("OPENAI_API_KEY", "test-key"); vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0");