diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index 867aea478..4007a3a7a 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -68,6 +68,7 @@ import { import { GuideDialog, GuideTrigger, UtilityDrawer } from "@/components/clinical-dashboard/dashboard-shell"; import { sanitizeAnswerDisplayText, sanitizeDisplayText } from "@/components/clinical-dashboard/display-text"; import { + isPreformattedGroundedAnswer, NaturalLanguageAnswer, ScopeAndGovernanceNotice, UserQuestionBubble, @@ -498,9 +499,13 @@ function answerTimedOutError() { } /** - * Read-only surface for a previous turn in the answer thread. Renders the - * question bubble and the natural-language answer with its source capsule; - * evidence drawers, clinical notes, and feedback stay on the latest turn only. + * Renders a collapsible, read-only view of a previous answer-thread turn with its question, answer, sources, and source-review notice. + * + * @param turn - The previous question and answer turn to display + * @param copied - Whether the turn's answer has been copied + * @param collapsed - Whether the answer content is collapsed + * @param onToggleCollapsed - Called when the answer visibility is toggled + * @param onCopy - Called with the answer text when copying is requested */ function PriorAnswerTurnSurface({ turn, @@ -519,7 +524,11 @@ function PriorAnswerTurnSurface({ () => buildAnswerRenderModel(turn.answer, { sources: turn.sources }), [turn.answer, turn.sources], ); - const safeText = useMemo(() => sanitizeAnswerDisplayText(turn.answer.answer), [turn.answer.answer]); + const turnPreformatted = isPreformattedGroundedAnswer(turn.answer); + const safeText = useMemo( + () => sanitizeAnswerDisplayText(turn.answer.answer, { preformatted: turnPreformatted }), + [turn.answer.answer, turnPreformatted], + ); const sourceCount = renderModel.primarySources.length || turn.sources.length || @@ -558,7 +567,8 @@ function PriorAnswerTurnSurface({ ) : ( <> new Map(sources.map((source) => [source.id, source])), [sources]); - const safeAnswerText = useMemo(() => sanitizeAnswerDisplayText(answer?.answer ?? ""), [answer?.answer]); + const answerPreformatted = isPreformattedGroundedAnswer(answer); + const safeAnswerText = useMemo( + () => sanitizeAnswerDisplayText(answer?.answer ?? "", { preformatted: answerPreformatted }), + [answer?.answer, answerPreformatted], + ); const answerFollowUpSuggestions = useMemo(() => { if (!answer || !latestAnswerQuery) return []; const priorQueries = [...priorAnswerTurns.map((turn) => turn.query), latestAnswerQuery]; @@ -2859,7 +2881,11 @@ export function ClinicalDashboard({ return (answer?.answerSections ?? []) .map((section) => { const heading = sanitizeDisplayText(section.heading, { minLength: 1, minTokens: 1 }); - const body = sanitizeAnswerDisplayText(section.body, { minLength: 8, minTokens: 2 }); + const body = sanitizeAnswerDisplayText(section.body, { + minLength: 8, + minTokens: 2, + preformatted: answerPreformatted, + }); if (!heading || !body) return null; const citationSources: SearchResult[] = []; @@ -2880,7 +2906,7 @@ export function ClinicalDashboard({ }; }) .filter((section): section is AnswerSection & { citationSources: SearchResult[] } => section !== null); - }, [answer?.answerSections, sourceLookup]); + }, [answer?.answerSections, answerPreformatted, sourceLookup]); const answerEvidenceMapRows = useMemo(() => { if (!answerRenderModel?.allowedBlocks.includes("evidenceMap")) return []; return evidenceMapRowsFromRenderModel(answerRenderModel).slice(0, answerRenderModel.trust === "high" ? 8 : 6); @@ -3613,7 +3639,6 @@ export function ClinicalDashboard({ can render it. + preserveBold?: boolean; +}; + +/** + * Reports whether an answer is a server-`preformatted`, grounded answer whose + * display text should bypass prose sanitization and be shown as-is. + * + * @param answer - The answer to check, or `null`/`undefined` when unavailable + * @returns `true` when the answer is preformatted and grounded + */ +export function isPreformattedGroundedAnswer(answer: Pick | null | undefined) { + return Boolean(answer?.preformatted && answer?.grounded); +} + +// Fragments carrying a safety-critical signal must never be dropped by the +// compact 3-fragment / 85-word cap — a withhold/threshold/escalation caveat +// hidden from the primary prose is a clinical-safety regression. +// Covers the common withhold / withdrawal / contraindication / negation / +// escalation directives so a short safety caveat is never dropped from the +// compact primary answer. Kept deliberately broad (matching a non-safety +// fragment only preserves it verbatim — the safe direction). +const primaryAnswerSafetySignalPattern = + /\b(?:withhold|withheld|stop|cease|discontinue\w*|suspend\w*|hold|held|threshold|escalat\w*|urgent|immediately|never|avoid|contraindicat\w*|toxic|red\s*zone|amber|(?:do|must|should|will)\s+not|not\s+recommended)\b/i; + +// Test against a de-bolded copy so server bold markers inside a phrase +// ("do **not** administer", "red **zone**") on the preserveBold path can never +// defeat the safety match and let a caveat be dropped by the compact cap. +function isPrimaryAnswerSafetyFragment(fragment: string) { + return primaryAnswerSafetySignalPattern.test(fragment.replace(/\*\*/g, "")); +} + +// Shared tail of the sanitize path: run the display sanitizer, then strip the +// synthetic-demo notice both plainAnswerText and primaryAnswerDisplayText need +// removed before the text reaches the screen. +function sanitizeAndStripSyntheticNotice(value: string, options: AnswerDisplayTextOptions) { + return sanitizeAnswerDisplayText(value, { + minLength: 8, + minTokens: 2, + preformatted: options.preformatted, + preserveBold: options.preserveBold, + }) .replace(/(?:\s*\n\s*)?Synthetic demo only:.*$/i, "") .trim(); } -function primaryAnswerDisplayText(value: string) { - const cleaned = plainAnswerText(value); +/** + * Produces sanitized, display-ready text for an answer. + * + * @param value - The answer text to sanitize + * @param options - Controls preformatted handling and bold-text preservation + * @returns The sanitized answer text + */ +export function plainAnswerText(value: string, options: AnswerDisplayTextOptions = {}) { + // clinicalProseUsefulness runs the source-noise stripper, so preformatted + // answers bypass it and go straight to the lossless display path. + const base = options.preformatted ? value : clinicalProseUsefulness(value).text || value; + return sanitizeAndStripSyntheticNotice(base, options); +} + +/** + * Selects and compacts the primary answer text while preserving safety-critical guidance. + * + * @param value - The answer text to prepare for display + * @param options - Formatting options, including preformatted mode + * @returns The display-ready answer text + */ +export function primaryAnswerDisplayText(value: string, options: AnswerDisplayTextOptions = {}) { + // Deterministic preformatted answers are already concise and display-ready; + // the fragment-level usefulness pass below would re-strip the very names/codes + // the preformatted path just preserved, so return them as-is. + if (options.preformatted) return plainAnswerText(value, options); + // Skip whole-text clinicalProseUsefulness: its 3-token floor drops short + // safety sentences ("Stop lithium.") before the fragment-level safety + // bypass below can rescue them. + const cleaned = sanitizeAndStripSyntheticNotice(value, { preformatted: false, preserveBold: options.preserveBold }); const fragments = cleaned .split(/\r?\n+/) .flatMap((line: string) => @@ -242,20 +321,44 @@ function primaryAnswerDisplayText(value: string) { ) .trim(), ) - .map((fragment: string) => clinicalProseUsefulness(fragment).text || fragment) + // Safety-bearing fragments pass through untouched and are never dropped by + // the usefulness/length gate — a short caveat like "Contraindicated in + // pregnancy" (under the 8-word floor) must still reach the display. + .map((fragment: string) => + isPrimaryAnswerSafetyFragment(fragment) ? fragment : clinicalProseUsefulness(fragment).text || fragment, + ) .filter((fragment: string) => { if (!fragment) return false; + if (isPrimaryAnswerSafetyFragment(fragment)) return true; const useful = clinicalProseUsefulness(fragment); return useful.useful || fragment.split(/\s+/).length >= 8; }); const uniqueFragments = Array.from(new Set(fragments)); - const selected = uniqueFragments.slice(0, 3).join(" "); - const words = selected.split(/\s+/).filter(Boolean); - if (words.length <= 85) return selected || cleaned; - return `${words - .slice(0, 85) - .join(" ") - .replace(/[;,:-]\s*$/, "")}...`; + const selected: string[] = []; + let nonSafetyKept = 0; + let wordBudget = 85; + for (const fragment of uniqueFragments) { + if (isPrimaryAnswerSafetyFragment(fragment)) { + selected.push(fragment); + continue; + } + if (nonSafetyKept >= 3 || wordBudget <= 0) continue; + nonSafetyKept += 1; + const words = fragment.split(/\s+/).filter(Boolean); + if (words.length <= wordBudget) { + selected.push(fragment); + wordBudget -= words.length; + } else { + selected.push( + `${words + .slice(0, wordBudget) + .join(" ") + .replace(/[;,:-]\s*$/, "")}...`, + ); + wordBudget = 0; + } + } + return selected.join(" ") || cleaned; } // One compact "Sources" pill in every state: the amber Source-only pill and the @@ -537,8 +640,23 @@ function SourcePreviewContent({ ); } +/** + * Displays a sanitized clinical answer with source status, source previews, and copy actions. + * + * @param text - The raw answer text to display. + * @param preformatted - Whether to preserve the supplied formatting during display processing. + * @param sourceCount - The number of direct sources associated with the answer. + * @param sourceOnly - Whether to show a notice that the answer was assembled solely from source passages. + * @param bestSource - The highest-priority source recommendation, when available. + * @param sources - Search results used to build the source preview. + * @param sourceLinks - Source links and snippets associated with the answer. + * @param copied - Whether the answer has been copied. + * @param onCopy - Callback invoked to copy the answer with source status. + * @returns The rendered answer section, or `null` when the answer has no displayable text. + */ export function NaturalLanguageAnswer({ text, + preformatted = false, sourceCount, sourceOnly, bestSource, @@ -547,7 +665,10 @@ export function NaturalLanguageAnswer({ copied, onCopy, }: { + // Raw answer text (server bold intact); this component owns display + // sanitization so can render the high-yield emphasis. text: string; + preformatted?: boolean; sourceCount: number; sourceOnly: boolean; bestSource: BestSourceRecommendation | null; @@ -567,7 +688,7 @@ export function NaturalLanguageAnswer({ if (copySourceQuoteTimerRef.current !== null) window.clearTimeout(copySourceQuoteTimerRef.current); }; }, []); - const cleaned = primaryAnswerDisplayText(text); + const cleaned = primaryAnswerDisplayText(text, { preformatted, preserveBold: true }); if (!cleaned) return null; const capsuleDisplay = sourceCapsuleDisplay({ sourceCount }); const previewSources = capsulePreviewSources(bestSource, sources, sourceLinks); diff --git a/src/components/clinical-dashboard/answer-result-surface.tsx b/src/components/clinical-dashboard/answer-result-surface.tsx index 735d53069..752886401 100644 --- a/src/components/clinical-dashboard/answer-result-surface.tsx +++ b/src/components/clinical-dashboard/answer-result-surface.tsx @@ -1,4 +1,4 @@ -"use client"; +"use client"; import Link from "next/link"; import { memo, type RefObject, useCallback, useEffect, useRef, useState } from "react"; @@ -7,7 +7,11 @@ import { ClipboardCheck, ExternalLink, Layers, ShieldAlert } from "lucide-react" import { type AnswerFeedbackType } from "@/lib/answer-feedback"; import { AnswerFollowUpSuggestions } from "@/components/clinical-dashboard/answer-follow-up-suggestions"; import { CrossModeLinksSection } from "@/components/clinical-dashboard/cross-mode-links"; -import { NaturalLanguageAnswer, UserQuestionBubble } from "@/components/clinical-dashboard/answer-content"; +import { + isPreformattedGroundedAnswer, + NaturalLanguageAnswer, + UserQuestionBubble, +} from "@/components/clinical-dashboard/answer-content"; import { AnswerSupportSummaryCard, answerHasCentralTable, @@ -38,10 +42,14 @@ import type { } from "@/lib/types"; import { type AnswerEvidenceMapRow, type AnswerViewMode } from "@/lib/ward-output"; +/** + * Renders a staged answer with inline content and optional clinical notes, evidence, safety findings, and follow-up interfaces. + * + * @returns The staged answer surface. + */ function StagedAnswerResultSurfaceImpl({ answer, query, - safeAnswerText, bestSource, sourceGovernanceWarnings, sourceSummary, @@ -68,7 +76,6 @@ function StagedAnswerResultSurfaceImpl({ }: { answer: RagAnswer; query: string; - safeAnswerText: string; bestSource: BestSourceRecommendation | null; sourceGovernanceWarnings: SourceGovernanceWarning[]; sourceSummary?: EvidenceSummary; @@ -211,7 +218,8 @@ function StagedAnswerResultSurfaceImpl({ >
can render it. + preserveBold?: boolean; }; +/** + * Normalizes display text by trimming surrounding whitespace and collapsing internal whitespace. + * + * @param value - The display text to normalize + * @returns The normalized display text + */ export function normalizeDisplayText(value: string) { return value.trim().replace(/\s+/g, " "); } @@ -182,6 +195,12 @@ export function compactSourceSnippet(value: string, options: CompactSourceSnippe return text; } +/** + * Compacts table fact fields into a deduplicated display representation. + * + * @param fact - The table fact whose fields should be compacted + * @returns The fact identifier and cleaned fields, or `null` when no displayable fields remain + */ export function compactTableFact(fact: NonNullable[number]) { const fields = [ { label: "Table", value: fact.table_title }, @@ -201,8 +220,22 @@ export function compactTableFact(fact: NonNullable[ return cleanedFields.length ? { id: fact.id, fields: cleanedFields } : null; } +/** + * Sanitizes answer text for display while removing embedded artifacts and low-information content. + * + * @param options - Controls minimum content thresholds, preformatted text handling, and bold formatting preservation. + * @returns The cleaned answer text, or an empty string when the input is empty, invalid, or artifact-like. + */ export function sanitizeAnswerDisplayText(value: string, options: DisplayTextSanitizeOptions = {}) { - const normalized = polishClinicalAnswerProse(sourceTextForClinicalProsePreservingBreaks(value)).trim(); + const normalized = ( + options.preformatted + ? normalizePreformattedDisplayText(value, { + preserveBold: options.preserveBold, + }) + : polishClinicalAnswerProse(sourceTextForClinicalProsePreservingBreaks(value), { + preserveBold: options.preserveBold, + }) + ).trim(); if (!normalized) return ""; const artifactStart = normalizeDisplayText(normalized).search( /\{\s*"(?:answer|heading|body|grounded|confidence|citations?|answerSections?|citation_chunk_ids|source_chunk_ids|chunk_id|conflictsOrGaps|quoteCards?)\s*:/i, diff --git a/src/components/clinical-dashboard/evidence-panels.tsx b/src/components/clinical-dashboard/evidence-panels.tsx index bf17b8726..32e5461f0 100644 --- a/src/components/clinical-dashboard/evidence-panels.tsx +++ b/src/components/clinical-dashboard/evidence-panels.tsx @@ -27,6 +27,7 @@ import { import { type AnswerFeedbackType } from "@/lib/answer-feedback"; import { ClinicalOutputPanel } from "@/components/clinical-dashboard/output-panel"; import { + isPreformattedGroundedAnswer, keyClinicalItemsFromSections, keyClinicalItemsFromTable, plainAnswerText, @@ -573,10 +574,17 @@ function clinicalNotesAvailableTabs(sections: ClinicalDetailSection[]) { .filter((tab) => tab.count > 0); } +/** + * Builds the non-empty clinical detail sections used by the clinical notes view. + * + * @param answer - The answer from which to derive clinical detail sections. + * @param viewMode - Selects the standard or high-yield section set. + * @returns The sorted clinical detail sections with display-ready items. + */ function clinicalNotesDetailSectionsForAnswer(answer: RagAnswer, viewMode: AnswerViewMode) { const sections = viewMode === "high_yield" ? buildHighYieldClinicalOutputSections(answer) : buildClinicalOutputSections(answer); - const primaryAnswer = plainAnswerText(answer.answer); + const primaryAnswer = plainAnswerText(answer.answer, { preformatted: isPreformattedGroundedAnswer(answer) }); const keepVerifySource = answer.answerQualityTier === "source_only" || answer.grounded === false; return sortClinicalDetailSections( sections diff --git a/src/components/clinical-dashboard/output-panel.tsx b/src/components/clinical-dashboard/output-panel.tsx index 65e0bce81..7e21d0989 100644 --- a/src/components/clinical-dashboard/output-panel.tsx +++ b/src/components/clinical-dashboard/output-panel.tsx @@ -4,7 +4,7 @@ import { CircleCheck, ListChecks } from "lucide-react"; import { AccessibleTable } from "@/components/AccessibleTable"; import { SafeBoldText } from "@/components/SafeBoldText"; -import { plainAnswerText } from "@/components/clinical-dashboard/answer-content"; +import { isPreformattedGroundedAnswer, plainAnswerText } from "@/components/clinical-dashboard/answer-content"; import { SectionHeading, UtilityDrawer } from "@/components/clinical-dashboard/dashboard-shell"; import { AnswerViewModeControl, @@ -26,6 +26,16 @@ import { buildHighYieldClinicalOutputSections, } from "@/lib/ward-output"; +/** + * Renders a source-backed clinical answer with structured details, lead content, or an evidence map. + * + * @param answer - The clinical answer to display + * @param collapsed - Whether to wrap the content in a collapsed utility drawer + * @param showLead - Whether to display the concise lead section + * @param viewMode - The presentation mode for the clinical output + * @param onViewModeChange - Callback invoked when the presentation mode changes + * @param evidenceMapRows - Evidence map rows to display instead of computing them from the answer + */ export function ClinicalOutputPanel({ answer, collapsed = false, @@ -46,7 +56,7 @@ export function ClinicalOutputPanel({ const rows = evidenceMapRows ?? buildAnswerEvidenceMap(answer); if (sections.length === 0 && (viewMode !== "evidence_map" || rows.length === 0)) return null; const leadSection = sections.find((section) => section.id === "bottom-line") ?? sections[0]; - const primaryAnswer = plainAnswerText(answer.answer); + const primaryAnswer = plainAnswerText(answer.answer, { preformatted: isPreformattedGroundedAnswer(answer) }); const detailSections = sections .filter((section) => section.id !== "verify-source") .filter((section) => (showLead ? section.id !== leadSection?.id : section.id !== "bottom-line")) diff --git a/src/lib/answer-render-policy.ts b/src/lib/answer-render-policy.ts index ed8e43576..9f198c588 100644 --- a/src/lib/answer-render-policy.ts +++ b/src/lib/answer-render-policy.ts @@ -544,6 +544,13 @@ export function formatAnswerRenderCopyText(args: { .trim(); } +/** + * Builds a trust-gated render model for a clinical answer, including sources, evidence, warnings, and copy-ready text. + * + * @param answer - The clinical answer and associated evidence to render + * @param options - Optional source overrides and diagnostic decision details + * @returns The structured answer render model + */ export function buildAnswerRenderModel( answer: RagAnswer, options: BuildAnswerRenderModelOptions = {}, @@ -575,6 +582,10 @@ export function buildAnswerRenderModel( }); const allowedBlocks = blockOrder.filter((block) => decisions[block].shown); const answerText = answer.answer.trim(); + // The copy/paste draft is plain text for clinical notes — strip the server's + // high-yield bold markers so a pasted draft never contains literal "**". + // (Preformatted answers carry no bold, so this is a no-op for them.) + const copyAnswerText = answerText.replace(/\*\*/g, ""); return { answerText, @@ -588,7 +599,7 @@ export function buildAnswerRenderModel( relatedDocuments, bestSource, warnings, - copyText: formatAnswerRenderCopyText({ answerText, trust, primarySources, warnings }), + copyText: formatAnswerRenderCopyText({ answerText: copyAnswerText, trust, primarySources, warnings }), debugReasons: options.includeDebugReasons ? decisions : undefined, }; } diff --git a/src/lib/rag-answer-text.ts b/src/lib/rag-answer-text.ts index 2898ae3f7..7996b4370 100644 --- a/src/lib/rag-answer-text.ts +++ b/src/lib/rag-answer-text.ts @@ -197,17 +197,20 @@ function removeBadAnswerFragments(value: string) { // immediately followed by a community schedule with no sentence break (e.g. "...monitored daily // for inpatients for community patients weekly..."). The synthesis prompt handles most of these, // but this is a narrow deterministic safety-net for the clearest recurring pattern. It also -// consumes the original comma so it cannot produce a double comma. +/** + * Separates flattened inpatient and community patient schedule phrases into distinct sentences. + * + * @param value - Clinical prose containing a setting-related run-on phrase + * @returns The prose with the targeted setting run-on replaced by separate sentences + */ function separateSettingRunOns(value: string): string { return value .replace(/\bfor inpatients,?\s+for community patients,?/gi, "for inpatients. For community patients,") .replace(/\bfor community patients,?\s+for inpatients,?/gi, "for community patients. For inpatients,"); } -export function polishClinicalAnswerProse(value: string) { - // Bold markers come off before bullet normalization so an emphasized item - // ("o **Reduce dose**") still reads as a sub-bullet to the "o" matcher. - const cleaned = normalizeInlineBulletGlyphs(normalizeSectionText(value).replace(/\*\*([^*]+)\*\*/g, "$1")) +function stripAnswerArtifactNoise(text: string) { + return text .replace(productCatalogueFragmentPattern, " ") .replace(brandOrFormularyFragmentPattern, " ") .replace(imprestLocationPattern, " ") @@ -224,10 +227,49 @@ export function polishClinicalAnswerProse(value: string) { .replace(/(?:\.\s*){2,}/g, ". ") .replace(/\s+/g, " ") .trim(); +} + +function restoreBoldSpans(text: string, boldSpans: string[]) { + let restored = text; + for (const span of boldSpans) { + if (restored.includes(span) && !restored.includes(`**${span}**`)) { + restored = restored.replace(span, `**${span}**`); + } + } + return restored; +} - return normalizeGenericMedicationCase( +/** + * Cleans clinical answer text by removing artifacts, normalizing formatting, and improving readability. + * + * @param value - The clinical answer text to polish + * @param options - Formatting options + * @param options.preserveBold - Whether to preserve inline bold markers + * @returns The polished clinical answer text + */ +export function polishClinicalAnswerProse(value: string, options: { preserveBold?: boolean } = {}) { + // Bold markers normally come off before bullet normalization so an emphasized + // item ("o **Reduce dose**") still reads as a sub-bullet to the "o" matcher. + // The display path passes preserveBold so can render the + // server's high-yield emphasis (and the un-bold unverified-number safety + // signal); the sub-bullet matcher tolerates a leading "**" so bolded + // sub-bullets still normalize. The server answer-gen path keeps the default + // (strip), so its quality gates are unchanged. + const normalized = normalizeSectionText(value); + const boldSpans = [...normalized.matchAll(/\*\*([^*]+)\*\*/g)].map((match) => match[1]); + const bulletNormalized = normalizeInlineBulletGlyphs( + options.preserveBold ? normalized : normalized.replace(/\*\*([^*]+)\*\*/g, "$1"), + ); + // Artifact regexes were written for unbolded catalogue/source noise; run + // them on a de-bolded copy so preserveBold still strips Lithicarb-style junk. + const deBolded = bulletNormalized.replace(/\*\*([^*]+)\*\*/g, "$1"); + const cleaned = stripAnswerArtifactNoise(deBolded); + + const polished = normalizeGenericMedicationCase( separateSettingRunOns(removeOrphanAnswerHeadings(removeBadAnswerFragments(cleaned))), ); + + return options.preserveBold ? restoreBoldSpans(polished, boldSpans) : polished; } export function sanitizeAnswerText(value: string) { diff --git a/src/lib/rag.ts b/src/lib/rag.ts index fe0fc6229..1e23815af 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -1349,7 +1349,14 @@ function mergeSearchResults(primary: SearchResult[], secondary: SearchResult[]) return Array.from(merged.values()); } -/** Search text chunk candidates. */ +/** + * Retrieves lexical document chunk candidates for the supplied query variants. + * + * @param queryVariants - Query variants used for strict matching and recall-oriented fallback searches. + * @param matchCount - Maximum number of candidates requested for the primary query. + * @param telemetry - Optional retrieval telemetry updated with RPC failures and text-relaxation usage. + * @returns Matching document chunks, using corrected or relaxed queries when strict matching produces no candidates. + */ async function searchTextChunkCandidates(args: { supabase: ReturnType; queryVariants: string[]; @@ -1366,6 +1373,10 @@ async function searchTextChunkCandidates(args: { document_filters: args.documentIds ?? undefined, owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds, args.allowGlobalSearch), }); + // Report the error before returning empty so a schema drift on this + // most-terminal lexical layer surfaces in hybrid_rpc_errors telemetry + // instead of silently degrading to zero candidates. Return value unchanged. + if (error) recordHybridRpcError(args.telemetry, "match_document_chunks_text", error); return error || !data?.length ? ([] as SearchResult[]) : (data as SearchResult[]); }; @@ -1630,7 +1641,12 @@ async function fetchDocumentTitleAliasRows(args: { })); } -/** Search document lookup fast path. */ +/** + * Retrieves and ranks document chunks for document-focused queries. + * + * @param args - Search configuration, including the required owner scope and optional document restrictions. + * @returns Ranked document lookup results, limited to `matchCount`. + */ async function searchDocumentLookupFastPath(args: { supabase: ReturnType; query: string; @@ -1638,6 +1654,7 @@ async function searchDocumentLookupFastPath(args: { ownerId?: string; documentIds?: string[]; matchCount: number; + telemetry?: SearchTelemetry; }): Promise { if (!args.ownerId) return [] as SearchResult[]; const variants = (args.queryVariants?.length ? args.queryVariants : [buildClinicalTextSearchQuery(args.query)]).slice( @@ -1651,6 +1668,7 @@ async function searchDocumentLookupFastPath(args: { match_count: index === 0 ? 12 : 8, owner_filter: requireOwnerScope(args.ownerId), }); + if (error) recordHybridRpcError(args.telemetry, "match_documents_for_query", error); if (error || !data?.length) return [] as DocumentLookupRow[]; return data as DocumentLookupRow[]; }), @@ -1926,7 +1944,12 @@ async function loadChunksForSignalMatches(args: { .filter(Boolean) as SearchResult[]; } -/** Search table fact candidates. */ +/** + * Retrieves document chunks containing table facts relevant to a query. + * + * @param args - Retrieval options, including query variants, scope filters, result limit, and optional telemetry. + * @returns Search results containing the highest-ranked table facts grouped by source chunk. + */ async function searchTableFactCandidates(args: { supabase: ReturnType; query: string; @@ -1935,6 +1958,7 @@ async function searchTableFactCandidates(args: { documentIds?: string[]; allowGlobalSearch?: boolean; matchCount: number; + telemetry?: SearchTelemetry; }) { const variants = (args.queryVariants?.length ? args.queryVariants : [buildClinicalTextSearchQuery(args.query)]).slice( 0, @@ -1948,6 +1972,7 @@ async function searchTableFactCandidates(args: { document_filters: args.documentIds ?? undefined, owner_filter: ownerScopeForDocumentFilteredRetrieval(args.ownerId, args.documentIds, args.allowGlobalSearch), }); + if (error) recordHybridRpcError(args.telemetry, "match_document_table_facts_text", error); if (error || !data?.length) return [] as TableFactRpcRow[]; return data as TableFactRpcRow[]; }), @@ -2889,7 +2914,12 @@ function shouldUseMemoryBeforeFastPath(queryClass: RagQueryClass) { return queryClass === "table_threshold" || queryClass === "medication_dose_risk" || queryClass === "comparison"; } -/** Search chunks with telemetry. */ +/** + * Retrieves and ranks document chunks using lexical, structured, memory, and embedding-based evidence, while recording retrieval telemetry. + * + * @param args - Retrieval options, including the query, scope, search mode, and embedding preferences. + * @returns The ranked search results and telemetry describing the retrieval process. + */ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { assertGlobalSearchAllowed(args); throwIfAborted(args.signal); @@ -3146,6 +3176,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { documentIds: documentFilterList, allowGlobalSearch: args.allowGlobalSearch, matchCount: Math.min(candidateCount, 48), + telemetry, }); const tableFactLatencyMs = Date.now() - tableFactStartedAt; telemetry.supabase_rpc_latency_ms += tableFactLatencyMs; @@ -3167,6 +3198,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { ownerId: args.ownerId, documentIds: documentFilterList, matchCount: candidateCount, + telemetry, }); const documentLookupLatencyMs = Date.now() - documentLookupStartedAt; telemetry.supabase_rpc_latency_ms += documentLookupLatencyMs; diff --git a/src/lib/source-text-sanitizer.ts b/src/lib/source-text-sanitizer.ts index efa3866ae..848152681 100644 --- a/src/lib/source-text-sanitizer.ts +++ b/src/lib/source-text-sanitizer.ts @@ -288,10 +288,40 @@ export function sourceTextForClinicalProse(text: string) { return compactWhitespace(stripLowYieldSourceNoise(stripInternalImageDataBlocks(text))); } +/** + * Produces clinical prose with meaningful line breaks preserved. + * + * @returns The cleaned text with normalized whitespace and low-yield source noise removed. + */ export function sourceTextForClinicalProsePreservingBreaks(text: string) { return readableWhitespace(stripLowYieldSourceNoise(sourceTextForDisplayPreservingBreaks(text))); } +// Lossless display normalization for server-`preformatted` answers (document +// support lists, table/visual source references). These are well-formed by +// construction and their "source-noise"-looking tokens — facility codes like +// "MP-0123", all-caps guideline titles — ARE the payload, so the noise-stripping +// prose sanitizer must NOT run on them (it would delete the document names and +// leave garble). Apply only glyph repair + whitespace collapse, matching the +// server's own `finalizeRagAnswerQuality` exemption for `preformatted && grounded`. +// When preserveBold is false, strip **bold** markers so literal Markdown markers +/** + * Normalizes preformatted text for display while preserving its readable structure. + * + * @param options - Controls whether Markdown bold markers are preserved. + * @returns The normalized text, with bold markers removed unless `preserveBold` is `true`. + */ +export function normalizePreformattedDisplayText(text: string, options: { preserveBold?: boolean } = {}) { + const normalized = readableWhitespace(text); + return options.preserveBold ? normalized : normalized.replace(/\*\*([^*]+)\*\*/g, "$1"); +} + +/** + * Determines whether source text contains little clinically useful content. + * + * @param text - Source text to assess. + * @returns `true` if the text is predominantly source noise or contains only a short remainder, `false` otherwise. + */ export function isLowYieldClinicalText(text: string) { const cleaned = sourceTextForClinicalProse(text); if (!cleaned) return true; @@ -554,7 +584,10 @@ const inlineBulletGlyphPattern = /\s*[•◦▪‣●]+\s*/g; // "Dose:\no Start 750 mg"), hence the start/newline alternatives. Lowercase // followers ("o pregnancy") are deliberately NOT converted: the risk of // deleting a genuine clinical "o" value outweighs the cosmetic artifact. -const subBulletOGlyphPattern = /(?<=^|[\r\n]|[^\d\s]\s)o(?=\s+(?:\d|[A-Z][a-z0-9]|[A-Z]{2,}))/g; +// The optional "**" before the follower lets a bolded sub-bullet ("o **Reduce +// dose**") still be recognised when the display path preserves bold markers; a +// superset of the previous match, so non-bold behaviour is unchanged. +const subBulletOGlyphPattern = /(?<=^|[\r\n]|[^\d\s]\s)o(?=\s+(?:\*\*)?(?:\d|[A-Z][a-z0-9]|[A-Z]{2,}))/g; // Blood-group exemptions: "blood group o RhD negative" / "Blood Type: o // Negative" (any casing, optional colon), or a bare "group/type o" directly // followed by an Rh/positive/negative value, keep their lowercase "o" — it @@ -566,15 +599,25 @@ const groupTypeLabelTailPattern = /\b(?:group|type):?\s$/i; // A word qualifying group/type ("risk group:", "test type:") marks a list // label, not a blood label — its positive/negative items are list content. const qualifiedGroupTypeTailPattern = /\b[A-Za-z][\w-]*\s+(?:group|type):?\s$/i; -const rhValueHeadPattern = /^\s+rh(?:d)?\b/i; -const posNegValueHeadPattern = /^\s+(?:pos(?:itive)?|neg(?:ative)?)\b/i; +const rhValueHeadPattern = /^\s+(?:\*\*)?rh(?:d)?\b/i; +const posNegValueHeadPattern = /^\s+(?:\*\*)?(?:pos(?:itive)?|neg(?:ative)?)\b/i; // An entire line that is just the ABO value ("o RhD negative", "o Negative", -// "o Rh positive") — the strongest signal that the "o" is the group itself. +// "o Rh positive", "o **RhD negative**", "o **Negative**") — the strongest +// signal that the "o" is the group itself. Bold markers around the value are +// allowed so high-yield emphasis from the server never causes a blood value to +// be misclassified as a bullet. const standaloneBloodValueLinePattern = - /^o\s+(?:rh(?:d)?(?:\s+(?:pos(?:itive)?|neg(?:ative)?))?|pos(?:itive)?|neg(?:ative)?)$/i; + /^o\s+(?:\*\*)?(?:rh(?:d)?(?:\s+(?:pos(?:itive)?|neg(?:ative)?))?|pos(?:itive)?|neg(?:ative)?)(?:\*\*)?$/i; const bloodValueWithNounTailLinePattern = - /^o\s+(?:rh(?:d)?\s+)?(?:pos(?:itive)?|neg(?:ative)?)\s+(?:blood(?!\s+(?:cultures?|tests?|screens?|samples?|results?)\b)|red\s+cells?)\b/i; - + /^o\s+(?:\*\*)?(?:rh(?:d)?\s+)?(?:pos(?:itive)?|neg(?:ative)?)(?:\*\*)?\s+(?:blood(?!\s+(?:cultures?|tests?|screens?|samples?|results?)\b)|red\s+cells?)\b/i; + +/** + * Replaces OCR-style lowercase `o` sub-bullets while preserving blood-group value text. + * + * @param text - Text containing potential OCR sub-bullet markers + * @param joiner - Text used to replace recognized sub-bullet markers + * @returns Text with eligible sub-bullet markers replaced + */ function replaceSubBulletOGlyphs(text: string, joiner: string) { return text.replace(subBulletOGlyphPattern, (match, offset: number) => { const before = text.slice(0, offset); diff --git a/supabase/migrations/20260712000000_forward_codify_retrieval_owner_matches.sql b/supabase/migrations/20260712000000_forward_codify_retrieval_owner_matches.sql new file mode 100644 index 000000000..61dd9b10e --- /dev/null +++ b/supabase/migrations/20260712000000_forward_codify_retrieval_owner_matches.sql @@ -0,0 +1,105 @@ +-- Forward-codify the live retrieval RPC bodies so committed migrations reproduce +-- production. Drift backlog item from 20260705210000_retrieval_owner_filter_sentinel.sql. +-- +-- VERIFIED LIVE STATE (read-only pg_get_functiondef inspection, 2026-07-12) +-- ------------------------------------------------------------------------------ +-- ALL EIGHT primary retrieval RPCs on live ALREADY gate ownership with the +-- fail-closed + sentinel-aware helper retrieval_owner_matches(owner_filter, +-- d.owner_id) — none carry the legacy fail-open inline predicate: +-- match_document_chunks_hybrid(vector,text,integer,double precision,uuid[],uuid) +-- match_document_chunks_text(text,integer,uuid[],uuid) +-- match_document_lookup_chunks_text(text,uuid[],integer,uuid) +-- match_document_table_facts_text(text,integer,uuid[],uuid) +-- match_document_embedding_fields_hybrid(vector,text,integer,double precision,uuid[],uuid) +-- match_document_index_units_hybrid(vector,text,integer,double precision,uuid[],uuid) +-- match_documents_for_query(text,integer,uuid) +-- match_document_chunks(vector,integer,double precision,uuid,uuid) +-- So there is NO live tenancy hole. The gap is reproducibility only: the +-- COMMITTED migration bodies are stale (fail-open + sentinel-blind), so any +-- environment rebuilt from source — Supabase preview branch, `db reset`, DR +-- restore, a new region, or the golden retrieval eval against a fresh DB — gets +-- empty public retrieval plus a fail-open NULL branch that live does not have. +-- +-- TWO NON-PRIMARY functions still carry a fail-open-shaped inline predicate on +-- live (assess separately — NOT part of the primary-retrieval codification): +-- * match_document_embedding_fields_text(text,integer,double precision,uuid[],uuid) +-- — sole gate is `(owner_filter is null or d.owner_id = owner_filter)`: +-- genuine latent fail-open on NULL + sentinel-blind. Confirm whether the +-- app calls this text-only variant (the app uses the _hybrid variant); +-- if live, route it through retrieval_owner_matches too. +-- * get_related_document_metadata(uuid[],uuid) +-- — the primary `documents d` join is fail-closed via +-- retrieval_owner_matches, but the label (l) and summary (s) joins keep +-- `(owner_filter is null or …)`. Backstopped by the fail-closed d gate +-- (NULL → no d rows → no l/s rows), so not a live escape, but tidy up +-- for consistency. +-- +-- WHY THIS FILE IS NOT HAND-FILLED WITH THE RPC BODIES +-- ------------------------------------------------------------------------------ +-- The live bodies total ~26 KB across 8 functions and have diverged FORWARD from +-- committed source (hnsw.ef_search wrapper on match_document_chunks; richer +-- multi-strategy _text/_table_facts_text bodies; left-join quality_score on the +-- hybrid path — see 20260705210000). They must be codified BYTE-FOR-BYTE from +-- live: a transcription error or a stale-source rebuild would REGRESS retrieval, +-- the exact trap 20260705210000 was neutralized to prevent. That requires a +-- direct DB dump (the environment running this review has no Postgres connection +-- string — only PostgREST keys — so it cannot produce a byte-perfect dump). This +-- is owner-driven, matching the documented `supabase migration repair` drift +-- workflow. +-- +-- OWNER COMPLETION (byte-perfect, no hand transcription; each step confirmed): +-- ------------------------------------------------------------------------------ +-- 1. With the linked project + DB connection string ($SUPABASE_DB_URL), append +-- the EXACT live bodies to this file: +-- +-- psql "$SUPABASE_DB_URL" -X -q -At -o /tmp/rpc_bodies.sql -c " +-- select string_agg(pg_get_functiondef(p.oid) || E';\n', E'\n' order by p.proname, p.oid) +-- from pg_proc p join pg_namespace n on n.oid = p.pronamespace +-- where n.nspname = 'public' and p.proname in ( +-- 'match_document_chunks_hybrid','match_document_chunks_text', +-- 'match_document_lookup_chunks_text','match_document_table_facts_text', +-- 'match_document_embedding_fields_hybrid','match_document_index_units_hybrid', +-- 'match_documents_for_query','match_document_chunks');" +-- cat /tmp/rpc_bodies.sql >> supabase/migrations/20260712000000_forward_codify_retrieval_owner_matches.sql +-- +-- (`create or replace function` is idempotent; applying on live is a no-op +-- because the bodies already match. On fresh rebuilds it installs them.) +-- 2. Apply on a Supabase PREVIEW BRANCH. The DO $verify$ guard below must pass, +-- then run: npm run eval:retrieval:quality -- --fail-on-threshold (36/36) +-- to prove the rebuilt retrieval matches live before this reaches main. +-- 3. Only after a green preview + golden eval, land + apply. + +set search_path = public, extensions, pg_temp; + +-- Safe, idempotent guard: re-assert the fail-closed + sentinel-aware truth table +-- for retrieval_owner_matches so a bad redefinition (or an environment where the +-- helper drifted back to fail-open) fails this migration loudly instead of +-- silently shipping a tenancy regression. Mirrors 20260708160001. +do $verify$ +declare + a uuid := '11111111-1111-1111-1111-111111111111'; + b uuid := '22222222-2222-2222-2222-222222222222'; + sentinel uuid := '00000000-0000-0000-0000-000000000000'; +begin + if public.retrieval_owner_matches(null, a) is not false then + raise exception 'retrieval_owner_matches(NULL, uuid) must be FALSE (fail-closed)'; + end if; + if public.retrieval_owner_matches(null, null) is not false then + raise exception 'retrieval_owner_matches(NULL, NULL) must be FALSE (fail-closed)'; + end if; + if public.retrieval_owner_matches(sentinel, null) is not true then + raise exception 'retrieval_owner_matches(sentinel, NULL) must be TRUE (public row)'; + end if; + if public.retrieval_owner_matches(sentinel, a) is not false then + raise exception 'retrieval_owner_matches(sentinel, owned) must be FALSE (public-only)'; + end if; + if public.retrieval_owner_matches(a, a) is not true then + raise exception 'retrieval_owner_matches(owner, same owner) must be TRUE'; + end if; + if public.retrieval_owner_matches(a, b) is not false then + raise exception 'retrieval_owner_matches(owner, other owner) must be FALSE'; + end if; +end +$verify$; + +-- >>> OWNER: append the byte-perfect live RPC bodies below this line (step 1). <<< diff --git a/tests/answer-content.test.ts b/tests/answer-content.test.ts new file mode 100644 index 000000000..7a4b0eb05 --- /dev/null +++ b/tests/answer-content.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; +import { primaryAnswerDisplayText } from "../src/components/clinical-dashboard/answer-content"; + +describe("primaryAnswerDisplayText", () => { + it("keeps a safety cue in a long leading fragment beyond the compact word budget", () => { + const lead = `${Array.from({ length: 90 }, (_, index) => `detail${index + 1}`).join(" ")} Do not administer the medicine.`; + + expect(primaryAnswerDisplayText(lead)).toContain("Do not administer the medicine."); + }); + + it("keeps a full-word escalation caveat that appears beyond the compact head", () => { + // A bare `escalat\b` stem never matched "Escalate", so this caveat (the + // fourth fragment) was silently dropped before the fix. + const answer = + "Give paracetamol for ongoing pain. Review the observations hourly overnight. Document the management plan clearly in the notes. Escalate to the senior doctor if the patient deteriorates."; + expect(primaryAnswerDisplayText(answer)).toContain("Escalate to the senior doctor"); + }); + + it("keeps a short contraindication caveat under the 8-word usefulness floor", () => { + // "Contraindicated in pregnancy and severe renal impairment" is 7 words, so + // the usefulness/length filter dropped it before the safety-aware exception. + const answer = + "Offer oral rehydration first. Reassess fluid balance after two hours. Record the intake and output totals. Contraindicated in pregnancy and severe renal impairment."; + expect(primaryAnswerDisplayText(answer)).toContain("Contraindicated in pregnancy"); + }); + + it("keeps a short stop instruction after normal leading prose", () => { + const answer = + "Give paracetamol for ongoing pain. Review the observations hourly overnight. Document the management plan clearly in the notes. Stop lithium."; + expect(primaryAnswerDisplayText(answer)).toContain("Stop lithium."); + }); + + it("keeps a passive held caveat beyond the compact head", () => { + const answer = + "Give paracetamol for ongoing pain. Review the observations hourly overnight. Document the management plan clearly in the notes. Clozapine should be held."; + expect(primaryAnswerDisplayText(answer)).toContain("should be held"); + }); + + it("keeps an avoid directive after normal leading prose", () => { + const answer = + "Give paracetamol for ongoing pain. Review the observations hourly overnight. Document the management plan clearly in the notes. Avoid lithium in pregnancy."; + expect(primaryAnswerDisplayText(answer)).toContain("Avoid lithium in pregnancy"); + }); + + it("keeps a caveat whose safety keyword is server-bolded on the preserveBold path", () => { + // "Do **not** administer" — bold markers inside the phrase must not defeat + // the safety match, or the compact cap could drop the withhold instruction. + const answer = + "Give paracetamol for ongoing pain. Review the observations hourly overnight. Document the management plan clearly in the notes. Do **not** administer lithium."; + expect(primaryAnswerDisplayText(answer, { preserveBold: true })).toContain("administer lithium"); + }); + + it("keeps a should-not / must-not contraindication directive beyond the compact head", () => { + const answer = + "Give paracetamol for ongoing pain. Review the observations hourly overnight. Document the management plan clearly in the notes. Clozapine should not be used in this patient."; + expect(primaryAnswerDisplayText(answer)).toContain("should not be used"); + }); + + it("is unchanged for a short answer with no safety signal", () => { + const answer = "Offer simple analgesia and reassess in one hour."; + expect(primaryAnswerDisplayText(answer)).toBe(answer); + }); +}); diff --git a/tests/answer-render-policy.test.ts b/tests/answer-render-policy.test.ts index 5cf6f2a00..cb36d2766 100644 --- a/tests/answer-render-policy.test.ts +++ b/tests/answer-render-policy.test.ts @@ -125,6 +125,18 @@ function answer(overrides: Partial = {}): RagAnswer { } describe("answer render policy", () => { + it("strips high-yield bold markers from the copy/paste clinical draft", () => { + const model = buildAnswerRenderModel( + answer({ + answer: "For red-range results, **withhold clozapine** and recheck ANC **within 24 hours**.", + }), + ); + // The pasted draft is plain text for clinical notes — no literal "**". + expect(model.copyText).not.toContain("**"); + expect(model.copyText).toContain("withhold clozapine"); + expect(model.copyText).toContain("within 24 hours"); + }); + it("limits unsupported answers to source review and warnings even when raw extras are present", () => { const model = buildAnswerRenderModel( answer({ diff --git a/tests/display-text.test.ts b/tests/display-text.test.ts index f5119d299..be5e5567f 100644 --- a/tests/display-text.test.ts +++ b/tests/display-text.test.ts @@ -7,6 +7,56 @@ import { } from "../src/components/clinical-dashboard/display-text"; describe("clinical dashboard display text", () => { + it("preserves document names and facility codes for preformatted answers", () => { + // A server-`preformatted` document-support-list answer: its all-caps titles + // and facility codes ARE the payload and must survive display. + const docList = + "I found 2 indexed documents that support this query: LITHIUM CLOZAPINE CO-PRESCRIBING GUIDELINE MP-0123; and Acute Behavioural Disturbance Protocol (NOCC)."; + + const rendered = sanitizeAnswerDisplayText(docList, { preformatted: true }); + expect(rendered).toContain("LITHIUM CLOZAPINE CO-PRESCRIBING GUIDELINE"); + expect(rendered).toContain("MP-0123"); + expect(rendered).toContain("(NOCC)"); + + // Without the flag the prose sanitizer deletes exactly those tokens — this is + // the mangling the preformatted path exists to prevent. + const mangled = sanitizeAnswerDisplayText(docList); + expect(mangled).not.toContain("LITHIUM CLOZAPINE CO-PRESCRIBING GUIDELINE"); + }); + + it("keeps high-yield bold when preserveBold is set, strips it otherwise", () => { + const bolded = "Withhold clozapine if ANC is **below 1.5**."; + expect(sanitizeAnswerDisplayText(bolded, { preserveBold: true })).toContain("**below 1.5**"); + const stripped = sanitizeAnswerDisplayText(bolded); + expect(stripped).not.toContain("**"); + expect(stripped).toContain("below 1.5"); + }); + + it("handles bold markers correctly in preformatted answers", () => { + const preformattedWithBold = + "The following documents are relevant: **LITHIUM THERAPY GUIDELINE MP-0123**; Clozapine Monitoring Protocol."; + + // When preserveBold is true, keep the markers for SafeBoldText rendering. + const withBold = sanitizeAnswerDisplayText(preformattedWithBold, { + preformatted: true, + preserveBold: true, + }); + expect(withBold).toContain("**LITHIUM THERAPY GUIDELINE MP-0123**"); + + // When preserveBold is false or omitted, strip the markers so literal Markdown + // markers never leak through when PriorAnswerTurnSurface renders collapsed text. + const withoutBold = sanitizeAnswerDisplayText(preformattedWithBold, { + preformatted: true, + preserveBold: false, + }); + expect(withoutBold).not.toContain("**"); + expect(withoutBold).toContain("LITHIUM THERAPY GUIDELINE MP-0123"); + + const defaultBehavior = sanitizeAnswerDisplayText(preformattedWithBold, { preformatted: true }); + expect(defaultBehavior).not.toContain("**"); + expect(defaultBehavior).toContain("LITHIUM THERAPY GUIDELINE MP-0123"); + }); + it("polishes cached generated answer prose before rendering", () => { const noisy = "Lithium Carbonate 250 mg Tablet – Lithicarb®. Imprest location: Formulary One DOSAGE & DOSAGE ADJUSTMENTS Therapy with lithium should always begin with conventional tablets (Lithium Carbonate 250 mg) to stabilise the do. Lithium MONITORING Baseline Tests1."; diff --git a/tests/rag-answer-text.test.ts b/tests/rag-answer-text.test.ts index 481d3c877..ca7c46bfe 100644 --- a/tests/rag-answer-text.test.ts +++ b/tests/rag-answer-text.test.ts @@ -9,6 +9,20 @@ import { } from "../src/lib/rag-answer-text"; describe("RAG answer text helpers", () => { + it("strips bold by default but preserves it under preserveBold", () => { + const input = "Escalate if **ANC below 1.5**."; + expect(polishClinicalAnswerProse(input)).not.toContain("**"); + expect(polishClinicalAnswerProse(input, { preserveBold: true })).toContain("**ANC below 1.5**"); + }); + + it("normalizes a bolded sub-bullet while preserving the bold", () => { + const out = polishClinicalAnswerProse("Actions: o **Reduce dose** o **Recheck ANC**", { preserveBold: true }); + expect(out).toContain("**Reduce dose**"); + expect(out).toContain("**Recheck ANC**"); + // The leading "o" sub-bullet glyph is normalized away, not left as a literal. + expect(out).not.toMatch(/\bo\s+\*\*Reduce/); + }); + it("normalizes balanced word tokens for query/source matching", () => { expect(splitBalancedWords("Clozapine: red-range blood results!")).toEqual([ "clozapine", @@ -74,6 +88,17 @@ describe("RAG answer text helpers", () => { expect(cleaned).not.toMatch(/Lithicarb|Quilonum|Imprest|DOSAGE|Dose evidence|Tests1/i); }); + it("strips bolded catalogue noise when preserveBold keeps clinical emphasis", () => { + const noisy = + "Dose evidence: **Lithium** Carbonate **250 mg** Tablet - Lithicarb®. Therapy with **lithium** should always begin with conventional tablets (**lithium** carbonate **250 mg**)."; + + const cleaned = polishClinicalAnswerProse(noisy, { preserveBold: true }); + + expect(cleaned).toContain("Therapy with **lithium**"); + expect(cleaned).toContain("**250 mg**"); + expect(cleaned).not.toMatch(/Lithicarb|Dose evidence/i); + }); + it("flags source-inventory wording and truncated clinical fragments as answer quality issues", () => { expect( hasClinicalAnswerQualityIssue( diff --git a/tests/rag-offline-answer.test.ts b/tests/rag-offline-answer.test.ts index be322688a..f6a817215 100644 --- a/tests/rag-offline-answer.test.ts +++ b/tests/rag-offline-answer.test.ts @@ -76,16 +76,26 @@ class EmptyQuery implements PromiseLike<{ data: unknown[]; error: null }> { } } -async function answerOffline(query: string, textSources: SearchResult[]) { +type RpcResult = { data: unknown; error: unknown }; + +async function answerOffline( + query: string, + textSources: SearchResult[], + rpcHandler?: (name: string) => RpcResult | Promise, +) { // offline provider mode forces source-only behaviour regardless of key presence. vi.stubEnv("RAG_PROVIDER_MODE", "offline"); vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0"); vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0"); - const rpc = vi.fn(async (name: string) => { - if (name === "match_document_chunks_text") return { data: textSources, error: null }; - return { data: [], error: null }; - }); + const rpc = vi.fn( + rpcHandler + ? async (name: string) => rpcHandler(name) + : async (name: string) => { + if (name === "match_document_chunks_text") return { data: textSources, error: null }; + return { data: [], error: null }; + }, + ); vi.doMock("@/lib/supabase/admin", () => ({ createAdminClient: () => ({ rpc, from: vi.fn(() => new EmptyQuery()) }), })); @@ -152,6 +162,27 @@ describe("source-only / offline answers", () => { expect(answer.fallbackReason).toContain("source_only"); }); + it("stays fail-closed (does not throw) when a lexical retrieval RPC errors", async () => { + // F8: the terminal lexical layer now records the RPC error in retrieval + // telemetry via recordHybridRpcError before returning empty. The return + // value is unchanged, so the path must still degrade to a source-gap answer + // rather than throw. + const rpcError = { + code: "42P01", + message: "relation match_document_chunks_text does not exist", + hint: null, + }; + const { answer, generateStructuredTextResult } = await answerOffline( + "What ANC threshold should withhold clozapine?", + [], + (name) => (name === "match_document_chunks_text" ? { data: null, error: rpcError } : { data: [], error: null }), + ); + + expect(generateStructuredTextResult).not.toHaveBeenCalled(); + expect(answer.grounded).toBe(false); + expect(answer.confidence).toBe("unsupported"); + }); + it("fails closed to a source-gap answer when there is no usable evidence", async () => { const { answer, generateStructuredTextResult } = await answerOffline( "What is the duress response procedure for the community team?", diff --git a/tests/source-text-sanitizer.test.ts b/tests/source-text-sanitizer.test.ts index 8bd43b99d..a0936234a 100644 --- a/tests/source-text-sanitizer.test.ts +++ b/tests/source-text-sanitizer.test.ts @@ -6,6 +6,7 @@ import { lowYieldSourceNoiseScore, normalizeExtractedGlyphs, normalizeInlineBulletGlyphs, + normalizePreformattedDisplayText, polishStoredSynopsis, repairTruncatedCompactTail, fenceSourceEvidence, @@ -20,6 +21,19 @@ import { } from "../src/lib/source-text-sanitizer"; describe("source text sanitizer", () => { + it("normalizePreformattedDisplayText is lossless for document names and codes", () => { + // Preformatted answers rely on this: it must repair glyphs/whitespace but + // NOT strip facility codes, all-caps titles, or parenthetical codes. + const docList = + "I found 2 indexed documents that support this query: LITHIUM CLOZAPINE GUIDELINE MP-0123;\n\nand Behaviour Protocol (NOCC)."; + const out = normalizePreformattedDisplayText(docList); + expect(out).toContain("LITHIUM CLOZAPINE GUIDELINE"); + expect(out).toContain("MP-0123"); + expect(out).toContain("(NOCC)"); + // Collapses runs of spaces but keeps the content intact. + expect(out).not.toMatch(/ {2,}/); + }); + it("removes complete and partial internal image metadata from display text", () => { const text = "Source mentions: [[IMAGE_DATA_START]] Image ID: img-1; Source kind: table_crop; Image type: clinical_table; Table text: | Dose | Route | [[IMAGE_DATA_END]] Continue oral medication when indicated."; @@ -508,6 +522,34 @@ describe("normalizeInlineBulletGlyphs", () => { expect(normalizeInlineBulletGlyphs("donor group o RhD negative")).toBe("donor group o RhD negative"); }); + it("keeps blood-group values with bold markers intact", () => { + // Line-start forms: "o **RhD negative**" is a blood value, not a bullet. + expect(normalizeInlineBulletGlyphs("o **RhD negative**")).toBe("o **RhD negative**"); + expect(normalizeInlineBulletGlyphs("o **Negative**")).toBe("o **Negative**"); + expect(normalizeInlineBulletGlyphs("o **Rh positive**")).toBe("o **Rh positive**"); + + // Label-prefixed forms: "group/type: o **Negative**" is a blood value. + expect(normalizeInlineBulletGlyphs("blood group: o **RhD negative**")).toBe("blood group: o **RhD negative**"); + expect(normalizeInlineBulletGlyphs("Blood Type: o **Negative**")).toBe("Blood Type: o **Negative**"); + expect(normalizeInlineBulletGlyphs("group o **Negative** units available")).toBe( + "group o **Negative** units available", + ); + + // Blood-value + noun forms with bold markers. + expect(normalizeInlineBulletGlyphs("o **Negative** blood should be used")).toBe( + "o **Negative** blood should be used", + ); + expect(normalizeInlineBulletGlyphs("o **RhD negative** red cells are required")).toBe( + "o **RhD negative** red cells are required", + ); + + // Non-blood bullets with bold content should still convert. + expect(normalizeInlineBulletGlyphs("o **Reduce dose** if ANC falls")).toBe("**Reduce dose** if ANC falls"); + expect(normalizeInlineBulletGlyphs("Risk: o **Positive symptoms** require urgent review")).toBe( + "Risk: **Positive symptoms** require urgent review", + ); + }); + it("repairs a label colon followed by a converted sub-bullet ('Label:; item' → 'Label: item')", () => { expect(normalizeInlineBulletGlyphs("Acute Mania: o IR product: 750 to 1000mg daily")).toBe( "Acute Mania: IR product: 750 to 1000mg daily",