From 6d20b6ef1487171d400641c41a227db7cd987834 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:17:53 +0800 Subject: [PATCH 01/14] fix(rag): recover grounded Australian answers --- src/lib/australian-source-priority.ts | 117 ++++++++ src/lib/rag-context-selection.ts | 14 +- src/lib/rag.ts | 293 +++++++++++++------- src/lib/source-authority-registry.ts | 385 ++++++++++++++++++++++++++ src/lib/source-metadata.ts | 6 + tests/rag-answer-fallback.test.ts | 179 +++++++++++- tests/rag-context-budget.test.ts | 152 +++++++++- tests/source-metadata.test.ts | 185 +++++++++++++ 8 files changed, 1216 insertions(+), 115 deletions(-) create mode 100644 src/lib/australian-source-priority.ts create mode 100644 src/lib/source-authority-registry.ts diff --git a/src/lib/australian-source-priority.ts b/src/lib/australian-source-priority.ts new file mode 100644 index 000000000..5fcc4668e --- /dev/null +++ b/src/lib/australian-source-priority.ts @@ -0,0 +1,117 @@ +import { classifySourceAuthority, type AustralianSourceTier } from "@/lib/source-authority-registry"; +import type { SearchResult } from "@/lib/types"; + +export type { AustralianSourceTier, SourceAuthorityClassification } from "@/lib/source-authority-registry"; + +export type AustralianSourceSelectionSummary = { + candidateCount: number; + selectedCount: number; + australianCandidateCount: number; + australianSelectedCount: number; + waCandidateCount: number; + waSelectedCount: number; + authoritativeAustralianDocumentCount: number; + authorityConflictCount: number; + usedSupplementaryFallback: boolean; +}; + +const tierRank: Record = { + wa_validated: 0, + australian_national: 1, + australian_state: 2, + supplementary: 3, +}; + +const relevanceRank = { + direct: 0, + partial: 1, + nearby: 2, + none: 3, +} as const; + +export function australianSourceClassification(result: Pick) { + return classifySourceAuthority(result.source_metadata); +} + +export function australianSourceTier(result: Pick): AustralianSourceTier { + return australianSourceClassification(result).tier; +} + +export function isAustralianSourceTier(tier: AustralianSourceTier) { + return tier !== "supplementary"; +} + +function capClinicalContextPerDocument(results: SearchResult[], maxPerDocument: number) { + if (new Set(results.map((result) => result.document_id)).size < 2) return results; + + const documentCounts = new Map(); + return results.filter((result) => { + const count = documentCounts.get(result.document_id) ?? 0; + if (count >= maxPerDocument) return false; + documentCounts.set(result.document_id, count + 1); + return true; + }); +} + +/** + * Select a bounded clinical context while preferring authoritative Australian + * evidence inside the same relevance band. Supplementary material remains + * available until Australian evidence is sufficient on its own. + */ +export function selectAustralianClinicalContext( + results: SearchResult[], + options: { limit?: number; maxPerDocument?: number; sufficientAustralianChunks?: number } = {}, +) { + const limit = options.limit ?? 6; + const maxPerDocument = options.maxPerDocument ?? 2; + const sufficientAustralianChunks = options.sufficientAustralianChunks ?? 4; + const ranked = results + .map((result, index) => ({ result, index, tier: australianSourceTier(result) })) + .filter(({ result }) => result.relevance?.verdict !== "none") + .sort((left, right) => { + const leftRelevance = left.result.relevance?.verdict + ? relevanceRank[left.result.relevance.verdict] + : relevanceRank.direct; + const rightRelevance = right.result.relevance?.verdict + ? relevanceRank[right.result.relevance.verdict] + : relevanceRank.direct; + return leftRelevance - rightRelevance || tierRank[left.tier] - tierRank[right.tier] || left.index - right.index; + }); + const capped = capClinicalContextPerDocument( + ranked.map(({ result }) => result), + maxPerDocument, + ); + const australian = capped.filter((result) => isAustralianSourceTier(australianSourceTier(result))); + const australianDocumentCount = new Set(australian.map((result) => result.document_id)).size; + + if (australian.length >= sufficientAustralianChunks && australianDocumentCount >= 2) { + return australian.slice(0, limit); + } + + return capped.slice(0, limit); +} + +export function summarizeAustralianSourceSelection( + candidates: SearchResult[], + selected: SearchResult[], +): AustralianSourceSelectionSummary { + const candidateClassifications = candidates.map(australianSourceClassification); + const selectedTiers = selected.map(australianSourceTier); + const authoritativeDocuments = new Set( + selected + .filter((result) => isAustralianSourceTier(australianSourceTier(result))) + .map((result) => result.document_id), + ); + + return { + candidateCount: candidates.length, + selectedCount: selected.length, + australianCandidateCount: candidateClassifications.filter((item) => isAustralianSourceTier(item.tier)).length, + australianSelectedCount: selectedTiers.filter(isAustralianSourceTier).length, + waCandidateCount: candidateClassifications.filter((item) => item.tier === "wa_validated").length, + waSelectedCount: selectedTiers.filter((tier) => tier === "wa_validated").length, + authoritativeAustralianDocumentCount: authoritativeDocuments.size, + authorityConflictCount: candidateClassifications.filter((item) => item.conflict).length, + usedSupplementaryFallback: selectedTiers.some((tier) => tier === "supplementary"), + }; +} diff --git a/src/lib/rag-context-selection.ts b/src/lib/rag-context-selection.ts index 58143d41e..d4f05e7d0 100644 --- a/src/lib/rag-context-selection.ts +++ b/src/lib/rag-context-selection.ts @@ -1,4 +1,7 @@ import type { RagAnswer, RagQueryClass, SearchResult } from "@/lib/types"; +import { selectAustralianClinicalContext } from "@/lib/australian-source-priority"; + +export { summarizeAustralianSourceSelection } from "@/lib/australian-source-priority"; const fastRoutineModelContextLimit = 4; @@ -28,15 +31,12 @@ export function selectModelContextResults(args: { crossDocument: boolean; results: SearchResult[]; }) { + if (args.queryClass === "medication_dose_risk" || args.queryClass === "table_threshold") { + return selectAustralianClinicalContext(args.results); + } const results = capPerDocumentCrowding(args.results); if (args.routeMode !== "fast") return results; - if ( - args.crossDocument || - args.queryClass === "comparison" || - args.queryClass === "broad_summary" || - args.queryClass === "medication_dose_risk" || - args.queryClass === "table_threshold" - ) { + if (args.crossDocument || args.queryClass === "comparison" || args.queryClass === "broad_summary") { return results; } return results.slice(0, fastRoutineModelContextLimit); diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 7db8736a1..8eabe515d 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -26,8 +26,12 @@ import { } from "@/lib/rag-quote-verification"; import { applyNumericVerification } from "@/lib/answer-verification"; export { applyNumericVerification, unboldUnverifiedNumbers } from "@/lib/answer-verification"; -import { capPerDocumentCrowding, selectModelContextResults } from "@/lib/rag-context-selection"; -export { capPerDocumentCrowding, selectModelContextResults } from "@/lib/rag-context-selection"; +import { selectModelContextResults, summarizeAustralianSourceSelection } from "@/lib/rag-context-selection"; +export { + capPerDocumentCrowding, + selectModelContextResults, + summarizeAustralianSourceSelection, +} from "@/lib/rag-context-selection"; import { buildExtractiveAnswer, cleanAnswerSectionHeading, @@ -3870,6 +3874,42 @@ function annotateAnswerWithDiagnostics( }; } +function buildContextDerivedArtifacts(query: string, results: SearchResult[]) { + const quoteCards = extractQuoteCards(results, query); + const memoryCardsUsed = collectMemoryCards(results); + return { + relevance: buildEvidenceRelevance(query, results), + quoteCards, + documentBreakdown: buildDocumentBreakdown(results, quoteCards), + smartPanel: buildSmartPanel(query, results), + evidenceSummary: buildEvidenceSummary(results, quoteCards), + sourceCoverage: buildSourceCoverage(results), + conflictsOrGaps: detectConflictsOrGaps(results), + visualEvidence: buildVisualEvidence(results), + bestSource: selectBestSourceRecommendation(results, quoteCards), + memoryCardsUsed, + indexingQuality: buildIndexingQuality(results, memoryCardsUsed), + scoreExplanations: buildAnswerScoreExplanations(results), + }; +} + +export function isCacheableGroundedGenerationFallback( + answer: Pick< + RagAnswer, + "routingMode" | "routingReason" | "grounded" | "confidence" | "citations" | "unverifiedNumericTokens" + >, +) { + return ( + answer.routingMode === "extractive" && + answer.grounded && + answer.confidence !== "unsupported" && + answer.citations.length > 0 && + (answer.unverifiedNumericTokens?.length ?? 0) === 0 && + /(?:source_backed_extractive_fallback|comparison_source_safe_fallback)/.test(answer.routingReason ?? "") && + !/source_backed_review_fallback/.test(answer.routingReason ?? "") + ); +} + /** Answer question. */ export async function answerQuestion(query: string, documentId?: string) { return answerQuestionWithScope({ query, documentId, allowGlobalSearch: true }); @@ -4021,7 +4061,6 @@ async function answerQuestionWithScopeUncoalesced( const results = annotateSearchResults(answerFocusQuery, answerRanking.rankedResults); const crossDocumentPlan = buildCrossDocumentSynthesisPlan(answerFocusQuery, results, queryClass); const answerInputResults = crossDocumentPlan.enabled ? crossDocumentPlan.results : results; - const relevance = buildEvidenceRelevance(answerFocusQuery, answerInputResults); const crossDocumentFusionBrief = crossDocumentPlan.enabled ? buildCrossDocumentFusionBrief(answerFocusQuery, answerInputResults) : null; @@ -4039,16 +4078,20 @@ async function answerQuestionWithScopeUncoalesced( cross_document_fusion_source_chunk_ids: crossDocumentFusionBrief?.sourceChunkIds ?? [], }; const searchLatencyMs = Date.now() - searchStartedAt; - const quoteCards = extractQuoteCards(answerInputResults, answerFocusQuery); - const documentBreakdown = buildDocumentBreakdown(answerInputResults, quoteCards); - const smartPanel = buildSmartPanel(answerFocusQuery, answerInputResults); - const evidenceSummary = buildEvidenceSummary(answerInputResults, quoteCards); - const sourceCoverage = buildSourceCoverage(answerInputResults); - const conflictsOrGaps = detectConflictsOrGaps(answerInputResults); - const visualEvidence = buildVisualEvidence(answerInputResults); - const bestSource = selectBestSourceRecommendation(answerInputResults, quoteCards); - const memoryCardsUsed = collectMemoryCards(answerInputResults); - const indexingQuality = buildIndexingQuality(answerInputResults, memoryCardsUsed); + const { + relevance, + quoteCards, + documentBreakdown, + smartPanel, + evidenceSummary, + sourceCoverage, + conflictsOrGaps, + visualEvidence, + bestSource, + memoryCardsUsed, + indexingQuality, + scoreExplanations: answerScoreExplanations, + } = buildContextDerivedArtifacts(answerFocusQuery, answerInputResults); const memoryLogMetadata = { memory_card_count: memoryCardsUsed.length, memory_top_score: Number( @@ -4062,7 +4105,6 @@ async function answerQuestionWithScopeUncoalesced( indexing_extraction_quality: indexingQuality.extractionQuality, indexing_stale: indexingQuality.stale, }; - const answerScoreExplanations = buildAnswerScoreExplanations(answerInputResults); const scoreLogMetadata = scoreExplanationLogMetadata(answerScoreExplanations); const emptyPanel = buildSmartPanel(answerFocusQuery, []); const relatedDocumentsPromise = buildRelatedDocumentsSafe({ @@ -4712,16 +4754,21 @@ ${qualityRetryInstruction}` async function buildGenerationFallbackAnswer( error: unknown, relatedDocuments: RelatedDocument[], + fallbackResults: SearchResult[], + fallbackArtifacts: ReturnType, ): Promise { - const hasSources = answerInputResults.length > 0; - const fallbackCitations = compactCitations(answerInputResults); + const hasSources = fallbackResults.length > 0; + const fallbackCitations = compactCitations(fallbackResults); const sanitizedReason = summarizeGenerationFailureReason(error); - const fallbackBestSource = hasSources - ? (selectBestSourceRecommendation(answerInputResults, quoteCards) ?? bestSource) - : null; + const fallbackBestSource = hasSources ? fallbackArtifacts.bestSource : null; const fallbackSmartPanel = hasSources - ? { ...smartPanel, relevance, bestSource: fallbackBestSource, relatedDocuments } - : { ...emptyPanel, relevance, relatedDocuments }; + ? { + ...fallbackArtifacts.smartPanel, + relevance: fallbackArtifacts.relevance, + bestSource: fallbackBestSource, + relatedDocuments, + } + : { ...emptyPanel, relevance: fallbackArtifacts.relevance, relatedDocuments }; return { answer: boldHighYieldClinicalText( @@ -4731,9 +4778,9 @@ ${qualityRetryInstruction}` args.query, ), grounded: false, - confidence: hasSources ? deriveConfidence(answerInputResults, fallbackCitations) : "unsupported", + confidence: hasSources ? deriveConfidence(fallbackResults, fallbackCitations) : "unsupported", citations: hasSources ? fallbackCitations : [], - sources: answerInputResults, + sources: fallbackResults, modelUsed: null, openAIRequestIds, openAIUsage: hasOpenAIUsage(openAIUsage) ? openAIUsage : undefined, @@ -4741,7 +4788,8 @@ ${qualityRetryInstruction}` routingReason: `${route.reason}; generation_fallback:${sanitizedReason}`, queryClass, queryAnalysis, - responseMode: buildCurrentSmartApiPlan("unsupported", `${route.reason}; generation_fallback`).displayMode, + responseMode: buildCurrentSmartApiPlan("unsupported", `${route.reason}; generation_fallback`, fallbackResults) + .displayMode, latencyTimings: { search_cache_hit: search.telemetry.search_cache_hit, shared_cache_hit: search.telemetry.shared_cache_hit, @@ -4771,21 +4819,21 @@ ${qualityRetryInstruction}` total_latency_ms: Date.now() - startedAt, }, answerSections: [], - quoteCards: hasSources ? reconcileQuoteCards(quoteCards, answerInputResults, args.query) : [], - visualEvidence: hasSources ? visualEvidence : [], + quoteCards: hasSources ? reconcileQuoteCards(fallbackArtifacts.quoteCards, fallbackResults, args.query) : [], + visualEvidence: hasSources ? fallbackArtifacts.visualEvidence : [], bestSource: hasSources ? fallbackBestSource : null, - documentBreakdown: hasSources ? documentBreakdown : [], - evidenceSummary: hasSources ? evidenceSummary : emptyPanel.evidenceSummary, - sourceCoverage: hasSources ? sourceCoverage : emptyPanel.sourceCoverage, - conflictsOrGaps: hasSources ? conflictsOrGaps : [], + documentBreakdown: hasSources ? fallbackArtifacts.documentBreakdown : [], + evidenceSummary: hasSources ? fallbackArtifacts.evidenceSummary : emptyPanel.evidenceSummary, + sourceCoverage: hasSources ? fallbackArtifacts.sourceCoverage : emptyPanel.sourceCoverage, + conflictsOrGaps: hasSources ? fallbackArtifacts.conflictsOrGaps : [], smartPanel: fallbackSmartPanel, relatedDocuments, - relevance, - memoryCardsUsed: hasSources ? memoryCardsUsed : [], + relevance: fallbackArtifacts.relevance, + memoryCardsUsed: hasSources ? fallbackArtifacts.memoryCardsUsed : [], indexingVersion: ragDeepMemoryVersion, - indexingQuality, - smartApiPlan: buildCurrentSmartApiPlan("unsupported", `${route.reason}; generation_fallback`), - scoreExplanations: answerScoreExplanations, + indexingQuality: fallbackArtifacts.indexingQuality, + smartApiPlan: buildCurrentSmartApiPlan("unsupported", `${route.reason}; generation_fallback`, fallbackResults), + scoreExplanations: fallbackArtifacts.scoreExplanations, } satisfies RagAnswer; } @@ -4795,6 +4843,18 @@ ${qualityRetryInstruction}` crossDocument: crossDocumentPlan.enabled, results: answerInputResults, }); + const strongRetryContextResults = selectModelContextResults({ + routeMode: "strong", + queryClass, + crossDocument: crossDocumentPlan.enabled, + results: answerInputResults, + }); + const generationFallbackResults = strongRetryContextResults; + const generationFallbackArtifacts = buildContextDerivedArtifacts(answerFocusQuery, generationFallbackResults); + const generationFallbackSelectionSummary = summarizeAustralianSourceSelection( + answerInputResults, + generationFallbackResults, + ); try { await args.onProgress?.({ stage: "generating", @@ -4833,7 +4893,7 @@ ${qualityRetryInstruction}` args.onRevising?.(retryReason); // Widen the retry context from the trimmed fast set to the full result set, but keep the P9 // per-document crowding cap — the strong-initial route is capped, so the retry must be too. - packedContextResults = await packContextForGeneration(capPerDocumentCrowding(answerInputResults)); + packedContextResults = await packContextForGeneration(strongRetryContextResults); // Boost the cap: a max_output_tokens truncation retried on the SAME budget with MORE // reasoning (strong) just re-truncates. This is the truncation self-heal. generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults, { @@ -4915,7 +4975,7 @@ ${qualityRetryInstruction}` }); args.onRevising?.(retryReason); // Same as the truncation retry above: widen but keep the P9 per-document crowding cap. - packedContextResults = await packContextForGeneration(capPerDocumentCrowding(answerInputResults)); + packedContextResults = await packContextForGeneration(strongRetryContextResults); // Strong spends more reasoning tokens than the fast attempt it is replacing, so it needs // the boosted cap to avoid truncating (and degrading to unsupported) on the escalation. generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults, { @@ -5148,66 +5208,96 @@ ${qualityRetryInstruction}` mode: "unsupported", reason: "generation_fallback", }); - const baseFallbackAnswer = await buildGenerationFallbackAnswer(error, relatedDocuments); + const baseFallbackAnswer = await buildGenerationFallbackAnswer( + error, + relatedDocuments, + generationFallbackResults, + generationFallbackArtifacts, + ); const sanitizedReason = summarizeGenerationFailureReason(error); const comparisonFallbackAnswer = queryClass === "comparison" ? (buildComparisonAnswer({ query: args.query, - results: answerInputResults, + results: generationFallbackResults, selectedDocuments: explicitlySelectedComparisonDocuments, routeReason: `${route.reason}; generation_fallback:${sanitizedReason}; comparison_source_safe_fallback`, timings: baseFallbackAnswer.latencyTimings, }) ?? buildComparisonEvidenceGapAnswer({ query: args.query, - results: answerInputResults, + results: generationFallbackResults, selectedDocuments: explicitlySelectedComparisonDocuments, routeReason: `${route.reason}; generation_fallback:${sanitizedReason}; comparison_evidence_gap`, timings: baseFallbackAnswer.latencyTimings, })) : null; const canRecoverGenerationErrorExtractively = - queryClass !== "comparison" && - answerInputResults.length > 0 && - baseFallbackAnswer.citations.length > 0 && - !/(?:max_output_tokens|incomplete)/i.test(sanitizedReason); - const extractiveFallbackAnswer = canRecoverGenerationErrorExtractively - ? { - ...buildExtractiveAnswer({ - query: args.query, - queryClass, - results: answerInputResults, - quoteCards, - documentBreakdown, - evidenceSummary, - sourceCoverage, - conflictsOrGaps, - visualEvidence, - bestSource, - smartPanel: { ...smartPanel, relevance, bestSource, relatedDocuments }, + queryClass !== "comparison" && generationFallbackResults.length > 0 && baseFallbackAnswer.citations.length > 0; + const extractiveFallbackRouteReason = `${route.reason}; generation_fallback:${sanitizedReason}; source_backed_extractive_fallback`; + const buildExtractiveFallbackCandidate = (candidateResults: SearchResult[]) => { + const candidateArtifacts = + candidateResults === generationFallbackResults + ? generationFallbackArtifacts + : buildContextDerivedArtifacts(answerFocusQuery, candidateResults); + const candidatePlan = buildCurrentSmartApiPlan("extractive", extractiveFallbackRouteReason, candidateResults); + return { + ...buildExtractiveAnswer({ + query: args.query, + queryClass, + results: candidateResults, + quoteCards: candidateArtifacts.quoteCards, + documentBreakdown: candidateArtifacts.documentBreakdown, + evidenceSummary: candidateArtifacts.evidenceSummary, + sourceCoverage: candidateArtifacts.sourceCoverage, + conflictsOrGaps: candidateArtifacts.conflictsOrGaps, + visualEvidence: candidateArtifacts.visualEvidence, + bestSource: candidateArtifacts.bestSource, + smartPanel: { + ...candidateArtifacts.smartPanel, + relevance: candidateArtifacts.relevance, + bestSource: candidateArtifacts.bestSource, relatedDocuments, - routeReason: `${route.reason}; generation_fallback:${sanitizedReason}; source_backed_extractive_fallback`, - timings: baseFallbackAnswer.latencyTimings, - }), - openAIRequestIds, - openAIUsage: hasOpenAIUsage(openAIUsage) ? openAIUsage : undefined, - queryAnalysis, - memoryCardsUsed, - indexingVersion: ragDeepMemoryVersion, - indexingQuality, - smartApiPlan: buildCurrentSmartApiPlan( - "extractive", - `${route.reason}; generation_fallback:${sanitizedReason}; source_backed_extractive_fallback`, - ), - responseMode: buildCurrentSmartApiPlan( - "extractive", - `${route.reason}; generation_fallback:${sanitizedReason}; source_backed_extractive_fallback`, - ).displayMode, - relevance, - scoreExplanations: answerScoreExplanations, - } + }, + relatedDocuments, + routeReason: extractiveFallbackRouteReason, + timings: baseFallbackAnswer.latencyTimings, + }), + openAIRequestIds, + openAIUsage: hasOpenAIUsage(openAIUsage) ? openAIUsage : undefined, + queryAnalysis, + memoryCardsUsed: candidateArtifacts.memoryCardsUsed, + indexingVersion: ragDeepMemoryVersion, + indexingQuality: candidateArtifacts.indexingQuality, + smartApiPlan: candidatePlan, + responseMode: candidatePlan.displayMode, + relevance: candidateArtifacts.relevance, + scoreExplanations: candidateArtifacts.scoreExplanations, + } satisfies RagAnswer; + }; + const isSafeExtractiveFallbackCandidate = (candidate: RagAnswer) => { + if (!candidate.grounded || candidate.confidence === "unsupported") return false; + if (generatedAnswerQualityFailureReason(candidate, args.query, queryClass)) return false; + const verified = applyNumericVerification(cloneAnswer(candidate)); + return ( + verified.grounded && + verified.confidence !== "unsupported" && + (verified.unverifiedNumericTokens?.length ?? 0) === 0 + ); + }; + let extractiveFallbackAnswer = canRecoverGenerationErrorExtractively + ? buildExtractiveFallbackCandidate(generationFallbackResults) : null; + if ( + extractiveFallbackAnswer && + (queryClass === "medication_dose_risk" || queryClass === "table_threshold") && + !isSafeExtractiveFallbackCandidate(extractiveFallbackAnswer) + ) { + extractiveFallbackAnswer = + generationFallbackResults + .map((result) => buildExtractiveFallbackCandidate([result])) + .find(isSafeExtractiveFallbackCandidate) ?? extractiveFallbackAnswer; + } const extractiveFallbackQualityReason = extractiveFallbackAnswer ? generatedAnswerQualityFailureReason(extractiveFallbackAnswer, args.query, queryClass) : null; @@ -5219,23 +5309,28 @@ ${qualityRetryInstruction}` const generationFallbackAnswer = comparisonFallbackAnswer ? { ...comparisonFallbackAnswer, - quoteCards, - documentBreakdown, - evidenceSummary, - sourceCoverage, - conflictsOrGaps, - visualEvidence, - bestSource, + quoteCards: generationFallbackArtifacts.quoteCards, + documentBreakdown: generationFallbackArtifacts.documentBreakdown, + evidenceSummary: generationFallbackArtifacts.evidenceSummary, + sourceCoverage: generationFallbackArtifacts.sourceCoverage, + conflictsOrGaps: generationFallbackArtifacts.conflictsOrGaps, + visualEvidence: generationFallbackArtifacts.visualEvidence, + bestSource: generationFallbackArtifacts.bestSource, relatedDocuments, - smartPanel: { ...smartPanel, relevance, bestSource, relatedDocuments }, + smartPanel: { + ...generationFallbackArtifacts.smartPanel, + relevance: generationFallbackArtifacts.relevance, + bestSource: generationFallbackArtifacts.bestSource, + relatedDocuments, + }, openAIRequestIds, openAIUsage: hasOpenAIUsage(openAIUsage) ? openAIUsage : undefined, queryAnalysis, - memoryCardsUsed, + memoryCardsUsed: generationFallbackArtifacts.memoryCardsUsed, indexingVersion: ragDeepMemoryVersion, - indexingQuality, - relevance, - scoreExplanations: answerScoreExplanations, + indexingQuality: generationFallbackArtifacts.indexingQuality, + relevance: generationFallbackArtifacts.relevance, + scoreExplanations: generationFallbackArtifacts.scoreExplanations, } : extractiveFallbackAnswer && sourceBackedReviewReason ? (() => { @@ -5250,15 +5345,15 @@ ${qualityRetryInstruction}` ...baseFallbackAnswer, answer: boldHighYieldClinicalText(sourceBackedGenerationTimeoutAnswer(args.query), args.query), grounded: true, - confidence: deriveConfidence(answerInputResults, baseFallbackAnswer.citations), + confidence: deriveConfidence(generationFallbackResults, baseFallbackAnswer.citations), routingMode: "extractive", routingReason: reviewRouteReason, queryAnalysis, responseMode: reviewPlan.displayMode, smartApiPlan: reviewPlan, answerSections: [], - relevance, - scoreExplanations: answerScoreExplanations, + relevance: generationFallbackArtifacts.relevance, + scoreExplanations: generationFallbackArtifacts.scoreExplanations, } satisfies RagAnswer; })() : (extractiveFallbackAnswer ?? baseFallbackAnswer); @@ -5272,7 +5367,7 @@ ${qualityRetryInstruction}` owner_id: args.ownerId ?? null, query: args.query, answer: fallbackAnswer.answer, - source_chunk_ids: answerInputResults.map((result) => result.id), + source_chunk_ids: generationFallbackResults.map((result) => result.id), model: null, metadata: { document_id: args.documentId ?? null, @@ -5298,6 +5393,12 @@ ${qualityRetryInstruction}` ...memoryLogMetadata, ...scoreLogMetadata, ...searchTelemetryDecisionMetadata(), + source_authority_candidate_count: generationFallbackSelectionSummary.candidateCount, + source_authority_selected_count: generationFallbackSelectionSummary.selectedCount, + australian_source_count: generationFallbackSelectionSummary.australianSelectedCount, + wa_source_count: generationFallbackSelectionSummary.waSelectedCount, + source_authority_conflict_count: generationFallbackSelectionSummary.authorityConflictCount, + used_supplementary_fallback: generationFallbackSelectionSummary.usedSupplementaryFallback, cited_chunk_count: fallbackAnswer.citations.length, quote_count: fallbackAnswer.quoteCards?.length ?? 0, visual_evidence_count: fallbackAnswer.visualEvidence?.length ?? 0, @@ -5325,7 +5426,9 @@ ${qualityRetryInstruction}` }, }); - await setCachedAnswer(args, fallbackAnswer, { indexingVersionAtRetrievalStart }); + if (isCacheableGroundedGenerationFallback(fallbackAnswer)) { + await setCachedAnswer(args, fallbackAnswer, { indexingVersionAtRetrievalStart }); + } return fallbackAnswer; } } diff --git a/src/lib/source-authority-registry.ts b/src/lib/source-authority-registry.ts new file mode 100644 index 000000000..21f9a6dcd --- /dev/null +++ b/src/lib/source-authority-registry.ts @@ -0,0 +1,385 @@ +import { normalizeSourceMetadata } from "@/lib/source-metadata"; +import type { ClinicalSourceMetadata } from "@/lib/types"; + +export type AustralianSourceTier = "wa_validated" | "australian_national" | "australian_state" | "supplementary"; + +export type SourceAuthorityScope = "wa" | "australian_national" | "australian_state" | "international"; + +export type SourceAuthorityDefinition = { + key: string; + codes: readonly string[]; + publisher: string; + publisherAliases: readonly string[]; + jurisdictions: readonly string[]; + scope: SourceAuthorityScope; + tier: Exclude | "supplementary"; +}; + +export type SourceAuthorityConflict = "publisher_mismatch" | "jurisdiction_mismatch"; + +export type SourceAuthorityClassification = { + tier: AustralianSourceTier; + authorityTier: SourceAuthorityDefinition["tier"] | null; + authority: SourceAuthorityDefinition | null; + matchedBy: "publisher_code" | "publisher_alias" | "none"; + codeKnown: boolean; + conflict: boolean; + conflicts: SourceAuthorityConflict[]; + eligibilityReasons: string[]; +}; + +const waJurisdictions = ["Australia/WA", "Australia/Western Australia", "Western Australia", "WA"] as const; +const nationalJurisdictions = [ + "Australia", + "Australia/National", + "Australia/Commonwealth", + "Australian national", + "Commonwealth of Australia", +] as const; + +function authority( + definition: Omit & { publisherAliases?: readonly string[] }, +): SourceAuthorityDefinition { + return { + ...definition, + publisherAliases: [definition.publisher, ...(definition.publisherAliases ?? [])], + }; +} + +/** + * Canonical authority registry shared by runtime selection and metadata tooling. + * + * Runtime classification is deliberately metadata-only: titles, filenames and + * document prose are never accepted as authority evidence here. + */ +export const sourceAuthorityRegistry = [ + authority({ + key: "wa-health", + codes: ["WAHEALTH", "DOHWA"], + publisher: "WA Health", + publisherAliases: [ + "WA Department of Health", + "Western Australian Department of Health", + "Department of Health Western Australia", + "Government of Western Australia Department of Health", + ], + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "armadale-kalamunda-group", + codes: ["AKG"], + publisher: "Armadale Kalamunda Group", + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "camhs-wa", + codes: ["CAMHS"], + publisher: "Child and Adolescent Mental Health Service", + publisherAliases: ["Child and Adolescent Mental Health Services"], + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "east-metropolitan-health-service", + codes: ["EMHS"], + publisher: "East Metropolitan Health Service", + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "fiona-stanley-fremantle-hospitals-group", + codes: ["FSH", "FSFH", "FSFHG"], + publisher: "Fiona Stanley Fremantle Hospitals Group", + publisherAliases: ["Fiona Stanley Hospital", "Fiona Stanley Fremantle Hospitals"], + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "king-edward-memorial-hospital", + codes: ["KEMH", "KEMHS"], + publisher: "King Edward Memorial Hospital", + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "north-metropolitan-health-service", + codes: ["NMHS"], + publisher: "North Metropolitan Health Service", + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "peel-health-campus", + codes: ["PHC"], + publisher: "Peel Health Campus", + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "rockingham-peel-group", + codes: ["RKPG"], + publisher: "Rockingham Peel Group", + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "royal-perth-bentley-group", + codes: ["RPBG"], + publisher: "Royal Perth Bentley Group", + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "south-metropolitan-health-service", + codes: ["SMHS"], + publisher: "South Metropolitan Health Service", + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "wa-country-health-service", + codes: ["WACHS"], + publisher: "WA Country Health Service", + publisherAliases: ["Western Australia Country Health Service"], + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), + authority({ + key: "acsqhc", + codes: ["ACSQHC"], + publisher: "Australian Commission on Safety and Quality in Health Care", + publisherAliases: ["Australian Commission on Safety and Quality in Healthcare"], + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), + authority({ + key: "australian-department-of-health", + codes: ["AUSDOH", "DOHA"], + publisher: "Australian Government Department of Health and Aged Care", + publisherAliases: ["Australian Department of Health and Aged Care", "Australian Government Department of Health"], + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), + authority({ + key: "nhmrc", + codes: ["NHMRC"], + publisher: "National Health and Medical Research Council", + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), + authority({ + key: "nps-medicinewise", + codes: ["NPS"], + publisher: "NPS MedicineWise", + publisherAliases: ["National Prescribing Service"], + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), + authority({ + key: "pbs", + codes: ["PBS"], + publisher: "Pharmaceutical Benefits Scheme", + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), + authority({ + key: "racgp", + codes: ["RACGP"], + publisher: "Royal Australian College of General Practitioners", + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), + authority({ + key: "ranzcp", + codes: ["RANZCP"], + publisher: "Royal Australian and New Zealand College of Psychiatrists", + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), + authority({ + key: "tga", + codes: ["TGA"], + publisher: "Therapeutic Goods Administration", + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), + ...[ + ["act-health", "ACTHEALTH", "ACT Health", "Australia/ACT"], + ["nsw-health", "NSWHEALTH", "NSW Health", "Australia/NSW"], + ["nt-health", "NTHEALTH", "NT Health", "Australia/NT"], + ["queensland-health", "QLDHEALTH", "Queensland Health", "Australia/QLD"], + ["sa-health", "SAHEALTH", "SA Health", "Australia/SA"], + ["tasmania-health", "TASHEALTH", "Tasmanian Department of Health", "Australia/TAS"], + ["victoria-health", "VICHEALTH", "Victorian Department of Health", "Australia/VIC"], + ].map(([key, code, publisher, jurisdiction]) => + authority({ + key, + codes: [code], + publisher, + jurisdictions: [jurisdiction], + scope: "australian_state", + tier: "australian_state", + }), + ), + authority({ + key: "bmj-best-practice", + codes: ["BMJ"], + publisher: "BMJ Best Practice", + publisherAliases: ["BMJ Publishing Group"], + jurisdictions: ["International", "Global", "United Kingdom", "UK"], + scope: "international", + tier: "supplementary", + }), + authority({ + key: "nice", + codes: ["NICE"], + publisher: "National Institute for Health and Care Excellence", + jurisdictions: ["United Kingdom", "UK", "England"], + scope: "international", + tier: "supplementary", + }), + authority({ + key: "world-health-organization", + codes: ["WHO"], + publisher: "World Health Organization", + jurisdictions: ["International", "Global"], + scope: "international", + tier: "supplementary", + }), +] satisfies SourceAuthorityDefinition[]; + +export function normalizeSourceAuthorityText(value: string | null | undefined) { + return (value ?? "") + .normalize("NFKC") + .trim() + .toLowerCase() + .replace(/&/g, " and ") + .replace(/[^a-z0-9/]+/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function normalizePublisherCode(value: string | null | undefined) { + return (value ?? "").trim().toUpperCase(); +} + +const authorityByCode = new Map( + sourceAuthorityRegistry.flatMap((entry) => entry.codes.map((code) => [normalizePublisherCode(code), entry] as const)), +); +const authorityByPublisher = new Map( + sourceAuthorityRegistry.flatMap((entry) => + entry.publisherAliases.map((publisher) => [normalizeSourceAuthorityText(publisher), entry] as const), + ), +); +const genericWaPublishers = new Set( + sourceAuthorityRegistry + .find((entry) => entry.key === "wa-health")! + .publisherAliases.map((publisher) => normalizeSourceAuthorityText(publisher)), +); + +export function sourceAuthorityForPublisherCode(code: string | null | undefined) { + return authorityByCode.get(normalizePublisherCode(code)) ?? null; +} + +export function sourceAuthorityForPublisher(publisher: string | null | undefined) { + return authorityByPublisher.get(normalizeSourceAuthorityText(publisher)) ?? null; +} + +function publisherCompatible(authorityEntry: SourceAuthorityDefinition, publisher: string) { + const normalizedPublisher = normalizeSourceAuthorityText(publisher); + if (!normalizedPublisher) return true; + if (authorityEntry.publisherAliases.some((alias) => normalizeSourceAuthorityText(alias) === normalizedPublisher)) { + return true; + } + return authorityEntry.scope === "wa" && genericWaPublishers.has(normalizedPublisher); +} + +function jurisdictionCompatible(authorityEntry: SourceAuthorityDefinition, jurisdiction: string) { + const normalizedJurisdiction = normalizeSourceAuthorityText(jurisdiction); + if (!normalizedJurisdiction) return true; + return authorityEntry.jurisdictions.some( + (candidate) => normalizeSourceAuthorityText(candidate) === normalizedJurisdiction, + ); +} + +function isCurrentUsableDocument(metadata: ClinicalSourceMetadata) { + return ( + metadata.source_kind !== "registry_record" && + metadata.document_status === "current" && + metadata.extraction_quality === "good" + ); +} + +function isLocallyValidated(metadata: ClinicalSourceMetadata) { + return ( + metadata.clinical_validation_status === "approved" || metadata.clinical_validation_status === "locally_reviewed" + ); +} + +export function classifySourceAuthority(input: unknown): SourceAuthorityClassification { + const metadata = normalizeSourceMetadata(input); + const code = normalizePublisherCode(metadata.publisher_code); + const codeAuthority = sourceAuthorityForPublisherCode(code); + const publisherAuthority = sourceAuthorityForPublisher(metadata.publisher); + const authorityEntry = codeAuthority ?? publisherAuthority; + const matchedBy = codeAuthority ? "publisher_code" : publisherAuthority ? "publisher_alias" : "none"; + const conflicts: SourceAuthorityConflict[] = []; + const eligibilityReasons: string[] = []; + + if (authorityEntry && metadata.publisher && !publisherCompatible(authorityEntry, metadata.publisher)) { + conflicts.push("publisher_mismatch"); + } + if (authorityEntry && metadata.jurisdiction && !jurisdictionCompatible(authorityEntry, metadata.jurisdiction)) { + conflicts.push("jurisdiction_mismatch"); + } + + if (!authorityEntry) eligibilityReasons.push("unrecognized_authority"); + if (!codeAuthority && publisherAuthority && !metadata.jurisdiction) { + eligibilityReasons.push("publisher_alias_requires_jurisdiction"); + } + if (conflicts.length > 0) eligibilityReasons.push("authority_metadata_conflict"); + if (!isCurrentUsableDocument(metadata)) eligibilityReasons.push("source_not_current_usable_document"); + if (authorityEntry?.tier === "wa_validated" && !isLocallyValidated(metadata)) { + eligibilityReasons.push("wa_source_not_locally_validated"); + } + + const eligible = + Boolean(authorityEntry) && + conflicts.length === 0 && + (Boolean(codeAuthority) || Boolean(metadata.jurisdiction)) && + isCurrentUsableDocument(metadata) && + (authorityEntry?.tier !== "wa_validated" || isLocallyValidated(metadata)); + + return { + tier: eligible ? (authorityEntry?.tier ?? "supplementary") : "supplementary", + authorityTier: authorityEntry?.tier ?? null, + authority: authorityEntry ?? null, + matchedBy, + codeKnown: Boolean(codeAuthority), + conflict: conflicts.length > 0, + conflicts, + eligibilityReasons, + }; +} diff --git a/src/lib/source-metadata.ts b/src/lib/source-metadata.ts index 79359f8d0..631087608 100644 --- a/src/lib/source-metadata.ts +++ b/src/lib/source-metadata.ts @@ -12,6 +12,10 @@ function enumOrDefault(value: unknown, allowed: Set, f return typeof value === "string" && allowed.has(value) ? (value as T) : fallback; } +function recordOrNull(value: unknown) { + return value && typeof value === "object" && !Array.isArray(value) ? { ...(value as Record) } : null; +} + export function normalizeSourceMetadata(input: unknown): ClinicalSourceMetadata { const value = input && typeof input === "object" ? (input as Record) : {}; @@ -23,6 +27,7 @@ export function normalizeSourceMetadata(input: unknown): ClinicalSourceMetadata registry_record_slug: stringOrNull(value.registry_record_slug), source_title: stringOrNull(value.source_title), publisher: stringOrNull(value.publisher), + publisher_code: stringOrNull(value.publisher_code), jurisdiction: stringOrNull(value.jurisdiction), version: stringOrNull(value.version), publication_date: stringOrNull(value.publication_date), @@ -32,6 +37,7 @@ export function normalizeSourceMetadata(input: unknown): ClinicalSourceMetadata uploaded_by: stringOrNull(value.uploaded_by), document_status: enumOrDefault(value.document_status, knownStatuses, "unknown"), clinical_validation_status: enumOrDefault(value.clinical_validation_status, knownValidation, "unverified"), + clinical_validation_evidence: recordOrNull(value.clinical_validation_evidence), extraction_quality: enumOrDefault(value.extraction_quality, knownExtraction, "unknown"), }; } diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 80c395b2a..8c53c3eba 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1516,15 +1516,96 @@ describe("RAG structured-output fallback", () => { expect(answer.answer).not.toMatch(/^Dosage and monitoring/i); }); - it("treats max-output truncation as a distinct retry and fallback reason", async () => { + it("recovers lithium dosing from validated WA evidence when both generated drafts hit max output", async () => { vi.stubEnv("OPENAI_API_KEY", "test-key"); vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0"); vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0"); + const waSource = (overrides: Partial, publisherCode: "FSH" | "EMHS") => + source({ + ...overrides, + source_metadata: { + source_title: overrides.title ?? "Lithium guideline", + publisher: + publisherCode === "FSH" ? "Fiona Stanley Fremantle Hospitals Group" : "East Metropolitan Health Service", + publisher_code: publisherCode, + jurisdiction: "Australia/WA", + version: "1", + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "current", + clinical_validation_status: "locally_reviewed", + extraction_quality: "good", + }, + }); const sources = [ + waSource( + { + id: "fsh-lithium-1", + document_id: "fsh-lithium", + title: "Lithium Therapy - Initiation and Continuation Guideline", + section_heading: "Initiation", + content: + "For lithium initiation in adults, start lithium carbonate at 250 mg at night and titrate according to the serum lithium concentration.", + }, + "FSH", + ), + waSource( + { + id: "emhs-lithium-1", + document_id: "emhs-lithium", + title: "Lithium Clinical Guideline", + section_heading: "Target range", + content: + "The usual target serum lithium concentration is 0.6 to 0.8 mmol/L for maintenance treatment in adults.", + }, + "EMHS", + ), + waSource( + { + id: "fsh-lithium-2", + document_id: "fsh-lithium", + title: "Lithium Therapy - Initiation and Continuation Guideline", + section_heading: "Monitoring after dose changes", + content: + "Measure the serum lithium concentration 12 hours after the previous dose and repeat it 5 to 7 days after a dose change.", + }, + "FSH", + ), + waSource( + { + id: "emhs-lithium-2", + document_id: "emhs-lithium", + title: "Lithium Clinical Guideline", + section_heading: "Dose adjustment", + content: + "Use a lower lithium starting dose in older adults and people with impaired renal function, with closer serum monitoring.", + }, + "EMHS", + ), source({ - content: - "Inpatient approach details for agitation and arousal management include a stepwise approach based on rating severity, route, oral options, and intramuscular options.", + id: "bmj-paediatric-depression", + document_id: "bmj-paediatric-depression", + title: "Depression in children", + content: "Psychological therapy is considered for depression in children and young people.", + source_metadata: { + source_title: "Depression in children", + publisher: "BMJ Best Practice", + publisher_code: "BMJ", + jurisdiction: "International", + version: null, + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "current", + clinical_validation_status: "unverified", + extraction_quality: "good", + }, }), ]; const rpc = vi.fn(async (name: string) => { @@ -1534,7 +1615,7 @@ describe("RAG structured-output fallback", () => { }); let requestIndex = 0; const generateStructuredTextResult = vi.fn(async () => ({ - text: '{"answer":"Use a stepwise approach', + text: '{"answer":"Lithium dosing requires', model: "gpt-5.4-mini", operation: "answer", latencyMs: 12, @@ -1556,23 +1637,32 @@ describe("RAG structured-output fallback", () => { generateStructuredTextResult, })); - const { answerQuestionWithScope } = await import("../src/lib/rag"); + const { answerQuestionWithScope, isCacheableGroundedGenerationFallback } = await import("../src/lib/rag"); const answer = await answerQuestionWithScope({ - query: "Summarize inpatient approach", + query: "Lithium dosing", ownerId: undefined, logQuery: false, skipCache: true, }); expect(generateStructuredTextResult).toHaveBeenCalledTimes(2); - expect(answer.routingMode).toBe("unsupported"); + const generationCalls = generateStructuredTextResult.mock.calls as unknown as Array<[string]>; + expect(generationCalls.every((call) => call[0].includes("start lithium carbonate at 250 mg"))).toBe(true); + expect(generationCalls.every((call) => !call[0].includes("Psychological therapy is considered"))).toBe(true); + expect(answer.routingMode).toBe("extractive"); + expect(answer.grounded).toBe(true); + expect(answer.confidence).not.toBe("unsupported"); + expect(answer.citations.length).toBeGreaterThan(0); expect(answer.routingReason).toContain("generation_fallback:provider_incomplete_max_output_tokens"); + expect(answer.routingReason).toContain("source_backed_extractive_fallback"); expect(answer.routingReason).not.toContain("OpenAI generation incomplete"); expect(answer.answerQualityTier).toBe("source_only"); - expect(answer.degradedMode).toMatchObject({ - active: true, - reason: "generation_fallback:provider_incomplete_max_output_tokens", - }); + expect(answer.answer.replace(/\*\*/g, "")).toMatch(/lithium|250 mg/i); + expect(answer.answer).not.toContain("could not generate a finalized answer"); + expect(answer.unverifiedNumericTokens ?? []).toEqual([]); + // Numeric fallback intentionally narrows the returned support to one complete + // claim/citation when a multi-source synthesis would mix figures across chunks. + expect(new Set(answer.sources.map((result) => result.document_id))).toEqual(new Set(["fsh-lithium"])); expect(answer.latencyTimings?.answer_retry_count).toBe(2); expect(answer.latencyTimings?.answer_retry_reasons).toEqual([ "fast_max_output_tokens_retry_strong", @@ -1580,6 +1670,73 @@ describe("RAG structured-output fallback", () => { ]); expect(answer.openAIRequestIds).toEqual(["req_truncated_1", "req_truncated_2"]); expect(answer.openAIUsage).toMatchObject({ output_tokens: 1300, total_tokens: 1500 }); + expect(isCacheableGroundedGenerationFallback(answer)).toBe(true); + }); + + it("fails closed instead of leaking another medication's numeric dose after generation failure", async () => { + const answer = await answerFromTextSources( + "What is the maximum sertraline dose?", + [ + source({ + id: "sertraline-source-gap", + document_id: "sertraline-source-gap-doc", + title: "Antidepressant Dose Overview", + file_name: "antidepressant-dose-overview.pdf", + section_heading: "Maximum doses", + content: + "The table lists fluoxetine 60 mg and citalopram 40 mg, but it does not state a maximum sertraline dose.", + similarity: 0.96, + hybrid_score: 0.96, + text_rank: 1.3, + }), + ], + new Error("OpenAI generation incomplete: max_output_tokens"), + ); + const { isCacheableGroundedGenerationFallback } = await import("../src/lib/rag"); + + expect(answer.answer).not.toMatch(/fluoxetine|citalopram|60 mg|40 mg/i); + expect(answer.answer).toMatch(/source|guidance|support|evidence/i); + expect(answer.routingReason).toContain("generation_fallback:provider_incomplete_max_output_tokens"); + expect(answer.routingReason).toContain("source_backed_review_fallback"); + expect(answer.unverifiedNumericTokens ?? []).toEqual([]); + expect(isCacheableGroundedGenerationFallback(answer)).toBe(false); + }); + + it("never marks the generic source-review fallback as cacheable", async () => { + const { isCacheableGroundedGenerationFallback } = await import("../src/lib/rag"); + + expect( + isCacheableGroundedGenerationFallback({ + routingMode: "unsupported", + routingReason: "strong_generation; generation_fallback:provider_timeout", + grounded: false, + confidence: "low", + citations: [], + unverifiedNumericTokens: [], + }), + ).toBe(false); + expect( + isCacheableGroundedGenerationFallback({ + routingMode: "extractive", + routingReason: + "strong_generation; generation_fallback:provider_timeout; source_backed_review_fallback; extractive_quality_gate:weak", + grounded: true, + confidence: "low", + citations: [ + { + chunk_id: "source-1", + document_id: "document-1", + title: "Source", + file_name: "source.pdf", + page_number: 1, + chunk_index: 0, + source_metadata: null, + provenance: "deterministic_support", + }, + ], + unverifiedNumericTokens: [], + }), + ).toBe(false); }); it("keeps the confidence gate active for weak ambiguous retrieval", async () => { diff --git a/tests/rag-context-budget.test.ts b/tests/rag-context-budget.test.ts index 781be094f..5cf54f90a 100644 --- a/tests/rag-context-budget.test.ts +++ b/tests/rag-context-budget.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { capPerDocumentCrowding, packedContextCacheKey, selectModelContextResults } from "../src/lib/rag"; import type { RagQueryClass, SearchResult } from "../src/lib/types"; -function source(index: number): SearchResult { +function source(index: number, overrides: Partial = {}): SearchResult { return { id: `chunk-${index}`, document_id: `doc-${index}`, @@ -16,9 +16,40 @@ function source(index: number): SearchResult { similarity: 0.9 - index * 0.01, hybrid_score: 0.9 - index * 0.01, images: [], + ...overrides, }; } +function governedSource( + index: number, + args: { + documentId: string; + publisherCode: string; + publisher: string; + jurisdiction: string; + validation?: "unverified" | "locally_reviewed" | "approved"; + }, +) { + return source(index, { + document_id: args.documentId, + source_metadata: { + source_title: `Guideline ${index}`, + publisher: args.publisher, + publisher_code: args.publisherCode, + jurisdiction: args.jurisdiction, + version: null, + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "current", + clinical_validation_status: args.validation ?? "approved", + extraction_quality: "good", + }, + }); +} + const results = Array.from({ length: 12 }, (_, index) => source(index + 1)); function select(args: { @@ -44,10 +75,23 @@ describe("RAG model context budgeting", () => { it("keeps broader context for synthesis-heavy fast routes", () => { expect(select({ routeMode: "fast", queryClass: "comparison" })).toHaveLength(12); expect(select({ routeMode: "fast", queryClass: "broad_summary" })).toHaveLength(12); - expect(select({ routeMode: "fast", queryClass: "medication_dose_risk" })).toHaveLength(12); expect(select({ routeMode: "fast", queryClass: "document_lookup", crossDocument: true })).toHaveLength(12); }); + it("bounds high-risk supplementary-only context without dropping all available evidence", () => { + const selected = select({ routeMode: "fast", queryClass: "medication_dose_risk" }); + + expect(selected).toHaveLength(6); + expect(selected.map((result) => result.id)).toEqual([ + "chunk-1", + "chunk-2", + "chunk-3", + "chunk-4", + "chunk-5", + "chunk-6", + ]); + }); + it("does not limit strong generation or non-model routes", () => { expect(select({ routeMode: "strong", queryClass: "document_lookup" })).toHaveLength(12); expect(select({ routeMode: "extractive", queryClass: "document_lookup" })).toHaveLength(12); @@ -88,6 +132,110 @@ describe("RAG model context budgeting", () => { expect(capPerDocumentCrowding(singleDoc)).toHaveLength(6); }); + it("uses four validated Australian passages across two documents without supplementary padding", () => { + const highRiskResults = [ + governedSource(1, { + documentId: "wa-doc-1", + publisherCode: "FSH", + publisher: "Fiona Stanley Fremantle Hospitals Group", + jurisdiction: "Australia/WA", + }), + governedSource(2, { + documentId: "wa-doc-2", + publisherCode: "EMHS", + publisher: "East Metropolitan Health Service", + jurisdiction: "Australia/WA", + }), + governedSource(3, { + documentId: "wa-doc-1", + publisherCode: "FSH", + publisher: "WA Health", + jurisdiction: "Australia/WA", + }), + governedSource(4, { + documentId: "wa-doc-2", + publisherCode: "EMHS", + publisher: "East Metropolitan Health Service", + jurisdiction: "Australia/WA", + }), + governedSource(5, { + documentId: "bmj-doc", + publisherCode: "BMJ", + publisher: "BMJ Best Practice", + jurisdiction: "International", + validation: "unverified", + }), + ]; + + const selected = selectModelContextResults({ + routeMode: "strong", + queryClass: "medication_dose_risk", + crossDocument: false, + results: highRiskResults, + }); + + expect(selected.map((result) => result.id)).toEqual(["chunk-1", "chunk-2", "chunk-3", "chunk-4"]); + expect(new Set(selected.map((result) => result.document_id))).toEqual(new Set(["wa-doc-1", "wa-doc-2"])); + }); + + it("keeps supplementary evidence when authoritative Australian coverage is not sufficient", () => { + const selected = selectModelContextResults({ + routeMode: "strong", + queryClass: "table_threshold", + crossDocument: false, + results: [ + governedSource(1, { + documentId: "wa-doc", + publisherCode: "WACHS", + publisher: "WA Country Health Service", + jurisdiction: "Australia/WA", + }), + governedSource(2, { + documentId: "national-doc", + publisherCode: "TGA", + publisher: "Therapeutic Goods Administration", + jurisdiction: "Australia/National", + validation: "unverified", + }), + governedSource(3, { + documentId: "bmj-doc", + publisherCode: "BMJ", + publisher: "BMJ Best Practice", + jurisdiction: "International", + validation: "unverified", + }), + ], + }); + + expect(selected.map((result) => result.id)).toEqual(["chunk-1", "chunk-2", "chunk-3"]); + }); + + it("does not promote a known international code with conflicting WA metadata", () => { + const conflict = governedSource(1, { + documentId: "conflict-doc", + publisherCode: "BMJ", + publisher: "BMJ Best Practice", + jurisdiction: "Australia/WA", + }); + const waSources = [2, 3, 4, 5].map((index) => + governedSource(index, { + documentId: index % 2 === 0 ? "wa-doc-1" : "wa-doc-2", + publisherCode: index % 2 === 0 ? "FSH" : "EMHS", + publisher: index % 2 === 0 ? "Fiona Stanley Fremantle Hospitals Group" : "East Metropolitan Health Service", + jurisdiction: "Australia/WA", + }), + ); + + const selected = selectModelContextResults({ + routeMode: "strong", + queryClass: "medication_dose_risk", + crossDocument: false, + results: [conflict, ...waSources], + }); + + expect(selected.map((result) => result.id)).toEqual(["chunk-2", "chunk-3", "chunk-4", "chunk-5"]); + }); + it("uses a stable context pack cache key for matching retry inputs", () => { const key = packedContextCacheKey(results, "broad_summary", { crossDocument: true }); const sameInputs = packedContextCacheKey([...results], "broad_summary", { crossDocument: true }); diff --git a/tests/source-metadata.test.ts b/tests/source-metadata.test.ts index 3c72e4875..ea4375011 100644 --- a/tests/source-metadata.test.ts +++ b/tests/source-metadata.test.ts @@ -7,6 +7,7 @@ import { sourceStatusLabel, validationStatusLabel, } from "../src/lib/source-metadata"; +import { classifySourceAuthority } from "../src/lib/source-authority-registry"; describe("source metadata helpers", () => { it("normalizes missing legacy metadata to explicit unknown labels without suppressing content", () => { @@ -57,6 +58,19 @@ describe("source metadata helpers", () => { expect(metadata.registry_record_slug).toBe("perth-adult-mental-health"); }); + it("preserves source authority and validation evidence metadata", () => { + const metadata = normalizeSourceMetadata({ + publisher_code: "WACHS", + clinical_validation_evidence: { reviewer: "clinical-governance", reference: "review-123" }, + }); + + expect(metadata.publisher_code).toBe("WACHS"); + expect(metadata.clinical_validation_evidence).toEqual({ + reviewer: "clinical-governance", + reference: "review-123", + }); + }); + it("preserves stale status labels for registry summaries", () => { const metadata = normalizeSourceMetadata({ source_kind: "registry_record", @@ -98,3 +112,174 @@ describe("source metadata helpers", () => { expect(line).toContain("Jurisdiction: Unknown"); }); }); + +describe("source authority classification", () => { + const usable = { + document_status: "current", + clinical_validation_status: "approved", + extraction_quality: "good", + } as const; + + it.each([ + { + label: "known WA code", + metadata: { + ...usable, + publisher_code: "FSH", + publisher: "Fiona Stanley Hospital", + jurisdiction: "Australia/WA", + }, + tier: "wa_validated", + matchedBy: "publisher_code", + }, + { + label: "generic WA Health alias", + metadata: { ...usable, publisher: "WA Health", jurisdiction: "Western Australia" }, + tier: "wa_validated", + matchedBy: "publisher_alias", + }, + { + label: "WA department alias", + metadata: { + ...usable, + publisher: "Western Australian Department of Health", + jurisdiction: "Australia/WA", + }, + tier: "wa_validated", + matchedBy: "publisher_alias", + }, + { + label: "WACHS alias with an unrecognised code", + metadata: { + ...usable, + publisher_code: "LOCAL", + publisher: "WA Country Health Service", + jurisdiction: "Australia/WA", + }, + tier: "wa_validated", + matchedBy: "publisher_alias", + }, + { + label: "Australian national code", + metadata: { + ...usable, + publisher_code: "TGA", + publisher: "Therapeutic Goods Administration", + jurisdiction: "Australia/National", + clinical_validation_status: "unverified", + }, + tier: "australian_national", + matchedBy: "publisher_code", + }, + { + label: "Australian national alias", + metadata: { + ...usable, + publisher: "Australian Commission on Safety and Quality in Healthcare", + jurisdiction: "Commonwealth of Australia", + }, + tier: "australian_national", + matchedBy: "publisher_alias", + }, + { + label: "other Australian state authority", + metadata: { ...usable, publisher: "NSW Health", jurisdiction: "Australia/NSW" }, + tier: "australian_state", + matchedBy: "publisher_alias", + }, + { + label: "international source", + metadata: { + ...usable, + publisher_code: "BMJ", + publisher: "BMJ Best Practice", + jurisdiction: "International", + }, + tier: "supplementary", + matchedBy: "publisher_code", + }, + ])("classifies $label from exact metadata", ({ metadata, tier, matchedBy }) => { + expect(classifySourceAuthority(metadata)).toMatchObject({ tier, matchedBy, conflict: false }); + }); + + it.each([ + { + label: "international code with WA jurisdiction", + metadata: { + ...usable, + publisher_code: "BMJ", + publisher: "BMJ Best Practice", + jurisdiction: "Australia/WA", + }, + conflicts: ["jurisdiction_mismatch"], + }, + { + label: "international code with a WA publisher and jurisdiction", + metadata: { ...usable, publisher_code: "BMJ", publisher: "WA Health", jurisdiction: "Australia/WA" }, + conflicts: ["publisher_mismatch", "jurisdiction_mismatch"], + }, + { + label: "WA code with a conflicting known WA publisher", + metadata: { + ...usable, + publisher_code: "FSH", + publisher: "WA Country Health Service", + jurisdiction: "Australia/WA", + }, + conflicts: ["publisher_mismatch"], + }, + { + label: "trusted alias with an incompatible jurisdiction", + metadata: { ...usable, publisher: "NSW Health", jurisdiction: "Australia/WA" }, + conflicts: ["jurisdiction_mismatch"], + }, + ])("fails closed for $label", ({ metadata, conflicts }) => { + const classification = classifySourceAuthority(metadata); + + expect(classification.tier).toBe("supplementary"); + expect(classification.conflict).toBe(true); + expect(classification.conflicts).toEqual(conflicts); + }); + + it.each([ + { + label: "jurisdiction without a trusted authority", + metadata: { ...usable, jurisdiction: "Australia/WA" }, + reason: "unrecognized_authority", + }, + { + label: "authority text only in source title", + metadata: { + ...usable, + source_title: "WA Country Health Service lithium guideline", + jurisdiction: "Australia/WA", + }, + reason: "unrecognized_authority", + }, + { + label: "publisher alias without jurisdiction", + metadata: { ...usable, publisher: "WA Health" }, + reason: "publisher_alias_requires_jurisdiction", + }, + { + label: "review-due source", + metadata: { ...usable, publisher_code: "WACHS", document_status: "review_due" }, + reason: "source_not_current_usable_document", + }, + { + label: "partial extraction", + metadata: { ...usable, publisher_code: "WACHS", extraction_quality: "partial" }, + reason: "source_not_current_usable_document", + }, + { + label: "unvalidated WA source", + metadata: { ...usable, publisher_code: "WACHS", clinical_validation_status: "unverified" }, + reason: "wa_source_not_locally_validated", + }, + ])("does not promote $label", ({ metadata, reason }) => { + const classification = classifySourceAuthority(metadata); + + expect(classification.tier).toBe("supplementary"); + expect(classification.eligibilityReasons).toContain(reason); + }); +}); From ae3ef9c8f3edfdfa4c48be17fcdc4d861a4d9f06 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:39:20 +0800 Subject: [PATCH 02/14] fix(rag): preserve relevant supplementary evidence --- src/lib/australian-source-priority.ts | 40 ++++++++---- src/lib/source-metadata.ts | 5 -- tests/rag-context-budget.test.ts | 94 +++++++++++++++++++++++++++ tests/source-metadata.test.ts | 13 ---- 4 files changed, 120 insertions(+), 32 deletions(-) diff --git a/src/lib/australian-source-priority.ts b/src/lib/australian-source-priority.ts index 5fcc4668e..7ae107c9c 100644 --- a/src/lib/australian-source-priority.ts +++ b/src/lib/australian-source-priority.ts @@ -53,6 +53,26 @@ function capClinicalContextPerDocument(results: SearchResult[], maxPerDocument: }); } +function resultRelevanceRank(result: SearchResult) { + return result.relevance?.verdict ? relevanceRank[result.relevance.verdict] : relevanceRank.direct; +} + +function isSupplementaryPadding(args: { + candidate: { result: SearchResult; tier: AustralianSourceTier }; + ranked: Array<{ result: SearchResult; tier: AustralianSourceTier }>; + sufficientAustralianChunks: number; +}) { + if (isAustralianSourceTier(args.candidate.tier)) return false; + const candidateRelevance = resultRelevanceRank(args.candidate.result); + const australianAtLeastAsRelevant = args.ranked.filter( + ({ result, tier }) => isAustralianSourceTier(tier) && resultRelevanceRank(result) <= candidateRelevance, + ); + return ( + australianAtLeastAsRelevant.length >= args.sufficientAustralianChunks && + new Set(australianAtLeastAsRelevant.map(({ result }) => result.document_id)).size >= 2 + ); +} + /** * Select a bounded clinical context while preferring authoritative Australian * evidence inside the same relevance band. Supplementary material remains @@ -69,25 +89,17 @@ export function selectAustralianClinicalContext( .map((result, index) => ({ result, index, tier: australianSourceTier(result) })) .filter(({ result }) => result.relevance?.verdict !== "none") .sort((left, right) => { - const leftRelevance = left.result.relevance?.verdict - ? relevanceRank[left.result.relevance.verdict] - : relevanceRank.direct; - const rightRelevance = right.result.relevance?.verdict - ? relevanceRank[right.result.relevance.verdict] - : relevanceRank.direct; + const leftRelevance = resultRelevanceRank(left.result); + const rightRelevance = resultRelevanceRank(right.result); return leftRelevance - rightRelevance || tierRank[left.tier] - tierRank[right.tier] || left.index - right.index; }); + const withoutSupplementaryPadding = ranked.filter( + (candidate) => !isSupplementaryPadding({ candidate, ranked, sufficientAustralianChunks }), + ); const capped = capClinicalContextPerDocument( - ranked.map(({ result }) => result), + withoutSupplementaryPadding.map(({ result }) => result), maxPerDocument, ); - const australian = capped.filter((result) => isAustralianSourceTier(australianSourceTier(result))); - const australianDocumentCount = new Set(australian.map((result) => result.document_id)).size; - - if (australian.length >= sufficientAustralianChunks && australianDocumentCount >= 2) { - return australian.slice(0, limit); - } - return capped.slice(0, limit); } diff --git a/src/lib/source-metadata.ts b/src/lib/source-metadata.ts index 631087608..c649e76c9 100644 --- a/src/lib/source-metadata.ts +++ b/src/lib/source-metadata.ts @@ -12,10 +12,6 @@ function enumOrDefault(value: unknown, allowed: Set, f return typeof value === "string" && allowed.has(value) ? (value as T) : fallback; } -function recordOrNull(value: unknown) { - return value && typeof value === "object" && !Array.isArray(value) ? { ...(value as Record) } : null; -} - export function normalizeSourceMetadata(input: unknown): ClinicalSourceMetadata { const value = input && typeof input === "object" ? (input as Record) : {}; @@ -37,7 +33,6 @@ export function normalizeSourceMetadata(input: unknown): ClinicalSourceMetadata uploaded_by: stringOrNull(value.uploaded_by), document_status: enumOrDefault(value.document_status, knownStatuses, "unknown"), clinical_validation_status: enumOrDefault(value.clinical_validation_status, knownValidation, "unverified"), - clinical_validation_evidence: recordOrNull(value.clinical_validation_evidence), extraction_quality: enumOrDefault(value.extraction_quality, knownExtraction, "unknown"), }; } diff --git a/tests/rag-context-budget.test.ts b/tests/rag-context-budget.test.ts index 5cf54f90a..ef9e82bbe 100644 --- a/tests/rag-context-budget.test.ts +++ b/tests/rag-context-budget.test.ts @@ -50,6 +50,49 @@ function governedSource( }); } +function withRelevance(result: SearchResult, verdict: NonNullable["verdict"]): SearchResult { + return { + ...result, + relevance: { + verdict, + label: verdict, + matchedTerms: verdict === "none" ? [] : ["lithium"], + missingTerms: [], + directSourceCount: verdict === "direct" ? 1 : 0, + weakSourceCount: verdict === "direct" ? 0 : 1, + score: verdict === "direct" ? 1 : verdict === "partial" ? 0.7 : verdict === "nearby" ? 0.4 : 0, + supportReason: `${verdict} test evidence`, + isSourceBacked: verdict === "direct" || verdict === "partial", + coverageScore: verdict === "direct" ? 1 : 0.5, + rankScore: verdict === "direct" ? 1 : 0.5, + titleMatchedTerms: [], + contentMatchedTerms: verdict === "none" ? [] : ["lithium"], + metadataMatchedTerms: [], + chips: [], + }, + }; +} + +function waGovernedSource(index: number, documentId: string) { + const fsh = documentId === "wa-doc-1"; + return governedSource(index, { + documentId, + publisherCode: fsh ? "FSH" : "EMHS", + publisher: fsh ? "Fiona Stanley Fremantle Hospitals Group" : "East Metropolitan Health Service", + jurisdiction: "Australia/WA", + }); +} + +function bmjSupplementarySource(index: number) { + return governedSource(index, { + documentId: "bmj-doc", + publisherCode: "BMJ", + publisher: "BMJ Best Practice", + jurisdiction: "International", + validation: "unverified", + }); +} + const results = Array.from({ length: 12 }, (_, index) => source(index + 1)); function select(args: { @@ -178,6 +221,57 @@ describe("RAG model context budgeting", () => { expect(new Set(selected.map((result) => result.document_id))).toEqual(new Set(["wa-doc-1", "wa-doc-2"])); }); + it("retains a direct supplementary passage when four Australian passages are only nearby", () => { + const selected = selectModelContextResults({ + routeMode: "strong", + queryClass: "medication_dose_risk", + crossDocument: false, + results: [ + withRelevance(bmjSupplementarySource(1), "direct"), + withRelevance(waGovernedSource(2, "wa-doc-1"), "nearby"), + withRelevance(waGovernedSource(3, "wa-doc-2"), "nearby"), + withRelevance(waGovernedSource(4, "wa-doc-1"), "nearby"), + withRelevance(waGovernedSource(5, "wa-doc-2"), "nearby"), + ], + }); + + expect(selected.map((result) => result.id)).toEqual(["chunk-1", "chunk-2", "chunk-3", "chunk-4", "chunk-5"]); + }); + + it("excludes supplementary padding before a 3+1 Australian distribution is diversity-capped", () => { + const selected = selectModelContextResults({ + routeMode: "strong", + queryClass: "medication_dose_risk", + crossDocument: false, + results: [ + withRelevance(waGovernedSource(1, "wa-doc-1"), "partial"), + withRelevance(waGovernedSource(2, "wa-doc-1"), "partial"), + withRelevance(waGovernedSource(3, "wa-doc-1"), "partial"), + withRelevance(waGovernedSource(4, "wa-doc-2"), "partial"), + withRelevance(bmjSupplementarySource(5), "partial"), + ], + }); + + expect(selected.map((result) => result.id)).toEqual(["chunk-1", "chunk-2", "chunk-4"]); + }); + + it("uses Australian-only context when four passages match supplementary relevance", () => { + const selected = selectModelContextResults({ + routeMode: "strong", + queryClass: "table_threshold", + crossDocument: false, + results: [ + withRelevance(waGovernedSource(1, "wa-doc-1"), "direct"), + withRelevance(waGovernedSource(2, "wa-doc-2"), "direct"), + withRelevance(waGovernedSource(3, "wa-doc-1"), "direct"), + withRelevance(waGovernedSource(4, "wa-doc-2"), "direct"), + withRelevance(bmjSupplementarySource(5), "direct"), + ], + }); + + expect(selected.map((result) => result.id)).toEqual(["chunk-1", "chunk-2", "chunk-3", "chunk-4"]); + }); + it("keeps supplementary evidence when authoritative Australian coverage is not sufficient", () => { const selected = selectModelContextResults({ routeMode: "strong", diff --git a/tests/source-metadata.test.ts b/tests/source-metadata.test.ts index ea4375011..3aa656508 100644 --- a/tests/source-metadata.test.ts +++ b/tests/source-metadata.test.ts @@ -58,19 +58,6 @@ describe("source metadata helpers", () => { expect(metadata.registry_record_slug).toBe("perth-adult-mental-health"); }); - it("preserves source authority and validation evidence metadata", () => { - const metadata = normalizeSourceMetadata({ - publisher_code: "WACHS", - clinical_validation_evidence: { reviewer: "clinical-governance", reference: "review-123" }, - }); - - expect(metadata.publisher_code).toBe("WACHS"); - expect(metadata.clinical_validation_evidence).toEqual({ - reviewer: "clinical-governance", - reference: "review-123", - }); - }); - it("preserves stale status labels for registry summaries", () => { const metadata = normalizeSourceMetadata({ source_kind: "registry_record", From 60283072ce80e5404441f52bcd8da6540853fad6 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:35:16 +0800 Subject: [PATCH 03/14] feat(answer): add safe progress lifecycle --- src/app/api/answer/stream/route.ts | 31 +++- src/components/ClinicalDashboard.tsx | 128 +++++++------ .../clinical-dashboard/answer-progress.ts | 106 +++++++++++ .../clinical-dashboard/answer-status.tsx | 151 ++++++++++++++- src/lib/answer-progress-public.ts | 102 +++++++++++ src/lib/rag.ts | 34 +++- tests/answer-progress-ui-smoke.spec.ts | 172 ++++++++++++++++++ tests/answer-progress.test.ts | 68 +++++++ tests/private-access-routes.test.ts | 151 +++++++++++++-- tests/rag-answer-fallback.test.ts | 29 +++ 10 files changed, 898 insertions(+), 74 deletions(-) create mode 100644 src/components/clinical-dashboard/answer-progress.ts create mode 100644 src/lib/answer-progress-public.ts create mode 100644 tests/answer-progress-ui-smoke.spec.ts create mode 100644 tests/answer-progress.test.ts diff --git a/src/app/api/answer/stream/route.ts b/src/app/api/answer/stream/route.ts index d47e89346..9db27577a 100644 --- a/src/app/api/answer/stream/route.ts +++ b/src/app/api/answer/stream/route.ts @@ -30,6 +30,7 @@ import { logger } from "@/lib/logger"; import { safeErrorLogDetails } from "@/lib/privacy"; import { startSseHeartbeat } from "@/lib/sse-heartbeat"; import { parseJsonBody } from "@/lib/validation/body"; +import { toPublicAnswerProgressEvent } from "@/lib/answer-progress-public"; import type { RagAnswer } from "@/lib/types"; export const runtime = "nodejs"; @@ -130,6 +131,8 @@ function streamAnswer(body: AnswerBody, accessScope: RetrievalAccessScope, signa return new Response( new ReadableStream({ async start(controller) { + const streamStartedAt = Date.now(); + let completionSent = false; const send = (event: string, data: unknown) => { try { controller.enqueue(encoder.encode(encodeSse(event, data))); @@ -138,18 +141,31 @@ function streamAnswer(body: AnswerBody, accessScope: RetrievalAccessScope, signa // stream is closed there is no remaining consumer for this frame. } }; + const sendProgress = (event: unknown) => { + const publicEvent = toPublicAnswerProgressEvent(event); + if (!publicEvent || (publicEvent.stage === "complete" && completionSent)) return; + if (publicEvent.stage === "complete") completionSent = true; + send("progress", publicEvent); + }; + const sendComplete = () => { + sendProgress({ stage: "complete", elapsedMs: Date.now() - streamStartedAt }); + }; + const sendFinal = (data: unknown) => { + sendComplete(); + send("final", data); + }; // Generation can go silent for long stretches (strong-route reasoning // before the first output token); heartbeat comments keep the // connection visibly alive for proxies and the client's stall watchdog. const stopHeartbeat = startSseHeartbeat((frame) => controller.enqueue(encoder.encode(frame))); - const onProgress = (event: AnswerProgressEvent) => send("progress", event); + const onProgress = (event: AnswerProgressEvent) => sendProgress(event); // Stream the answer prose as it generates (content-preserving) and signal a reset when a // provisional answer is being revised by the quality gates. const onToken = (delta: string) => send("token", { delta }); - const onRevising = (reason: string) => send("revising", { reason }); + const onRevising = () => send("revising", {}); try { - send("progress", { stage: "retrieving", message: "Searching indexed documents." }); + sendProgress({ stage: "scoping" }); const scope = isDemoMode() ? null : await resolveSearchScope({ @@ -158,8 +174,9 @@ function streamAnswer(body: AnswerBody, accessScope: RetrievalAccessScope, signa documentIds: body.documentIds ?? (body.documentId ? [body.documentId] : undefined), filters: body.filters, }); + sendProgress({ stage: "retrieving" }); if (scope?.documentIds?.length === 0) { - send("final", { + sendFinal({ answer: "The selected filters did not match any indexed documents, so I cannot generate an answer for that scope.", grounded: false, @@ -217,7 +234,7 @@ function streamAnswer(body: AnswerBody, accessScope: RetrievalAccessScope, signa }, }); } - send("final", { + sendFinal({ answer: sourceGovernanceRefusalAnswer, grounded: false, confidence: "unsupported", @@ -234,7 +251,7 @@ function streamAnswer(body: AnswerBody, accessScope: RetrievalAccessScope, signa logAnswerDiagnostics({ supabase: createAdminClient(), query: body.query, ownerId, answer }); } - send("final", { + sendFinal({ // Boundary trim only — governance warnings and diagnostics above // consumed the full answer (see answer-client-payload.ts). ...toClientAnswerPayload(answer), @@ -249,7 +266,7 @@ function streamAnswer(body: AnswerBody, accessScope: RetrievalAccessScope, signa // error — the UI's answer search uses this route, not /api/answer. const fallbackReason = nonProductionSupabaseDemoFallbackReason(error); if (fallbackReason) { - send("final", buildDemoStreamAnswer(body, fallbackReason)); + sendFinal(buildDemoStreamAnswer(body, fallbackReason)); return; } const streamError = streamErrorPayload(error); diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index aa661acf7..d9d2ed0f8 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -73,7 +73,12 @@ import { ScopeAndGovernanceNotice, UserQuestionBubble, } from "@/components/clinical-dashboard/answer-content"; -import { AnswerEmptyState, AnswerSkeleton } from "@/components/clinical-dashboard/answer-status"; +import { AnswerEmptyState, AnswerProgressStepper, AnswerSkeleton } from "@/components/clinical-dashboard/answer-status"; +import { + normalizeAnswerProgressEvent, + type AnswerProgressUpdate, + type TimedAnswerProgressUpdate, +} from "@/components/clinical-dashboard/answer-progress"; import { evidenceMapRowsFromRenderModel } from "@/components/clinical-dashboard/evidence-map-model"; import { MasterSearchHeader } from "@/components/clinical-dashboard/master-search-header"; import { useScrollHideReporter } from "@/components/clinical-dashboard/use-hide-on-scroll"; @@ -337,12 +342,6 @@ function isAbortError(error: unknown): boolean { return error instanceof DOMException && error.name === "AbortError"; } -function answerStreamProgressMessage(data: unknown) { - if (!data || typeof data !== "object") return null; - const message = (data as { message?: unknown }).message; - return typeof message === "string" && message.trim() ? message.trim() : null; -} - function findSseSeparator(buffer: string) { const match = /\r?\n\r?\n/.exec(buffer); return match ? { index: match.index, length: match[0].length } : null; @@ -350,7 +349,7 @@ function findSseSeparator(buffer: string) { async function readAnswerStream( response: Response, - onProgress: (message: string) => void, + onProgress: (progress: AnswerProgressUpdate) => void, onToken?: (delta: string) => void, onRevising?: () => void, onActivity?: () => void, @@ -375,8 +374,11 @@ async function readAnswerStream( const data = parseSseData(dataLines); if (data === null) return; if (event === "progress") { - const message = answerStreamProgressMessage(data); - if (message) onProgress(message); + const progress = normalizeAnswerProgressEvent(data); + if (progress) { + onProgress(progress); + if (progress.stage === "fallback") onRevising?.(); + } return; } if (event === "token") { @@ -861,6 +863,8 @@ export function ClinicalDashboard({ const [bulkActionBusy, setBulkActionBusy] = useState(false); const [loading, setLoading] = useState(false); const [answerProgress, setAnswerProgress] = useState(null); + const [answerProgressEvents, setAnswerProgressEvents] = useState([]); + const [answerProgressStartedAt, setAnswerProgressStartedAt] = useState(null); // In-progress streamed answer prose (content-preserving — the final committed answer still comes // from the parsed `final` payload). null between searches; `{ text, revising }` while generating. // `revising` = the quality gates dropped a provisional answer and are re-generating, so a @@ -1981,7 +1985,7 @@ export function ClinicalDashboard({ queryText: string, filtersOverride: SearchScopeFilters = scopeFilters, queryModeOverride: ClinicalQueryMode = requestQueryMode, - onProgress: (message: string) => void = setAnswerProgress, + onProgress: (progress: AnswerProgressUpdate) => void, signal?: AbortSignal, onStreamActivity?: () => void, ) { @@ -2092,6 +2096,8 @@ export function ClinicalDashboard({ setStreamingAnswer(null); setLoading(false); setAnswerProgress(null); + setAnswerProgressEvents([]); + setAnswerProgressStartedAt(null); dispatchAnswerLifecycle({ type: "cancel" }); } @@ -2164,6 +2170,7 @@ export function ClinicalDashboard({ ? (persistPrivateSearchScope(window.sessionStorage, auth.session.user.id, selectedDocumentIds) ?? undefined) : undefined); const isDifferentialsMode = modeSearch.resultKind === "differentials"; + const isAnswerRequest = modeSearch.resultKind === "answer"; // Note: no automatic mode-default label scope for Services/Forms. Applying // one on every search routed resolveSearchScope's label path over the whole // library, whose single `document_labels.in()` request produces an @@ -2233,6 +2240,29 @@ export function ClinicalDashboard({ const onProgress = (message: string | null) => { if (requestIsCurrent()) setAnswerProgress(message); }; + const onAnswerProgress = (progress: AnswerProgressUpdate) => { + if (!requestIsCurrent()) return; + setAnswerProgress(progress.message); + setAnswerProgressEvents((current) => { + const latest = current.at(-1); + if ( + latest?.stage === progress.stage && + latest.message === progress.message && + latest.resultCount === progress.resultCount && + latest.selectedContextCount === progress.selectedContextCount && + latest.australianSourceCount === progress.australianSourceCount && + latest.waSourceCount === progress.waSourceCount && + latest.usedSupplementaryFallback === progress.usedSupplementaryFallback + ) { + return current; + } + return [...current, { ...progress, receivedAt: Date.now() }].slice(-16); + }); + }; + const onRetryProgress = (message: string) => { + if (isAnswerRequest) onAnswerProgress({ stage: "retrying", message }); + else onProgress(message); + }; // A newer search already invalidated any prior request via requestId; abort // its network work too so the server stops generating, then own the signal. searchAbortRef.current?.abort(); @@ -2250,14 +2280,28 @@ export function ClinicalDashboard({ setSearchScope(null); setSourceGovernanceWarnings([]); setAnswerViewMode("high_yield"); - onProgress(modeSearch.progressLabel); + if (isAnswerRequest) { + const startedAt = Date.now(); + setAnswerProgressStartedAt(startedAt); + setAnswerProgressEvents([ + { + stage: "scoping", + message: "Preparing the clinical search scope.", + receivedAt: startedAt, + }, + ]); + setAnswerProgress("Preparing the clinical search scope."); + } else { + setAnswerProgressStartedAt(null); + setAnswerProgressEvents([]); + onProgress(modeSearch.progressLabel); + } rememberRecentQuery(trimmedQuery); // Answer-mode follow-ups: the API takes a single query string, so a short // ambiguous follow-up ("what about renal impairment?") is wrapped with the // previous turn's question before retrieval. The raw text the user typed // is what the thread displays (via displayQuery below). - const isAnswerRequest = modeSearch.resultKind === "answer"; if (isAnswerRequest) dispatchAnswerLifecycle({ type: "start", query: trimmedQuery }); const priorTurnQuery = isAnswerRequest && !replaceExistingAnswer ? latestAnswerTurnRef.current?.query : undefined; const isAnswerFollowUp = isAnswerRequest && Boolean(priorTurnQuery); @@ -2293,7 +2337,10 @@ export function ClinicalDashboard({ let emptyDifferentialsPayload: SearchResultModePayload | null = null; for (const entry of queryPlan) { - if (entry.isKeyword) onProgress("Trying keyword-based search..."); + if (entry.isKeyword) { + if (isAnswerRequest) onAnswerProgress({ stage: "retrieving", message: "Trying keyword-based search..." }); + else onProgress("Trying keyword-based search..."); + } try { const payload = @@ -2316,11 +2363,11 @@ export function ClinicalDashboard({ entry.query, filtersOverride, targetQueryMode, - onProgress, + onAnswerProgress, abortController.signal, answerWatchdog.touch, ), - onProgress, + onRetryProgress, abortController.signal, ); @@ -3251,6 +3298,11 @@ export function ClinicalDashboard({ const showAnswerHome = activeModeResultKind === "answer" && !answer && !loading && !submittedAnswerSearchActive; const showAnswerPending = activeModeResultKind === "answer" && !answer && (loading || (submittedAnswerSearchActive && !error)); + const answerProgressCompleted = answerProgressEvents.at(-1)?.stage === "complete"; + const showAnswerProgress = + activeModeResultKind === "answer" && + answerProgressEvents.length > 0 && + (loading || (Boolean(answer) && answerProgressCompleted)); const showDesktopHomeComposer = !error && (activeModeResultKind === "tools" || @@ -3787,41 +3839,15 @@ export function ClinicalDashboard({ ) : null} {searchMode !== "prescribing" && - (activeModeResultKind === "answer" && (loading || answer) ? ( - // Answer result view keeps this status slot mounted through the - // whole loading→answer swap so its height never collapses and the - // answer below it doesn't jump up (CLS). The accent chrome only - // appears while a progress message is live; otherwise it's a - // height-reserved, visually empty spacer. -
- {loading && answerProgress ? ( - <> -
+ (activeModeResultKind === "answer" ? ( + showAnswerProgress ? ( + + ) : null ) : loading && answerProgress ? (
([ + "scoping", + "retrieving", + "retrieved", + "ranking", + "generating", + "retrying", + "fallback", + "verifying", + "cached", + "complete", +]); + +function inferLegacyStage(message: string): PublicAnswerProgressStage { + if (/\b(?:scope|prepar)/i.test(message)) return "scoping"; + if (/\b(?:search|retriev|indexed documents?)/i.test(message)) return "retrieving"; + if (/\b(?:rank|select|australian|evidence)/i.test(message)) return "ranking"; + if (/\b(?:fallback|source-backed|source based)/i.test(message)) return "fallback"; + if (/\b(?:retry|revis)/i.test(message)) return "retrying"; + if (/\b(?:draft|generat|answer route)/i.test(message)) return "generating"; + if (/\b(?:check|verif|citation|finaliz)/i.test(message)) return "verifying"; + if (/\bcach/i.test(message)) return "cached"; + if (/\bready|complete/i.test(message)) return "complete"; + return "retrieving"; +} + +function finiteCount(value: unknown) { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : undefined; +} + +export function normalizeAnswerProgressEvent(data: unknown): AnswerProgressUpdate | null { + if (typeof data === "string" && data.trim()) { + const message = data.trim(); + return { stage: inferLegacyStage(message), message }; + } + if (!data || typeof data !== "object") return null; + + const value = data as Record; + const message = typeof value.message === "string" && value.message.trim() ? value.message.trim() : ""; + if (!message) return null; + const stage = + typeof value.stage === "string" && answerProgressStages.has(value.stage as PublicAnswerProgressStage) + ? (value.stage as PublicAnswerProgressStage) + : inferLegacyStage(message); + + return { + stage, + message, + resultCount: finiteCount(value.resultCount), + selectedContextCount: finiteCount(value.selectedContextCount), + australianSourceCount: finiteCount(value.australianSourceCount), + waSourceCount: finiteCount(value.waSourceCount), + usedSupplementaryFallback: + typeof value.usedSupplementaryFallback === "boolean" ? value.usedSupplementaryFallback : undefined, + elapsedMs: finiteCount(value.elapsedMs), + }; +} + +export function answerProgressStepIndex(stage: PublicAnswerProgressStage) { + if (stage === "scoping") return 0; + if (stage === "retrieving" || stage === "retrieved") return 1; + if (stage === "ranking") return 2; + if (stage === "generating" || stage === "retrying" || stage === "fallback") return 3; + return 4; +} + +/** UI copy is derived from the public stage/counts and never from an incoming message. */ +export function answerProgressDisplayMessage(progress: AnswerProgressUpdate) { + if (progress.stage === "scoping") return "Preparing the clinical search scope."; + if (progress.stage === "retrieved" && progress.resultCount !== undefined) { + return `Found ${progress.resultCount} candidate source passage${progress.resultCount === 1 ? "" : "s"}.`; + } + if (progress.stage === "retrieving" || progress.stage === "retrieved") { + return "Searching indexed clinical documents."; + } + if (progress.stage === "ranking") { + if (progress.australianSourceCount) { + const waDetail = progress.waSourceCount ? `, including ${progress.waSourceCount} WA` : ""; + return `Prioritising ${progress.australianSourceCount} Australian source passage${progress.australianSourceCount === 1 ? "" : "s"}${waDetail}.`; + } + return "Selecting the most relevant source passages."; + } + if (progress.stage === "retrying") { + return "The draft needs another pass; revising it against the evidence."; + } + if (progress.stage === "fallback") { + return "Building a source-backed answer from the selected passages."; + } + if (progress.stage === "generating") return "Drafting a cited answer from the selected passages."; + if (progress.stage === "verifying") return "Checking citations, clinical numbers, and source metadata."; + if (progress.stage === "cached") return "Loading a recent cited answer."; + return "Answer ready."; +} diff --git a/src/components/clinical-dashboard/answer-status.tsx b/src/components/clinical-dashboard/answer-status.tsx index c48802f20..3f6b48f4f 100644 --- a/src/components/clinical-dashboard/answer-status.tsx +++ b/src/components/clinical-dashboard/answer-status.tsx @@ -1,7 +1,26 @@ "use client"; -import { Clipboard, ClipboardCheck, History, MessageSquareText, Search, ShieldCheck, Upload } from "lucide-react"; +import { + Check, + Circle, + Clipboard, + ClipboardCheck, + History, + Loader2, + MessageSquareText, + Search, + ShieldCheck, + Square, + Upload, +} from "lucide-react"; +import { useEffect, useState } from "react"; +import { + answerProgressDisplayMessage, + answerProgressStepIndex, + answerProgressSteps, + type TimedAnswerProgressUpdate, +} from "@/components/clinical-dashboard/answer-progress"; import { AnswerSuggestionChips } from "@/components/clinical-dashboard/answer-suggestion-chips"; import { ModeHomeTemplate, ModeHomeVerificationFooter } from "@/components/mode-home-template"; import { cn, floatingControl, sourceCard } from "@/components/ui-primitives"; @@ -127,3 +146,133 @@ export function AnswerSkeleton() {
); } + +function elapsedLabel(elapsedMs: number) { + const seconds = Math.max(0, Math.floor(elapsedMs / 1_000)); + return seconds < 1 ? "<1s" : `${seconds}s`; +} + +export function AnswerProgressStepper({ + events, + startedAt, + active, + onStop, +}: { + events: TimedAnswerProgressUpdate[]; + startedAt: number | null; + active: boolean; + onStop: () => void; +}) { + const [now, setNow] = useState(() => Date.now()); + const latest = events.at(-1) ?? null; + const finished = latest?.stage === "complete"; + const currentStep = latest ? answerProgressStepIndex(latest.stage) : 0; + + useEffect(() => { + if (!active || finished || !startedAt) return; + const interval = window.setInterval(() => setNow(Date.now()), 1_000); + return () => window.clearInterval(interval); + }, [active, finished, startedAt]); + + const clientElapsedMs = startedAt ? Math.max(0, (finished ? (latest?.receivedAt ?? now) : now) - startedAt) : 0; + const elapsedMs = finished && latest?.elapsedMs !== undefined ? latest.elapsedMs : clientElapsedMs; + const details = events + .map((event) => ({ ...event, displayMessage: answerProgressDisplayMessage(event) })) + .filter((event, index, all) => index === 0 || event.displayMessage !== all[index - 1]?.displayMessage) + .slice(-8); + + return ( +
+
+ {finished ? ( + + + + ) : ( + + )} +

+ {finished + ? `Answer ready in ${elapsedLabel(elapsedMs)}` + : latest + ? answerProgressDisplayMessage(latest) + : "Preparing the clinical search scope."} +

+ {!finished ? ( + + {elapsedLabel(elapsedMs)} + + ) : null} + {active && !finished ? ( + + ) : null} +
+ + {!finished ? ( +
+
    + {answerProgressSteps.map((step, index) => { + const complete = index < currentStep; + const current = index === currentStep; + return ( +
  1. + {complete ? ( + + ) : current ? ( + + ) : ( + + )} + {step.label} +
    2. + ); + })} +
+
+ ) : null} + +
+ + Processing details + +
    + {details.map((event, index) => ( +
  1. + {event.displayMessage} +
    2. + ))} +
+
+
+ ); +} diff --git a/src/lib/answer-progress-public.ts b/src/lib/answer-progress-public.ts new file mode 100644 index 000000000..8ea2f080a --- /dev/null +++ b/src/lib/answer-progress-public.ts @@ -0,0 +1,102 @@ +export type PublicAnswerProgressStage = + | "scoping" + | "retrieving" + | "retrieved" + | "ranking" + | "generating" + | "retrying" + | "fallback" + | "verifying" + | "cached" + | "complete"; + +export type PublicAnswerProgressEvent = { + stage: PublicAnswerProgressStage; + message: string; + resultCount?: number; + selectedContextCount?: number; + australianSourceCount?: number; + waSourceCount?: number; + usedSupplementaryFallback?: boolean; + elapsedMs?: number; +}; + +function safeProgressNumber(value: unknown) { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.floor(value) : undefined; +} + +/** Convert internal RAG progress into the minimal, stable DTO allowed at the browser boundary. */ +export function toPublicAnswerProgressEvent(event: unknown): PublicAnswerProgressEvent | null { + if (!event || typeof event !== "object") return null; + const value = event as Record; + const resultCount = safeProgressNumber(value.resultCount); + const selectedContextCount = safeProgressNumber(value.selectedContextCount); + const australianSourceCount = safeProgressNumber(value.australianSourceCount); + const waSourceCount = safeProgressNumber(value.waSourceCount); + const elapsedMs = safeProgressNumber(value.elapsedMs); + const usedSupplementaryFallback = + typeof value.usedSupplementaryFallback === "boolean" ? value.usedSupplementaryFallback : undefined; + + let stage: PublicAnswerProgressStage; + let message: string; + switch (value.stage) { + case "scoping": + stage = "scoping"; + message = "Preparing the clinical search scope."; + break; + case "retrieving": + stage = "retrieving"; + message = "Searching indexed clinical documents."; + break; + case "retrieved": + stage = "retrieved"; + message = + resultCount === undefined + ? "Source passages found." + : `Found ${resultCount} candidate source passage${resultCount === 1 ? "" : "s"}.`; + break; + case "ranking": + case "routing": + stage = "ranking"; + message = "Selecting the most relevant source passages."; + break; + case "generating": + stage = "generating"; + message = "Drafting a cited answer from the selected passages."; + break; + case "retrying": + stage = "retrying"; + message = "The draft needs another pass; revising it against the evidence."; + break; + case "fallback": + stage = "fallback"; + message = "Building a source-backed answer from the selected passages."; + break; + case "verifying": + case "finalizing": + stage = "verifying"; + message = "Checking citations, clinical numbers, and source metadata."; + break; + case "cached": + stage = "cached"; + message = "Loading a recent cited answer."; + break; + case "complete": + stage = "complete"; + message = "Answer ready."; + break; + default: + return null; + } + + return { + stage, + message, + ...(resultCount === undefined ? {} : { resultCount }), + ...(selectedContextCount === undefined ? {} : { selectedContextCount }), + ...(australianSourceCount === undefined ? {} : { australianSourceCount }), + ...(waSourceCount === undefined ? {} : { waSourceCount }), + ...(usedSupplementaryFallback === undefined ? {} : { usedSupplementaryFallback }), + ...(elapsedMs === undefined ? {} : { elapsedMs }), + }; +} diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 8eabe515d..738d53271 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -403,9 +403,23 @@ function throwIfAborted(signal?: AbortSignal) { } export type AnswerProgressEvent = { - stage: "retrieved" | "routing" | "generating" | "retrying" | "finalizing" | "cached"; + stage: + | "retrieved" + | "ranking" + | "routing" + | "generating" + | "retrying" + | "fallback" + | "verifying" + | "finalizing" + | "cached" + | "complete"; message: string; resultCount?: number; + selectedContextCount?: number; + australianSourceCount?: number; + waSourceCount?: number; + usedSupplementaryFallback?: boolean; visibleSourceCount?: number; directSourceCount?: number; weakSourceCount?: number; @@ -4855,6 +4869,15 @@ ${qualityRetryInstruction}` answerInputResults, generationFallbackResults, ); + const modelContextSelectionSummary = summarizeAustralianSourceSelection(answerInputResults, modelContextResults); + await args.onProgress?.({ + stage: "ranking", + message: "Selected governed source passages for answer generation.", + selectedContextCount: modelContextSelectionSummary.selectedCount, + australianSourceCount: modelContextSelectionSummary.australianSelectedCount, + waSourceCount: modelContextSelectionSummary.waSelectedCount, + usedSupplementaryFallback: modelContextSelectionSummary.usedSupplementaryFallback, + }); try { await args.onProgress?.({ stage: "generating", @@ -5036,7 +5059,7 @@ ${qualityRetryInstruction}` retrievalDiagnostics, ); } - await args.onProgress?.({ stage: "finalizing", message: "Checking citations and source metadata." }); + await args.onProgress?.({ stage: "verifying", message: "Checking citations and source metadata." }); const relatedDocuments = await relatedDocumentsPromise; const answerTimings = { @@ -5203,10 +5226,14 @@ ${qualityRetryInstruction}` if ((error instanceof DOMException && error.name === "AbortError") || args.signal?.aborted) throw error; const relatedDocuments = await relatedDocumentsPromise; await args.onProgress?.({ - stage: "finalizing", + stage: "fallback", message: "Generation failed, returning source-based fallback answer.", mode: "unsupported", reason: "generation_fallback", + selectedContextCount: generationFallbackSelectionSummary.selectedCount, + australianSourceCount: generationFallbackSelectionSummary.australianSelectedCount, + waSourceCount: generationFallbackSelectionSummary.waSelectedCount, + usedSupplementaryFallback: generationFallbackSelectionSummary.usedSupplementaryFallback, }); const baseFallbackAnswer = await buildGenerationFallbackAnswer( error, @@ -5362,6 +5389,7 @@ ${qualityRetryInstruction}` args.query, queryClass, ); + await args.onProgress?.({ stage: "verifying", message: "Checking citations and source metadata." }); if (args.logQuery !== false) await logRagQuery({ owner_id: args.ownerId ?? null, diff --git a/tests/answer-progress-ui-smoke.spec.ts b/tests/answer-progress-ui-smoke.spec.ts new file mode 100644 index 000000000..7f5d014dc --- /dev/null +++ b/tests/answer-progress-ui-smoke.spec.ts @@ -0,0 +1,172 @@ +import { expect, test, type Page } from "playwright/test"; +import { demoAnswer, demoDocuments } from "../src/lib/demo-data"; + +const readySetupChecks = [ + { id: "env", label: ".env.local configured", status: "ready", detail: "Test environment ready." }, + { id: "project", label: "Clinical KB Database target", status: "ready", detail: "Test project ready." }, + { id: "schema", label: "supabase/schema.sql applied", status: "ready", detail: "Test schema ready." }, + { id: "search", label: "Search RPC and vector indexes", status: "ready", detail: "Test search ready." }, + { id: "openai", label: "Answer provider", status: "ready", detail: "Mock stream ready." }, +]; + +async function mockDashboardApis(page: Page) { + await page.route("**/*", async (route) => { + const url = new URL(route.request().url()); + if ( + (url.protocol === "http:" || url.protocol === "https:") && + !["localhost", "127.0.0.1", "::1"].includes(url.hostname) + ) { + await route.abort("blockedbyclient"); + return; + } + if (!url.pathname.startsWith("/api/")) { + await route.fallback(); + return; + } + if (url.pathname === "/api/local-project-id") { + await route.fulfill({ + json: { + appName: "Clinical KB", + projectId: "test-project", + identityPath: "/api/local-project-id", + localServer: { safeLocalOrigin: true }, + }, + }); + return; + } + if (url.pathname === "/api/setup-status") { + await route.fulfill({ json: { demoMode: true, checks: readySetupChecks } }); + return; + } + if (url.pathname === "/api/documents") { + await route.fulfill({ + json: { + documents: demoDocuments, + demoMode: true, + pagination: { + limit: 150, + offset: 0, + total: demoDocuments.length, + nextOffset: demoDocuments.length, + hasMore: false, + }, + }, + }); + return; + } + if (url.pathname === "/api/ingestion/jobs") { + await route.fulfill({ json: { jobs: [], demoMode: true } }); + return; + } + if (url.pathname === "/api/ingestion/batches") { + await route.fulfill({ json: { batches: [], demoMode: true } }); + return; + } + if (url.pathname === "/api/ingestion/quality") { + await route.fulfill({ json: { items: [], demoMode: true } }); + return; + } + await route.fulfill({ json: { demoMode: true } }); + }); +} + +async function installTimedAnswerStream(page: Page) { + const finalAnswer = { ...demoAnswer("Lithium dosing"), demoMode: true }; + await page.addInitScript( + ({ answer }) => { + const originalFetch = window.fetch.bind(window); + window.fetch = async (input, init) => { + const rawUrl = typeof input === "string" ? input : input instanceof Request ? input.url : String(input); + const pathname = new URL(rawUrl, window.location.href).pathname; + if (pathname !== "/api/answer/stream") return originalFetch(input, init); + + const encoder = new TextEncoder(); + const events: Array<{ delay: number; event: string; data: unknown }> = [ + { delay: 0, event: "progress", data: { stage: "scoping", message: "Preparing scope." } }, + { delay: 250, event: "progress", data: { stage: "retrieving", message: "Searching documents." } }, + { + delay: 600, + event: "progress", + data: { stage: "retrieved", message: "Retrieved candidates.", resultCount: 12 }, + }, + { + delay: 900, + event: "progress", + data: { + stage: "ranking", + message: "private-model-marker private-route-marker", + selectedContextCount: 4, + australianSourceCount: 4, + waSourceCount: 4, + usedSupplementaryFallback: false, + }, + }, + { delay: 1_600, event: "progress", data: { stage: "generating", message: "private-draft-marker" } }, + { delay: 2_000, event: "token", data: { delta: "Provisional lithium draft" } }, + { delay: 2_400, event: "revising", data: { reason: "private-provider-reason-marker" } }, + { delay: 2_450, event: "progress", data: { stage: "fallback", message: "private-fallback-marker" } }, + { delay: 3_100, event: "progress", data: { stage: "verifying", message: "private-check-marker" } }, + { + delay: 3_700, + event: "progress", + data: { stage: "complete", message: "private-ready-marker", elapsedMs: 3_700 }, + }, + { delay: 3_800, event: "final", data: answer }, + ]; + + return new Response( + new ReadableStream({ + start(controller) { + for (const item of events) { + window.setTimeout(() => { + controller.enqueue(encoder.encode(`event: ${item.event}\ndata: ${JSON.stringify(item.data)}\n\n`)); + if (item.event === "final") controller.close(); + }, item.delay); + } + }, + }), + { status: 200, headers: { "Content-Type": "text/event-stream; charset=utf-8" } }, + ); + }; + }, + { answer: finalAnswer }, + ); +} + +test("answer progress remains user-safe through fallback and keeps a compact completed state", async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await mockDashboardApis(page); + await installTimedAnswerStream(page); + await page.goto("/?mode=answer", { waitUntil: "domcontentloaded" }); + + const input = page.locator('[aria-label^="Search indexed guidelines by question or keyword"]:visible').first(); + const submit = page.locator('[aria-label="Generate source-backed answer"]:visible').first(); + await expect(input).toBeEditable({ timeout: 30_000 }); + await input.fill("Lithium dosing"); + await submit.click(); + + const progress = page.getByTestId("answer-progress-stepper"); + await expect(progress).toBeVisible(); + for (const label of ["Prepare scope", "Search sources", "Select evidence", "Draft answer", "Check answer"]) { + await expect(progress.getByText(label, { exact: true })).toBeVisible(); + } + await expect(progress).toContainText("Prioritising 4 Australian source passages, including 4 WA", { + timeout: 3_000, + }); + + await expect(page.getByTestId("answer-streaming")).toContainText("Provisional lithium draft", { + timeout: 4_000, + }); + await expect(progress).toContainText("Building a source-backed answer", { timeout: 5_000 }); + await expect(page.getByTestId("answer-streaming-revising")).toBeVisible(); + await expect(page.getByText("Provisional lithium draft")).toHaveCount(0); + + await expect(progress).toHaveAttribute("data-progress-state", "complete", { timeout: 6_000 }); + await expect(progress).toContainText("Answer ready in 3s"); + await expect(progress.getByText("Processing details", { exact: true })).toBeVisible(); + await expect(page.getByTestId("stop-answer")).toHaveCount(0); + await expect(page.getByText(/In the synthetic lithium document/i)).toBeVisible({ timeout: 8_000 }); + await expect(page.locator("body")).not.toContainText( + /private-(?:model|route|provider-reason|fallback|draft|check|ready)-marker/i, + ); +}); diff --git a/tests/answer-progress.test.ts b/tests/answer-progress.test.ts new file mode 100644 index 000000000..bf48a51df --- /dev/null +++ b/tests/answer-progress.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { + answerProgressDisplayMessage, + answerProgressStepIndex, + normalizeAnswerProgressEvent, +} from "../src/components/clinical-dashboard/answer-progress"; +import { toPublicAnswerProgressEvent } from "../src/lib/answer-progress-public"; + +describe("answer progress events", () => { + it("keeps only safe, normalized Australian source counts at the public boundary", () => { + expect( + toPublicAnswerProgressEvent({ + stage: "ranking", + message: "private model route marker", + selectedContextCount: 4.9, + australianSourceCount: 4, + waSourceCount: 3, + usedSupplementaryFallback: false, + model: "private-model", + reason: "private-reason", + smartApiPlan: { private: true }, + }), + ).toEqual({ + stage: "ranking", + message: "Selecting the most relevant source passages.", + selectedContextCount: 4, + australianSourceCount: 4, + waSourceCount: 3, + usedSupplementaryFallback: false, + }); + }); + + it("accepts legacy message-only progress while rendering stable copy", () => { + const progress = normalizeAnswerProgressEvent({ message: "Selected fast route using private-model-marker." }); + + expect(progress).toMatchObject({ stage: "ranking" }); + expect(answerProgressDisplayMessage(progress!)).toBe("Selecting the most relevant source passages."); + expect(answerProgressDisplayMessage(progress!)).not.toMatch(/fast|private|model|route/i); + }); + + it("renders truthful Australian priority and fallback copy", () => { + const progress = normalizeAnswerProgressEvent({ + stage: "ranking", + message: "Selected governed passages.", + selectedContextCount: 4, + australianSourceCount: 4, + waSourceCount: 4, + usedSupplementaryFallback: false, + }); + + expect(answerProgressDisplayMessage(progress!)).toBe("Prioritising 4 Australian source passages, including 4 WA."); + expect(answerProgressStepIndex("fallback")).toBe(3); + expect(answerProgressDisplayMessage({ stage: "fallback", message: "private" })).toContain("source-backed answer"); + }); + + it("rejects invalid progress objects and clamps safe counts", () => { + expect(normalizeAnswerProgressEvent(null)).toBeNull(); + expect(normalizeAnswerProgressEvent({ stage: "ranking", message: "" })).toBeNull(); + expect( + normalizeAnswerProgressEvent({ + stage: "retrieved", + message: "Found passages.", + resultCount: 2.8, + selectedContextCount: -1, + }), + ).toMatchObject({ resultCount: 2, selectedContextCount: undefined }); + }); +}); diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index e01528796..070efc48e 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -288,6 +288,7 @@ function mockRuntime( publicWorkspaceOwnerId?: string; maxConcurrentUploads?: number; maxInFlightUploadMb?: number; + demoMode?: boolean; } = {}, ) { vi.resetModules(); @@ -317,7 +318,7 @@ function mockRuntime( WORKER_STALE_AFTER_MINUTES: 10, WORKER_MAX_ATTEMPTS: 3, }, - isDemoMode: () => false, + isDemoMode: () => Boolean(options.demoMode), isLocalNoAuthMode: () => Boolean(options.localNoAuth), publicWorkspaceOwnerId: () => options.publicWorkspaceOwnerId ?? null, publicUploadsEnabled: () => Boolean(options.publicUploadsEnabled), @@ -397,6 +398,15 @@ function ssePayload(body: string, eventName: string) { return JSON.parse(dataLine!.slice("data: ".length)) as Record; } +function expectSingleCompletionBeforeFinal(body: string) { + const blocks = body.split("\n\n").filter(Boolean); + const completions = blocks.filter( + (block) => block.includes("event: progress\n") && block.includes('"stage":"complete"'), + ); + expect(completions).toHaveLength(1); + expect(body.indexOf(completions[0]!)).toBeLessThan(body.indexOf("event: final\n")); +} + afterEach(() => { vi.unstubAllEnvs(); vi.restoreAllMocks(); @@ -3834,11 +3844,129 @@ describe("private document API access", () => { expect(secondPayload.telemetry).toMatchObject({ coalesced: true }); }); - it("streams answer progress before the final answer for authenticated users", async () => { + it("streams only public progress details and exactly one completion before the final answer", async () => { + const answerQuestionWithScope = vi.fn( + async (args: { + onProgress?: (event: unknown) => void | Promise; + onRevising?: (reason: string) => void; + }) => { + await args.onProgress?.({ + stage: "routing", + message: "private-message-marker", + resultCount: 2, + visibleSourceCount: 1, + timingMs: 42, + selectedContextCount: 4.9, + australianSourceCount: 4, + waSourceCount: 3, + usedSupplementaryFallback: false, + model: "private-model-marker", + mode: "private-mode-marker", + reason: "private-reason-marker", + smartApiPlan: { marker: "private-plan-marker" }, + relevance: { marker: "private-relevance-marker" }, + privateMarker: "private-direct-marker", + }); + args.onRevising?.("private-revising-marker"); + await args.onProgress?.({ stage: "complete", message: "private-complete-marker" }); + await args.onProgress?.({ stage: "complete", message: "private-duplicate-complete-marker" }); + return { + answer: "Owned evidence.", + grounded: true, + confidence: "medium", + citations: [], + sources: [], + }; + }, + ); + const client = createSupabaseMock(); + mockRuntime(client, { answerQuestionWithScope }); + const { POST } = await import("../src/app/api/answer/stream/route"); + + const response = await POST( + authenticatedRequest("/api/answer/stream", { + method: "POST", + body: JSON.stringify({ query: "monitoring", documentId: otherDocumentId }), + }), + ); + const body = await response.text(); + + expect(response.status).toBe(200); + const eventBlocks = body.split("\n\n").filter(Boolean); + const progressBlocks = eventBlocks.filter((block) => block.includes("event: progress\n")); + const completionBlocks = progressBlocks.filter((block) => block.includes('"stage":"complete"')); + const rankingBlock = progressBlocks.find((block) => block.includes('"stage":"ranking"')); + const rankingPayload = JSON.parse( + rankingBlock! + .split("\n") + .find((line) => line.startsWith("data: "))! + .slice("data: ".length), + ); + + expect(body.indexOf("event: progress")).toBeGreaterThanOrEqual(0); + expect(body.indexOf("event: final")).toBeGreaterThan(body.lastIndexOf('"stage":"complete"')); + expect(completionBlocks).toHaveLength(1); + expectSingleCompletionBeforeFinal(body); + expect(rankingPayload).toEqual({ + stage: "ranking", + message: "Selecting the most relevant source passages.", + resultCount: 2, + selectedContextCount: 4, + australianSourceCount: 4, + waSourceCount: 3, + usedSupplementaryFallback: false, + }); + expect(body).toContain("event: revising\ndata: {}"); + for (const privateMarker of [ + "private-message-marker", + "private-model-marker", + "private-mode-marker", + "private-reason-marker", + "private-plan-marker", + "private-relevance-marker", + "private-direct-marker", + "private-revising-marker", + "private-complete-marker", + "private-duplicate-complete-marker", + ]) { + expect(body).not.toContain(privateMarker); + } + expect(answerQuestionWithScope).toHaveBeenCalledWith( + expect.objectContaining({ ownerId: userId, documentId: otherDocumentId, onProgress: expect.any(Function) }), + ); + expect(client.auth.getUser).toHaveBeenCalledWith(token); + }); + + it("completes demo answers before their successful final event", async () => { + const answerQuestionWithScope = vi.fn(); + const client = createSupabaseMock(); + mockRuntime(client, { answerQuestionWithScope }, { demoMode: true }); + const { POST } = await import("../src/app/api/answer/stream/route"); + + const response = await POST( + request("/api/answer/stream", { + method: "POST", + body: JSON.stringify({ query: "Lithium dosing" }), + }), + ); + const body = await response.text(); + + expect(response.status).toBe(200); + expectSingleCompletionBeforeFinal(body); + expect(ssePayload(body, "final")).toMatchObject({ demoMode: true }); + expect(answerQuestionWithScope).not.toHaveBeenCalled(); + }); + + it("completes cached answers after safe cached progress and before final", async () => { const answerQuestionWithScope = vi.fn(async (args: { onProgress?: (event: unknown) => void | Promise }) => { - await args.onProgress?.({ stage: "retrieved", message: "Retrieved 2 candidate sources." }); + await args.onProgress?.({ + stage: "cached", + message: "private-cache-message-marker", + model: "private-cache-model-marker", + reason: "private-cache-reason-marker", + }); return { - answer: "Owned evidence.", + answer: "Cached owned evidence.", grounded: true, confidence: "medium", citations: [], @@ -3858,13 +3986,10 @@ describe("private document API access", () => { const body = await response.text(); expect(response.status).toBe(200); - expect(body.indexOf("event: progress")).toBeGreaterThanOrEqual(0); - expect(body.indexOf("event: final")).toBeGreaterThan(body.indexOf("event: progress")); - expect(body).toContain("Retrieved 2 candidate sources."); - expect(answerQuestionWithScope).toHaveBeenCalledWith( - expect.objectContaining({ ownerId: userId, documentId: otherDocumentId, onProgress: expect.any(Function) }), - ); - expect(client.auth.getUser).toHaveBeenCalledWith(token); + expect(body.indexOf('"stage":"cached"')).toBeLessThan(body.indexOf('"stage":"complete"')); + expect(body).toContain('"message":"Loading a recent cited answer."'); + expect(body).not.toContain("private-cache"); + expectSingleCompletionBeforeFinal(body); }); it("emits a structured SSE error when authenticated streaming answers are rate limited", async () => { @@ -4021,9 +4146,11 @@ describe("private document API access", () => { body: JSON.stringify({ query: "monitoring", documentId }), }), ); - const finalPayload = ssePayload(await response.text(), "final"); + const responseBody = await response.text(); + const finalPayload = ssePayload(responseBody, "final"); expect(response.status).toBe(200); + expectSingleCompletionBeforeFinal(responseBody); expect(finalPayload.grounded).toBe(false); expect(finalPayload.confidence).toBe("unsupported"); expect(finalPayload.citations).toEqual([]); diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index 8c53c3eba..f95657fa1 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1638,11 +1638,21 @@ describe("RAG structured-output fallback", () => { })); const { answerQuestionWithScope, isCacheableGroundedGenerationFallback } = await import("../src/lib/rag"); + const progressEvents: Array<{ + stage: string; + selectedContextCount?: number; + australianSourceCount?: number; + waSourceCount?: number; + usedSupplementaryFallback?: boolean; + }> = []; const answer = await answerQuestionWithScope({ query: "Lithium dosing", ownerId: undefined, logQuery: false, skipCache: true, + onProgress: (event) => { + progressEvents.push(event); + }, }); expect(generateStructuredTextResult).toHaveBeenCalledTimes(2); @@ -1671,6 +1681,25 @@ describe("RAG structured-output fallback", () => { expect(answer.openAIRequestIds).toEqual(["req_truncated_1", "req_truncated_2"]); expect(answer.openAIUsage).toMatchObject({ output_tokens: 1300, total_tokens: 1500 }); expect(isCacheableGroundedGenerationFallback(answer)).toBe(true); + expect(progressEvents).toContainEqual( + expect.objectContaining({ + stage: "ranking", + selectedContextCount: 4, + australianSourceCount: 4, + waSourceCount: 4, + usedSupplementaryFallback: false, + }), + ); + expect(progressEvents).toContainEqual( + expect.objectContaining({ + stage: "fallback", + selectedContextCount: 4, + australianSourceCount: 4, + waSourceCount: 4, + usedSupplementaryFallback: false, + }), + ); + expect(progressEvents).toContainEqual(expect.objectContaining({ stage: "verifying" })); }); it("fails closed instead of leaking another medication's numeric dose after generation failure", async () => { From bf8458afd1f20f87b2e62fb73c86c2815a88e107 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 19:57:08 +0800 Subject: [PATCH 04/14] fix(answer): validate final before completion --- src/components/ClinicalDashboard.tsx | 19 ++++- .../clinical-dashboard/answer-progress.ts | 2 - src/lib/answer-progress-public.ts | 4 - tests/answer-progress-ui-smoke.spec.ts | 81 ++++++++++++++++++- tests/answer-progress.test.ts | 29 ++++--- tests/private-access-routes.test.ts | 4 +- 6 files changed, 111 insertions(+), 28 deletions(-) diff --git a/src/components/ClinicalDashboard.tsx b/src/components/ClinicalDashboard.tsx index d9d2ed0f8..680cc63e3 100644 --- a/src/components/ClinicalDashboard.tsx +++ b/src/components/ClinicalDashboard.tsx @@ -359,6 +359,7 @@ async function readAnswerStream( const reader = response.body.getReader(); const decoder = new TextDecoder(); let buffer = ""; + let pendingCompletion: AnswerProgressUpdate | null = null; function processEvent(block: string) { const lines = block.split(/\r?\n/); @@ -376,8 +377,12 @@ async function readAnswerStream( if (event === "progress") { const progress = normalizeAnswerProgressEvent(data); if (progress) { - onProgress(progress); - if (progress.stage === "fallback") onRevising?.(); + if (progress.stage === "complete") { + pendingCompletion = progress; + } else { + onProgress(progress); + if (progress.stage === "fallback") onRevising?.(); + } } return; } @@ -391,6 +396,7 @@ async function readAnswerStream( return; } if (event === "error") { + pendingCompletion = null; const message = data && typeof data === "object" ? (data as { error?: unknown }).error : null; const details = data && typeof data === "object" ? (data as { details?: { message?: unknown } | unknown }).details : null; @@ -414,8 +420,13 @@ async function readAnswerStream( } if (event === "final") { if (!isAnswerPayload(data)) { + pendingCompletion = null; throw makeSearchError("Answer stream returned an invalid final payload.", 502, true); } + if (pendingCompletion) { + onProgress(pendingCompletion); + pendingCompletion = null; + } return data; } @@ -446,6 +457,7 @@ async function readAnswerStream( const finalPayload = buffer.trim() ? processEvent(buffer.trim()) : null; if (finalPayload) return finalPayload; + pendingCompletion = null; throw makeSearchError("Answer stream ended before a final answer was received.", undefined, true); } @@ -2251,8 +2263,7 @@ export function ClinicalDashboard({ latest.resultCount === progress.resultCount && latest.selectedContextCount === progress.selectedContextCount && latest.australianSourceCount === progress.australianSourceCount && - latest.waSourceCount === progress.waSourceCount && - latest.usedSupplementaryFallback === progress.usedSupplementaryFallback + latest.waSourceCount === progress.waSourceCount ) { return current; } diff --git a/src/components/clinical-dashboard/answer-progress.ts b/src/components/clinical-dashboard/answer-progress.ts index 3d81daa7f..0b0cab180 100644 --- a/src/components/clinical-dashboard/answer-progress.ts +++ b/src/components/clinical-dashboard/answer-progress.ts @@ -63,8 +63,6 @@ export function normalizeAnswerProgressEvent(data: unknown): AnswerProgressUpdat selectedContextCount: finiteCount(value.selectedContextCount), australianSourceCount: finiteCount(value.australianSourceCount), waSourceCount: finiteCount(value.waSourceCount), - usedSupplementaryFallback: - typeof value.usedSupplementaryFallback === "boolean" ? value.usedSupplementaryFallback : undefined, elapsedMs: finiteCount(value.elapsedMs), }; } diff --git a/src/lib/answer-progress-public.ts b/src/lib/answer-progress-public.ts index 8ea2f080a..5df11a4cb 100644 --- a/src/lib/answer-progress-public.ts +++ b/src/lib/answer-progress-public.ts @@ -17,7 +17,6 @@ export type PublicAnswerProgressEvent = { selectedContextCount?: number; australianSourceCount?: number; waSourceCount?: number; - usedSupplementaryFallback?: boolean; elapsedMs?: number; }; @@ -34,8 +33,6 @@ export function toPublicAnswerProgressEvent(event: unknown): PublicAnswerProgres const australianSourceCount = safeProgressNumber(value.australianSourceCount); const waSourceCount = safeProgressNumber(value.waSourceCount); const elapsedMs = safeProgressNumber(value.elapsedMs); - const usedSupplementaryFallback = - typeof value.usedSupplementaryFallback === "boolean" ? value.usedSupplementaryFallback : undefined; let stage: PublicAnswerProgressStage; let message: string; @@ -96,7 +93,6 @@ export function toPublicAnswerProgressEvent(event: unknown): PublicAnswerProgres ...(selectedContextCount === undefined ? {} : { selectedContextCount }), ...(australianSourceCount === undefined ? {} : { australianSourceCount }), ...(waSourceCount === undefined ? {} : { waSourceCount }), - ...(usedSupplementaryFallback === undefined ? {} : { usedSupplementaryFallback }), ...(elapsedMs === undefined ? {} : { elapsedMs }), }; } diff --git a/tests/answer-progress-ui-smoke.spec.ts b/tests/answer-progress-ui-smoke.spec.ts index 7f5d014dc..a2ce149b1 100644 --- a/tests/answer-progress-ui-smoke.spec.ts +++ b/tests/answer-progress-ui-smoke.spec.ts @@ -98,7 +98,7 @@ async function installTimedAnswerStream(page: Page) { selectedContextCount: 4, australianSourceCount: 4, waSourceCount: 4, - usedSupplementaryFallback: false, + usedSupplementaryFallback: true, }, }, { delay: 1_600, event: "progress", data: { stage: "generating", message: "private-draft-marker" } }, @@ -133,6 +133,59 @@ async function installTimedAnswerStream(page: Page) { ); } +async function installSuccessfulThenInvalidAnswerStreams(page: Page) { + const firstAnswer = { ...demoAnswer("Lithium dosing"), demoMode: true }; + await page.addInitScript( + ({ answer }) => { + const originalFetch = window.fetch.bind(window); + let answerRequestCount = 0; + window.fetch = async (input, init) => { + const rawUrl = typeof input === "string" ? input : input instanceof Request ? input.url : String(input); + const pathname = new URL(rawUrl, window.location.href).pathname; + if (pathname !== "/api/answer/stream") return originalFetch(input, init); + + answerRequestCount += 1; + const encoder = new TextEncoder(); + const events = + answerRequestCount === 1 + ? [ + { delay: 0, event: "progress", data: { stage: "scoping", message: "Preparing scope." } }, + { + delay: 40, + event: "progress", + data: { stage: "complete", message: "Answer ready.", elapsedMs: 40 }, + }, + { delay: 80, event: "final", data: answer }, + ] + : [ + { delay: 0, event: "progress", data: { stage: "retrieving", message: "Searching." } }, + { + delay: 40, + event: "progress", + data: { stage: "complete", message: "Answer ready.", elapsedMs: 40 }, + }, + { delay: 80, event: "final", data: { answer: 42 } }, + ]; + + return new Response( + new ReadableStream({ + start(controller) { + for (const item of events) { + window.setTimeout(() => { + controller.enqueue(encoder.encode(`event: ${item.event}\ndata: ${JSON.stringify(item.data)}\n\n`)); + if (item.event === "final") controller.close(); + }, item.delay); + } + }, + }), + { status: 200, headers: { "Content-Type": "text/event-stream; charset=utf-8" } }, + ); + }; + }, + { answer: firstAnswer }, + ); +} + test("answer progress remains user-safe through fallback and keeps a compact completed state", async ({ page }) => { await page.setViewportSize({ width: 390, height: 844 }); await mockDashboardApis(page); @@ -170,3 +223,29 @@ test("answer progress remains user-safe through fallback and keeps a compact com /private-(?:model|route|provider-reason|fallback|draft|check|ready)-marker/i, ); }); + +test("a completion frame cannot mark a previous answer complete when final is invalid", async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await mockDashboardApis(page); + await installSuccessfulThenInvalidAnswerStreams(page); + await page.goto("/?mode=answer", { waitUntil: "domcontentloaded" }); + + const input = page.locator('[aria-label^="Search indexed guidelines by question or keyword"]:visible').first(); + const submit = page.locator('[aria-label="Generate source-backed answer"]:visible').first(); + await expect(input).toBeEditable({ timeout: 30_000 }); + await input.fill("Lithium dosing"); + await submit.click(); + + await expect(page.getByText(/In the synthetic lithium document/i)).toBeVisible({ timeout: 8_000 }); + await expect(page.getByTestId("answer-progress-stepper")).toHaveAttribute("data-progress-state", "complete"); + + await input.fill("What about monitoring?"); + await submit.click(); + + await expect(page.getByTestId("answer-error")).toContainText("Answer stream returned an invalid final payload", { + timeout: 10_000, + }); + await expect(page.locator('[data-progress-state="complete"]')).toHaveCount(0); + await expect(page.getByText(/Answer ready in/)).toHaveCount(0); + await expect(page.getByText(/In the synthetic lithium document/i)).toBeVisible(); +}); diff --git a/tests/answer-progress.test.ts b/tests/answer-progress.test.ts index bf48a51df..eb212a066 100644 --- a/tests/answer-progress.test.ts +++ b/tests/answer-progress.test.ts @@ -8,26 +8,26 @@ import { toPublicAnswerProgressEvent } from "../src/lib/answer-progress-public"; describe("answer progress events", () => { it("keeps only safe, normalized Australian source counts at the public boundary", () => { - expect( - toPublicAnswerProgressEvent({ - stage: "ranking", - message: "private model route marker", - selectedContextCount: 4.9, - australianSourceCount: 4, - waSourceCount: 3, - usedSupplementaryFallback: false, - model: "private-model", - reason: "private-reason", - smartApiPlan: { private: true }, - }), - ).toEqual({ + const publicEvent = toPublicAnswerProgressEvent({ + stage: "ranking", + message: "private model route marker", + selectedContextCount: 4.9, + australianSourceCount: 4, + waSourceCount: 3, + usedSupplementaryFallback: true, + model: "private-model", + reason: "private-reason", + smartApiPlan: { private: true }, + }); + + expect(publicEvent).toEqual({ stage: "ranking", message: "Selecting the most relevant source passages.", selectedContextCount: 4, australianSourceCount: 4, waSourceCount: 3, - usedSupplementaryFallback: false, }); + expect(publicEvent).not.toHaveProperty("usedSupplementaryFallback"); }); it("accepts legacy message-only progress while rendering stable copy", () => { @@ -45,7 +45,6 @@ describe("answer progress events", () => { selectedContextCount: 4, australianSourceCount: 4, waSourceCount: 4, - usedSupplementaryFallback: false, }); expect(answerProgressDisplayMessage(progress!)).toBe("Prioritising 4 Australian source passages, including 4 WA."); diff --git a/tests/private-access-routes.test.ts b/tests/private-access-routes.test.ts index 070efc48e..5e92c48d4 100644 --- a/tests/private-access-routes.test.ts +++ b/tests/private-access-routes.test.ts @@ -3859,7 +3859,7 @@ describe("private document API access", () => { selectedContextCount: 4.9, australianSourceCount: 4, waSourceCount: 3, - usedSupplementaryFallback: false, + usedSupplementaryFallback: true, model: "private-model-marker", mode: "private-mode-marker", reason: "private-reason-marker", @@ -3914,8 +3914,8 @@ describe("private document API access", () => { selectedContextCount: 4, australianSourceCount: 4, waSourceCount: 3, - usedSupplementaryFallback: false, }); + expect(body).not.toContain("usedSupplementaryFallback"); expect(body).toContain("event: revising\ndata: {}"); for (const privateMarker of [ "private-message-marker", From 7713b4b249d8e9d6b6f329a024e204ec031001c9 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:38:22 +0800 Subject: [PATCH 05/14] feat(governance): add safe source locality repair --- scripts/audit-source-governance.ts | 70 +++++-- scripts/backfill-source-metadata.ts | 224 +++++++++++++------- scripts/eval-rag.ts | 41 +++- src/lib/rag-eval-diagnostics.ts | 156 ++++++++++++++ src/lib/source-authority-metadata.ts | 255 +++++++++++++++++++++++ tests/rag-eval-source-governance.test.ts | 150 +++++++++++++ tests/source-authority-tooling.test.ts | 149 +++++++++++++ 7 files changed, 954 insertions(+), 91 deletions(-) create mode 100644 src/lib/rag-eval-diagnostics.ts create mode 100644 src/lib/source-authority-metadata.ts create mode 100644 tests/rag-eval-source-governance.test.ts create mode 100644 tests/source-authority-tooling.test.ts diff --git a/scripts/audit-source-governance.ts b/scripts/audit-source-governance.ts index f4a1a7d81..37419f6c4 100644 --- a/scripts/audit-source-governance.ts +++ b/scripts/audit-source-governance.ts @@ -1,15 +1,14 @@ import { readFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; import * as nextEnv from "@next/env"; +import { auditSourceAuthorityDocuments } from "@/lib/source-authority-metadata"; import type { DocumentLabel } from "@/lib/types"; const loadEnvConfig = nextEnv.loadEnvConfig ?? (nextEnv as unknown as { default?: { loadEnvConfig?: typeof nextEnv.loadEnvConfig } }).default?.loadEnvConfig; -if (!loadEnvConfig) throw new Error("Unable to load @next/env loadEnvConfig."); -loadEnvConfig(process.cwd()); - type AuditArgs = { json: boolean; help: boolean; @@ -43,6 +42,7 @@ type DocumentRow = { id: string; title: string; file_name: string; + source_path: string | null; status: string; metadata: Record | null; }; @@ -84,7 +84,7 @@ async function loadAdminClient() { return createAdminClient(); } -function parseArgs(argv: string[]): AuditArgs { +export function parseSourceGovernanceAuditArgs(argv: string[]): AuditArgs { const args: AuditArgs = { json: false, help: false }; for (let index = 0; index < argv.length; index += 1) { const token = argv[index]; @@ -108,6 +108,11 @@ function parseArgs(argv: string[]): AuditArgs { return args; } +function loadProjectEnvironment() { + if (!loadEnvConfig) throw new Error("Unable to load @next/env loadEnvConfig."); + loadEnvConfig(process.cwd()); +} + function usage() { return [ "Usage: npm run audit:source-governance -- [options]", @@ -243,17 +248,21 @@ async function loadDebtPolicy(path: string): Promise { }; } -async function main() { - const args = parseArgs(process.argv.slice(2)); +export async function main(argv = process.argv.slice(2)) { + const args = parseSourceGovernanceAuditArgs(argv); if (args.help) { console.log(usage()); return; } + loadProjectEnvironment(); const debtPolicy = args.debtPolicyPath ? await loadDebtPolicy(args.debtPolicyPath) : undefined; const supabase = await loadAdminClient(); - const documents = await fetchAll(supabase, "documents", "id,title,file_name,status,metadata", (query) => - query.eq("status", "indexed"), + const documents = await fetchAll( + supabase, + "documents", + "id,title,file_name,source_path,status,metadata", + (query) => query.eq("status", "indexed"), ); const labels = await fetchAll(supabase, "document_labels", "id,document_id,label_type,source", (query) => query.eq("source", "generated"), @@ -289,6 +298,7 @@ async function main() { const missingGeneratedLabelDocuments = documents.filter((document) => !generatedLabelDocumentIds.has(document.id)); const missingSmartV2LabelDocuments = documents.filter((document) => !smartV2DocumentIds.has(document.id)); const requiredMetadataMissingTotal = [...requiredMissingCounts.values()].reduce((total, count) => total + count, 0); + const sourceAuthorityAudit = auditSourceAuthorityDocuments(documents); const debtCounts = { review_due: statusCounts.get("review_due") ?? 0, unknown_status: statusCounts.get("unknown") ?? 0, @@ -395,6 +405,8 @@ async function main() { })), indexed_document_id_count: indexedDocumentIds.size, passed_required_metadata_gate: requiredMetadataMissingTotal === 0, + source_authority: sourceAuthorityAudit, + passed_source_authority_gate: sourceAuthorityAudit.passed, debt_policy: debtPolicy ? { path: debtPolicy.path, @@ -437,6 +449,32 @@ async function main() { console.log( `Smart-v2 labels: missing=${report.smart_v2_label_coverage.indexed_without_smart_v2_labels}, covered=${report.smart_v2_label_coverage.documents_with_smart_v2_labels}`, ); + console.log( + `Source authority: recognised=${report.source_authority.recognized_documents}, Australian candidates=${report.source_authority.australian_authority_candidates}, conflicts=${report.source_authority.authority_conflict_count}, missing locality=${report.source_authority.missing_australian_locality_count}, safe corrections=${report.source_authority.proposed_locality_correction_count}`, + ); + if (report.source_authority.authority_conflict_count) { + console.log( + `Source authority conflict reasons: ${JSON.stringify(report.source_authority.authority_conflict_reason_counts)}`, + ); + } + if (report.source_authority.conflicts.length) { + console.log("Source authority conflicts:"); + for (const document of report.source_authority.conflicts) { + console.log(`- ${document.title} (${document.file_name}): ${document.conflicts.join(", ")}`); + } + } + if (report.source_authority.missing_australian_locality.length) { + console.log("Australian sources missing locality metadata:"); + for (const document of report.source_authority.missing_australian_locality) { + console.log(`- ${document.title} (${document.file_name}): ${document.missing_keys.join(", ")}`); + } + } + if (report.source_authority.proposed_locality_corrections.length) { + console.log("Proposed safe locality corrections:"); + for (const document of report.source_authority.proposed_locality_corrections) { + console.log(`- ${document.title} (${document.file_name}): ${JSON.stringify(document.changes)}`); + } + } if (report.missing_smart_v2_label_documents.length) { console.log("Documents missing smart-v2 labels:"); for (const document of report.missing_smart_v2_label_documents) { @@ -448,6 +486,11 @@ async function main() { ? "PASS: required source governance metadata is complete." : "FAIL: required source governance metadata has gaps.", ); + console.log( + report.passed_source_authority_gate + ? "PASS: recognised source authority metadata is complete and compatible." + : "FAIL: recognised source authority metadata has missing or conflicting locality fields.", + ); if (report.debt_policy) { console.log( report.debt_policy.passed @@ -459,10 +502,13 @@ async function main() { } if (!report.passed_required_metadata_gate) process.exitCode = 1; + if (!report.passed_source_authority_gate) process.exitCode = 1; if (report.debt_policy && !report.debt_policy.passed) process.exitCode = 1; } -main().catch((error) => { - console.error(error instanceof Error ? error.message : error); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/backfill-source-metadata.ts b/scripts/backfill-source-metadata.ts index ea61868cd..42a8aa5c6 100644 --- a/scripts/backfill-source-metadata.ts +++ b/scripts/backfill-source-metadata.ts @@ -1,4 +1,13 @@ +import { pathToFileURL } from "node:url"; + import { loadEnvConfig } from "@next/env"; +import { + analyzeSourceLocality, + assertLocalityMetadataPatch, + auditSourceAuthorityDocuments, + inferSourceAuthorityFromIdentity, +} from "@/lib/source-authority-metadata"; +import { sourceAuthorityForPublisherCode } from "@/lib/source-authority-registry"; import type { Json } from "@/lib/supabase/database.types"; import { UNKNOWN_STATUS_DERIVATION_VERSION, @@ -6,8 +15,6 @@ import { unknownStatusDerivationBasis, } from "@/lib/unknown-status-derivation"; -loadEnvConfig(process.cwd()); - type DocumentRow = { id: string; title: string; @@ -42,38 +49,75 @@ type ClinicalValidationEvidence = { evidence_text: string | null; }; -const APPLY = process.argv.includes("--apply"); -const EVAL_ONLY = process.argv.includes("--eval-only"); const BACKFILL_VERSION = "source_metadata_backfill_2026_06_30_v1"; -function backfillAsOfDate() { - const index = process.argv.indexOf("--as-of"); - if (index < 0) return new Date(); - const raw = process.argv[index + 1]; - if (!raw || raw.startsWith("--")) throw new Error("Missing value for --as-of"); +export type BackfillSourceMetadataArgs = { + apply: boolean; + confirm: boolean; + localityOnly: boolean; + evalOnly: boolean; + help: boolean; + asOf: Date; +}; + +function parseBackfillAsOfDate(raw: string | undefined) { + if (!raw) return new Date(); const date = /^\d{4}-\d{2}-\d{2}$/.test(raw) ? new Date(`${raw}T00:00:00+08:00`) : new Date(raw); if (Number.isNaN(date.getTime())) throw new Error(`Invalid --as-of date: ${raw}`); return date; } -const NOW = backfillAsOfDate(); - -const publisherByCode: Record = { - AKG: { publisher: "Armadale Kalamunda Group", jurisdiction: "Australia/WA" }, - BMJ: { publisher: "BMJ Best Practice", jurisdiction: "International" }, - CAMHS: { publisher: "Child and Adolescent Mental Health Service", jurisdiction: "Australia/WA" }, - EMHS: { publisher: "East Metropolitan Health Service", jurisdiction: "Australia/WA" }, - FSH: { publisher: "Fiona Stanley Fremantle Hospitals Group", jurisdiction: "Australia/WA" }, - FSFH: { publisher: "Fiona Stanley Fremantle Hospitals Group", jurisdiction: "Australia/WA" }, - FSFHG: { publisher: "Fiona Stanley Fremantle Hospitals Group", jurisdiction: "Australia/WA" }, - KEMH: { publisher: "King Edward Memorial Hospital", jurisdiction: "Australia/WA" }, - KEMHS: { publisher: "King Edward Memorial Hospital", jurisdiction: "Australia/WA" }, - NMHS: { publisher: "North Metropolitan Health Service", jurisdiction: "Australia/WA" }, - PHC: { publisher: "Peel Health Campus", jurisdiction: "Australia/WA" }, - RKPG: { publisher: "Rockingham Peel Group", jurisdiction: "Australia/WA" }, - RPBG: { publisher: "Royal Perth Bentley Group", jurisdiction: "Australia/WA" }, - SMHS: { publisher: "South Metropolitan Health Service", jurisdiction: "Australia/WA" }, -}; +export function parseBackfillSourceMetadataArgs(argv: string[]): BackfillSourceMetadataArgs { + const parsed = { + apply: false, + confirm: false, + localityOnly: false, + evalOnly: false, + help: false, + asOfRaw: undefined as string | undefined, + }; + + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (token === "--apply") parsed.apply = true; + else if (token === "--confirm") parsed.confirm = true; + else if (token === "--locality-only") parsed.localityOnly = true; + else if (token === "--eval-only") parsed.evalOnly = true; + else if (token === "--help" || token === "-h") parsed.help = true; + else if (token === "--as-of") { + const raw = argv[index + 1]; + if (!raw || raw.startsWith("--")) throw new Error("Missing value for --as-of"); + parsed.asOfRaw = raw; + index += 1; + } else { + throw new Error(`Unknown option: ${token}`); + } + } + + const args: BackfillSourceMetadataArgs = { ...parsed, asOf: parseBackfillAsOfDate(parsed.asOfRaw) }; + if (args.help) return args; + if (args.apply && (!args.localityOnly || !args.confirm)) { + throw new Error("Refusing metadata writes: --apply requires both --locality-only and --confirm."); + } + if (args.confirm && !args.apply) throw new Error("--confirm is only valid together with --apply."); + if (args.localityOnly && args.evalOnly) throw new Error("--locality-only cannot be combined with --eval-only."); + return args; +} + +function usage() { + return [ + "Usage: npm run backfill:source-metadata -- [options]", + "", + "Dry-run source metadata derivation by default. Production writes are restricted to deterministic locality fields.", + "", + "Options:", + " --locality-only Analyse only publisher_code, publisher, and jurisdiction.", + " --apply --confirm Apply locality-only patches after reviewing the dry-run.", + " --eval-only Restrict the legacy dry-run to retrieval eval documents.", + " --as-of Set the date used by the legacy status dry-run.", + " --help Show this help without loading provider configuration.", + ].join("\n"); +} function metadataRecord(value: unknown) { return value && typeof value === "object" && !Array.isArray(value) ? { ...(value as Record) } : {}; @@ -87,19 +131,9 @@ function titleWithoutExtension(fileName: string) { return fileName.replace(/\.[^.]+$/, ""); } -function publisherCodeFor(document: DocumentRow, text = "") { - const haystack = `${document.file_name} ${document.title} ${document.source_path ?? ""}`; - const parentheticalCodes = [...haystack.matchAll(/\(([A-Z]{2,8})\)/g)].map((match) => match[1]); - for (const code of parentheticalCodes) { - if (publisherByCode[code]) return code; - } - for (const code of Object.keys(publisherByCode).sort((a, b) => b.length - a.length)) { - if (new RegExp(`(?:^|[\\\\/\\s])${code}(?:[\\\\/\\s]|$)`, "i").test(haystack)) return code; - } - if (/\bBM[J)]\s+Best\s+Practice\b/i.test(text) || /\bStraight\s+to\s+the\s+point\s+of\s+care\b/i.test(text)) { - return "BMJ"; - } - return null; +function publisherCodeFor(document: DocumentRow) { + const identityMatch = inferSourceAuthorityFromIdentity(document); + return identityMatch.conflict ? null : identityMatch.code; } function sourceTypeFor(document: DocumentRow, text: string) { @@ -112,7 +146,7 @@ function sourceTypeFor(document: DocumentRow, text: string) { if (/\bguideline\b/.test(haystack)) return "guideline"; if (/\bshared care\b/.test(haystack)) return "shared_care_guideline"; if (/\bform\b|\bchart\b|\bplan\b/.test(haystack)) return "form_or_plan"; - if (publisherCodeFor(document, text) === "BMJ") return "clinical_reference"; + if (publisherCodeFor(document) === "BMJ") return "clinical_reference"; return "document"; } @@ -344,13 +378,13 @@ function extractDates(text: string) { return { review, publication, lastUpdated }; } -function documentStatusFor(dates: ReturnType, publisherCode: string | null) { +function documentStatusFor(dates: ReturnType, publisherCode: string | null, now: Date) { if (dates.review?.date) { - return new Date(`${dates.review.date}T23:59:59+08:00`) >= NOW ? "current" : "review_due"; + return new Date(`${dates.review.date}T23:59:59+08:00`) >= now ? "current" : "review_due"; } if (publisherCode === "BMJ" && dates.lastUpdated?.date) { const updated = new Date(`${dates.lastUpdated.date}T00:00:00+08:00`); - const threeYearsAgo = new Date(NOW); + const threeYearsAgo = new Date(now); threeYearsAgo.setFullYear(threeYearsAgo.getFullYear() - 3); return updated >= threeYearsAgo ? "current" : "review_due"; } @@ -378,7 +412,7 @@ function clinicalValidationEvidenceFor(args: { evidence_text: null, }; } - if (!args.publisherCode || publisherByCode[args.publisherCode]?.jurisdiction !== "Australia/WA") { + if (!args.publisherCode || sourceAuthorityForPublisherCode(args.publisherCode)?.scope !== "wa") { return { status: "unverified", basis: "not a local WA source", @@ -478,14 +512,19 @@ function setIfChanged(metadata: Record, key: string, value: unk changed.push(key); } -function deriveMetadata(document: DocumentRow, text: string, quality: QualityRow | undefined): DerivedMetadata { +function deriveMetadata( + document: DocumentRow, + text: string, + quality: QualityRow | undefined, + now: Date, +): DerivedMetadata { const metadata = metadataRecord(document.metadata); const changedKeys: string[] = []; - const publisherCode = publisherCodeFor(document, text); - const publisher = publisherCode ? publisherByCode[publisherCode] : null; + const publisherCode = publisherCodeFor(document); + const publisher = sourceAuthorityForPublisherCode(publisherCode); const extractedDates = extractDates(text); const dates = publisherCode === "BMJ" ? { ...extractedDates, review: null } : extractedDates; - const documentStatus = documentStatusFor(dates, publisherCode); + const documentStatus = documentStatusFor(dates, publisherCode, now); // Automatic review-cycle inference: when no explicit/derivable review date // exists but a publication date does, infer the status from the standard // review cycle (same logic as `scripts/derive-unknown-status.ts`). Documents @@ -493,7 +532,7 @@ function deriveMetadata(document: DocumentRow, text: string, quality: QualityRow // hidden or marked outdated. const inferredUnknownStatus = documentStatus === "unknown" && dates.publication?.date - ? deriveUnknownStatus(dates.publication.date, { now: NOW }) + ? deriveUnknownStatus(dates.publication.date, { now }) : null; const derivedStatus = inferredUnknownStatus?.kind === "derived" ? inferredUnknownStatus.status : documentStatus; const inferredReviewDate = inferredUnknownStatus?.kind === "derived" ? inferredUnknownStatus.reviewDate : null; @@ -509,7 +548,7 @@ function deriveMetadata(document: DocumentRow, text: string, quality: QualityRow setIfChanged(metadata, "source_title", titleWithoutExtension(document.file_name), changedKeys); setIfChanged(metadata, "publisher_code", publisherCode, changedKeys); setIfChanged(metadata, "publisher", publisher?.publisher, changedKeys); - setIfChanged(metadata, "jurisdiction", publisher?.jurisdiction, changedKeys); + setIfChanged(metadata, "jurisdiction", publisher?.jurisdictions[0], changedKeys); setIfChanged(metadata, "source_type", sourceTypeFor(document, text), changedKeys); setIfChanged(metadata, "category", categoryFor(document), changedKeys); setIfChanged(metadata, "publication_date", dates.publication?.date, changedKeys); @@ -621,8 +660,8 @@ async function loadPageText(documentIds: string[]) { return textByDocument; } -async function evalDocumentIds() { - if (!EVAL_ONLY) return null; +async function evalDocumentIds(evalOnly: boolean) { + if (!evalOnly) return null; const { readFileSync } = await import("node:fs"); const report = JSON.parse(readFileSync("output/evals/retrieval-quality-2026-06-30T09-18-10-598Z.json", "utf8")) as { retrieval?: { results?: Array<{ topResults?: Array<{ file_name: string }> }> }; @@ -634,28 +673,68 @@ async function evalDocumentIds() { return topFiles; } -async function main() { +async function runLocalityOnlyBackfill( + args: BackfillSourceMetadataArgs, + documents: DocumentRow[], + supabase: Awaited>, +) { + const analysed = documents.map((document) => ({ document, analysis: analyzeSourceLocality(document) })); + const changed = analysed.filter(({ analysis }) => !analysis.unresolvedConflict && analysis.changedKeys.length > 0); + const authorityAudit = auditSourceAuthorityDocuments(documents); + const summary = { + mode: args.apply ? "apply" : "dry-run", + scope: "locality-only", + allowed_metadata_keys: ["publisher_code", "publisher", "jurisdiction"], + documents_seen: documents.length, + documents_with_changes: changed.length, + ...authorityAudit, + }; + console.log(JSON.stringify(summary, null, 2)); + + if (!args.apply) return; + + for (const item of changed) { + const patch = item.analysis.changes as Record; + assertLocalityMetadataPatch(patch); + const { error } = await supabase.rpc("apply_document_metadata_patch", { + p_document_id: item.document.id, + p_metadata_patch: patch as Json, + }); + if (error) throw new Error(`Failed to patch ${item.document.file_name}: ${error.message}`); + } + console.log(`Applied ${changed.length} bounded locality metadata patch(es).`); +} + +export async function main(argv = process.argv.slice(2)) { + const args = parseBackfillSourceMetadataArgs(argv); + if (args.help) { + console.log(usage()); + return; + } + loadEnvConfig(process.cwd()); const { createAdminClient } = await import("@/lib/supabase/admin"); const supabase = createAdminClient(); - const [documents, qualityByDocument, evalFiles] = await Promise.all([ - loadAllDocuments(), - loadQualityRows(), - evalDocumentIds(), - ]); + const documents = await loadAllDocuments(); + if (args.localityOnly) { + await runLocalityOnlyBackfill(args, documents, supabase); + return; + } + + const [qualityByDocument, evalFiles] = await Promise.all([loadQualityRows(), evalDocumentIds(args.evalOnly)]); const targetDocuments = evalFiles ? documents.filter((document) => evalFiles.has(document.file_name)) : documents; const pageText = await loadPageText(targetDocuments.map((document) => document.id)); const derived = targetDocuments.map((document) => { const text = normalizeWhitespace(pageText.get(document.id) ?? ""); return { document, - ...deriveMetadata(document, text, qualityByDocument.get(document.id)), + ...deriveMetadata(document, text, qualityByDocument.get(document.id), args.asOf), }; }); const changed = derived.filter((item) => item.changedKeys.length > 0); const summary = { - mode: APPLY ? "apply" : "dry-run", - scope: EVAL_ONLY ? "eval-top-result-documents" : "all-documents", + mode: "dry-run", + scope: args.evalOnly ? "eval-top-result-documents" : "all-documents", documents_seen: targetDocuments.length, documents_with_changes: changed.length, status_counts: changed.reduce>((counts, item) => { @@ -681,20 +760,11 @@ async function main() { })), }; console.log(JSON.stringify(summary, null, 2)); - - if (!APPLY) return; - - for (const item of changed) { - const { error } = await supabase - .from("documents") - .update({ metadata: item.metadata as Json }) - .eq("id", item.document.id); - if (error) throw new Error(`Failed to update ${item.document.file_name}: ${error.message}`); - } - console.log(`Applied source metadata backfill to ${changed.length} documents.`); } -main().catch((error) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/eval-rag.ts b/scripts/eval-rag.ts index f85d2b4ee..ae93b9d41 100644 --- a/scripts/eval-rag.ts +++ b/scripts/eval-rag.ts @@ -1,4 +1,9 @@ import { loadEnvConfig } from "@next/env"; +import { + buildRagEvaluationDiagnostics, + evaluateAustralianRagExpectation, + type RagEvalProgressDiagnosticEvent, +} from "@/lib/rag-eval-diagnostics"; import { selectRagEvalCases, type RagEvalCase } from "@/lib/rag-eval-cases"; import type { RagAnswer } from "@/lib/types"; import { @@ -19,6 +24,7 @@ type EvalArgs = { question?: string; json: boolean; failOnThreshold: boolean; + expectAustralian: boolean; }; type EvalResult = { @@ -61,6 +67,8 @@ type EvalResult = { reasoningOutputTokens: number; totalTokens: number; estimatedCostUsd: number | null; + australianExpectationWarnings: string[]; + governanceDiagnostics: ReturnType; }; function evalSourceDiagnostics(answer: RagAnswer): EvalResult["retrievedSources"] { @@ -85,6 +93,7 @@ function parseArgs(argv: string[]): EvalArgs { ownerId: process.env.RAG_EVAL_OWNER_ID ?? process.env.LOCAL_NO_AUTH_OWNER_ID, json: false, failOnThreshold: false, + expectAustralian: false, }; for (let index = 0; index < argv.length; index += 1) { @@ -99,6 +108,10 @@ function parseArgs(argv: string[]): EvalArgs { args.failOnThreshold = true; continue; } + if (token === "--expect-australian") { + args.expectAustralian = true; + continue; + } const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); @@ -113,6 +126,7 @@ function parseArgs(argv: string[]): EvalArgs { if (args.limit !== undefined && (!Number.isInteger(args.limit) || args.limit <= 0)) { throw new Error("--limit must be a positive integer."); } + if (args.expectAustralian && !args.question) throw new Error("--expect-australian requires --question."); return args; } @@ -220,16 +234,24 @@ async function main() { if (!args.json) console.log(`Running ${cases.length} RAG eval case(s), scope=${scope}.`); for (const testCase of cases) { + const progress: RagEvalProgressDiagnosticEvent[] = []; const answer = (await withProviderBackoff(`rag:${testCase.id}`, () => answerQuestionWithScope({ query: testCase.question, ownerId, logQuery: false, skipCache: true, + onProgress: (event) => { + progress.push(event); + }, }), )) as RagAnswer; const latencyMs = answer.latencyTimings?.total_latency_ms ?? 0; const validation = validateRagAnswer(testCase, answer); + const governanceDiagnostics = buildRagEvaluationDiagnostics(answer, progress); + const australianExpectation = args.expectAustralian + ? evaluateAustralianRagExpectation(governanceDiagnostics) + : { passed: true, failures: [], warnings: [] }; const result: EvalResult = { id: testCase.id, question: testCase.question, @@ -245,7 +267,7 @@ async function main() { model: answer.modelUsed ?? null, citations: answer.citations.length, visualEvidence: answer.visualEvidence?.length ?? 0, - failures: validation.failures, + failures: [...validation.failures, ...australianExpectation.failures], retrievedSources: evalSourceDiagnostics(answer), retrievalIntent: answer.smartApiPlan?.answerPlan.retrievalIntent, sourceSelection: answer.smartApiPlan?.answerPlan.sourceSelection, @@ -264,6 +286,8 @@ async function main() { cachedInputTokens: answer.openAIUsage?.cached_input_tokens ?? 0, outputTokens: answer.openAIUsage?.output_tokens ?? 0, }), + australianExpectationWarnings: australianExpectation.warnings, + governanceDiagnostics, }; results.push(result); @@ -279,6 +303,13 @@ async function main() { ); console.log(` Q: ${testCase.question}`); if (citationSummary) console.log(` Sources: ${citationSummary}`); + if (args.expectAustralian) { + console.log( + ` Australian evidence: passages=${governanceDiagnostics.australian_candidate_passage_count}, documents=${governanceDiagnostics.australian_candidate_document_count}, valid citations=${governanceDiagnostics.valid_australian_citation_count}, conflicts=${governanceDiagnostics.authority_conflict_count}, supplementary violation=${governanceDiagnostics.supplementary_selected_despite_sufficient_australian ? "yes" : "no"}`, + ); + console.log(` Progress: ${governanceDiagnostics.progress_sequence.join(" -> ") || "none"}`); + for (const warning of result.australianExpectationWarnings) console.log(` Warning: ${warning}`); + } if (result.failures.length > 0) { console.log( [ @@ -306,7 +337,13 @@ async function main() { const thresholdFailures = summarizeFailures(results); if (args.json) { - console.log(JSON.stringify({ scope, results, thresholdFailures }, null, 2)); + console.log( + JSON.stringify( + { scope, expectations: { australian: args.expectAustralian }, results, thresholdFailures }, + null, + 2, + ), + ); } else { printHumanSummary(results); if (thresholdFailures.length > 0) { diff --git a/src/lib/rag-eval-diagnostics.ts b/src/lib/rag-eval-diagnostics.ts new file mode 100644 index 000000000..6e2a0f5cc --- /dev/null +++ b/src/lib/rag-eval-diagnostics.ts @@ -0,0 +1,156 @@ +import { + australianSourceClassification, + australianSourceTier, + isAustralianSourceTier, + type AustralianSourceTier, +} from "@/lib/australian-source-priority"; +import type { RagAnswer, SearchResult } from "@/lib/types"; + +export type RagEvalProgressDiagnosticEvent = { + stage: string; + mode?: RagAnswer["routingMode"]; + selectedContextCount?: number; + australianSourceCount?: number; + waSourceCount?: number; + usedSupplementaryFallback?: boolean; + timingMs?: number; +}; + +function emptyTierCounts(): Record { + return { + wa_validated: 0, + australian_national: 0, + australian_state: 0, + supplementary: 0, + }; +} + +function sourceTierCounts(sources: Array>) { + return sources.reduce>((counts, source) => { + counts[australianSourceTier(source)] += 1; + return counts; + }, emptyTierCounts()); +} + +function sourceForCitation(answer: RagAnswer, chunkId: string) { + return answer.sources.find((source) => source.id === chunkId) ?? null; +} + +function genericFinalizationFailure(answer: string) { + return /(?:could not|unable to) generate (?:a )?finali[sz]ed answer|review the source snippets below/i.test(answer); +} + +export function buildRagEvaluationDiagnostics(answer: RagAnswer, progress: RagEvalProgressDiagnosticEvent[] = []) { + const sourceClassifications = answer.sources.map((source) => ({ + source, + classification: australianSourceClassification(source), + })); + const australianSources = sourceClassifications.filter(({ classification }) => + isAustralianSourceTier(classification.tier), + ); + const citationIsValid = (citation: RagAnswer["citations"][number]) => { + const source = sourceForCitation(answer, citation.chunk_id); + return Boolean(source && source.document_id === citation.document_id); + }; + const validCitations = answer.citations.filter(citationIsValid); + const validCitationSources = validCitations + .map((citation) => sourceForCitation(answer, citation.chunk_id)) + .filter((source): source is SearchResult => Boolean(source)); + const invalidCitations = answer.citations.filter((citation) => !citationIsValid(citation)); + const validAustralianCitations = validCitationSources.filter((source) => + isAustralianSourceTier(australianSourceTier(source)), + ); + const australianDocumentCount = new Set(australianSources.map(({ source }) => source.document_id)).size; + const sufficientAustralianCandidates = australianSources.length >= 4 && australianDocumentCount >= 2; + const supplementaryContextSelected = progress.some( + (event) => (event.stage === "ranking" || event.stage === "fallback") && event.usedSupplementaryFallback === true, + ); + const supplementaryCitationCount = validCitationSources.filter( + (source) => australianSourceTier(source) === "supplementary", + ).length; + const authorityConflicts = sourceClassifications + .filter(({ classification }) => classification.conflict) + .map(({ source, classification }) => ({ + chunk_id: source.id, + document_id: source.document_id, + file_name: source.file_name, + conflicts: classification.conflicts, + })); + const generationRoutes = [ + ...progress + .filter((event) => event.stage === "generating" || event.stage === "retrying" || event.stage === "fallback") + .map((event) => event.mode) + .filter((mode): mode is NonNullable => Boolean(mode)), + ...(answer.routingMode ? [answer.routingMode] : []), + ]; + + return { + grounded: answer.grounded, + source_tier_counts: sourceTierCounts(answer.sources), + citation_tier_counts: sourceTierCounts(validCitationSources), + australian_candidate_passage_count: australianSources.length, + australian_candidate_document_count: australianDocumentCount, + authority_conflict_count: authorityConflicts.length, + authority_conflicts: authorityConflicts, + citation_count: answer.citations.length, + valid_citation_count: validCitations.length, + invalid_citation_count: invalidCitations.length, + invalid_citation_chunk_ids: invalidCitations.map((citation) => citation.chunk_id), + valid_australian_citation_count: validAustralianCitations.length, + unverified_numeric_token_count: answer.unverifiedNumericTokens?.length ?? 0, + unverified_numeric_tokens: answer.unverifiedNumericTokens ?? [], + supplementary_context_selected: supplementaryContextSelected, + supplementary_citation_count: supplementaryCitationCount, + sufficient_australian_candidates: sufficientAustralianCandidates, + supplementary_selected_despite_sufficient_australian: + sufficientAustralianCandidates && (supplementaryContextSelected || supplementaryCitationCount > 0), + generic_finalization_failure: genericFinalizationFailure(answer.answer), + route: answer.routingMode ?? "none", + generation_routes: [...new Set(generationRoutes)], + provider_mode: answer.providerMode ?? null, + response_mode: answer.responseMode ?? null, + answer_quality_tier: answer.answerQualityTier ?? null, + fallback_reason: answer.fallbackReason ?? null, + degraded_mode: answer.degradedMode ?? null, + latency_timings: answer.latencyTimings ?? null, + progress_sequence: progress.map((event) => event.stage), + progress_events: progress.map((event) => ({ + stage: event.stage, + mode: event.mode ?? null, + selected_context_count: event.selectedContextCount ?? null, + australian_source_count: event.australianSourceCount ?? null, + wa_source_count: event.waSourceCount ?? null, + used_supplementary_fallback: event.usedSupplementaryFallback ?? null, + timing_ms: event.timingMs ?? null, + })), + }; +} + +export function evaluateAustralianRagExpectation(diagnostics: ReturnType) { + const failures: string[] = []; + const warnings: string[] = []; + + if (!diagnostics.grounded) failures.push("answer was not grounded"); + if (diagnostics.valid_australian_citation_count < 1) failures.push("no valid Australian citation"); + if (diagnostics.invalid_citation_count > 0) { + failures.push(`invalid citations ${diagnostics.invalid_citation_count}`); + } + if (diagnostics.authority_conflict_count > 0) { + failures.push(`source authority conflicts ${diagnostics.authority_conflict_count}`); + } + if (diagnostics.unverified_numeric_token_count > 0) { + failures.push(`unverified numeric tokens ${diagnostics.unverified_numeric_token_count}`); + } + if (diagnostics.generic_finalization_failure) failures.push("generic finalization failure returned"); + if (diagnostics.supplementary_selected_despite_sufficient_australian) { + failures.push("supplementary evidence selected despite sufficient Australian evidence"); + } + if (diagnostics.australian_candidate_passage_count < 4) { + warnings.push(`Australian candidate passages ${diagnostics.australian_candidate_passage_count}/4`); + } + if (diagnostics.australian_candidate_document_count < 2) { + warnings.push(`Australian candidate documents ${diagnostics.australian_candidate_document_count}/2`); + } + + return { passed: failures.length === 0, failures, warnings }; +} diff --git a/src/lib/source-authority-metadata.ts b/src/lib/source-authority-metadata.ts new file mode 100644 index 000000000..73543f0f5 --- /dev/null +++ b/src/lib/source-authority-metadata.ts @@ -0,0 +1,255 @@ +import { + classifySourceAuthority, + sourceAuthorityForPublisher, + sourceAuthorityForPublisherCode, + sourceAuthorityRegistry, + type SourceAuthorityDefinition, +} from "@/lib/source-authority-registry"; + +export const localityMetadataKeys = ["publisher_code", "publisher", "jurisdiction"] as const; + +export type LocalityMetadataKey = (typeof localityMetadataKeys)[number]; + +export type SourceAuthorityDocumentIdentity = { + id?: string; + title: string; + file_name: string; + source_path?: string | null; +}; + +export type SourceAuthorityDocument = SourceAuthorityDocumentIdentity & { + metadata: Record | null; +}; + +export type SourceIdentityAuthorityMatch = { + authority: SourceAuthorityDefinition | null; + code: string | null; + codes: string[]; + conflict: boolean; + authorityKeys: string[]; +}; + +export type SourceLocalityAnalysis = { + authority: SourceAuthorityDefinition | null; + matchedBy: "publisher_code" | "identity_code" | "publisher_alias" | "none"; + targetCode: string | null; + conflicts: string[]; + unresolvedConflict: boolean; + missingLocalityKeys: LocalityMetadataKey[]; + changes: Partial>; + changedKeys: LocalityMetadataKey[]; +}; + +function metadataRecord(value: unknown) { + return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; +} + +function metadataString(value: unknown) { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function normalizedCode(value: unknown) { + return metadataString(value)?.toUpperCase() ?? null; +} + +function escapeRegExp(value: string) { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +const registeredCodes = sourceAuthorityRegistry + .flatMap((authority) => authority.codes.map((code) => ({ code: code.toUpperCase(), authority }))) + .sort((left, right) => right.code.length - left.code.length || left.code.localeCompare(right.code)); + +function identityFields(identity: SourceAuthorityDocumentIdentity) { + return [identity.file_name, identity.source_path ?? "", identity.title]; +} + +/** + * Infer an authority only from exact registered code tokens in document identity + * fields. Publisher names and clinical document prose are never authority evidence. + */ +export function inferSourceAuthorityFromIdentity( + identity: SourceAuthorityDocumentIdentity, +): SourceIdentityAuthorityMatch { + const matches: Array<{ code: string; authority: SourceAuthorityDefinition }> = []; + + for (const field of identityFields(identity)) { + for (const candidate of registeredCodes) { + const token = new RegExp(`(?:^|[^A-Z0-9])${escapeRegExp(candidate.code)}(?=$|[^A-Z0-9])`, "i"); + if (token.test(field) && !matches.some((match) => match.code === candidate.code)) matches.push(candidate); + } + } + + const authorityKeys = [...new Set(matches.map((match) => match.authority.key))]; + const conflict = authorityKeys.length > 1; + const first = matches[0] ?? null; + + return { + authority: conflict ? null : (first?.authority ?? null), + code: conflict ? null : (first?.code ?? null), + codes: matches.map((match) => match.code), + conflict, + authorityKeys, + }; +} + +function canonicalJurisdiction(authority: SourceAuthorityDefinition) { + return authority.jurisdictions[0] ?? null; +} + +function valuesDiffer(current: unknown, next: string | null) { + if (!next) return false; + return metadataString(current) !== next; +} + +export function analyzeSourceLocality(document: SourceAuthorityDocument): SourceLocalityAnalysis { + const metadata = metadataRecord(document.metadata); + const existingCode = normalizedCode(metadata.publisher_code); + const codeAuthority = sourceAuthorityForPublisherCode(existingCode); + const publisherAuthority = sourceAuthorityForPublisher(metadataString(metadata.publisher)); + const identityMatch = inferSourceAuthorityFromIdentity(document); + const classification = classifySourceAuthority(metadata); + const identityClassification = identityMatch.code + ? classifySourceAuthority({ ...metadata, publisher_code: identityMatch.code }) + : null; + const conflicts: string[] = [...classification.conflicts]; + let unresolvedConflict = identityMatch.conflict; + + if (identityMatch.conflict) conflicts.push(`identity_multiple_authorities:${identityMatch.authorityKeys.join(",")}`); + if ( + codeAuthority && + publisherAuthority && + codeAuthority.key !== publisherAuthority.key && + classification.conflicts.includes("publisher_mismatch") + ) { + conflicts.push(`publisher_code_publisher_conflict:${codeAuthority.key}/${publisherAuthority.key}`); + unresolvedConflict = true; + } + if (codeAuthority && identityMatch.authority && codeAuthority.key !== identityMatch.authority.key) { + conflicts.push(`publisher_code_identity_conflict:${codeAuthority.key}/${identityMatch.authority.key}`); + unresolvedConflict = true; + } + if ( + !codeAuthority && + identityMatch.authority && + publisherAuthority && + identityMatch.authority.key !== publisherAuthority.key && + identityClassification?.conflicts.includes("publisher_mismatch") + ) { + conflicts.push(`publisher_identity_conflict:${publisherAuthority.key}/${identityMatch.authority.key}`); + unresolvedConflict = true; + } + + const authority = unresolvedConflict ? null : (codeAuthority ?? identityMatch.authority ?? publisherAuthority); + if (authority && existingCode && !codeAuthority) conflicts.push(`publisher_code_unrecognized:${existingCode}`); + const matchedBy = codeAuthority + ? "publisher_code" + : identityMatch.authority + ? "identity_code" + : publisherAuthority + ? "publisher_alias" + : "none"; + const targetCode = authority + ? codeAuthority?.key === authority.key && existingCode + ? existingCode + : identityMatch.authority?.key === authority.key && identityMatch.code + ? identityMatch.code + : (authority.codes[0] ?? null) + : null; + const targetJurisdiction = authority ? canonicalJurisdiction(authority) : null; + const changes: Partial> = {}; + + if (authority && targetCode && valuesDiffer(metadata.publisher_code, targetCode)) changes.publisher_code = targetCode; + if (authority && valuesDiffer(metadata.publisher, authority.publisher)) changes.publisher = authority.publisher; + if (authority && targetJurisdiction && valuesDiffer(metadata.jurisdiction, targetJurisdiction)) { + changes.jurisdiction = targetJurisdiction; + } + + const missingLocalityKeys = + authority && authority.scope !== "international" + ? localityMetadataKeys.filter((key) => !metadataString(metadata[key])) + : []; + + return { + authority, + matchedBy, + targetCode, + conflicts: [...new Set(conflicts)], + unresolvedConflict, + missingLocalityKeys, + changes, + changedKeys: localityMetadataKeys.filter((key) => changes[key] !== undefined), + }; +} + +export function assertLocalityMetadataPatch(patch: Record) { + const invalidKeys = Object.keys(patch).filter( + (key): key is string => !localityMetadataKeys.includes(key as LocalityMetadataKey), + ); + if (invalidKeys.length > 0) { + throw new Error(`Locality-only metadata patch contains disallowed keys: ${invalidKeys.sort().join(", ")}`); + } + for (const [key, value] of Object.entries(patch)) { + if (typeof value !== "string" || !value.trim()) { + throw new Error(`Locality-only metadata patch ${key} must be a non-empty string.`); + } + } +} + +function compactAuthorityDocument(document: SourceAuthorityDocument, analysis: SourceLocalityAnalysis) { + const metadata = metadataRecord(document.metadata); + return { + id: document.id ?? null, + title: document.title, + file_name: document.file_name, + publisher_code: metadataString(metadata.publisher_code), + publisher: metadataString(metadata.publisher), + jurisdiction: metadataString(metadata.jurisdiction), + authority_key: analysis.authority?.key ?? null, + authority_scope: analysis.authority?.scope ?? null, + matched_by: analysis.matchedBy, + }; +} + +export function auditSourceAuthorityDocuments(documents: SourceAuthorityDocument[]) { + const analyses = documents.map((document) => ({ document, analysis: analyzeSourceLocality(document) })); + const recognized = analyses.filter(({ analysis }) => analysis.authority); + const australianCandidates = recognized.filter(({ analysis }) => analysis.authority?.scope !== "international"); + const conflicts = analyses.filter(({ analysis }) => analysis.conflicts.length > 0); + const missingLocality = australianCandidates.filter(({ analysis }) => analysis.missingLocalityKeys.length > 0); + const proposedCorrections = analyses.filter( + ({ analysis }) => !analysis.unresolvedConflict && analysis.changedKeys.length > 0, + ); + const conflictReasonCounts = conflicts.reduce>((counts, { analysis }) => { + for (const conflict of analysis.conflicts) counts[conflict] = (counts[conflict] ?? 0) + 1; + return counts; + }, {}); + + return { + recognized_documents: recognized.length, + australian_authority_candidates: australianCandidates.length, + international_authority_documents: recognized.length - australianCandidates.length, + authority_conflict_count: conflicts.length, + authority_conflict_reason_counts: Object.fromEntries( + Object.entries(conflictReasonCounts).sort(([left], [right]) => left.localeCompare(right)), + ), + unresolved_authority_conflict_count: conflicts.filter(({ analysis }) => analysis.unresolvedConflict).length, + missing_australian_locality_count: missingLocality.length, + proposed_locality_correction_count: proposedCorrections.length, + passed: conflicts.length === 0 && missingLocality.length === 0, + conflicts: conflicts.map(({ document, analysis }) => ({ + ...compactAuthorityDocument(document, analysis), + conflicts: analysis.conflicts, + safe_correction_available: !analysis.unresolvedConflict && analysis.changedKeys.length > 0, + })), + missing_australian_locality: missingLocality.map(({ document, analysis }) => ({ + ...compactAuthorityDocument(document, analysis), + missing_keys: analysis.missingLocalityKeys, + })), + proposed_locality_corrections: proposedCorrections.map(({ document, analysis }) => ({ + ...compactAuthorityDocument(document, analysis), + changed_keys: analysis.changedKeys, + changes: analysis.changes, + })), + }; +} diff --git a/tests/rag-eval-source-governance.test.ts b/tests/rag-eval-source-governance.test.ts new file mode 100644 index 000000000..4f3ef3aab --- /dev/null +++ b/tests/rag-eval-source-governance.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it } from "vitest"; +import { buildRagEvaluationDiagnostics, evaluateAustralianRagExpectation } from "@/lib/rag-eval-diagnostics"; +import type { Citation, RagAnswer, SearchResult } from "@/lib/types"; + +function source(args: { + id: string; + documentId: string; + publisherCode?: string; + publisher?: string; + jurisdiction?: string; +}): SearchResult { + const publisherCode = args.publisherCode ?? "WACHS"; + return { + id: args.id, + document_id: args.documentId, + title: `${publisherCode} lithium guidance`, + file_name: `${publisherCode}-lithium-${args.documentId}.pdf`, + page_number: 1, + chunk_index: 0, + section_heading: "Lithium", + content: "Lithium guidance with a directly supported monitoring statement.", + image_ids: [], + similarity: 0.9, + source_metadata: { + source_title: `${publisherCode} lithium guidance`, + publisher_code: publisherCode, + publisher: args.publisher ?? (publisherCode === "BMJ" ? "BMJ Best Practice" : "WA Country Health Service"), + jurisdiction: args.jurisdiction ?? (publisherCode === "BMJ" ? "International" : "Australia/WA"), + version: null, + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "current", + clinical_validation_status: publisherCode === "BMJ" ? "unverified" : "locally_reviewed", + extraction_quality: "good", + }, + images: [], + }; +} + +function citationFor(item: SearchResult): Citation { + return { + chunk_id: item.id, + document_id: item.document_id, + title: item.title, + file_name: item.file_name, + page_number: item.page_number, + chunk_index: item.chunk_index, + source_metadata: item.source_metadata, + }; +} + +function answer(sources: SearchResult[], citations: Citation[]): RagAnswer { + return { + answer: "A finalized source-backed lithium answer.", + grounded: true, + confidence: "high", + sources, + citations, + routingMode: "fast", + responseMode: "clinical_pathway", + answerQualityTier: "model_synthesis", + providerMode: "openai", + fallbackReason: null, + latencyTimings: { generation_latency_ms: 1200, total_latency_ms: 1800 }, + unverifiedNumericTokens: [], + }; +} + +describe("Australian RAG evaluation diagnostics", () => { + it("passes a grounded answer with valid Australian citations and sufficient local candidates", () => { + const sources = [ + source({ id: "wa-1", documentId: "wa-a" }), + source({ id: "wa-2", documentId: "wa-a" }), + source({ id: "wa-3", documentId: "wa-b" }), + source({ id: "wa-4", documentId: "wa-b" }), + ]; + const diagnostics = buildRagEvaluationDiagnostics(answer(sources, [citationFor(sources[0])]), [ + { stage: "retrieved" }, + { stage: "ranking", usedSupplementaryFallback: false, australianSourceCount: 4 }, + { stage: "generating", mode: "fast" }, + { stage: "verifying" }, + ]); + const expectation = evaluateAustralianRagExpectation(diagnostics); + + expect(expectation).toEqual({ passed: true, failures: [], warnings: [] }); + expect(diagnostics.source_tier_counts.wa_validated).toBe(4); + expect(diagnostics.valid_australian_citation_count).toBe(1); + expect(diagnostics.progress_sequence).toEqual(["retrieved", "ranking", "generating", "verifying"]); + expect(diagnostics.generation_routes).toEqual(["fast"]); + expect(diagnostics.latency_timings?.total_latency_ms).toBe(1800); + }); + + it("fails when supplementary evidence is selected despite four Australian passages across two documents", () => { + const australian = [ + source({ id: "wa-1", documentId: "wa-a" }), + source({ id: "wa-2", documentId: "wa-a" }), + source({ id: "wa-3", documentId: "wa-b" }), + source({ id: "wa-4", documentId: "wa-b" }), + ]; + const supplementary = source({ id: "bmj-1", documentId: "bmj", publisherCode: "BMJ" }); + const diagnostics = buildRagEvaluationDiagnostics( + answer([...australian, supplementary], [citationFor(australian[0]), citationFor(supplementary)]), + [{ stage: "ranking", usedSupplementaryFallback: true, australianSourceCount: 4 }], + ); + + expect(diagnostics.supplementary_selected_despite_sufficient_australian).toBe(true); + expect(evaluateAustralianRagExpectation(diagnostics).failures).toContain( + "supplementary evidence selected despite sufficient Australian evidence", + ); + }); + + it("fails authority, citation, numeric, and generic-finalization defects", () => { + const conflicted = source({ + id: "wa-conflict", + documentId: "wa-a", + publisher: "NPS MedicineWise", + }); + const invalidCitation = { ...citationFor(conflicted), chunk_id: "missing-chunk" }; + const defective = { + ...answer([conflicted], [invalidCitation]), + answer: "I found matching indexed passages, but could not generate a finalized answer right now.", + unverifiedNumericTokens: ["900 mg"], + }; + const diagnostics = buildRagEvaluationDiagnostics(defective); + const expectation = evaluateAustralianRagExpectation(diagnostics); + + expect(expectation.passed).toBe(false); + expect(expectation.failures).toEqual( + expect.arrayContaining([ + "no valid Australian citation", + "invalid citations 1", + "source authority conflicts 1", + "unverified numeric tokens 1", + "generic finalization failure returned", + ]), + ); + }); + + it("warns without failing when Australian coverage is valid but sparse", () => { + const local = source({ id: "wa-1", documentId: "wa-a" }); + const diagnostics = buildRagEvaluationDiagnostics(answer([local], [citationFor(local)])); + const expectation = evaluateAustralianRagExpectation(diagnostics); + + expect(expectation.passed).toBe(true); + expect(expectation.warnings).toEqual(["Australian candidate passages 1/4", "Australian candidate documents 1/2"]); + }); +}); diff --git a/tests/source-authority-tooling.test.ts b/tests/source-authority-tooling.test.ts new file mode 100644 index 000000000..988c2ee48 --- /dev/null +++ b/tests/source-authority-tooling.test.ts @@ -0,0 +1,149 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; +import { + analyzeSourceLocality, + assertLocalityMetadataPatch, + auditSourceAuthorityDocuments, + inferSourceAuthorityFromIdentity, + type SourceAuthorityDocument, +} from "@/lib/source-authority-metadata"; +import { parseBackfillSourceMetadataArgs } from "../scripts/backfill-source-metadata"; + +function document( + overrides: Partial & Pick, +): SourceAuthorityDocument { + return { + id: overrides.id ?? "document-id", + title: overrides.title ?? overrides.file_name, + file_name: overrides.file_name, + source_path: overrides.source_path ?? null, + metadata: overrides.metadata ?? {}, + }; +} + +describe("source authority metadata tooling", () => { + it("infers WA and national authorities from exact registry code tokens", () => { + expect(inferSourceAuthorityFromIdentity(document({ file_name: "WACHS-lithium-guideline.pdf" }))).toMatchObject({ + code: "WACHS", + authority: { key: "wa-country-health-service", scope: "wa" }, + conflict: false, + }); + expect( + inferSourceAuthorityFromIdentity(document({ file_name: "TGA/lithium-product-information.pdf" })), + ).toMatchObject({ + code: "TGA", + authority: { key: "tga", scope: "australian_national" }, + conflict: false, + }); + }); + + it("does not require locality metadata for unknown or international documents", () => { + const report = auditSourceAuthorityDocuments([ + document({ file_name: "unknown-clinical-reference.pdf" }), + document({ file_name: "BMJ-lithium.pdf", metadata: { publisher_code: "BMJ" } }), + ]); + + expect(report.missing_australian_locality_count).toBe(0); + expect(report.passed).toBe(true); + }); + + it("gates inferable Australian sources with missing locality and proposes only safe fields", () => { + const candidate = document({ file_name: "WACHS-lithium-guideline.pdf" }); + const analysis = analyzeSourceLocality(candidate); + const report = auditSourceAuthorityDocuments([candidate]); + + expect(analysis.missingLocalityKeys).toEqual(["publisher_code", "publisher", "jurisdiction"]); + expect(analysis.changes).toEqual({ + publisher_code: "WACHS", + publisher: "WA Country Health Service", + jurisdiction: "Australia/WA", + }); + expect(report.passed).toBe(false); + expect(report.missing_australian_locality_count).toBe(1); + expect(report.proposed_locality_corrections[0]?.changed_keys).toEqual([ + "publisher_code", + "publisher", + "jurisdiction", + ]); + }); + + it("fails closed on cross-authority code and publisher claims", () => { + const conflicted = document({ + file_name: "lithium-guideline.pdf", + metadata: { + publisher_code: "WACHS", + publisher: "NPS MedicineWise", + jurisdiction: "Australia/WA", + }, + }); + const analysis = analyzeSourceLocality(conflicted); + const report = auditSourceAuthorityDocuments([conflicted]); + + expect(analysis.unresolvedConflict).toBe(true); + expect(analysis.conflicts).toContain( + "publisher_code_publisher_conflict:wa-country-health-service/nps-medicinewise", + ); + expect(analysis.changes).toEqual({}); + expect(report.authority_conflict_reason_counts).toMatchObject({ + publisher_mismatch: 1, + "publisher_code_publisher_conflict:wa-country-health-service/nps-medicinewise": 1, + }); + }); + + it("canonicalizes compatible aliases without broad metadata changes", () => { + const analysis = analyzeSourceLocality( + document({ + file_name: "WACHS-lithium-guideline.pdf", + metadata: { + publisher_code: "WACHS", + publisher: "WA Health", + jurisdiction: "WA", + clinical_validation_status: "approved", + }, + }), + ); + + expect(analysis.unresolvedConflict).toBe(false); + expect(analysis.changes).toEqual({ + publisher: "WA Country Health Service", + jurisdiction: "Australia/WA", + }); + expect(Object.keys(analysis.changes)).not.toContain("clinical_validation_status"); + }); + + it("enforces the locality patch allowlist", () => { + expect(() => + assertLocalityMetadataPatch({ + publisher_code: "WACHS", + publisher: "WA Country Health Service", + jurisdiction: "Australia/WA", + }), + ).not.toThrow(); + expect(() => assertLocalityMetadataPatch({ publisher_code: "WACHS", document_status: "current" })).toThrow( + /disallowed keys: document_status/, + ); + }); + + it("refuses every write form except confirmed locality-only apply", () => { + expect(() => parseBackfillSourceMetadataArgs(["--apply"])).toThrow(/requires both --locality-only and --confirm/); + expect(() => parseBackfillSourceMetadataArgs(["--apply", "--confirm"])).toThrow( + /requires both --locality-only and --confirm/, + ); + expect(() => parseBackfillSourceMetadataArgs(["--confirm"])).toThrow(/only valid together with --apply/); + expect(parseBackfillSourceMetadataArgs(["--locality-only", "--apply", "--confirm"])).toMatchObject({ + localityOnly: true, + apply: true, + confirm: true, + }); + }); + + it("makes both governance scripts consume the shared authority helper", () => { + const auditScript = readFileSync("scripts/audit-source-governance.ts", "utf8"); + const backfillScript = readFileSync("scripts/backfill-source-metadata.ts", "utf8"); + + expect(auditScript).toContain('from "@/lib/source-authority-metadata"'); + expect(backfillScript).toContain('from "@/lib/source-authority-metadata"'); + expect(backfillScript).not.toContain("const publisherByCode"); + }); +}); From c5fde11e64d8976e1c163d1b8618f58a52e0b8ff Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:23:00 +0800 Subject: [PATCH 06/14] fix(governance): exclude non-document authority signals --- scripts/audit-source-governance.ts | 17 +++-- src/lib/source-authority-metadata.ts | 68 +++++++++++++++++--- tests/source-authority-tooling.test.ts | 86 ++++++++++++++++++++++++++ 3 files changed, 156 insertions(+), 15 deletions(-) diff --git a/scripts/audit-source-governance.ts b/scripts/audit-source-governance.ts index 37419f6c4..93f2a011b 100644 --- a/scripts/audit-source-governance.ts +++ b/scripts/audit-source-governance.ts @@ -2,7 +2,7 @@ import { readFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; import * as nextEnv from "@next/env"; -import { auditSourceAuthorityDocuments } from "@/lib/source-authority-metadata"; +import { auditSourceAuthorityDocuments, isRegistryRecordSource } from "@/lib/source-authority-metadata"; import type { DocumentLabel } from "@/lib/types"; const loadEnvConfig = @@ -267,6 +267,8 @@ export async function main(argv = process.argv.slice(2)) { const labels = await fetchAll(supabase, "document_labels", "id,document_id,label_type,source", (query) => query.eq("source", "generated"), ); + const clinicalGovernanceDocuments = documents.filter((document) => !isRegistryRecordSource(document)); + const registryRecordsExcludedFromClinicalMetadataGate = documents.length - clinicalGovernanceDocuments.length; const statusCounts = new Map(); const validationCounts = new Map(); @@ -274,7 +276,7 @@ export async function main(argv = process.argv.slice(2)) { const requiredMissingCounts = new Map(); const missingRequiredDocuments: Array & { missing_keys: string[] }> = []; - for (const document of documents) { + for (const document of clinicalGovernanceDocuments) { const metadata = metadataRecord(document.metadata); increment(statusCounts, stringValue(metadata.document_status)); increment(validationCounts, stringValue(metadata.clinical_validation_status)); @@ -369,6 +371,8 @@ export async function main(argv = process.argv.slice(2)) { const report = { mode: "read-only", indexed_documents: documents.length, + clinical_governance_documents: clinicalGovernanceDocuments.length, + registry_records_excluded_from_clinical_metadata_gate: registryRecordsExcludedFromClinicalMetadataGate, required_metadata_missing_total: requiredMetadataMissingTotal, required_metadata_missing_counts: Object.fromEntries( requiredMetadataKeys.map((key) => [key, requiredMissingCounts.get(key) ?? 0]), @@ -385,15 +389,15 @@ export async function main(argv = process.argv.slice(2)) { indexed_without_smart_v2_labels: missingSmartV2LabelDocuments.length, }, debt_counts: debtCounts, - sample_review_due_documents: documents + sample_review_due_documents: clinicalGovernanceDocuments .filter((document) => metadataRecord(document.metadata).document_status === "review_due") .slice(0, 10) .map(compactDocument), - sample_unknown_status_documents: documents + sample_unknown_status_documents: clinicalGovernanceDocuments .filter((document) => metadataRecord(document.metadata).document_status === "unknown") .slice(0, 10) .map(compactDocument), - sample_unverified_documents: documents + sample_unverified_documents: clinicalGovernanceDocuments .filter((document) => metadataRecord(document.metadata).clinical_validation_status === "unverified") .slice(0, 10) .map(compactDocument), @@ -427,6 +431,9 @@ export async function main(argv = process.argv.slice(2)) { console.log("[Source Governance Audit]"); console.log(`Mode: ${report.mode}`); console.log(`Indexed documents: ${report.indexed_documents}`); + console.log( + `Clinical governance documents: ${report.clinical_governance_documents} (registry projections excluded: ${report.registry_records_excluded_from_clinical_metadata_gate})`, + ); console.log(`Required metadata missing: ${report.required_metadata_missing_total}`); console.log( `Document status: ${Object.entries(report.document_status_counts) diff --git a/src/lib/source-authority-metadata.ts b/src/lib/source-authority-metadata.ts index 73543f0f5..dd7de3ad5 100644 --- a/src/lib/source-authority-metadata.ts +++ b/src/lib/source-authority-metadata.ts @@ -32,6 +32,7 @@ export type SourceIdentityAuthorityMatch = { export type SourceLocalityAnalysis = { authority: SourceAuthorityDefinition | null; matchedBy: "publisher_code" | "identity_code" | "publisher_alias" | "none"; + excludedReason: "registry_record" | null; targetCode: string | null; conflicts: string[]; unresolvedConflict: boolean; @@ -60,8 +61,38 @@ const registeredCodes = sourceAuthorityRegistry .flatMap((authority) => authority.codes.map((code) => ({ code: code.toUpperCase(), authority }))) .sort((left, right) => right.code.length - left.code.length || left.code.localeCompare(right.code)); -function identityFields(identity: SourceAuthorityDocumentIdentity) { - return [identity.file_name, identity.source_path ?? "", identity.title]; +const caseSensitiveIdentityCodes = new Set(["WHO"]); + +function authorityMatchesInField(field: string) { + return registeredCodes.filter((candidate) => { + const flags = caseSensitiveIdentityCodes.has(candidate.code) ? "" : "i"; + const token = new RegExp(`(?:^|[^A-Za-z0-9])${escapeRegExp(candidate.code)}(?=$|[^A-Za-z0-9])`, flags); + return token.test(field); + }); +} + +function preferredIdentityMatches(identity: SourceAuthorityDocumentIdentity) { + const trailingParenthetical = identity.file_name.match(/\(([^()]*)\)\s*(?:\.[^./\\]+)?$/)?.[1]; + const trailingMatches = trailingParenthetical ? authorityMatchesInField(trailingParenthetical) : []; + if (trailingMatches.length > 0) return trailingMatches; + + const fileMatches = authorityMatchesInField(identity.file_name); + if (fileMatches.length > 0) return fileMatches; + + const titleMatches = authorityMatchesInField(identity.title); + if (titleMatches.length > 0) return titleMatches; + + const pathSegments = (identity.source_path ?? "") + .split(/[\\/]/) + .map((segment) => segment.trim()) + .filter(Boolean) + .reverse(); + for (const segment of pathSegments) { + const segmentMatches = authorityMatchesInField(segment); + if (segmentMatches.length > 0) return segmentMatches; + } + + return []; } /** @@ -71,14 +102,9 @@ function identityFields(identity: SourceAuthorityDocumentIdentity) { export function inferSourceAuthorityFromIdentity( identity: SourceAuthorityDocumentIdentity, ): SourceIdentityAuthorityMatch { - const matches: Array<{ code: string; authority: SourceAuthorityDefinition }> = []; - - for (const field of identityFields(identity)) { - for (const candidate of registeredCodes) { - const token = new RegExp(`(?:^|[^A-Z0-9])${escapeRegExp(candidate.code)}(?=$|[^A-Z0-9])`, "i"); - if (token.test(field) && !matches.some((match) => match.code === candidate.code)) matches.push(candidate); - } - } + const matches = preferredIdentityMatches(identity).filter( + (candidate, index, all) => all.findIndex((match) => match.code === candidate.code) === index, + ); const authorityKeys = [...new Set(matches.map((match) => match.authority.key))]; const conflict = authorityKeys.length > 1; @@ -102,8 +128,25 @@ function valuesDiffer(current: unknown, next: string | null) { return metadataString(current) !== next; } +export function isRegistryRecordSource(document: Pick) { + return metadataRecord(document.metadata).source_kind === "registry_record"; +} + export function analyzeSourceLocality(document: SourceAuthorityDocument): SourceLocalityAnalysis { const metadata = metadataRecord(document.metadata); + if (isRegistryRecordSource(document)) { + return { + authority: null, + matchedBy: "none", + excludedReason: "registry_record", + targetCode: null, + conflicts: [], + unresolvedConflict: false, + missingLocalityKeys: [], + changes: {}, + changedKeys: [], + }; + } const existingCode = normalizedCode(metadata.publisher_code); const codeAuthority = sourceAuthorityForPublisherCode(existingCode); const publisherAuthority = sourceAuthorityForPublisher(metadataString(metadata.publisher)); @@ -173,6 +216,7 @@ export function analyzeSourceLocality(document: SourceAuthorityDocument): Source return { authority, matchedBy, + excludedReason: null, targetCode, conflicts: [...new Set(conflicts)], unresolvedConflict, @@ -208,6 +252,8 @@ function compactAuthorityDocument(document: SourceAuthorityDocument, analysis: S authority_key: analysis.authority?.key ?? null, authority_scope: analysis.authority?.scope ?? null, matched_by: analysis.matchedBy, + source_kind: metadataString(metadata.source_kind), + excluded_reason: analysis.excludedReason, }; } @@ -224,11 +270,13 @@ export function auditSourceAuthorityDocuments(documents: SourceAuthorityDocument for (const conflict of analysis.conflicts) counts[conflict] = (counts[conflict] ?? 0) + 1; return counts; }, {}); + const excludedRegistryRecords = analyses.filter(({ analysis }) => analysis.excludedReason === "registry_record"); return { recognized_documents: recognized.length, australian_authority_candidates: australianCandidates.length, international_authority_documents: recognized.length - australianCandidates.length, + excluded_registry_record_count: excludedRegistryRecords.length, authority_conflict_count: conflicts.length, authority_conflict_reason_counts: Object.fromEntries( Object.entries(conflictReasonCounts).sort(([left], [right]) => left.localeCompare(right)), diff --git a/tests/source-authority-tooling.test.ts b/tests/source-authority-tooling.test.ts index 988c2ee48..11dfdfc72 100644 --- a/tests/source-authority-tooling.test.ts +++ b/tests/source-authority-tooling.test.ts @@ -6,6 +6,7 @@ import { assertLocalityMetadataPatch, auditSourceAuthorityDocuments, inferSourceAuthorityFromIdentity, + isRegistryRecordSource, type SourceAuthorityDocument, } from "@/lib/source-authority-metadata"; import { parseBackfillSourceMetadataArgs } from "../scripts/backfill-source-metadata"; @@ -38,6 +39,59 @@ describe("source authority metadata tooling", () => { }); }); + it("prefers the document identity over parent health-service path segments", () => { + expect( + inferSourceAuthorityFromIdentity( + document({ + file_name: "RPBG-lithium-guideline.pdf", + source_path: "WA Health/EMHS/Clinical resources/RPBG/RPBG-lithium-guideline.pdf", + }), + ), + ).toMatchObject({ + code: "RPBG", + authority: { key: "royal-perth-bentley-group" }, + conflict: false, + }); + + expect( + inferSourceAuthorityFromIdentity( + document({ + file_name: "lithium-guideline.pdf", + title: "FSH lithium guideline", + source_path: "WA Health/SMHS/Clinical resources/FSH/lithium-guideline.pdf", + }), + ), + ).toMatchObject({ + code: "FSH", + authority: { key: "fiona-stanley-fremantle-hospitals-group" }, + conflict: false, + }); + }); + + it("treats a trailing parenthetical code as publisher identity rather than subject acronyms", () => { + expect( + inferSourceAuthorityFromIdentity( + document({ file_name: "MDU Booking Process Pharmacy PBS and IPA Process (RPBG).pdf" }), + ), + ).toMatchObject({ code: "RPBG", authority: { key: "royal-perth-bentley-group" }, conflict: false }); + expect( + inferSourceAuthorityFromIdentity( + document({ file_name: "Governance of ACSQHC Clinical Care Standards (PHC).pdf" }), + ), + ).toMatchObject({ code: "PHC", authority: { key: "peel-health-campus" }, conflict: false }); + }); + + it("does not infer WHO from ordinary lowercase prose", () => { + expect( + inferSourceAuthorityFromIdentity( + document({ + file_name: "people-who-care.pdf", + title: "People who care for patients", + }), + ), + ).toEqual({ code: null, authority: null, codes: [], authorityKeys: [], conflict: false }); + }); + it("does not require locality metadata for unknown or international documents", () => { const report = auditSourceAuthorityDocuments([ document({ file_name: "unknown-clinical-reference.pdf" }), @@ -48,6 +102,37 @@ describe("source authority metadata tooling", () => { expect(report.passed).toBe(true); }); + it("excludes registry projections from document locality governance", () => { + const registryRecord = document({ + file_name: "emhs-crisis-service.json", + source_path: "WA Health/EMHS/Registry/emhs-crisis-service.json", + metadata: { + source_kind: "registry_record", + publisher: "Clinical KB registry", + jurisdiction: "WA/local clinical workspace", + }, + }); + const analysis = analyzeSourceLocality(registryRecord); + const report = auditSourceAuthorityDocuments([registryRecord]); + + expect(isRegistryRecordSource(registryRecord)).toBe(true); + expect(analysis).toMatchObject({ + authority: null, + matchedBy: "none", + excludedReason: "registry_record", + missingLocalityKeys: [], + changes: {}, + unresolvedConflict: false, + }); + expect(report).toMatchObject({ + excluded_registry_record_count: 1, + authority_conflict_count: 0, + missing_australian_locality_count: 0, + proposed_locality_correction_count: 0, + passed: true, + }); + }); + it("gates inferable Australian sources with missing locality and proposes only safe fields", () => { const candidate = document({ file_name: "WACHS-lithium-guideline.pdf" }); const analysis = analyzeSourceLocality(candidate); @@ -143,6 +228,7 @@ describe("source authority metadata tooling", () => { const backfillScript = readFileSync("scripts/backfill-source-metadata.ts", "utf8"); expect(auditScript).toContain('from "@/lib/source-authority-metadata"'); + expect(auditScript).toContain("documents.filter((document) => !isRegistryRecordSource(document))"); expect(backfillScript).toContain('from "@/lib/source-authority-metadata"'); expect(backfillScript).not.toContain("const publisherByCode"); }); From 19e1889aa0d976844797c4c1d897d062a0900cc9 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:07:51 +0800 Subject: [PATCH 07/14] docs(review): record lithium recovery handoff --- docs/branch-review-ledger.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index e05bc3c5a..6df60cd9b 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -54,3 +54,4 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-13 | codex/fix-48h-review-findings-current | 49735663370735a60870d065ed0de3b9d34e077f | last-48-hours PR remediation | Revalidated the last-48-hours findings on current main after PRs #538 and #540; retained only unique fixes across auth/cache isolation, stale-response protection, upload/routing/UI behavior, RAG coalescing, telemetry, worktree tooling, and SAST enforcement. No remaining high-confidence local defect was found in the changed scope. The approved live drift check reported only the five differences already explained by unapplied migrations from #540. | Focused Vitest 107/107; `npm run verify:pr-local` (1,762 passed, 1 skipped; production build and client-bundle scan; offline RAG 60/60); critical Chromium 8/8; live `check:drift`; `git diff --check`. Full Chromium remains advisory after the earlier runner hang; the required critical subset passed on current main. | | 2026-07-13 | codex/public-anonymous-access | 7f3eded3d17c9daf6a443c9cac3f0553e4e9321b | production UI design and accessibility review | Fixed the fullscreen clinical-table focus leak and divergent modal implementation, removed the non-native table-surface control, and lifted meaningful production metadata from 8-10px to the 11px floor with stronger muted contrast. No remaining high-confidence defect was found in the reviewed visual scope. | Baseline/final screenshots at 1440x1000 and 390x820; focused Chromium table expansion 3/3; focused Vitest 6/6; `npm run typecheck`; targeted ESLint; type-scale and focused Prettier checks; `git diff --check`. `verify:cheap` timed out in full lint/test execution; full `verify:ui` deferred under the API confirmation boundary. | | 2026-07-13 | main (PR #570 squash, glass header) | cc6bfc1c80902ca3c91e5ba2ebe78a80f3fd9e14 | post-merge review: CSS/visual/a11y/perf + logic/regression | No P0/P1. P2s confirmed and fixed in follow-up: build pipeline dropped ALL hand-authored backdrop-filter declarations (manual -webkit- duplicates confused Lightning CSS — header scrim, bottom dock, and composer pill were tint-only in every engine); scrim retuned to carry the bar's frost alone (header backdrop-root removed) with masks fading to true zero; private-scope alert made sticky inside main (was scroll-away in non-answer modes); scroll-hide reporter reset on breakpoint-gate change; non-forced fallback backgrounds layered to preserve the utility-wins contract. | Ran custom Playwright probe (`node scratchpad/probe-blur.mjs`, Chromium 390x844): baseline showed `getComputedStyle(.edge-glass-header-backdrop).backdropFilter === "none"` on all three passes, after fix `blur(14px)/blur(20px)/blur(26px)` — passed; Ran `node scripts/run-playwright.mjs tests/ui-smoke.spec.ts --project=chromium -g "glass header\|collapse hide\|private-scope alert\|phone (short\|long) answer stays"`: 6/6 passed; Ran `node scripts/run-playwright.mjs tests/ui-smoke.spec.ts --project=chromium` (full file): 71 passed, 4 failed (pre-existing `/privacy` heading test, fails identically on clean main); Ran `npm run verify:cheap`: exit 0 (1935 unit tests passed); Ran `npm run format:check`: passed; Not run: WebKit/Safari real-device check (no WebKit runner in this environment — served client chunk verified to pair `-webkit-backdrop-filter` with each declaration for Safari <= 17) | +| 2026-07-13 | codex/lithium-answer-recovery-pr | c5fde11e64d8976e1c163d1b8618f58a52e0b8ff | lithium answer recovery and source governance | Fixed the provider-failure path with a grounded Australian source-backed fallback, a public-safe progress lifecycle, centralised Australian authority/context selection, and fail-closed locality repair. The live audit exposed and the branch fixed hierarchy identity false conflicts plus registry projections entering clinical metadata gates. The corrected audit/backfill found zero proposals, so no production write was made. No high-confidence defect remains. Residual risk is provider latency; live generation timed out but the grounded fallback completed. | `verify:pr-local` passed: 1,988 tests passed/1 skipped, build/client scan, 36 fixtures, and 267 offline RAG tests; production readiness 5/5; authority tests 12/12; live audit 0 gaps/conflicts/proposals across 2,851 rows; backfill dry-run 0 changes; uncached live query passed with a grounded FSH citation and no threshold/safety failures; progress Chromium 2/2 and critical Chromium 9/9; `git diff --check`. Full `verify:ui` was not rerun after server loss; the isolated failed case passed after restart. | From 2476d24185cf3b3336d428cdccf589a02b3c89d4 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:18:07 +0800 Subject: [PATCH 08/14] docs(review): avoid ledger append conflict --- docs/branch-review-ledger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index 6df60cd9b..c44e37681 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -20,6 +20,7 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | Date | Branch or ref | Reviewed HEAD | Scope | Outcome | Checks | | ---------- | ------------------------------------------------------ | ---------------------------------------- | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2026-07-13 | codex/lithium-answer-recovery-pr | c5fde11e64d8976e1c163d1b8618f58a52e0b8ff | lithium answer recovery and source governance | Fixed the provider-failure path with a grounded Australian source-backed fallback, a public-safe progress lifecycle, centralised Australian authority/context selection, and fail-closed locality repair. The live audit exposed and the branch fixed hierarchy identity false conflicts plus registry projections entering clinical metadata gates. The corrected audit/backfill found zero proposals, so no production write was made. No high-confidence defect remains. Residual risk is provider latency; live generation timed out but the grounded fallback completed. | `verify:pr-local` passed: 1,988 tests passed/1 skipped, build/client scan, 36 fixtures, and 267 offline RAG tests; production readiness 5/5; authority tests 12/12; live audit 0 gaps/conflicts/proposals across 2,851 rows; backfill dry-run 0 changes; uncached live query passed with a grounded FSH citation and no threshold/safety failures; progress Chromium 2/2 and critical Chromium 9/9; `git diff --check`. Full `verify:ui` was not rerun after server loss; the isolated failed case passed after restart. | | 2026-07-10 | codex/design-ux-review-fixes | 648abfa3f7c91395b5eeca543f70e0b6ea59e9e0 | design-system + UX + design | Five issue groups confirmed; scoped fixes applied in the worktree. | `npm run check:type-scale`; focused Vitest (19/19); `npm run typecheck`; `npm run lint`; `npm run sitemap:check`; browser/API-backed checks awaiting approval | | 2026-07-11 | codex/design-ux-review-integration | 98093ec7b | branch-integration-review | Replayed the reviewed design and UX fixes onto current `origin/main`, preserved the lightweight evidence-panel boundary, and retained the merged quality fixes. | `npm run check:type-scale`; combined focused Vitest (8 files, 42 tests); runtime/action/sitemap/type-scale/lint stages of `verify:cheap`; typecheck blocked by stale worktree dependencies pending hosted clean install; `git diff --check` | | 2026-07-09 | example/branch | abc1234 | branch-cleanup | Example: already merged into `main`; no unique patch content. | `git log --right-only --cherry-pick main...example/branch`; `git diff --name-status main...example/branch` | @@ -54,4 +55,3 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-13 | codex/fix-48h-review-findings-current | 49735663370735a60870d065ed0de3b9d34e077f | last-48-hours PR remediation | Revalidated the last-48-hours findings on current main after PRs #538 and #540; retained only unique fixes across auth/cache isolation, stale-response protection, upload/routing/UI behavior, RAG coalescing, telemetry, worktree tooling, and SAST enforcement. No remaining high-confidence local defect was found in the changed scope. The approved live drift check reported only the five differences already explained by unapplied migrations from #540. | Focused Vitest 107/107; `npm run verify:pr-local` (1,762 passed, 1 skipped; production build and client-bundle scan; offline RAG 60/60); critical Chromium 8/8; live `check:drift`; `git diff --check`. Full Chromium remains advisory after the earlier runner hang; the required critical subset passed on current main. | | 2026-07-13 | codex/public-anonymous-access | 7f3eded3d17c9daf6a443c9cac3f0553e4e9321b | production UI design and accessibility review | Fixed the fullscreen clinical-table focus leak and divergent modal implementation, removed the non-native table-surface control, and lifted meaningful production metadata from 8-10px to the 11px floor with stronger muted contrast. No remaining high-confidence defect was found in the reviewed visual scope. | Baseline/final screenshots at 1440x1000 and 390x820; focused Chromium table expansion 3/3; focused Vitest 6/6; `npm run typecheck`; targeted ESLint; type-scale and focused Prettier checks; `git diff --check`. `verify:cheap` timed out in full lint/test execution; full `verify:ui` deferred under the API confirmation boundary. | | 2026-07-13 | main (PR #570 squash, glass header) | cc6bfc1c80902ca3c91e5ba2ebe78a80f3fd9e14 | post-merge review: CSS/visual/a11y/perf + logic/regression | No P0/P1. P2s confirmed and fixed in follow-up: build pipeline dropped ALL hand-authored backdrop-filter declarations (manual -webkit- duplicates confused Lightning CSS — header scrim, bottom dock, and composer pill were tint-only in every engine); scrim retuned to carry the bar's frost alone (header backdrop-root removed) with masks fading to true zero; private-scope alert made sticky inside main (was scroll-away in non-answer modes); scroll-hide reporter reset on breakpoint-gate change; non-forced fallback backgrounds layered to preserve the utility-wins contract. | Ran custom Playwright probe (`node scratchpad/probe-blur.mjs`, Chromium 390x844): baseline showed `getComputedStyle(.edge-glass-header-backdrop).backdropFilter === "none"` on all three passes, after fix `blur(14px)/blur(20px)/blur(26px)` — passed; Ran `node scripts/run-playwright.mjs tests/ui-smoke.spec.ts --project=chromium -g "glass header\|collapse hide\|private-scope alert\|phone (short\|long) answer stays"`: 6/6 passed; Ran `node scripts/run-playwright.mjs tests/ui-smoke.spec.ts --project=chromium` (full file): 71 passed, 4 failed (pre-existing `/privacy` heading test, fails identically on clean main); Ran `npm run verify:cheap`: exit 0 (1935 unit tests passed); Ran `npm run format:check`: passed; Not run: WebKit/Safari real-device check (no WebKit runner in this environment — served client chunk verified to pair `-webkit-backdrop-filter` with each declaration for Safari <= 17) | -| 2026-07-13 | codex/lithium-answer-recovery-pr | c5fde11e64d8976e1c163d1b8618f58a52e0b8ff | lithium answer recovery and source governance | Fixed the provider-failure path with a grounded Australian source-backed fallback, a public-safe progress lifecycle, centralised Australian authority/context selection, and fail-closed locality repair. The live audit exposed and the branch fixed hierarchy identity false conflicts plus registry projections entering clinical metadata gates. The corrected audit/backfill found zero proposals, so no production write was made. No high-confidence defect remains. Residual risk is provider latency; live generation timed out but the grounded fallback completed. | `verify:pr-local` passed: 1,988 tests passed/1 skipped, build/client scan, 36 fixtures, and 267 offline RAG tests; production readiness 5/5; authority tests 12/12; live audit 0 gaps/conflicts/proposals across 2,851 rows; backfill dry-run 0 changes; uncached live query passed with a grounded FSH citation and no threshold/safety failures; progress Chromium 2/2 and critical Chromium 9/9; `git diff --check`. Full `verify:ui` was not rerun after server loss; the isolated failed case passed after restart. | From 6f5162dc09a835c2733aa94e3f11ccd1a1d326e7 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:04:48 +0800 Subject: [PATCH 09/14] fix(rag): resolve retry and governance review findings Keep eval progress scoped to the successful provider attempt, bound locality audit output, and cover the metadata apply path. Verified with focused tests, verify:cheap, and production readiness. --- docs/branch-review-ledger.md | 2 +- scripts/backfill-source-metadata.ts | 7 +- scripts/eval-rag.ts | 25 ++++--- scripts/eval-utils.ts | 19 ++++++ src/lib/rag.ts | 10 +-- tests/eval-utils.test.ts | 26 +++++++ tests/source-authority-tooling.test.ts | 94 +++++++++++++++++++++++++- 7 files changed, 161 insertions(+), 22 deletions(-) diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index c44e37681..ac312fc4b 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -20,7 +20,7 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | Date | Branch or ref | Reviewed HEAD | Scope | Outcome | Checks | | ---------- | ------------------------------------------------------ | ---------------------------------------- | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 2026-07-13 | codex/lithium-answer-recovery-pr | c5fde11e64d8976e1c163d1b8618f58a52e0b8ff | lithium answer recovery and source governance | Fixed the provider-failure path with a grounded Australian source-backed fallback, a public-safe progress lifecycle, centralised Australian authority/context selection, and fail-closed locality repair. The live audit exposed and the branch fixed hierarchy identity false conflicts plus registry projections entering clinical metadata gates. The corrected audit/backfill found zero proposals, so no production write was made. No high-confidence defect remains. Residual risk is provider latency; live generation timed out but the grounded fallback completed. | `verify:pr-local` passed: 1,988 tests passed/1 skipped, build/client scan, 36 fixtures, and 267 offline RAG tests; production readiness 5/5; authority tests 12/12; live audit 0 gaps/conflicts/proposals across 2,851 rows; backfill dry-run 0 changes; uncached live query passed with a grounded FSH citation and no threshold/safety failures; progress Chromium 2/2 and critical Chromium 9/9; `git diff --check`. Full `verify:ui` was not rerun after server loss; the isolated failed case passed after restart. | +| 2026-07-13 | codex/lithium-answer-recovery-pr | c5fde11e64d8976e1c163d1b8618f58a52e0b8ff | lithium answer recovery and source governance | Fixed the provider-failure path with a grounded Australian source-backed fallback, a public-safe progress lifecycle, centralised Australian authority/context selection, and fail-closed locality repair. The live audit exposed and the branch fixed hierarchy identity false conflicts plus registry projections entering clinical metadata gates. The corrected audit/backfill found zero proposals, so no production write was made. No high-confidence defect remains. Residual risk is provider latency; live generation timed out but the grounded fallback completed. | `npm run verify:pr-local` passed: 1,988 tests passed/1 skipped, build/client scan, 36 fixtures, and 267 offline RAG tests. After review fixes, `npm run verify:cheap` passed with 1,992 tests passed/1 skipped. `npm run check:production-readiness` passed 5/5. `npm run test:e2e:critical` passed 9/9. `node scripts/run-playwright.mjs tests/answer-progress-ui-smoke.spec.ts --project=chromium` passed 2/2. `node scripts/run-eval-safe.mjs scripts/eval-rag.ts --question "Lithium dosing" --expect-australian --fail-on-threshold --json` exited 0 with one grounded FSH citation and no threshold/safety failures. `npm run audit:source-governance` reported 0 gaps/conflicts/proposals across 2,851 rows. `npm run backfill:source-metadata -- --locality-only` reported 0 changes. `git diff --check` passed. Full `npm run verify:ui` was not rerun after the aggregate runner lost its local server. | | 2026-07-10 | codex/design-ux-review-fixes | 648abfa3f7c91395b5eeca543f70e0b6ea59e9e0 | design-system + UX + design | Five issue groups confirmed; scoped fixes applied in the worktree. | `npm run check:type-scale`; focused Vitest (19/19); `npm run typecheck`; `npm run lint`; `npm run sitemap:check`; browser/API-backed checks awaiting approval | | 2026-07-11 | codex/design-ux-review-integration | 98093ec7b | branch-integration-review | Replayed the reviewed design and UX fixes onto current `origin/main`, preserved the lightweight evidence-panel boundary, and retained the merged quality fixes. | `npm run check:type-scale`; combined focused Vitest (8 files, 42 tests); runtime/action/sitemap/type-scale/lint stages of `verify:cheap`; typecheck blocked by stale worktree dependencies pending hosted clean install; `git diff --check` | | 2026-07-09 | example/branch | abc1234 | branch-cleanup | Example: already merged into `main`; no unique patch content. | `git log --right-only --cherry-pick main...example/branch`; `git diff --name-status main...example/branch` | diff --git a/scripts/backfill-source-metadata.ts b/scripts/backfill-source-metadata.ts index 42a8aa5c6..69fe8bbed 100644 --- a/scripts/backfill-source-metadata.ts +++ b/scripts/backfill-source-metadata.ts @@ -50,6 +50,7 @@ type ClinicalValidationEvidence = { }; const BACKFILL_VERSION = "source_metadata_backfill_2026_06_30_v1"; +const LOCALITY_AUDIT_DETAIL_LIMIT = 20; export type BackfillSourceMetadataArgs = { apply: boolean; @@ -673,7 +674,7 @@ async function evalDocumentIds(evalOnly: boolean) { return topFiles; } -async function runLocalityOnlyBackfill( +export async function runLocalityOnlyBackfill( args: BackfillSourceMetadataArgs, documents: DocumentRow[], supabase: Awaited>, @@ -688,6 +689,10 @@ async function runLocalityOnlyBackfill( documents_seen: documents.length, documents_with_changes: changed.length, ...authorityAudit, + detail_limit: LOCALITY_AUDIT_DETAIL_LIMIT, + conflicts: authorityAudit.conflicts.slice(0, LOCALITY_AUDIT_DETAIL_LIMIT), + missing_australian_locality: authorityAudit.missing_australian_locality.slice(0, LOCALITY_AUDIT_DETAIL_LIMIT), + proposed_locality_corrections: authorityAudit.proposed_locality_corrections.slice(0, LOCALITY_AUDIT_DETAIL_LIMIT), }; console.log(JSON.stringify(summary, null, 2)); diff --git a/scripts/eval-rag.ts b/scripts/eval-rag.ts index ae93b9d41..468bc07a1 100644 --- a/scripts/eval-rag.ts +++ b/scripts/eval-rag.ts @@ -12,7 +12,7 @@ import { percentile, resolveEvalOwnerId, validateRagAnswer, - withProviderBackoff, + withProviderBackoffProgress, } from "./eval-utils"; loadEnvConfig(process.cwd()); @@ -234,18 +234,17 @@ async function main() { if (!args.json) console.log(`Running ${cases.length} RAG eval case(s), scope=${scope}.`); for (const testCase of cases) { - const progress: RagEvalProgressDiagnosticEvent[] = []; - const answer = (await withProviderBackoff(`rag:${testCase.id}`, () => - answerQuestionWithScope({ - query: testCase.question, - ownerId, - logQuery: false, - skipCache: true, - onProgress: (event) => { - progress.push(event); - }, - }), - )) as RagAnswer; + const { result: answer, progress } = await withProviderBackoffProgress( + `rag:${testCase.id}`, + (onProgress) => + answerQuestionWithScope({ + query: testCase.question, + ownerId, + logQuery: false, + skipCache: true, + onProgress, + }), + ); const latencyMs = answer.latencyTimings?.total_latency_ms ?? 0; const validation = validateRagAnswer(testCase, answer); const governanceDiagnostics = buildRagEvaluationDiagnostics(answer, progress); diff --git a/scripts/eval-utils.ts b/scripts/eval-utils.ts index 09dcc020b..ba99ec510 100644 --- a/scripts/eval-utils.ts +++ b/scripts/eval-utils.ts @@ -77,6 +77,25 @@ export async function withProviderBackoff( throw new Error(`Provider retry loop exhausted for ${label}.`); } +export async function withProviderBackoffProgress( + label: string, + operation: (onProgress: (event: TProgress) => void) => Promise, + options: { maxAttempts?: number; initialDelayMs?: number; maxDelayMs?: number } = {}, +) { + let successfulProgress: TProgress[] = []; + const result = await withProviderBackoff( + label, + async () => { + const attemptProgress: TProgress[] = []; + const attemptResult = await operation((event) => attemptProgress.push(event)); + successfulProgress = attemptProgress; + return attemptResult; + }, + options, + ); + return { result, progress: successfulProgress }; +} + export async function findOwnerIdByEmail(supabase: SupabaseAdmin, email: string) { const normalized = email.trim().toLowerCase(); const perPage = 1000; diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 738d53271..dfad8f4af 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -4864,11 +4864,6 @@ ${qualityRetryInstruction}` results: answerInputResults, }); const generationFallbackResults = strongRetryContextResults; - const generationFallbackArtifacts = buildContextDerivedArtifacts(answerFocusQuery, generationFallbackResults); - const generationFallbackSelectionSummary = summarizeAustralianSourceSelection( - answerInputResults, - generationFallbackResults, - ); const modelContextSelectionSummary = summarizeAustralianSourceSelection(answerInputResults, modelContextResults); await args.onProgress?.({ stage: "ranking", @@ -5225,6 +5220,11 @@ ${qualityRetryInstruction}` } catch (error) { if ((error instanceof DOMException && error.name === "AbortError") || args.signal?.aborted) throw error; const relatedDocuments = await relatedDocumentsPromise; + const generationFallbackArtifacts = buildContextDerivedArtifacts(answerFocusQuery, generationFallbackResults); + const generationFallbackSelectionSummary = summarizeAustralianSourceSelection( + answerInputResults, + generationFallbackResults, + ); await args.onProgress?.({ stage: "fallback", message: "Generation failed, returning source-based fallback answer.", diff --git a/tests/eval-utils.test.ts b/tests/eval-utils.test.ts index b53869c95..b9c2ce6d5 100644 --- a/tests/eval-utils.test.ts +++ b/tests/eval-utils.test.ts @@ -7,6 +7,7 @@ import { resolveEvalOwnerId, validateRagAnswer, withProviderBackoff, + withProviderBackoffProgress, type SupabaseAdmin, } from "../scripts/eval-utils"; import type { RagEvalCase } from "../src/lib/rag-eval-cases"; @@ -177,6 +178,31 @@ describe("RAG eval source identity matching", () => { expect(isProviderRateLimitError(new Error("429 too many requests"))).toBe(true); }); + it("keeps progress only from the successful provider attempt", async () => { + let attempts = 0; + + const outcome = await withProviderBackoffProgress( + "test-progress-rate-limit", + async (onProgress) => { + attempts += 1; + onProgress(`attempt-${attempts}:retrieved`); + if (attempts === 1) { + onProgress("attempt-1:supplementary-selected"); + throw new Error("429 too many requests"); + } + onProgress("attempt-2:finalized"); + return "ok"; + }, + { maxAttempts: 2, initialDelayMs: 1, maxDelayMs: 1 }, + ); + + expect(outcome).toEqual({ + result: "ok", + progress: ["attempt-2:retrieved", "attempt-2:finalized"], + }); + expect(attempts).toBe(2); + }); + it("pauses between eval cases when configured", async () => { vi.useFakeTimers(); try { diff --git a/tests/source-authority-tooling.test.ts b/tests/source-authority-tooling.test.ts index 11dfdfc72..92bfec829 100644 --- a/tests/source-authority-tooling.test.ts +++ b/tests/source-authority-tooling.test.ts @@ -1,6 +1,6 @@ import { readFileSync } from "node:fs"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { analyzeSourceLocality, assertLocalityMetadataPatch, @@ -9,7 +9,7 @@ import { isRegistryRecordSource, type SourceAuthorityDocument, } from "@/lib/source-authority-metadata"; -import { parseBackfillSourceMetadataArgs } from "../scripts/backfill-source-metadata"; +import { parseBackfillSourceMetadataArgs, runLocalityOnlyBackfill } from "../scripts/backfill-source-metadata"; function document( overrides: Partial & Pick, @@ -223,6 +223,96 @@ describe("source authority metadata tooling", () => { }); }); + it("applies only bounded locality fields through the metadata patch RPC", async () => { + const rpc = vi.fn().mockResolvedValue({ error: null }); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const target = { + ...document({ + file_name: "WACHS-lithium-guideline.pdf", + metadata: { publisher_code: "WACHS" }, + }), + id: "document-1", + source_path: null, + status: "active", + }; + + try { + await runLocalityOnlyBackfill( + parseBackfillSourceMetadataArgs(["--locality-only", "--apply", "--confirm"]), + [target], + { rpc } as never, + ); + } finally { + log.mockRestore(); + } + + expect(rpc).toHaveBeenCalledOnce(); + expect(rpc).toHaveBeenCalledWith("apply_document_metadata_patch", { + p_document_id: "document-1", + p_metadata_patch: { + publisher: "WA Country Health Service", + jurisdiction: "Australia/WA", + }, + }); + }); + + it("fails closed when a locality metadata patch RPC is rejected", async () => { + const rpc = vi.fn().mockResolvedValue({ error: { message: "write denied" } }); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const target = { + ...document({ + file_name: "WACHS-lithium-guideline.pdf", + metadata: { publisher_code: "WACHS" }, + }), + id: "document-2", + source_path: null, + status: "active", + }; + + try { + await expect( + runLocalityOnlyBackfill( + parseBackfillSourceMetadataArgs(["--locality-only", "--apply", "--confirm"]), + [target], + { rpc } as never, + ), + ).rejects.toThrow("Failed to patch WACHS-lithium-guideline.pdf: write denied"); + } finally { + log.mockRestore(); + } + }); + + it("caps locality audit details without changing aggregate counts", async () => { + const logged: string[] = []; + const log = vi.spyOn(console, "log").mockImplementation((value) => logged.push(String(value))); + const targets = Array.from({ length: 25 }, (_, index) => ({ + ...document({ + file_name: `WACHS-conflict-${index}.pdf`, + metadata: { publisher_code: "TGA", publisher: "Therapeutic Goods Administration", jurisdiction: "Australia" }, + }), + id: `document-${index}`, + source_path: null, + status: "active", + })); + + try { + await runLocalityOnlyBackfill(parseBackfillSourceMetadataArgs(["--locality-only"]), targets, { + rpc: vi.fn(), + } as never); + } finally { + log.mockRestore(); + } + + const summary = JSON.parse(logged[0]) as { + authority_conflict_count: number; + conflicts: unknown[]; + detail_limit: number; + }; + expect(summary.authority_conflict_count).toBe(25); + expect(summary.detail_limit).toBe(20); + expect(summary.conflicts).toHaveLength(20); + }); + it("makes both governance scripts consume the shared authority helper", () => { const auditScript = readFileSync("scripts/audit-source-governance.ts", "utf8"); const backfillScript = readFileSync("scripts/backfill-source-metadata.ts", "utf8"); From 0d4f87f6c79ba12fc6cb494d36696951fad29928 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:09:20 +0800 Subject: [PATCH 10/14] fix(rag): narrow uncited numeric fallback evidence Keep deterministic dose and threshold recovery scoped to evidence cited by the final answer, including after the updated claim-scoped numeric verifier from main. --- src/lib/rag.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/lib/rag.ts b/src/lib/rag.ts index dfad8f4af..d00b03abe 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -5312,13 +5312,18 @@ ${qualityRetryInstruction}` (verified.unverifiedNumericTokens?.length ?? 0) === 0 ); }; + const hasUncitedExtractiveFallbackEvidence = (candidate: RagAnswer) => { + const citedChunkIds = new Set(candidate.citations.map((citation) => citation.chunk_id)); + return candidate.sources.some((source) => !citedChunkIds.has(source.id)); + }; let extractiveFallbackAnswer = canRecoverGenerationErrorExtractively ? buildExtractiveFallbackCandidate(generationFallbackResults) : null; if ( extractiveFallbackAnswer && (queryClass === "medication_dose_risk" || queryClass === "table_threshold") && - !isSafeExtractiveFallbackCandidate(extractiveFallbackAnswer) + (!isSafeExtractiveFallbackCandidate(extractiveFallbackAnswer) || + hasUncitedExtractiveFallbackEvidence(extractiveFallbackAnswer)) ) { extractiveFallbackAnswer = generationFallbackResults From 3e49c05f8dedd2f146f7e9b96142f9b4e1837517 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:23:33 +0800 Subject: [PATCH 11/14] fix(rag): prefer single-source numeric recovery When generation fails, keep deterministic dose and threshold recovery scoped to one complete evidence chunk instead of stitching figures across sources. --- src/lib/rag.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/lib/rag.ts b/src/lib/rag.ts index d00b03abe..7dba65a77 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -5312,19 +5312,13 @@ ${qualityRetryInstruction}` (verified.unverifiedNumericTokens?.length ?? 0) === 0 ); }; - const hasUncitedExtractiveFallbackEvidence = (candidate: RagAnswer) => { - const citedChunkIds = new Set(candidate.citations.map((citation) => citation.chunk_id)); - return candidate.sources.some((source) => !citedChunkIds.has(source.id)); - }; let extractiveFallbackAnswer = canRecoverGenerationErrorExtractively ? buildExtractiveFallbackCandidate(generationFallbackResults) : null; - if ( - extractiveFallbackAnswer && - (queryClass === "medication_dose_risk" || queryClass === "table_threshold") && - (!isSafeExtractiveFallbackCandidate(extractiveFallbackAnswer) || - hasUncitedExtractiveFallbackEvidence(extractiveFallbackAnswer)) - ) { + // Generated synthesis has already failed, so do not stitch dose or threshold figures + // across fallback chunks. Prefer the first individually complete candidate that passes + // every extractive and numeric safety gate. + if (extractiveFallbackAnswer && (queryClass === "medication_dose_risk" || queryClass === "table_threshold")) { extractiveFallbackAnswer = generationFallbackResults .map((result) => buildExtractiveFallbackCandidate([result])) From 13a3152ad913e7ecfb12b57cdbd2c1059afa8409 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:30:47 +0800 Subject: [PATCH 12/14] fix: retain only cited lithium fallback evidence --- src/lib/rag.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib/rag.ts b/src/lib/rag.ts index d00b03abe..bd428428a 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -5316,6 +5316,13 @@ ${qualityRetryInstruction}` const citedChunkIds = new Set(candidate.citations.map((citation) => citation.chunk_id)); return candidate.sources.some((source) => !citedChunkIds.has(source.id)); }; + const retainCitedExtractiveFallbackEvidence = (candidate: RagAnswer) => { + const citedChunkIds = new Set(candidate.citations.map((citation) => citation.chunk_id)); + return { + ...candidate, + sources: candidate.sources.filter((source) => citedChunkIds.has(source.id)), + }; + }; let extractiveFallbackAnswer = canRecoverGenerationErrorExtractively ? buildExtractiveFallbackCandidate(generationFallbackResults) : null; @@ -5327,7 +5334,7 @@ ${qualityRetryInstruction}` ) { extractiveFallbackAnswer = generationFallbackResults - .map((result) => buildExtractiveFallbackCandidate([result])) + .map((result) => retainCitedExtractiveFallbackEvidence(buildExtractiveFallbackCandidate([result]))) .find(isSafeExtractiveFallbackCandidate) ?? extractiveFallbackAnswer; } const extractiveFallbackQualityReason = extractiveFallbackAnswer From 237d6db8fbae744116b96c49e0c9d8588b0daa3d Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:50:07 +0800 Subject: [PATCH 13/14] fix: preserve narrowed fallback answer type --- src/lib/rag.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 9941fd851..a7b321704 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -5455,7 +5455,7 @@ ${qualityRetryInstruction}` (verified.unverifiedNumericTokens?.length ?? 0) === 0 ); }; - const retainCitedExtractiveFallbackEvidence = (candidate: RagAnswer) => { + const retainCitedExtractiveFallbackEvidence = (candidate: T): T => { const citedChunkIds = new Set(candidate.citations.map((citation) => citation.chunk_id)); return { ...candidate, From d6d2e17e2772a775d2fd9817e1dce1b983283590 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:50:09 +0800 Subject: [PATCH 14/14] fix(rag): type cited fallback boundary Keep the filtered fallback candidate at the RagAnswer boundary so current TypeScript inference accepts the single-source recovery assignment. --- src/lib/rag.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 9941fd851..f994594d8 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -5455,14 +5455,14 @@ ${qualityRetryInstruction}` (verified.unverifiedNumericTokens?.length ?? 0) === 0 ); }; - const retainCitedExtractiveFallbackEvidence = (candidate: RagAnswer) => { + const retainCitedExtractiveFallbackEvidence = (candidate: RagAnswer): RagAnswer => { const citedChunkIds = new Set(candidate.citations.map((citation) => citation.chunk_id)); return { ...candidate, sources: candidate.sources.filter((source) => citedChunkIds.has(source.id)), }; }; - let extractiveFallbackAnswer = canRecoverGenerationErrorExtractively + let extractiveFallbackAnswer: RagAnswer | null = canRecoverGenerationErrorExtractively ? buildExtractiveFallbackCandidate(generationFallbackResults) : null; // Generated synthesis has already failed, so do not stitch dose or threshold figures