From 0fb1fe4f1125a77dce45dfed53343e011b356347 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 18:29:21 +0800 Subject: [PATCH 1/3] feat(eval): answer-targeting metric + retrieval-eval process gate + quote truncation flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 1 of the safe-execution plan — measurement and guardrails, zero live behavior change. P3 (measurement, additive, opt-in): - scoreAnswerTargeting: structural per-intent targeting check (dose→figure/regimen, red-result→withhold action, monitoring→schedule, contraindication→avoid, referral→criteria, doc-lookup→named/cited document) — stronger than the loose keyword cue; informational, never a hard gate; unsupported/fail-closed cases are n/a. Existing scoreAnswerQualityEvalCase and its 5-metric contract untouched. - New standalone scripts/eval-answer-quality.ts (npm run eval:answer-quality) runs the previously dormant 30-case answerQualityEvalCases fixture and reports the 5 metrics + per-intent targeting. Separate/opt-in so it adds no spend to any existing eval run. P2 (process gate, docs only): - PR template: require eval:retrieval:quality (23/23) for retrieval/ranking/selection/chunking PRs and eval:rag + eval:quality for answer-generation PRs (CI can't run them — needs live keys). - process-hardening.md: document the gate + the measured PR-118 regression (governance metadata weighting buried correct docs 1.0->0.76) and the relevance-first standing constraint. P4c (provenance, additive display): - QuoteCard.isTruncated (optional) set by bestQuoteFromContent when a quote is cut at 340 chars; quote card shows a "truncated — open the source" note so a value past the cut is never read as the complete passage. verify:cheap green (984 passed); format:check clean. New unit tests for scoreAnswerTargeting. Co-Authored-By: Claude Opus 4.8 --- .github/pull_request_template.md | 2 + docs/process-hardening.md | 7 + package.json | 1 + scripts/eval-answer-quality.ts | 166 ++++++++++++++++++ .../clinical-dashboard/evidence-panels.tsx | 5 + src/lib/evidence.ts | 14 +- src/lib/rag-eval-cases.ts | 60 +++++++ src/lib/types.ts | 3 + tests/rag-eval-cases.test.ts | 59 +++++++ 9 files changed, 312 insertions(+), 5 deletions(-) create mode 100644 scripts/eval-answer-quality.ts diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 53e384435..95e697361 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,6 +8,8 @@ - [ ] `npm run verify:ui` when UI, routing, styling, browser behavior, reduced-motion, or forced-colors behavior changed - [ ] `npm run verify:release` before release or handoff confidence claims - [ ] `npm run format:check` +- [ ] **`npm run eval:retrieval:quality` (must stay 23/23) when retrieval, ranking, selection, chunking, or scoring behavior changed** — CI cannot run it (needs live keys), so run it locally and paste the summary. A metadata/governance-weighting change once buried correct docs (recall 1.0→0.76) and only this eval caught it. +- [ ] `npm run eval:rag -- --limit 15` + `npm run eval:quality -- --rag-only` when answer generation, the synthesis prompt, or answer post-processing changed (grounded-supported must not drop; citation-failure 0) - [ ] `npm run check:production-readiness` when clinical workflow, privacy, environment, Supabase, source governance, or deployment behavior changed - [ ] `npm run check:deployment-readiness` when deployment startup, hosting, or rollout behavior changed diff --git a/docs/process-hardening.md b/docs/process-hardening.md index bf9932630..e7da421f5 100644 --- a/docs/process-hardening.md +++ b/docs/process-hardening.md @@ -135,3 +135,10 @@ All approved render-surface modules are extracted. `ClinicalDashboard.tsx` went - **Root cause (confirmed live):** the soft-tail branch of `shouldShortCircuitUnsupportedSearch` (`src/lib/rag.ts`) fires on `analysis.queryClass === "unsupported_or_general"`, and that class is set by `analyzeQueryWithClassifierFallback`, which calls a **generative LLM classifier** (6s timeout, `reasoningEffort:"low"`, uncached) for low-confidence queries. That call is nondeterministic: it reclassifies "bipolar disorder" to a supported class on some runs (→ answered) and declines/times-out on others (→ short-circuited to 0). Reproduced with a same-process ×N probe. - **Why a clean Phase-1 fix does NOT exist:** the deterministic analyzer carries **no signal** distinguishing in-corpus topics from out-of-corpus (bipolar, anorexia, gout, DKA all produce identical `unsupported_or_general`, confidence ≈ 0.40, `canonicalTerms` = query tokens). Only the corpus can tell them apart. Removing the soft-tail short-circuit fixes the valid topics but **regresses `unsupported_correct_rate` 1.0 → 0.79** on `eval:quality --rag-only`: a lexical distinctive-term relevance gate in `chooseAnswerRoute` either over-refuses legitimate semantic/vector matches (whose exact term is not in the retrieved text — e.g. the `rag-routing.test.ts` "admission" fixtures) or under-refuses invented terms ("florbizone syndrome") that score strongly on generic scaffolding words ("syndrome"/"management"). The corpus is broad (gout 13 / crohn 49 / appendicitis 30 / angioplasty 32 / bipolar 719 chunks), so almost no common medical term is truly absent — grounded low-confidence answers, not fabrication, are the realistic worst case for real terms; only genuinely invented terms should refuse. - **Decision (2026-07-03):** the risky change (soft-tail removal + relevance gate + invented-term eval controls) was **reverted**; only a safe, independent hardening was kept — `fetchEnabledRagAliases` no longer caches `[]` on a transient `rag_aliases` read error (which would suppress alias expansion for the whole TTL). Finding #11 is **re-scoped into RAG optimisation Phase 2** ("fit retrieval to content"), where it should be fixed with corpus-grounded relevance — IDF/corpus-frequency weighting of query terms and/or the semantic relevance model, plus the data-driven query-understanding vocabulary (RC6) so the deterministic classifier recognises in-corpus topics up front. Any gate must keep `eval:quality --rag-only` `unsupported_correct_rate` at 1.0 (add invented-term controls like "florbizone syndrome management" / "quxbyria disorder treatment" once it can pass them) while letting valid bare topics answer. A cheaper interim option worth measuring first: **classifier-verdict memoization** to at least make the current behaviour deterministic per query. + +## Retrieval changes must pass the golden eval before merge (2026-07-03) + +- **Any PR that touches retrieval, ranking, selection, chunking, or scoring MUST run `npm run eval:retrieval:quality` (23/23) locally before merge** and paste the summary in the PR. CI cannot run it (it needs live Supabase + OpenAI keys), so it is a manual gate — now a checkbox in the PR template. +- **Why (measured):** PR #118 caught a main-side change (uncapped candidate score + blanket source-governance metadata weighting in `retrieval-selection.ts`) that regressed the golden set 23/23 → 16/23 (doc-recall@5 1.0 → 0.76) on the partially-enriched corpus. `verify:cheap` was green throughout — only the golden retrieval eval surfaced it. Unit tests do not exercise live ranking, so they cannot substitute. +- **Standing constraint (do not relearn):** source-governance metadata (`document_status`/`clinical_validation_status`/`extraction_quality`) must NOT weight retrieval **selection ordering**, and candidate relevance scores must stay clamped. Live scores saturate at 1.0 and the corpus is only partially enriched (unenriched → unknown/unverified), so metadata weighting buries correct documents. Governance belongs in ranking penalties and the answer/source-governance layer. See [[no-governance-weighting-in-retrieval-selection]] and `docs/rag-hybrid-findings-and-todo.md` (RC8). +- **Answer-generation changes** (synthesis prompt, post-processing) additionally run `eval:rag --limit 15` + `eval:quality --rag-only` (grounded-supported must not drop; citation-failure 0). A new opt-in `npm run eval:answer-quality` reports a structural per-intent **targeting** metric (informational) for measuring how precisely answers hit the asked question. diff --git a/package.json b/package.json index 757979240..8d3989118 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "supabase:recovery-status": "tsx scripts/supabase-recovery-status.ts", "promote:query-misses": "tsx scripts/promote-query-misses.ts", "eval:rag": "node scripts/run-eval-safe.mjs scripts/eval-rag.ts", + "eval:answer-quality": "node scripts/run-eval-safe.mjs scripts/eval-answer-quality.ts", "eval:quality": "node scripts/run-eval-safe.mjs scripts/eval-quality.ts", "eval:quality:release": "npm run eval:quality -- --fail-on-threshold --source-metadata-debt docs/release-source-metadata-debt-2026-06-30.json", "eval:retrieval": "node scripts/run-eval-safe.mjs scripts/eval-retrieval.ts", diff --git a/scripts/eval-answer-quality.ts b/scripts/eval-answer-quality.ts new file mode 100644 index 000000000..957ce0db1 --- /dev/null +++ b/scripts/eval-answer-quality.ts @@ -0,0 +1,166 @@ +// Answer-quality / targeting eval (P3). Runs the 30-case `answerQualityEvalCases` fixture through +// the live answer path and reports the five answer-quality metrics PLUS a structural per-intent +// `targeting` metric (dose→figure, red-result→withhold action, monitoring→schedule, etc.). +// +// This is a SEPARATE, opt-in script (npm run eval:answer-quality) — it does not touch eval:quality +// or add spend to any existing run. All metrics are reported informationally; the script exits 0 +// unless --fail-on-threshold is passed with an explicit floor, so it can be calibrated safely. +import { loadEnvConfig } from "@next/env"; +import { + answerQualityEvalCases, + answerQualityMetricLabels, + answerTargetingMetricLabel, + scoreAnswerQualityEvalCase, + scoreAnswerTargeting, + type AnswerQualityEvalCase, + type AnswerQualityMetric, +} from "@/lib/rag-eval-cases"; +import type { RagAnswer } from "@/lib/types"; +import { findOwnerIdByEmail, loadAdminClient, withProviderBackoff } from "./eval-utils"; + +loadEnvConfig(process.cwd()); + +type Args = { + ownerEmail?: string; + ownerId?: string; + limit?: number; + intent?: string; + json: boolean; + targetingFloor?: number; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + ownerEmail: process.env.RAG_EVAL_OWNER_EMAIL, + ownerId: process.env.RAG_EVAL_OWNER_ID ?? process.env.LOCAL_NO_AUTH_OWNER_ID, + json: false, + }; + for (let index = 0; index < argv.length; index += 1) { + const token = argv[index]; + if (!token.startsWith("--")) continue; + if (token === "--json") { + args.json = true; + continue; + } + const value = argv[index + 1]; + if (token === "--owner-id") args.ownerId = value; + if (token === "--owner-email") args.ownerEmail = value; + if (token === "--intent") args.intent = value; + if (token === "--limit") args.limit = Number.parseInt(value, 10); + if (token === "--targeting-floor") args.targetingFloor = Number.parseFloat(value); + } + return args; +} + +const METRICS: AnswerQualityMetric[] = ["relevance", "readability", "artifact_leaks", "intent_coverage", "fail_closed"]; + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const [{ requireOpenAIEnv, requireServerEnv }, { answerQuestionWithScope }, supabase] = await Promise.all([ + import("@/lib/env"), + import("@/lib/rag"), + loadAdminClient(), + ]); + requireServerEnv(); + requireOpenAIEnv(); + + const ownerId = args.ownerId ?? (args.ownerEmail ? await findOwnerIdByEmail(supabase, args.ownerEmail) : undefined); + let cases: AnswerQualityEvalCase[] = answerQualityEvalCases; + if (args.intent) cases = cases.filter((testCase) => testCase.expectedIntent === args.intent); + if (args.limit) cases = cases.slice(0, args.limit); + + const metricTotals: Record = { + relevance: 0, + readability: 0, + artifact_leaks: 0, + intent_coverage: 0, + fail_closed: 0, + }; + const targetingByIntent = new Map(); + let targetingApplicable = 0; + let targetingHit = 0; + const targetingMisses: Array<{ id: string; intent: string; reason: string; answer: string }> = []; + + for (const testCase of cases) { + const answer = (await withProviderBackoff(`answer-quality:${testCase.id}`, () => + answerQuestionWithScope({ query: testCase.question, ownerId, logQuery: false, skipCache: true }), + )) as RagAnswer; + + for (const score of scoreAnswerQualityEvalCase(testCase, answer)) { + metricTotals[score.metric] += score.score; + } + + const targeting = scoreAnswerTargeting(testCase, answer); + const bucket = targetingByIntent.get(testCase.expectedIntent) ?? { applicable: 0, hit: 0 }; + if (targeting.applicable) { + bucket.applicable += 1; + targetingApplicable += 1; + if (targeting.score === 1) { + bucket.hit += 1; + targetingHit += 1; + } else { + targetingMisses.push({ + id: testCase.id, + intent: testCase.expectedIntent, + reason: targeting.reason, + answer: (answer.answer ?? "").replace(/\s+/g, " ").slice(0, 160), + }); + } + } + targetingByIntent.set(testCase.expectedIntent, bucket); + } + + const caseCount = cases.length; + const metricRates = Object.fromEntries( + METRICS.map((metric) => [metric, caseCount ? Number((metricTotals[metric] / caseCount).toFixed(4)) : 0]), + ) as Record; + const targetingRate = targetingApplicable ? Number((targetingHit / targetingApplicable).toFixed(4)) : null; + const targetingByIntentRates = Object.fromEntries( + [...targetingByIntent.entries()].map(([intent, { applicable, hit }]) => [ + intent, + { applicable, hit, rate: applicable ? Number((hit / applicable).toFixed(4)) : null }, + ]), + ); + + const summary = { + case_count: caseCount, + metric_labels: answerQualityMetricLabels, + metric_rates: metricRates, + targeting_label: answerTargetingMetricLabel, + targeting_applicable: targetingApplicable, + targeting_rate: targetingRate, + targeting_by_intent: targetingByIntentRates, + targeting_misses: targetingMisses, + }; + + if (args.json) { + console.log(JSON.stringify(summary, null, 2)); + } else { + console.log(`Answer-quality eval: ${caseCount} case(s).`); + console.log(" metric_rates:"); + for (const metric of METRICS) console.log(` ${metric}=${metricRates[metric]}`); + console.log(` targeting_rate=${targetingRate} (applicable=${targetingApplicable})`); + console.log(" targeting_by_intent:"); + for (const [intent, value] of Object.entries(targetingByIntentRates)) { + console.log(` ${intent}=${value.rate} (${value.hit}/${value.applicable})`); + } + if (targetingMisses.length) { + console.log(" targeting_misses:"); + for (const miss of targetingMisses) { + console.log(` [${miss.intent}] ${miss.id}: ${miss.reason} :: "${miss.answer}"`); + } + } + } + + if (args.targetingFloor !== undefined && targetingRate !== null && targetingRate < args.targetingFloor) { + console.error(`FAIL: targeting_rate ${targetingRate} < floor ${args.targetingFloor}`); + process.exit(1); + } +} + +main() + .then(() => process.exit(0)) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/src/components/clinical-dashboard/evidence-panels.tsx b/src/components/clinical-dashboard/evidence-panels.tsx index 8b30a4b3d..d38db875e 100644 --- a/src/components/clinical-dashboard/evidence-panels.tsx +++ b/src/components/clinical-dashboard/evidence-panels.tsx @@ -1889,6 +1889,11 @@ export function QuoteCards({
“{quoteText}”
+ {quote.isTruncated ? ( +

+ Quote truncated for length — open the source to read the full passage. +

+ ) : null}
b.adjustedScore - a.adjustedScore || b.score - a.score || a.sentence.length - b.sentence.length, )[0]?.sentence ?? clean; - if (best.length <= 340) return best; - return `${best.slice(0, 337).trim()}...`; + if (best.length <= 340) return { quote: best, truncated: false }; + return { quote: `${best.slice(0, 337).trim()}...`, truncated: true }; } function normalizeText(text: string) { @@ -201,7 +204,7 @@ export function extractQuoteCards(results: SearchResult[], query: string, limit const quoteCards: QuoteCard[] = []; for (const result of results) { - const quote = bestQuoteFromContent(result.content, query); + const { quote, truncated } = bestQuoteFromContent(result.content, query); if (!quote) continue; const key = `${result.document_id}:${result.page_number}:${quote.toLowerCase()}`; if (seen.has(key)) continue; @@ -209,6 +212,7 @@ export function extractQuoteCards(results: SearchResult[], query: string, limit quoteCards.push({ ...citationFromResult(result), quote, + isTruncated: truncated, section_heading: result.section_heading, source_strength: result.source_strength ?? sourceStrengthForSimilarity(result.similarity), }); diff --git a/src/lib/rag-eval-cases.ts b/src/lib/rag-eval-cases.ts index 45a038f80..b2a243c29 100644 --- a/src/lib/rag-eval-cases.ts +++ b/src/lib/rag-eval-cases.ts @@ -108,6 +108,66 @@ export function scoreAnswerQualityEvalCase(testCase: AnswerQualityEvalCase, answ ] satisfies AnswerQualityMetricScore[]; } +export type AnswerTargetingScore = { + /** 1 = the answer carries the structural shape the intent demands (or the case is n/a). */ + score: 0 | 1; + /** false = not counted toward the targeting rate (unsupported/fail-closed or general intent). */ + applicable: boolean; + reason: string; +}; + +export const answerTargetingMetricLabel = + "Answer carries the structural shape its intent demands: dose→figure/regimen, red-result→withhold/stop action, monitoring→schedule/interval, contraindication→avoid cue, referral→criteria/pathway, document-lookup→a named/cited document."; + +// P3 structural targeting metric — stronger than the loose keyword `mustContainAny` cue. It measures +// how precisely a SUPPORTED answer hits the asked question (the "targeted / specific / high-yield" +// bar), by checking the answer carries the shape its intent demands. It is INFORMATIONAL: never a +// hard gate, so it can be calibrated against real answers without blocking anyone. Unsupported / +// correctly fail-closed cases are n/a (a precise refusal is on-target by construction) and do not +// count toward the applicable denominator. +export function scoreAnswerTargeting(testCase: AnswerQualityEvalCase, answer: RagAnswer): AnswerTargetingScore { + const unsupported = answer.confidence === "unsupported" || answer.grounded === false; + if (!testCase.supported || unsupported) { + return { score: 1, applicable: false, reason: "n/a: unsupported / fail-closed case" }; + } + const text = answerTextForQuality(answer).toLowerCase(); + const hasNumber = /\d/.test(text); + const hasDoseFigure = + /\b\d+(?:\.\d+)?\s?(?:mg|mcg|microgram|micrograms|g|ml|mmol\/l|mmol|units?|iu)\b/i.test(text) || + /\bmaximum\b|\btitrat|\bdivided doses\b/i.test(text); + + switch (testCase.expectedIntent) { + case "dose": + return hasDoseFigure + ? { score: 1, applicable: true, reason: "carries a dose figure/regimen" } + : { score: 0, applicable: true, reason: "no dose figure/regimen" }; + case "red_result_action": + return /\b(?:withhold|cease|stop|discontinu|hold)\w*/i.test(text) && hasNumber + ? { score: 1, applicable: true, reason: "states a withhold/stop action with a threshold" } + : { score: 0, applicable: true, reason: "no withhold/stop action with a threshold" }; + case "monitoring_schedule": + return /\b(?:weekly|monthly|annual|every|baseline|then|ongoing|fbc|anc|\d+\s*(?:week|month|day|hour)s?)\b/i.test( + text, + ) + ? { score: 1, applicable: true, reason: "carries a schedule/interval" } + : { score: 0, applicable: true, reason: "no schedule/interval" }; + case "contraindication": + return /\b(?:contraindicat|avoid|must not|do not|should not|not use|caution)\w*/i.test(text) + ? { score: 1, applicable: true, reason: "states a contraindication/avoid cue" } + : { score: 0, applicable: true, reason: "no contraindication cue" }; + case "pathway_referral": + return /\b(?:refer|referral|criteria|pathway|indicat|eligib)\w*/i.test(text) + ? { score: 1, applicable: true, reason: "names referral/pathway criteria" } + : { score: 0, applicable: true, reason: "no referral/pathway cue" }; + case "document_lookup": + return answer.citations.length > 0 || /\b(?:document|guideline|policy|procedure|form)\b/i.test(text) + ? { score: 1, applicable: true, reason: "names/cites a document" } + : { score: 0, applicable: true, reason: "no document named/cited" }; + default: + return { score: 1, applicable: false, reason: "n/a: general intent" }; + } +} + type CapturedEvalCaseRow = { id: string; query: string; diff --git a/src/lib/types.ts b/src/lib/types.ts index 3e481ac6e..a00bcf3e0 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -425,6 +425,9 @@ export type QuoteCard = Citation & { quote: string; section_heading: string | null; source_strength?: SourceStrength; + // True when the quote was cut at the display cap; the card shows a "truncated — open source" + // affordance so a value past the cut is never read as the complete passage. + isTruncated?: boolean; }; export type SourceStrength = "strong" | "moderate" | "limited"; diff --git a/tests/rag-eval-cases.test.ts b/tests/rag-eval-cases.test.ts index 0f8433b4d..a1c8c07e5 100644 --- a/tests/rag-eval-cases.test.ts +++ b/tests/rag-eval-cases.test.ts @@ -6,6 +6,8 @@ import { mapCapturedEvalCase, mergeRagEvalCases, scoreAnswerQualityEvalCase, + scoreAnswerTargeting, + type AnswerQualityEvalCase, } from "../src/lib/rag-eval-cases"; import type { RagAnswer } from "../src/lib/types"; @@ -243,4 +245,61 @@ describe("captured RAG eval cases", () => { ]); expect(scores.every((score) => score.score === 1)).toBe(true); }); + + describe("scoreAnswerTargeting (structural per-intent targeting)", () => { + const doseCase = { + id: "t-dose", + question: "What is the maximum sertraline dose?", + expectedIntent: "dose", + supported: true, + category: "routine", + expectedFiles: [], + allowedRoutes: ["fast"], + minCitations: 1, + latencyTargetMs: 20000, + } as unknown as AnswerQualityEvalCase; + + function grounded(text: string): RagAnswer { + return { + answer: text, + grounded: true, + confidence: "high", + citations: [{ chunk_id: "c1" } as never], + sources: [], + answerSections: [], + } as unknown as RagAnswer; + } + + it("passes a dose answer that carries a figure+unit", () => { + const result = scoreAnswerTargeting(doseCase, grounded("The maximum dose is 200 mg daily.")); + expect(result).toMatchObject({ applicable: true, score: 1 }); + }); + + it("fails a supported dose answer that carries no dose figure or regimen", () => { + const bare = scoreAnswerTargeting(doseCase, grounded("Sertraline is an SSRI used for depression.")); + expect(bare).toMatchObject({ applicable: true, score: 0 }); + }); + + it("treats a fail-closed/unsupported case as n/a (not counted)", () => { + const unsupported = { + answer: "No current source with dose guidance for this query was found.", + grounded: false, + confidence: "unsupported", + citations: [], + sources: [], + answerSections: [], + } as unknown as RagAnswer; + expect(scoreAnswerTargeting(doseCase, unsupported)).toMatchObject({ applicable: false, score: 1 }); + }); + + it("requires a withhold/stop action with a threshold for red_result_action", () => { + const redCase = { ...doseCase, expectedIntent: "red_result_action" } as AnswerQualityEvalCase; + expect(scoreAnswerTargeting(redCase, grounded("Withhold clozapine if the ANC falls below 1.5."))).toMatchObject({ + score: 1, + }); + expect(scoreAnswerTargeting(redCase, grounded("Neutropenia is a recognised clozapine risk."))).toMatchObject({ + score: 0, + }); + }); + }); }); From b364def0469c9aebfa44733f0a5fbbd6abc1fc05 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 19:29:16 +0800 Subject: [PATCH 2/3] fix(eval): tighten answer targeting review cases --- .../clinical-dashboard/evidence-panels.tsx | 5 +- src/lib/rag-eval-cases.ts | 77 ++++++++++++++++--- tests/evidence-panels.test.ts | 32 ++++++++ tests/rag-eval-cases.test.ts | 69 ++++++++++++++++- 4 files changed, 168 insertions(+), 15 deletions(-) create mode 100644 tests/evidence-panels.test.ts diff --git a/src/components/clinical-dashboard/evidence-panels.tsx b/src/components/clinical-dashboard/evidence-panels.tsx index d38db875e..367a24a6e 100644 --- a/src/components/clinical-dashboard/evidence-panels.tsx +++ b/src/components/clinical-dashboard/evidence-panels.tsx @@ -1933,9 +1933,12 @@ export function formatQuoteCardsForClipboard(quotes: QuoteCard[]) { // Clean the copied text the same way the card displays it, so clipboard // output never contains internal image-data blocks or glyph artifacts. `${index + 1}. "${sourceTextForVerbatimQuote(quote.quote)}"`, + quote.isTruncated ? "Warning: quote truncated for length; open the source to read the full passage." : null, `Source: ${formatCitationLabel(quote)}`, `Link: ${documentCitationHref(quote)}`, - ].join("\n"), + ] + .filter(Boolean) + .join("\n"), ) .join("\n\n"); } diff --git a/src/lib/rag-eval-cases.ts b/src/lib/rag-eval-cases.ts index b2a243c29..9760d230b 100644 --- a/src/lib/rag-eval-cases.ts +++ b/src/lib/rag-eval-cases.ts @@ -78,6 +78,35 @@ function containsNone(text: string, values: string[] | undefined) { return values.every((value) => !normalized.includes(value.toLowerCase())); } +function normalizeDocumentSignal(value: string) { + return value + .toLowerCase() + .replace(/\.[a-z0-9]+$/i, "") + .replace(/[^a-z0-9]+/g, " ") + .trim(); +} + +function citesOrNamesExpectedDocument(testCase: AnswerQualityEvalCase, answer: RagAnswer, text: string) { + const expectedDocuments = testCase.expectedFiles.map(normalizeDocumentSignal).filter(Boolean); + if (!expectedDocuments.length) return /\b(?:document|guideline|policy|procedure|form)\b/i.test(text); + + const citationSignals = answer.citations.flatMap((citation) => [ + normalizeDocumentSignal(citation.file_name ?? ""), + normalizeDocumentSignal(citation.title ?? ""), + ]).filter(Boolean); + + return expectedDocuments.some((expectedDocument) => { + const expectedTokens = expectedDocument.split(/\s+/).filter((token) => token.length > 2); + const hasExpectedCitation = citationSignals.some( + (signal) => signal === expectedDocument || signal.includes(expectedDocument) || expectedDocument.includes(signal), + ); + const namesExpectedDocument = + text.includes(expectedDocument) || + (expectedTokens.length > 0 && expectedTokens.every((token) => text.includes(token))); + return hasExpectedCitation || namesExpectedDocument; + }); +} + export function scoreAnswerQualityEvalCase(testCase: AnswerQualityEvalCase, answer: RagAnswer) { const text = answerTextForQuality(answer); const wordCount = text.split(/\s+/).filter(Boolean).length; @@ -134,7 +163,19 @@ export function scoreAnswerTargeting(testCase: AnswerQualityEvalCase, answer: Ra const hasNumber = /\d/.test(text); const hasDoseFigure = /\b\d+(?:\.\d+)?\s?(?:mg|mcg|microgram|micrograms|g|ml|mmol\/l|mmol|units?|iu)\b/i.test(text) || - /\bmaximum\b|\btitrat|\bdivided doses\b/i.test(text); + /\btitrat|\bdivided doses?\b/i.test(text); + const asksForThreshold = /\b(?:threshold|level|range|below|above|over|under|less than|greater than|cutoff|count)\b/i.test( + testCase.question, + ); + const hasRedAction = /\b(?:withhold|cease|stop|discontinu|hold|escalat|urgent|review|seek|refer)\w*/i.test(text); + const hasMonitoringSchedule = + /\b(?:weekly|monthly|annual|annually|every|baseline|then|ongoing|fbc|anc|\d+\s*(?:week|month|day|hour)s?)\b/i.test( + text, + ); + const asksForMonitoringRange = /\b(?:level|range|target|therapeutic|maintenance)\b/i.test(testCase.question); + const hasMonitoringRange = + /\b\d+(?:\.\d+)?\s*(?:-|to|–)\s*\d+(?:\.\d+)?\s*(?:mmol\/l|mmol|mg\/l|microgram\/l|mcg\/l|ng\/ml)\b/i.test(text) || + /\b(?:mmol\/l|mmol|mg\/l|microgram\/l|mcg\/l|ng\/ml)\b/i.test(text); switch (testCase.expectedIntent) { case "dose": @@ -142,15 +183,29 @@ export function scoreAnswerTargeting(testCase: AnswerQualityEvalCase, answer: Ra ? { score: 1, applicable: true, reason: "carries a dose figure/regimen" } : { score: 0, applicable: true, reason: "no dose figure/regimen" }; case "red_result_action": - return /\b(?:withhold|cease|stop|discontinu|hold)\w*/i.test(text) && hasNumber - ? { score: 1, applicable: true, reason: "states a withhold/stop action with a threshold" } - : { score: 0, applicable: true, reason: "no withhold/stop action with a threshold" }; + return hasRedAction && (!asksForThreshold || hasNumber) + ? { + score: 1, + applicable: true, + reason: asksForThreshold ? "states an action with a threshold" : "states a red-result action", + } + : { + score: 0, + applicable: true, + reason: asksForThreshold ? "no action with a threshold" : "no red-result action", + }; case "monitoring_schedule": - return /\b(?:weekly|monthly|annual|every|baseline|then|ongoing|fbc|anc|\d+\s*(?:week|month|day|hour)s?)\b/i.test( - text, - ) - ? { score: 1, applicable: true, reason: "carries a schedule/interval" } - : { score: 0, applicable: true, reason: "no schedule/interval" }; + return hasMonitoringSchedule || (asksForMonitoringRange && hasMonitoringRange) + ? { + score: 1, + applicable: true, + reason: hasMonitoringSchedule ? "carries a schedule/interval" : "carries a monitoring level/range", + } + : { + score: 0, + applicable: true, + reason: asksForMonitoringRange ? "no schedule/interval or monitoring range" : "no schedule/interval", + }; case "contraindication": return /\b(?:contraindicat|avoid|must not|do not|should not|not use|caution)\w*/i.test(text) ? { score: 1, applicable: true, reason: "states a contraindication/avoid cue" } @@ -160,9 +215,9 @@ export function scoreAnswerTargeting(testCase: AnswerQualityEvalCase, answer: Ra ? { score: 1, applicable: true, reason: "names referral/pathway criteria" } : { score: 0, applicable: true, reason: "no referral/pathway cue" }; case "document_lookup": - return answer.citations.length > 0 || /\b(?:document|guideline|policy|procedure|form)\b/i.test(text) + return citesOrNamesExpectedDocument(testCase, answer, text) ? { score: 1, applicable: true, reason: "names/cites a document" } - : { score: 0, applicable: true, reason: "no document named/cited" }; + : { score: 0, applicable: true, reason: "no expected document named/cited" }; default: return { score: 1, applicable: false, reason: "n/a: general intent" }; } diff --git a/tests/evidence-panels.test.ts b/tests/evidence-panels.test.ts new file mode 100644 index 000000000..3b843236f --- /dev/null +++ b/tests/evidence-panels.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; +import { formatQuoteCardsForClipboard } from "../src/components/clinical-dashboard/evidence-panels"; +import type { QuoteCard } from "../src/lib/types"; + +function quoteCard(overrides: Partial = {}): QuoteCard { + return { + chunk_id: "chunk-1", + document_id: "doc-1", + title: "Lithium Guideline", + file_name: "CG.MHSP.Lithium.pdf", + page_number: 4, + chunk_index: 0, + quote: "Maintenance range is 0.4-0.8 mmol/L.", + section_heading: null, + ...overrides, + }; +} + +describe("formatQuoteCardsForClipboard", () => { + it("marks copied quotes when the displayed quote was truncated", () => { + const text = formatQuoteCardsForClipboard([quoteCard({ isTruncated: true })]); + + expect(text).toContain("Warning: quote truncated for length; open the source to read the full passage."); + expect(text).toContain("Source: Lithium Guideline, p. 4"); + }); + + it("does not add a truncation warning for complete quotes", () => { + const text = formatQuoteCardsForClipboard([quoteCard({ isTruncated: false })]); + + expect(text).not.toContain("Warning: quote truncated"); + }); +}); diff --git a/tests/rag-eval-cases.test.ts b/tests/rag-eval-cases.test.ts index a1c8c07e5..2d83a47cd 100644 --- a/tests/rag-eval-cases.test.ts +++ b/tests/rag-eval-cases.test.ts @@ -259,12 +259,12 @@ describe("captured RAG eval cases", () => { latencyTargetMs: 20000, } as unknown as AnswerQualityEvalCase; - function grounded(text: string): RagAnswer { + function grounded(text: string, citation: Partial = {}): RagAnswer { return { answer: text, grounded: true, confidence: "high", - citations: [{ chunk_id: "c1" } as never], + citations: [{ chunk_id: "c1", ...citation } as RagAnswer["citations"][number]], sources: [], answerSections: [], } as unknown as RagAnswer; @@ -280,6 +280,11 @@ describe("captured RAG eval cases", () => { expect(bare).toMatchObject({ applicable: true, score: 0 }); }); + it("does not count maximum alone as dose targeting", () => { + const result = scoreAnswerTargeting(doseCase, grounded("The maximum sertraline dose is in the source.")); + expect(result).toMatchObject({ applicable: true, score: 0 }); + }); + it("treats a fail-closed/unsupported case as n/a (not counted)", () => { const unsupported = { answer: "No current source with dose guidance for this query was found.", @@ -293,7 +298,11 @@ describe("captured RAG eval cases", () => { }); it("requires a withhold/stop action with a threshold for red_result_action", () => { - const redCase = { ...doseCase, expectedIntent: "red_result_action" } as AnswerQualityEvalCase; + const redCase = { + ...doseCase, + question: "What ANC threshold should trigger clozapine withholding?", + expectedIntent: "red_result_action", + } as AnswerQualityEvalCase; expect(scoreAnswerTargeting(redCase, grounded("Withhold clozapine if the ANC falls below 1.5."))).toMatchObject({ score: 1, }); @@ -301,5 +310,59 @@ describe("captured RAG eval cases", () => { score: 0, }); }); + + it("allows red-result action answers without a numeric threshold when the question asks for action", () => { + const redCase = { + ...doseCase, + question: "What action is required for suspected lithium toxicity?", + expectedIntent: "red_result_action", + } as AnswerQualityEvalCase; + expect(scoreAnswerTargeting(redCase, grounded("Stop lithium and seek urgent medical review."))).toMatchObject({ + applicable: true, + score: 1, + }); + }); + + it("accepts monitoring level ranges for monitoring cases that ask for a range", () => { + const monitoringCase = { + ...doseCase, + question: "What lithium level range is used for maintenance monitoring?", + expectedIntent: "monitoring_schedule", + } as AnswerQualityEvalCase; + expect( + scoreAnswerTargeting(monitoringCase, grounded("The maintenance range is 0.4-0.8 mmol/L.")), + ).toMatchObject({ + applicable: true, + score: 1, + }); + }); + + it("requires document-lookup targeting to name or cite the expected document", () => { + const documentCase = { + ...doseCase, + question: "What documents support lithium monitoring?", + expectedIntent: "document_lookup", + expectedFiles: ["CG.MHSP.Lithium.pdf"], + } as AnswerQualityEvalCase; + + expect( + scoreAnswerTargeting( + documentCase, + grounded("The monitoring document supports regular review.", { + file_name: "Unrelated.Policy.pdf", + title: "Unrelated Policy", + }), + ), + ).toMatchObject({ applicable: true, score: 0 }); + expect( + scoreAnswerTargeting( + documentCase, + grounded("The lithium guideline supports regular review.", { + file_name: "CG.MHSP.Lithium.pdf", + title: "CG.MHSP.Lithium", + }), + ), + ).toMatchObject({ applicable: true, score: 1 }); + }); }); }); From 55b30e7f70b8c31f9fb502421cb8400e4af4545e Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:01:48 +0800 Subject: [PATCH 3/3] fix(eval): reuse document aliases for targeting --- scripts/eval-utils.ts | 112 +----------------------------- src/lib/eval-document-matching.ts | 110 +++++++++++++++++++++++++++++ src/lib/rag-eval-cases.ts | 49 ++++++------- tests/rag-eval-cases.test.ts | 23 +++++- 4 files changed, 155 insertions(+), 139 deletions(-) create mode 100644 src/lib/eval-document-matching.ts diff --git a/scripts/eval-utils.ts b/scripts/eval-utils.ts index 6b3f8d53f..907934e86 100644 --- a/scripts/eval-utils.ts +++ b/scripts/eval-utils.ts @@ -1,6 +1,9 @@ +import { expectedFileCoverage } from "@/lib/eval-document-matching"; import type { RagEvalCase } from "@/lib/rag-eval-cases"; import type { RagAnswer, SearchResult, VisualEvidenceCard } from "@/lib/types"; +export { expectedFileCoverage, type ExpectedFileCoverage } from "@/lib/eval-document-matching"; + export type SupabaseAdmin = Awaited>; export async function loadAdminClient() { @@ -79,115 +82,6 @@ export function percentile(values: number[], percentileValue: number) { return sorted[index]; } -export type ExpectedFileCoverage = { - expectedFiles: string[]; - matchedFiles: string[]; - missingFiles: string[]; - anyHit: boolean; - allHit: boolean; -}; - -function normalizedDocumentName(value: string) { - return value - .toLowerCase() - .replace(/[^a-z0-9]+/g, " ") - .replace(/\s+/g, " ") - .trim(); -} - -const clinicalDocumentAliases: Record = { - ActiveCommunityPtED: [ - "Active Community Patients in the Emergency Department", - "Active Community Patients Emergency Department", - ], - AdmissionCommunityPts: ["Admission of Community Patients", "Admission Community Patients"], - AgitationArousalPharmaMgt: [ - "Agitation and Arousal Pharmacological Management", - "Pharmacological Management of Acute Agitation and Arousal", - "Medication for Agitation and Arousal", - "Mental Health Pharmacological Management of Agitation and Arousal", - ], - AssessmentDocumentation: ["Assessment Documentation", "Clinical Assessment", "Mental Health Assessment"], - BestPracticePrescription: ["Best Practice Prescription", "Best Practice Prescribing", "Prescription"], - ClozapinePresAdminMonitor: [ - "Clozapine Prescribing Administration Monitoring", - "Clozapine Prescribing Administration and Monitoring", - "Clozapine Prescribing Administering Monitoring", - "Clozapine Prescribing Administering Monitoring and Capillary Sampling", - "Clozapine Prescribing", - "Clozapine Prescribing NMHS", - "Clozapine GP Shared Care", - "Clozapine Management by GP", - "Clozapine Therapy", - ], - CommunityHomeVisit: ["Community Home Visit", "Home Visit", "Community Visits"], - Discharge: [ - "Admission to Discharge for Mental Health Inpatients", - "Admission to Discharge for Community Mental Health", - "Referral Admission and Discharge Mental Health Hospital in the Home", - "Mental Health Hospital in the Home", - "Mental Health Medically Cleared for Discharge", - "Mental Health Inpatient Triage to Discharge", - "ACMHS and OACMHS Triage to Discharge", - // The synthetic MHSP.Discharge.pdf was superseded by real named discharge documents; - // "Discharge Planning for Community Patients / Inpatients" is the discharge counterpart - // that ranks for the admission-vs-discharge comparison, so recognize it as a real hit. - "Discharge Planning", - ], - Duress: ["Duress", "Duress Procedure", "Duress Response"], - ECTProcedure: ["ECT Procedure", "Electroconvulsive Therapy", "Electroconvulsive Therapy ECT"], - IllegalSubstances: ["Illegal Substances", "Substances", "Contraband"], - LongActingInjectable: [ - "Long Acting Injectable", - "Long-Acting Injectable", - "Depot", - "Olanzapine LAI", - "Long Acting Injectable Antipsychotic", - ], - MetabolicScreening: ["Metabolic Screening", "Metabolic Monitoring", "Physical Health Monitoring"], - MHATMHCTTreatmentTeamProcess: ["Mental Health Treatment Team Process", "Treatment Team Process", "MHAT", "MHCT"], - NeurolepticSideEffect: ["Neuroleptic Side Effects", "Neuroleptic Side Effect", "Neuroleptic Effects"], - NOCC: ["NOCC", "National Outcomes and Casemix Collection", "Outcome Measures Completion"], - PtSafetyPlan: ["Patient Safety Plan", "Safety Planning", "Safety Plan"], -}; - -function documentExpectationAlternatives(expectation: string) { - const normalizedExpectation = normalizedDocumentName(expectation); - const compactExpectation = normalizedExpectation.replace(/\s+/g, ""); - const aliasValues = Object.entries(clinicalDocumentAliases).flatMap(([key, values]) => { - const normalizedKey = normalizedDocumentName(key); - const compactKey = normalizedKey.replace(/\s+/g, ""); - if (!compactExpectation.includes(compactKey) && !normalizedExpectation.includes(normalizedKey)) return []; - return values; - }); - return Array.from(new Set([expectation, ...aliasValues].map(normalizedDocumentName).filter(Boolean))); -} - -function resultDocumentText(source: Pick) { - return normalizedDocumentName(`${source.title} ${source.file_name}`); -} - -export function expectedFileCoverage( - expectedFiles: string[], - sources: Array>, - limit = 3, -): ExpectedFileCoverage { - const topFiles = sources.slice(0, limit).map(resultDocumentText); - const matchedFiles = expectedFiles.filter((expected) => - documentExpectationAlternatives(expected).some((alternative) => - topFiles.some((file) => file.includes(alternative)), - ), - ); - - return { - expectedFiles, - matchedFiles, - missingFiles: expectedFiles.filter((expected) => !matchedFiles.includes(expected)), - anyHit: matchedFiles.length > 0, - allHit: expectedFiles.length > 0 && matchedFiles.length === expectedFiles.length, - }; -} - export function expectedFileHit( expectedFiles: string[], sources: Array>, diff --git a/src/lib/eval-document-matching.ts b/src/lib/eval-document-matching.ts new file mode 100644 index 000000000..afec16d2d --- /dev/null +++ b/src/lib/eval-document-matching.ts @@ -0,0 +1,110 @@ +import type { SearchResult } from "@/lib/types"; + +export type ExpectedFileCoverage = { + expectedFiles: string[]; + matchedFiles: string[]; + missingFiles: string[]; + anyHit: boolean; + allHit: boolean; +}; + +export function normalizedDocumentName(value: string) { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +const clinicalDocumentAliases: Record = { + ActiveCommunityPtED: [ + "Active Community Patients in the Emergency Department", + "Active Community Patients Emergency Department", + ], + AdmissionCommunityPts: ["Admission of Community Patients", "Admission Community Patients"], + AgitationArousalPharmaMgt: [ + "Agitation and Arousal Pharmacological Management", + "Pharmacological Management of Acute Agitation and Arousal", + "Medication for Agitation and Arousal", + "Mental Health Pharmacological Management of Agitation and Arousal", + ], + AssessmentDocumentation: ["Assessment Documentation", "Clinical Assessment", "Mental Health Assessment"], + BestPracticePrescription: ["Best Practice Prescription", "Best Practice Prescribing", "Prescription"], + ClozapinePresAdminMonitor: [ + "Clozapine Prescribing Administration Monitoring", + "Clozapine Prescribing Administration and Monitoring", + "Clozapine Prescribing Administering Monitoring", + "Clozapine Prescribing Administering Monitoring and Capillary Sampling", + "Clozapine Prescribing", + "Clozapine Prescribing NMHS", + "Clozapine GP Shared Care", + "Clozapine Management by GP", + "Clozapine Therapy", + ], + CommunityHomeVisit: ["Community Home Visit", "Home Visit", "Community Visits"], + Discharge: [ + "Admission to Discharge for Mental Health Inpatients", + "Admission to Discharge for Community Mental Health", + "Referral Admission and Discharge Mental Health Hospital in the Home", + "Mental Health Hospital in the Home", + "Mental Health Medically Cleared for Discharge", + "Mental Health Inpatient Triage to Discharge", + "ACMHS and OACMHS Triage to Discharge", + // The synthetic MHSP.Discharge.pdf was superseded by real named discharge documents; + // "Discharge Planning for Community Patients / Inpatients" is the discharge counterpart + // that ranks for the admission-vs-discharge comparison, so recognize it as a real hit. + "Discharge Planning", + ], + Duress: ["Duress", "Duress Procedure", "Duress Response"], + ECTProcedure: ["ECT Procedure", "Electroconvulsive Therapy", "Electroconvulsive Therapy ECT"], + IllegalSubstances: ["Illegal Substances", "Substances", "Contraband"], + LongActingInjectable: [ + "Long Acting Injectable", + "Long-Acting Injectable", + "Depot", + "Olanzapine LAI", + "Long Acting Injectable Antipsychotic", + ], + MetabolicScreening: ["Metabolic Screening", "Metabolic Monitoring", "Physical Health Monitoring"], + MHATMHCTTreatmentTeamProcess: ["Mental Health Treatment Team Process", "Treatment Team Process", "MHAT", "MHCT"], + NeurolepticSideEffect: ["Neuroleptic Side Effects", "Neuroleptic Side Effect", "Neuroleptic Effects"], + NOCC: ["NOCC", "National Outcomes and Casemix Collection", "Outcome Measures Completion"], + PtSafetyPlan: ["Patient Safety Plan", "Safety Planning", "Safety Plan"], +}; + +export function documentExpectationAlternatives(expectation: string) { + const normalizedExpectation = normalizedDocumentName(expectation); + const compactExpectation = normalizedExpectation.replace(/\s+/g, ""); + const aliasValues = Object.entries(clinicalDocumentAliases).flatMap(([key, values]) => { + const normalizedKey = normalizedDocumentName(key); + const compactKey = normalizedKey.replace(/\s+/g, ""); + if (!compactExpectation.includes(compactKey) && !normalizedExpectation.includes(normalizedKey)) return []; + return values; + }); + return Array.from(new Set([expectation, ...aliasValues].map(normalizedDocumentName).filter(Boolean))); +} + +function resultDocumentText(source: Pick) { + return normalizedDocumentName(`${source.title} ${source.file_name}`); +} + +export function expectedFileCoverage( + expectedFiles: string[], + sources: Array>, + limit = 3, +): ExpectedFileCoverage { + const topFiles = sources.slice(0, limit).map(resultDocumentText); + const matchedFiles = expectedFiles.filter((expected) => + documentExpectationAlternatives(expected).some((alternative) => + topFiles.some((file) => file.includes(alternative)), + ), + ); + + return { + expectedFiles, + matchedFiles, + missingFiles: expectedFiles.filter((expected) => !matchedFiles.includes(expected)), + anyHit: matchedFiles.length > 0, + allHit: expectedFiles.length > 0 && matchedFiles.length === expectedFiles.length, + }; +} diff --git a/src/lib/rag-eval-cases.ts b/src/lib/rag-eval-cases.ts index 9760d230b..a2f612a45 100644 --- a/src/lib/rag-eval-cases.ts +++ b/src/lib/rag-eval-cases.ts @@ -1,4 +1,9 @@ import { isDangerSourceGovernanceMessage } from "@/lib/source-governance"; +import { + documentExpectationAlternatives, + expectedFileCoverage, + normalizedDocumentName, +} from "@/lib/eval-document-matching"; import type { RagAnswer, RagQueryClass } from "@/lib/types"; export type RagEvalCategory = "routine" | "complex" | "unsupported"; @@ -78,33 +83,24 @@ function containsNone(text: string, values: string[] | undefined) { return values.every((value) => !normalized.includes(value.toLowerCase())); } -function normalizeDocumentSignal(value: string) { - return value - .toLowerCase() - .replace(/\.[a-z0-9]+$/i, "") - .replace(/[^a-z0-9]+/g, " ") - .trim(); -} - function citesOrNamesExpectedDocument(testCase: AnswerQualityEvalCase, answer: RagAnswer, text: string) { - const expectedDocuments = testCase.expectedFiles.map(normalizeDocumentSignal).filter(Boolean); - if (!expectedDocuments.length) return /\b(?:document|guideline|policy|procedure|form)\b/i.test(text); + if (!testCase.expectedFiles.length) return /\b(?:document|guideline|policy|procedure|form)\b/i.test(text); - const citationSignals = answer.citations.flatMap((citation) => [ - normalizeDocumentSignal(citation.file_name ?? ""), - normalizeDocumentSignal(citation.title ?? ""), - ]).filter(Boolean); + const expectedCoverage = expectedFileCoverage(testCase.expectedFiles, answer.citations, answer.citations.length); + if (expectedCoverage.anyHit) return true; - return expectedDocuments.some((expectedDocument) => { - const expectedTokens = expectedDocument.split(/\s+/).filter((token) => token.length > 2); - const hasExpectedCitation = citationSignals.some( - (signal) => signal === expectedDocument || signal.includes(expectedDocument) || expectedDocument.includes(signal), - ); - const namesExpectedDocument = - text.includes(expectedDocument) || - (expectedTokens.length > 0 && expectedTokens.every((token) => text.includes(token))); - return hasExpectedCitation || namesExpectedDocument; - }); + const normalizedText = normalizedDocumentName(text); + return testCase.expectedFiles.some((expectedDocument) => + documentExpectationAlternatives(expectedDocument).some( + (alternative) => + text.includes(alternative) || + normalizedText.includes(alternative) || + alternative + .split(/\s+/) + .filter((token) => token.length > 2) + .every((token) => text.includes(token)), + ), + ); } export function scoreAnswerQualityEvalCase(testCase: AnswerQualityEvalCase, answer: RagAnswer) { @@ -164,9 +160,8 @@ export function scoreAnswerTargeting(testCase: AnswerQualityEvalCase, answer: Ra const hasDoseFigure = /\b\d+(?:\.\d+)?\s?(?:mg|mcg|microgram|micrograms|g|ml|mmol\/l|mmol|units?|iu)\b/i.test(text) || /\btitrat|\bdivided doses?\b/i.test(text); - const asksForThreshold = /\b(?:threshold|level|range|below|above|over|under|less than|greater than|cutoff|count)\b/i.test( - testCase.question, - ); + const asksForThreshold = + /\b(?:threshold|level|range|below|above|over|under|less than|greater than|cutoff|count)\b/i.test(testCase.question); const hasRedAction = /\b(?:withhold|cease|stop|discontinu|hold|escalat|urgent|review|seek|refer)\w*/i.test(text); const hasMonitoringSchedule = /\b(?:weekly|monthly|annual|annually|every|baseline|then|ongoing|fbc|anc|\d+\s*(?:week|month|day|hour)s?)\b/i.test( diff --git a/tests/rag-eval-cases.test.ts b/tests/rag-eval-cases.test.ts index 2d83a47cd..12ef79dc3 100644 --- a/tests/rag-eval-cases.test.ts +++ b/tests/rag-eval-cases.test.ts @@ -329,9 +329,7 @@ describe("captured RAG eval cases", () => { question: "What lithium level range is used for maintenance monitoring?", expectedIntent: "monitoring_schedule", } as AnswerQualityEvalCase; - expect( - scoreAnswerTargeting(monitoringCase, grounded("The maintenance range is 0.4-0.8 mmol/L.")), - ).toMatchObject({ + expect(scoreAnswerTargeting(monitoringCase, grounded("The maintenance range is 0.4-0.8 mmol/L."))).toMatchObject({ applicable: true, score: 1, }); @@ -364,5 +362,24 @@ describe("captured RAG eval cases", () => { ), ).toMatchObject({ applicable: true, score: 1 }); }); + + it("reuses eval document aliases for document-lookup targeting", () => { + const documentCase = { + ...doseCase, + question: "What discharge documentation is required?", + expectedIntent: "document_lookup", + expectedFiles: ["MHSP.Discharge.pdf"], + } as AnswerQualityEvalCase; + + expect( + scoreAnswerTargeting( + documentCase, + grounded("The discharge planning document sets out documentation responsibilities.", { + file_name: "Admission to Discharge for Mental Health Inpatients (NMHS).pdf", + title: "Admission to Discharge for Mental Health Inpatients", + }), + ), + ).toMatchObject({ applicable: true, score: 1 }); + }); }); });