From fcc931ad113c41e9a6ab9f789f43dd863480f636 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:32:24 +0800 Subject: [PATCH 1/4] eval(retrieval): add content_mrr@10 passage-rank metric MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The golden retrieval eval measured document-level rank (mrr@10, over expectedDocumentSubstrings) and content-level *recall* (contentRecall@5, binary "is the answer passage in the top 5"), but nothing measured how HIGH the answer-bearing passage ranks. That left rerank/chunking changes that lift the right passage from #5 to #1 unprovable: content recall stays 1.0 across that move and doc-level mrr is blind to passage order within a document. Add contentReciprocalRankAt10 — the mean over a case's expected content terms of the reciprocal rank of the earliest top-10 result carrying that term (1.0 iff every term rides the rank-1 passage). Surface it in the summary as content_mrr_at_10, averaged only over cases that declare content terms (content_mrr_case_count) so structural zeros don't dilute the signal, plus a per-case contentRR@10 column and the human summary line. Purely additive: the summary type is inferred, so reindex-eval-gate and the other summary consumers are unaffected. Guarded by a focused test proving the metric drops to 0.5 when the answer passage is demoted below a distractor while content recall stays 1.0. Co-Authored-By: Claude Opus 4.8 --- scripts/eval-retrieval.ts | 42 ++++++++++++++++++++++++++++- tests/eval-quality.test.ts | 1 + tests/eval-retrieval.test.ts | 52 ++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 0a4332f2f..13ce3eaa5 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -53,6 +53,7 @@ export type GoldenRetrievalResult = { hitAtK: boolean; topK: number; reciprocalRankAt10: number; + contentReciprocalRankAt10: number; latencyMs: number; retrievalStrategy: string | null; retrievalPlan: string | null; @@ -344,6 +345,31 @@ function reciprocalRankAt10(expectedSubstrings: string[], results: SearchResult[ return index >= 0 ? 1 / (index + 1) : 0; } +/** + * Passage-level rank quality ("best-passage rank"): the mean over expected content terms of + * the reciprocal rank of the earliest top-10 result whose content carries that term. Where + * `contentRecallAt5` only asks *whether* an answer-bearing passage appears in the top 5, this + * asks *how high* it ranks — so a rerank/chunking change that lifts the right passage from #5 + * to #1 is measurable (content recall stays 1.0 across that move; doc-level `mrr@10` is blind + * to passage order within a document). 1.0 iff every expected term rides the rank-1 passage. + * Cases with no expected content terms return 0 and are excluded from the summary average. + */ +function contentReciprocalRankAt10( + expectedTerms: GoldenRetrievalCase["expectedContentTerms"], + results: SearchResult[], +) { + if (expectedTerms.length === 0) return 0; + const top = results.slice(0, 10); + const total = expectedTerms.reduce((sum, expectation) => { + const alternatives = contentExpectationAlternatives(expectation); + const index = top.findIndex((result) => + alternatives.some((term) => textContainsClinicalTerm(resultContentText(result), term)), + ); + return sum + (index >= 0 ? 1 / (index + 1) : 0); + }, 0); + return total / expectedTerms.length; +} + function hasTableEvidence(results: SearchResult[], limit = 5) { return results.slice(0, limit).some((result) => { if ((result.table_facts?.length ?? 0) > 0) return true; @@ -490,6 +516,7 @@ export function evaluateGoldenRetrievalCase(args: { hitAtK, topK, reciprocalRankAt10: reciprocalRankAt10(args.testCase.expectedDocumentSubstrings, args.results), + contentReciprocalRankAt10: contentReciprocalRankAt10(args.testCase.expectedContentTerms, args.results), latencyMs: args.latencyMs, retrievalStrategy: args.telemetry.retrieval_strategy ?? null, retrievalPlan: args.telemetry.retrieval_plan ?? null, @@ -520,6 +547,9 @@ export function evaluateGoldenRetrievalCase(args: { export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[]) { const documentRecallDenominator = Math.max(results.length, 1); const contentRecallDenominator = Math.max(results.length, 1); + // Passage-rank quality is only meaningful for cases that declare answer-bearing content + // terms; averaging over cases without them would dilute the signal with structural zeros. + const contentRankCases = results.filter((result) => result.expectedContentTerms.length > 0); const strategyCounts = results.reduce>((counts, result) => { const strategy = result.retrievalStrategy ?? "none"; counts[strategy] = (counts[strategy] ?? 0) + 1; @@ -562,6 +592,13 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[] mrr_at_10: Number( (results.reduce((sum, result) => sum + result.reciprocalRankAt10, 0) / Math.max(results.length, 1)).toFixed(4), ), + content_mrr_at_10: Number( + ( + contentRankCases.reduce((sum, result) => sum + result.contentReciprocalRankAt10, 0) / + Math.max(contentRankCases.length, 1) + ).toFixed(4), + ), + content_mrr_case_count: contentRankCases.length, median_latency_ms: percentile( results.map((result) => result.latencyMs), 50, @@ -646,6 +683,9 @@ function printHumanSummary(summary: ReturnType = {}): Golden hitAtK: true, topK: 8, reciprocalRankAt10: 1, + contentReciprocalRankAt10: 1, latencyMs: 120, retrievalStrategy: "hybrid", retrievalPlan: "table_threshold:table_facts_visual_units_then_chunks", diff --git a/tests/eval-retrieval.test.ts b/tests/eval-retrieval.test.ts index 930d6d333..f4190e408 100644 --- a/tests/eval-retrieval.test.ts +++ b/tests/eval-retrieval.test.ts @@ -55,9 +55,61 @@ describe("golden retrieval eval helpers", () => { expect(evaluated.documentRecallAt5).toBe(1); expect(evaluated.contentRecallAt5).toBe(1); expect(evaluated.reciprocalRankAt10).toBe(1); + expect(evaluated.contentReciprocalRankAt10).toBe(1); expect(evaluated.failures).toEqual([]); }); + it("content_mrr@10 penalises answer passages that rank below distractors even when recall is perfect", () => { + const testCase = { + id: "content-rank", + query: "What ANC threshold should withhold clozapine?", + expectedQueryClass: "table_threshold" as const, + expectedDocumentSubstrings: ["ClozapinePresAdminMonitor"], + expectedContentTerms: ["anc", "withhold"], + topK: 8, + expectTableEvidence: false, + }; + // A distractor with no matching content terms sits above the real answer passage. + const distractor = result({ + id: "chunk-distractor", + title: "Unrelated Ward Policy", + file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf", + section_heading: "General notes", + content: "This section covers unrelated logistics and has no matching clinical terms.", + retrieval_synopsis: undefined, + }); + const topRanked = evaluateGoldenRetrievalCase({ + testCase, + results: [result(), distractor], + telemetry: { query_class: "table_threshold", retrieval_strategy: "hybrid" }, + latencyMs: 100, + }); + const demoted = evaluateGoldenRetrievalCase({ + testCase, + results: [distractor, result()], + telemetry: { query_class: "table_threshold", retrieval_strategy: "hybrid" }, + latencyMs: 100, + }); + + // Content recall is blind to ordering — both keep the answer passage in the top 5. + expect(topRanked.contentRecallAt5).toBe(1); + expect(demoted.contentRecallAt5).toBe(1); + // Passage rank is not: demoting the answer passage to rank 2 halves each term's RR. + expect(topRanked.contentReciprocalRankAt10).toBe(1); + expect(demoted.contentReciprocalRankAt10).toBeCloseTo(0.5, 5); + + // The summary averages content_mrr only over cases that declare content terms. + const noContentCase = evaluateGoldenRetrievalCase({ + testCase: { ...testCase, id: "no-content", expectedContentTerms: [] }, + results: [result()], + telemetry: { query_class: "table_threshold", retrieval_strategy: "hybrid" }, + latencyMs: 100, + }); + const summary = summarizeGoldenRetrievalResults([demoted, noContentCase]); + expect(summary.content_mrr_case_count).toBe(1); + expect(summary.content_mrr_at_10).toBeCloseTo(0.5, 4); + }); + it("matches legacy compact expected document names to current corpus titles", () => { const evaluated = evaluateGoldenRetrievalCase({ testCase: { From 1f56972c0b58c416107fd5f968851f174b1cda64 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 20:57:53 +0800 Subject: [PATCH 2/4] Fix retrieval content MRR scoring --- scripts/eval-retrieval.ts | 26 +++++++++++---- tests/eval-retrieval.test.ts | 64 ++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 6 deletions(-) diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 13ce3eaa5..556c23b74 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -277,6 +277,20 @@ function resultDocumentEvidenceText(result: SearchResult) { } function resultContentText(result: SearchResult) { + return normalized( + [ + result.title, + result.file_name, + result.section_heading, + result.section_path?.join(" "), + resultContentEvidenceText(result), + ] + .filter(Boolean) + .join(" "), + ); +} + +function resultContentEvidenceText(result: SearchResult) { const tableFactText = (result.table_facts ?? []) .map((fact) => [fact.table_title, fact.row_label, fact.clinical_parameter, fact.threshold_value, fact.action] @@ -291,10 +305,6 @@ function resultContentText(result: SearchResult) { .join(" "); return normalized( [ - result.title, - result.file_name, - result.section_heading, - result.section_path?.join(" "), result.retrieval_synopsis, result.content, tableFactText, @@ -363,13 +373,17 @@ function contentReciprocalRankAt10( const total = expectedTerms.reduce((sum, expectation) => { const alternatives = contentExpectationAlternatives(expectation); const index = top.findIndex((result) => - alternatives.some((term) => textContainsClinicalTerm(resultContentText(result), term)), + alternatives.some((term) => textContainsClinicalTerm(resultContentEvidenceText(result), term)), ); return sum + (index >= 0 ? 1 / (index + 1) : 0); }, 0); return total / expectedTerms.length; } +export function retrievalLimitForGoldenCase(testCase: GoldenRetrievalCase) { + return Math.max(testCase.topK, 10); +} + function hasTableEvidence(results: SearchResult[], limit = 5) { return results.slice(0, limit).some((result) => { if ((result.table_facts?.length ?? 0) > 0) return true; @@ -780,7 +794,7 @@ async function main() { searchChunksWithTelemetry({ query: testCase.query, ownerId, - topK: testCase.topK, + topK: retrievalLimitForGoldenCase(testCase), minSimilarity: 0.12, skipCache: args.mode !== "latency", }), diff --git a/tests/eval-retrieval.test.ts b/tests/eval-retrieval.test.ts index f4190e408..9cb6f73d8 100644 --- a/tests/eval-retrieval.test.ts +++ b/tests/eval-retrieval.test.ts @@ -3,6 +3,7 @@ import { capturedRagCaseToGoldenCase, evaluateGoldenRetrievalCase, loadGoldenRetrievalCases, + retrievalLimitForGoldenCase, summarizeGoldenRetrievalResults, } from "../scripts/eval-retrieval"; import type { SearchResult } from "../src/lib/types"; @@ -110,6 +111,69 @@ describe("golden retrieval eval helpers", () => { expect(summary.content_mrr_at_10).toBeCloseTo(0.5, 4); }); + it("content_mrr@10 ignores document metadata when ranking answer-bearing passages", () => { + const testCase = { + id: "metadata-distractor", + query: "What is the patient property safety plan?", + expectedQueryClass: "document_lookup" as const, + expectedDocumentSubstrings: ["Patient Property Safety Plan"], + expectedContentTerms: ["patient", "property"], + topK: 8, + expectTableEvidence: false, + }; + const metadataOnlyDistractor = result({ + id: "metadata-only", + title: "Patient Property Safety Plan", + file_name: "PatientPropertySafetyPlan.pdf", + section_heading: "Patient property", + content: "This unrelated passage describes roster logistics without the expected evidence.", + retrieval_synopsis: undefined, + }); + const answerPassage = result({ + id: "answer-passage", + title: "Other Document", + file_name: "Other.pdf", + section_heading: "Unrelated heading", + content: "The patient property process requires documenting valuables in the safety plan.", + retrieval_synopsis: undefined, + }); + + const evaluated = evaluateGoldenRetrievalCase({ + testCase, + results: [metadataOnlyDistractor, answerPassage], + telemetry: { query_class: "document_lookup", retrieval_strategy: "hybrid" }, + latencyMs: 100, + }); + + expect(evaluated.contentRecallAt5).toBe(1); + expect(evaluated.contentReciprocalRankAt10).toBeCloseTo(0.5, 5); + }); + + it("fetches enough results to score content_mrr@10 even when hit@K uses a smaller topK", () => { + expect( + retrievalLimitForGoldenCase({ + id: "top-k-8", + query: "What ANC threshold should withhold clozapine?", + expectedQueryClass: "table_threshold", + expectedDocumentSubstrings: [], + expectedContentTerms: ["anc"], + topK: 8, + expectTableEvidence: false, + }), + ).toBe(10); + expect( + retrievalLimitForGoldenCase({ + id: "top-k-12", + query: "What ANC threshold should withhold clozapine?", + expectedQueryClass: "table_threshold", + expectedDocumentSubstrings: [], + expectedContentTerms: ["anc"], + topK: 12, + expectTableEvidence: false, + }), + ).toBe(12); + }); + it("matches legacy compact expected document names to current corpus titles", () => { const evaluated = evaluateGoldenRetrievalCase({ testCase: { From b4f273427b3915b1816d237be0214460f37c769d Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:00:54 +0800 Subject: [PATCH 3/4] style: prettier-format eval-retrieval content_mrr scoring The retrievalLimitForGoldenCase fix landed without a prettier pass, failing the required verify check (format:check). Formatting-only; no logic change. Co-Authored-By: Claude Opus 4.8 --- scripts/eval-retrieval.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 556c23b74..135569160 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -303,16 +303,7 @@ function resultContentEvidenceText(result: SearchResult) { [image.caption, image.tableTitle, image.tableLabel, image.tableTextSnippet].filter(Boolean).join(" "), ) .join(" "); - return normalized( - [ - result.retrieval_synopsis, - result.content, - tableFactText, - imageText, - ] - .filter(Boolean) - .join(" "), - ); + return normalized([result.retrieval_synopsis, result.content, tableFactText, imageText].filter(Boolean).join(" ")); } function expectedDocumentHits(expectedSubstrings: string[], results: SearchResult[], limit: number) { From 927a6ee9c6d22799aac8f7a4af6bc80cb1ff6b29 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 22:53:43 +0800 Subject: [PATCH 4/4] eval(retrieval): address Codex review on content_mrr@10 evidence text Two remaining P2 review findings on the passage-rank metric: - Quality-eval path truncation: runRetrievalQualityCases (eval-quality.ts) still fetched topK=testCase.topK, so content_mrr@10 silently degraded to content_mrr@topK for the many fixtures with topK<10. Now uses retrievalLimitForGoldenCase(=max(topK,10)), matching the standalone runner. hit@K semantics are unchanged (evaluateGoldenRetrievalCase still slices to the case's topK), so only positions 9-10 are exposed for the @10 metric. - Missing first-class table/visual evidence: resultContentEvidenceText matched table_facts and a few image fields but not typed index-unit content or the accessible table markdown / table rows, so a hit carrying the answer only in a medication-chart row / risk-matrix cell / flowchart step was invisible to contentReciprocalRankAt10. Now includes index_unit title+content and image accessibleTableMarkdown + flattened tableRows. The retrieval_synopsis stays in the evidence text: it is the stored first-class passage representation used in ranking/display, and the metadata-distractor test already guards the title/file/section vector that the earlier fix excluded. Co-Authored-By: Claude Opus 4.8 --- scripts/eval-quality.ts | 5 ++++- scripts/eval-retrieval.ts | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 52cbc50f2..69da42e5b 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -7,6 +7,7 @@ import { capturedRagCaseToGoldenCase, evaluateGoldenRetrievalCase, loadGoldenRetrievalCases, + retrievalLimitForGoldenCase, summarizeGoldenRetrievalResults, type GoldenRetrievalResult, } from "./eval-retrieval"; @@ -750,7 +751,9 @@ async function runRetrievalQualityCases(args: { searchChunksWithTelemetry({ query: testCase.query, ownerId: args.ownerId, - topK: testCase.topK, + // Fetch ≥10 ranked rows so content_mrr@10 is scored over a true top-10 in this path + // too (fixtures with topK<10 would otherwise silently degrade it to content_mrr@topK). + topK: retrievalLimitForGoldenCase(testCase), minSimilarity: 0.12, skipCache: true, }), diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 135569160..ed1dbf954 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -300,10 +300,27 @@ function resultContentEvidenceText(result: SearchResult) { .join(" "); const imageText = (result.images ?? []) .map((image) => - [image.caption, image.tableTitle, image.tableLabel, image.tableTextSnippet].filter(Boolean).join(" "), + [ + image.caption, + image.tableTitle, + image.tableLabel, + image.tableTextSnippet, + image.accessibleTableMarkdown, + (image.tableRows ?? []).flat().join(" "), + ] + .filter(Boolean) + .join(" "), ) .join(" "); - return normalized([result.retrieval_synopsis, result.content, tableFactText, imageText].filter(Boolean).join(" ")); + // First-class table/visual evidence: when a hit carries the answer in a typed index unit + // (medication-chart row, risk-matrix cell, flowchart step, …) it is not in `content`, so + // include it here — otherwise contentReciprocalRankAt10 would miss table/visual answers. + const indexUnitText = result.index_unit + ? [result.index_unit.title, result.index_unit.content].filter(Boolean).join(" ") + : ""; + return normalized( + [result.retrieval_synopsis, result.content, tableFactText, imageText, indexUnitText].filter(Boolean).join(" "), + ); } function expectedDocumentHits(expectedSubstrings: string[], results: SearchResult[], limit: number) {