eval(retrieval): add content_mrr@10 passage-rank metric

scripts/eval-quality.ts

@@ -7,6 +7,7 @@ import {
   capturedRagCaseToGoldenCase,
   evaluateGoldenRetrievalCase,
   loadGoldenRetrievalCases,
+  retrievalLimitForGoldenCase,
   summarizeGoldenRetrievalResults,
   type GoldenRetrievalResult,
 } from "./eval-retrieval";
@@ -750,7 +751,9 @@ async function runRetrievalQualityCases(args: {
       searchChunksWithTelemetry({
         query: testCase.query,
         ownerId: args.ownerId,
-        topK: testCase.topK,
+        // Fetch ≥10 ranked rows so content_mrr@10 is scored over a true top-10 in this path
+        // too (fixtures with topK<10 would otherwise silently degrade it to content_mrr@topK).
+        topK: retrievalLimitForGoldenCase(testCase),
         minSimilarity: 0.12,
         skipCache: true,
       }),

scripts/eval-retrieval.ts

@@ -53,6 +53,7 @@ export type GoldenRetrievalResult = {
   hitAtK: boolean;
   topK: number;
   reciprocalRankAt10: number;
+  contentReciprocalRankAt10: number;
   latencyMs: number;
   retrievalStrategy: string | null;
   retrievalPlan: string | null;
@@ -276,6 +277,20 @@ function resultDocumentEvidenceText(result: SearchResult) {
 }
 
 function resultContentText(result: SearchResult) {
+  return normalized(
+    [
+      result.title,
+      result.file_name,
+      result.section_heading,
+      result.section_path?.join(" "),
+      resultContentEvidenceText(result),
+    ]
+      .filter(Boolean)
+      .join(" "),
+  );
+}
+
+function resultContentEvidenceText(result: SearchResult) {
   const tableFactText = (result.table_facts ?? [])
     .map((fact) =>
       [fact.table_title, fact.row_label, fact.clinical_parameter, fact.threshold_value, fact.action]
@@ -285,22 +300,26 @@ function resultContentText(result: SearchResult) {
     .join(" ");
   const imageText = (result.images ?? [])
     .map((image) =>
-      [image.caption, image.tableTitle, image.tableLabel, image.tableTextSnippet].filter(Boolean).join(" "),
+      [
+        image.caption,
+        image.tableTitle,
+        image.tableLabel,
+        image.tableTextSnippet,
+        image.accessibleTableMarkdown,
+        (image.tableRows ?? []).flat().join(" "),
+      ]
+        .filter(Boolean)
+        .join(" "),
     )
     .join(" ");
+  // First-class table/visual evidence: when a hit carries the answer in a typed index unit
+  // (medication-chart row, risk-matrix cell, flowchart step, …) it is not in `content`, so
+  // include it here — otherwise contentReciprocalRankAt10 would miss table/visual answers.
+  const indexUnitText = result.index_unit
+    ? [result.index_unit.title, result.index_unit.content].filter(Boolean).join(" ")
+    : "";
   return normalized(
-    [
-      result.title,
-      result.file_name,
-      result.section_heading,
-      result.section_path?.join(" "),
-      result.retrieval_synopsis,
-      result.content,
-      tableFactText,
-      imageText,
-    ]
-      .filter(Boolean)
-      .join(" "),
+    [result.retrieval_synopsis, result.content, tableFactText, imageText, indexUnitText].filter(Boolean).join(" "),
   );
 }
 
@@ -344,6 +363,35 @@ function reciprocalRankAt10(expectedSubstrings: string[], results: SearchResult[
   return index >= 0 ? 1 / (index + 1) : 0;
 }
 
+/**
+ * Passage-level rank quality ("best-passage rank"): the mean over expected content terms of
+ * the reciprocal rank of the earliest top-10 result whose content carries that term. Where
+ * `contentRecallAt5` only asks *whether* an answer-bearing passage appears in the top 5, this
+ * asks *how high* it ranks — so a rerank/chunking change that lifts the right passage from #5
+ * to #1 is measurable (content recall stays 1.0 across that move; doc-level `mrr@10` is blind
+ * to passage order within a document). 1.0 iff every expected term rides the rank-1 passage.
+ * Cases with no expected content terms return 0 and are excluded from the summary average.
+ */
+function contentReciprocalRankAt10(
+  expectedTerms: GoldenRetrievalCase["expectedContentTerms"],
+  results: SearchResult[],
+) {
+  if (expectedTerms.length === 0) return 0;
+  const top = results.slice(0, 10);
+  const total = expectedTerms.reduce((sum, expectation) => {
+    const alternatives = contentExpectationAlternatives(expectation);
+    const index = top.findIndex((result) =>
+      alternatives.some((term) => textContainsClinicalTerm(resultContentEvidenceText(result), term)),
+    );
+    return sum + (index >= 0 ? 1 / (index + 1) : 0);
+  }, 0);
+  return total / expectedTerms.length;
+}
+
+export function retrievalLimitForGoldenCase(testCase: GoldenRetrievalCase) {
+  return Math.max(testCase.topK, 10);
+}
+
 function hasTableEvidence(results: SearchResult[], limit = 5) {
   return results.slice(0, limit).some((result) => {
     if ((result.table_facts?.length ?? 0) > 0) return true;
@@ -490,6 +538,7 @@ export function evaluateGoldenRetrievalCase(args: {
     hitAtK,
     topK,
     reciprocalRankAt10: reciprocalRankAt10(args.testCase.expectedDocumentSubstrings, args.results),
+    contentReciprocalRankAt10: contentReciprocalRankAt10(args.testCase.expectedContentTerms, args.results),
     latencyMs: args.latencyMs,
     retrievalStrategy: args.telemetry.retrieval_strategy ?? null,
     retrievalPlan: args.telemetry.retrieval_plan ?? null,
@@ -520,6 +569,9 @@ export function evaluateGoldenRetrievalCase(args: {
 export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[]) {
   const documentRecallDenominator = Math.max(results.length, 1);
   const contentRecallDenominator = Math.max(results.length, 1);
+  // Passage-rank quality is only meaningful for cases that declare answer-bearing content
+  // terms; averaging over cases without them would dilute the signal with structural zeros.
+  const contentRankCases = results.filter((result) => result.expectedContentTerms.length > 0);
   const strategyCounts = results.reduce<Record<string, number>>((counts, result) => {
     const strategy = result.retrievalStrategy ?? "none";
     counts[strategy] = (counts[strategy] ?? 0) + 1;
@@ -562,6 +614,13 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[]
     mrr_at_10: Number(
       (results.reduce((sum, result) => sum + result.reciprocalRankAt10, 0) / Math.max(results.length, 1)).toFixed(4),
     ),
+    content_mrr_at_10: Number(
+      (
+        contentRankCases.reduce((sum, result) => sum + result.contentReciprocalRankAt10, 0) /
+        Math.max(contentRankCases.length, 1)
+      ).toFixed(4),
+    ),
+    content_mrr_case_count: contentRankCases.length,
     median_latency_ms: percentile(
       results.map((result) => result.latencyMs),
       50,
@@ -646,6 +705,9 @@ function printHumanSummary(summary: ReturnType<typeof summarizeGoldenRetrievalRe
   console.log(`  content_recall@5=${summary.content_recall_at_5}`);
   console.log(`  top_k_hit_rate=${summary.top_k_hit_rate}`);
   console.log(`  mrr@10=${summary.mrr_at_10}`);
+  console.log(
+    `  content_mrr@10=${summary.content_mrr_at_10} (over ${summary.content_mrr_case_count} content-term case(s))`,
+  );
   console.log(`  median_latency_ms=${summary.median_latency_ms}`);
   console.log(`  p90_latency_ms=${summary.p90_latency_ms}`);
   console.log(`  retrieval_strategy_counts=${JSON.stringify(summary.retrieval_strategy_counts)}`);
@@ -740,7 +802,7 @@ async function main() {
       searchChunksWithTelemetry({
         query: testCase.query,
         ownerId,
-        topK: testCase.topK,
+        topK: retrievalLimitForGoldenCase(testCase),
         minSimilarity: 0.12,
         skipCache: args.mode !== "latency",
       }),
@@ -787,7 +849,7 @@ async function main() {
             ? "FAIL"
             : "PASS";
       console.log(
-        `${status} ${result.id} hit@${result.topK}=${result.hitAtK ? "1" : "0"} docRecall@5=${result.documentRecallAt5.toFixed(2)} contentRecall@5=${result.contentRecallAt5.toFixed(2)} rr@10=${result.reciprocalRankAt10.toFixed(2)} latency=${result.latencyMs}ms strategy=${result.retrievalStrategy ?? "none"} gate=${result.coverageGateReason ?? "none"} layers=${JSON.stringify(result.retrievalLayerCounts ?? {})}`,
+        `${status} ${result.id} hit@${result.topK}=${result.hitAtK ? "1" : "0"} docRecall@5=${result.documentRecallAt5.toFixed(2)} contentRecall@5=${result.contentRecallAt5.toFixed(2)} rr@10=${result.reciprocalRankAt10.toFixed(2)} contentRR@10=${result.contentReciprocalRankAt10.toFixed(2)} latency=${result.latencyMs}ms strategy=${result.retrievalStrategy ?? "none"} gate=${result.coverageGateReason ?? "none"} layers=${JSON.stringify(result.retrievalLayerCounts ?? {})}`,
       );
     }
   }

tests/eval-quality.test.ts

@@ -26,6 +26,7 @@ function retrievalResult(overrides: Partial<GoldenRetrievalResult> = {}): Golden
     hitAtK: true,
     topK: 8,
     reciprocalRankAt10: 1,
+    contentReciprocalRankAt10: 1,
     latencyMs: 120,
     retrievalStrategy: "hybrid",
     retrievalPlan: "table_threshold:table_facts_visual_units_then_chunks",

tests/eval-retrieval.test.ts

@@ -3,6 +3,7 @@ import {
   capturedRagCaseToGoldenCase,
   evaluateGoldenRetrievalCase,
   loadGoldenRetrievalCases,
+  retrievalLimitForGoldenCase,
   summarizeGoldenRetrievalResults,
 } from "../scripts/eval-retrieval";
 import type { SearchResult } from "../src/lib/types";
@@ -55,9 +56,124 @@ describe("golden retrieval eval helpers", () => {
     expect(evaluated.documentRecallAt5).toBe(1);
     expect(evaluated.contentRecallAt5).toBe(1);
     expect(evaluated.reciprocalRankAt10).toBe(1);
+    expect(evaluated.contentReciprocalRankAt10).toBe(1);
     expect(evaluated.failures).toEqual([]);
   });
 
+  it("content_mrr@10 penalises answer passages that rank below distractors even when recall is perfect", () => {
+    const testCase = {
+      id: "content-rank",
+      query: "What ANC threshold should withhold clozapine?",
+      expectedQueryClass: "table_threshold" as const,
+      expectedDocumentSubstrings: ["ClozapinePresAdminMonitor"],
+      expectedContentTerms: ["anc", "withhold"],
+      topK: 8,
+      expectTableEvidence: false,
+    };
+    // A distractor with no matching content terms sits above the real answer passage.
+    const distractor = result({
+      id: "chunk-distractor",
+      title: "Unrelated Ward Policy",
+      file_name: "CG.MHSP.ClozapinePresAdminMonitor.pdf",
+      section_heading: "General notes",
+      content: "This section covers unrelated logistics and has no matching clinical terms.",
+      retrieval_synopsis: undefined,
+    });
+    const topRanked = evaluateGoldenRetrievalCase({
+      testCase,
+      results: [result(), distractor],
+      telemetry: { query_class: "table_threshold", retrieval_strategy: "hybrid" },
+      latencyMs: 100,
+    });
+    const demoted = evaluateGoldenRetrievalCase({
+      testCase,
+      results: [distractor, result()],
+      telemetry: { query_class: "table_threshold", retrieval_strategy: "hybrid" },
+      latencyMs: 100,
+    });
+
+    // Content recall is blind to ordering — both keep the answer passage in the top 5.
+    expect(topRanked.contentRecallAt5).toBe(1);
+    expect(demoted.contentRecallAt5).toBe(1);
+    // Passage rank is not: demoting the answer passage to rank 2 halves each term's RR.
+    expect(topRanked.contentReciprocalRankAt10).toBe(1);
+    expect(demoted.contentReciprocalRankAt10).toBeCloseTo(0.5, 5);
+
+    // The summary averages content_mrr only over cases that declare content terms.
+    const noContentCase = evaluateGoldenRetrievalCase({
+      testCase: { ...testCase, id: "no-content", expectedContentTerms: [] },
+      results: [result()],
+      telemetry: { query_class: "table_threshold", retrieval_strategy: "hybrid" },
+      latencyMs: 100,
+    });
+    const summary = summarizeGoldenRetrievalResults([demoted, noContentCase]);
+    expect(summary.content_mrr_case_count).toBe(1);
+    expect(summary.content_mrr_at_10).toBeCloseTo(0.5, 4);
+  });
+
+  it("content_mrr@10 ignores document metadata when ranking answer-bearing passages", () => {
+    const testCase = {
+      id: "metadata-distractor",
+      query: "What is the patient property safety plan?",
+      expectedQueryClass: "document_lookup" as const,
+      expectedDocumentSubstrings: ["Patient Property Safety Plan"],
+      expectedContentTerms: ["patient", "property"],
+      topK: 8,
+      expectTableEvidence: false,
+    };
+    const metadataOnlyDistractor = result({
+      id: "metadata-only",
+      title: "Patient Property Safety Plan",
+      file_name: "PatientPropertySafetyPlan.pdf",
+      section_heading: "Patient property",
+      content: "This unrelated passage describes roster logistics without the expected evidence.",
+      retrieval_synopsis: undefined,
+    });
+    const answerPassage = result({
+      id: "answer-passage",
+      title: "Other Document",
+      file_name: "Other.pdf",
+      section_heading: "Unrelated heading",
+      content: "The patient property process requires documenting valuables in the safety plan.",
+      retrieval_synopsis: undefined,
+    });
+
+    const evaluated = evaluateGoldenRetrievalCase({
+      testCase,
+      results: [metadataOnlyDistractor, answerPassage],
+      telemetry: { query_class: "document_lookup", retrieval_strategy: "hybrid" },
+      latencyMs: 100,
+    });
+
+    expect(evaluated.contentRecallAt5).toBe(1);
+    expect(evaluated.contentReciprocalRankAt10).toBeCloseTo(0.5, 5);
+  });
+
+  it("fetches enough results to score content_mrr@10 even when hit@K uses a smaller topK", () => {
+    expect(
+      retrievalLimitForGoldenCase({
+        id: "top-k-8",
+        query: "What ANC threshold should withhold clozapine?",
+        expectedQueryClass: "table_threshold",
+        expectedDocumentSubstrings: [],
+        expectedContentTerms: ["anc"],
+        topK: 8,
+        expectTableEvidence: false,
+      }),
+    ).toBe(10);
+    expect(
+      retrievalLimitForGoldenCase({
+        id: "top-k-12",
+        query: "What ANC threshold should withhold clozapine?",
+        expectedQueryClass: "table_threshold",
+        expectedDocumentSubstrings: [],
+        expectedContentTerms: ["anc"],
+        topK: 12,
+        expectTableEvidence: false,
+      }),
+    ).toBe(12);
+  });
+
   it("matches legacy compact expected document names to current corpus titles", () => {
     const evaluated = evaluateGoldenRetrievalCase({
       testCase: {