From c4868031b2294d26a5d35b06f9ecbb4c9b55ac28 Mon Sep 17 00:00:00 2001
From: BigSimmo <87357024+BigSimmo@users.noreply.github.com>
Date: Mon, 13 Jul 2026 15:30:06 +0800
Subject: [PATCH] fix(eval): require expected-doc coverage for acceptSourceOnly
 quality pass

Addresses the Codex P2 on PR #538: scoreAnswerQualityEvalCase's acceptSourceOnly
relevance branch accepted any source-only/unsupported answer without checking
that it still surfaced the expected documents, so a retrieval regression that
stopped returning e.g. MHSP.Duress.pdf / MHSP.Discharge.pdf could hide in the
quality canary (contradicting the "expectedFiles keeps the retrieval guard"
intent). Wire the existing citesOrNamesExpectedDocument helper into the
acceptSourceOnly branch so a source-only answer must still cite or name the
expected doc. Updates the discharge test to assert both the cited (relevant)
and uncited (not relevant) cases.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 src/lib/rag-eval-cases.ts    |  7 +++++--
 tests/rag-eval-cases.test.ts | 34 +++++++++++++++++++++++++++++-----
 2 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/src/lib/rag-eval-cases.ts b/src/lib/rag-eval-cases.ts
index 88ae8224f..b29a31344 100644
--- a/src/lib/rag-eval-cases.ts
+++ b/src/lib/rag-eval-cases.ts
@@ -128,8 +128,11 @@ export function scoreAnswerQualityEvalCase(testCase: AnswerQualityEvalCase, answ
   const expectedClassOk = !testCase.expectedQueryClass || answer.queryClass === testCase.expectedQueryClass;
   const relevanceOk = testCase.supported
     ? testCase.acceptSourceOnly
-      ? // Diffuse question: a grounded synthesis OR a source-only/unsupported answer is acceptable.
-        (answer.grounded || unsupported) && expectedClassOk
+      ? // Diffuse question: a grounded synthesis OR a source-only/unsupported answer is acceptable,
+        // but it must still surface the expected documents (cite or name them). Without this a
+        // source-only answer that stopped retrieving the expected doc would still score relevant,
+        // hiding a retrieval regression in this canary.
+        (answer.grounded || unsupported) && expectedClassOk && citesOrNamesExpectedDocument(testCase, answer, text)
       : answer.grounded && answer.citations.length >= testCase.minCitations && expectedClassOk
     : unsupported;
   const readabilityOk = wordCount >= 5 && wordCount <= 220 && !fragmentPattern.test(text);
diff --git a/tests/rag-eval-cases.test.ts b/tests/rag-eval-cases.test.ts
index 708604570..37c1c4303 100644
--- a/tests/rag-eval-cases.test.ts
+++ b/tests/rag-eval-cases.test.ts
@@ -249,9 +249,34 @@ describe("captured RAG eval cases", () => {
     expect(quality?.expectedFiles).toEqual(["MHSP.Duress.pdf"]);
   });
 
-  it("scores an acceptSourceOnly case as relevant for a clean source-only answer", () => {
+  it("scores an acceptSourceOnly source-only answer as relevant only when it still cites the expected doc", () => {
     const testCase = answerQualityEvalCases.find((item) => item.id === "quality-discharge-documentation")!;
-    const sourceOnly = {
+
+    const citingExpected = {
+      answer: "The uploaded discharge documents are cited below — review them directly.",
+      grounded: false,
+      confidence: "unsupported",
+      citations: [
+        {
+          chunk_id: "discharge-1",
+          document_id: "discharge-doc",
+          title: "Discharge",
+          file_name: "MHSP.Discharge.pdf",
+          page_number: 1,
+          chunk_index: 0,
+        },
+      ],
+      sources: [],
+      routingMode: "extractive",
+      queryClass: "document_lookup",
+      answerSections: [],
+    } satisfies RagAnswer;
+    const withCite = scoreAnswerQualityEvalCase(testCase, citingExpected).find((s) => s.metric === "relevance");
+    expect(withCite?.score).toBe(1);
+
+    // A source-only answer that no longer surfaces the expected document must NOT score relevant —
+    // otherwise a retrieval regression that stops returning MHSP.Discharge.pdf would hide here.
+    const withoutCite = {
       answer: "No current indexed document directly supporting this request was found.",
       grounded: false,
       confidence: "unsupported",
@@ -261,9 +286,8 @@ describe("captured RAG eval cases", () => {
       queryClass: "document_lookup",
       answerSections: [],
     } satisfies RagAnswer;
-
-    const relevance = scoreAnswerQualityEvalCase(testCase, sourceOnly).find((score) => score.metric === "relevance");
-    expect(relevance?.score).toBe(1);
+    const noCite = scoreAnswerQualityEvalCase(testCase, withoutCite).find((s) => s.metric === "relevance");
+    expect(noCite?.score).toBe(0);
   });
 
   it("scores answer quality for relevance, readability, artifacts, intent coverage, and fail-closed behavior", () => {