fix(eval): require expected-doc coverage for acceptSourceOnly quality pass

src/lib/rag-eval-cases.ts

@@ -128,8 +128,11 @@ export function scoreAnswerQualityEvalCase(testCase: AnswerQualityEvalCase, answ
   const expectedClassOk = !testCase.expectedQueryClass || answer.queryClass === testCase.expectedQueryClass;
   const relevanceOk = testCase.supported
     ? testCase.acceptSourceOnly
-      ? // Diffuse question: a grounded synthesis OR a source-only/unsupported answer is acceptable.
-        (answer.grounded || unsupported) && expectedClassOk
+      ? // Diffuse question: a grounded synthesis OR a source-only/unsupported answer is acceptable,
+        // but it must still surface the expected documents (cite or name them). Without this a
+        // source-only answer that stopped retrieving the expected doc would still score relevant,
+        // hiding a retrieval regression in this canary.
+        (answer.grounded || unsupported) && expectedClassOk && citesOrNamesExpectedDocument(testCase, answer, text)
       : answer.grounded && answer.citations.length >= testCase.minCitations && expectedClassOk
     : unsupported;
   const readabilityOk = wordCount >= 5 && wordCount <= 220 && !fragmentPattern.test(text);

tests/rag-eval-cases.test.ts

@@ -249,9 +249,34 @@ describe("captured RAG eval cases", () => {
     expect(quality?.expectedFiles).toEqual(["MHSP.Duress.pdf"]);
   });
 
-  it("scores an acceptSourceOnly case as relevant for a clean source-only answer", () => {
+  it("scores an acceptSourceOnly source-only answer as relevant only when it still cites the expected doc", () => {
     const testCase = answerQualityEvalCases.find((item) => item.id === "quality-discharge-documentation")!;
-    const sourceOnly = {
+
+    const citingExpected = {
+      answer: "The uploaded discharge documents are cited below — review them directly.",
+      grounded: false,
+      confidence: "unsupported",
+      citations: [
+        {
+          chunk_id: "discharge-1",
+          document_id: "discharge-doc",
+          title: "Discharge",
+          file_name: "MHSP.Discharge.pdf",
+          page_number: 1,
+          chunk_index: 0,
+        },
+      ],
+      sources: [],
+      routingMode: "extractive",
+      queryClass: "document_lookup",
+      answerSections: [],
+    } satisfies RagAnswer;
+    const withCite = scoreAnswerQualityEvalCase(testCase, citingExpected).find((s) => s.metric === "relevance");
+    expect(withCite?.score).toBe(1);
+
+    // A source-only answer that no longer surfaces the expected document must NOT score relevant —
+    // otherwise a retrieval regression that stops returning MHSP.Discharge.pdf would hide here.
+    const withoutCite = {
       answer: "No current indexed document directly supporting this request was found.",
       grounded: false,
       confidence: "unsupported",
@@ -261,9 +286,8 @@ describe("captured RAG eval cases", () => {
       queryClass: "document_lookup",
       answerSections: [],
     } satisfies RagAnswer;
-
-    const relevance = scoreAnswerQualityEvalCase(testCase, sourceOnly).find((score) => score.metric === "relevance");
-    expect(relevance?.score).toBe(1);
+    const noCite = scoreAnswerQualityEvalCase(testCase, withoutCite).find((s) => s.metric === "relevance");
+    expect(noCite?.score).toBe(0);
   });
 
   it("scores answer quality for relevance, readability, artifacts, intent coverage, and fail-closed behavior", () => {