From f7fb001c5ceb98874f45b4c8dad91e1b7e9c3f38 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:39:40 +0800 Subject: [PATCH] fix(eval): require citation coverage (not prose) for acceptSourceOnly relevance Follow-up to #562 (Codex P2). The acceptSourceOnly relevance guard used citesOrNamesExpectedDocument, whose answer-text fallback matches bare topic tokens (documentExpectationAlternatives("MHSP.Duress.pdf") includes "duress"), so an uncited gap answer that merely mentions the topic still scored relevant, defeating the retrieval-regression guard. Require citation coverage via expectedFileCoverage(...).anyHit instead. Strengthens the test so a topic mention without a citation scores 0. Co-Authored-By: Claude Opus 4.8 --- src/lib/rag-eval-cases.ts | 12 ++++++++---- tests/rag-eval-cases.test.ts | 4 +++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/lib/rag-eval-cases.ts b/src/lib/rag-eval-cases.ts index b29a31344..a6d028a8c 100644 --- a/src/lib/rag-eval-cases.ts +++ b/src/lib/rag-eval-cases.ts @@ -129,10 +129,14 @@ export function scoreAnswerQualityEvalCase(testCase: AnswerQualityEvalCase, answ const relevanceOk = testCase.supported ? testCase.acceptSourceOnly ? // Diffuse question: a grounded synthesis OR a source-only/unsupported answer is acceptable, - // but it must still surface the expected documents (cite or name them). Without this a - // source-only answer that stopped retrieving the expected doc would still score relevant, - // hiding a retrieval regression in this canary. - (answer.grounded || unsupported) && expectedClassOk && citesOrNamesExpectedDocument(testCase, answer, text) + // but it must still CITE the expected documents. A prose mention is not enough — the doc-name + // alternatives include bare topic tokens (e.g. "duress"), so a gap answer that merely names + // the topic would slip through. Require citation coverage, otherwise a source-only answer + // that stopped retrieving the expected doc would still score relevant and hide a retrieval + // regression in this canary. + (answer.grounded || unsupported) && + expectedClassOk && + expectedFileCoverage(testCase.expectedFiles, answer.citations, answer.citations.length).anyHit : answer.grounded && answer.citations.length >= testCase.minCitations && expectedClassOk : unsupported; const readabilityOk = wordCount >= 5 && wordCount <= 220 && !fragmentPattern.test(text); diff --git a/tests/rag-eval-cases.test.ts b/tests/rag-eval-cases.test.ts index 37c1c4303..7f3358d29 100644 --- a/tests/rag-eval-cases.test.ts +++ b/tests/rag-eval-cases.test.ts @@ -276,8 +276,10 @@ describe("captured RAG eval cases", () => { // A source-only answer that no longer surfaces the expected document must NOT score relevant — // otherwise a retrieval regression that stops returning MHSP.Discharge.pdf would hide here. + // A prose mention of the topic ("discharge") must NOT rescue an uncited answer — coverage is + // citation-based, not answer-text based (the doc-name alternatives include bare topic tokens). const withoutCite = { - answer: "No current indexed document directly supporting this request was found.", + answer: "No current source with discharge documentation guidance was found.", grounded: false, confidence: "unsupported", citations: [],