diff --git a/src/lib/rag-eval-cases.ts b/src/lib/rag-eval-cases.ts index b29a31344..a6d028a8c 100644 --- a/src/lib/rag-eval-cases.ts +++ b/src/lib/rag-eval-cases.ts @@ -129,10 +129,14 @@ export function scoreAnswerQualityEvalCase(testCase: AnswerQualityEvalCase, answ const relevanceOk = testCase.supported ? testCase.acceptSourceOnly ? // Diffuse question: a grounded synthesis OR a source-only/unsupported answer is acceptable, - // but it must still surface the expected documents (cite or name them). Without this a - // source-only answer that stopped retrieving the expected doc would still score relevant, - // hiding a retrieval regression in this canary. - (answer.grounded || unsupported) && expectedClassOk && citesOrNamesExpectedDocument(testCase, answer, text) + // but it must still CITE the expected documents. A prose mention is not enough — the doc-name + // alternatives include bare topic tokens (e.g. "duress"), so a gap answer that merely names + // the topic would slip through. Require citation coverage, otherwise a source-only answer + // that stopped retrieving the expected doc would still score relevant and hide a retrieval + // regression in this canary. + (answer.grounded || unsupported) && + expectedClassOk && + expectedFileCoverage(testCase.expectedFiles, answer.citations, answer.citations.length).anyHit : answer.grounded && answer.citations.length >= testCase.minCitations && expectedClassOk : unsupported; const readabilityOk = wordCount >= 5 && wordCount <= 220 && !fragmentPattern.test(text); diff --git a/tests/rag-eval-cases.test.ts b/tests/rag-eval-cases.test.ts index 37c1c4303..7f3358d29 100644 --- a/tests/rag-eval-cases.test.ts +++ b/tests/rag-eval-cases.test.ts @@ -276,8 +276,10 @@ describe("captured RAG eval cases", () => { // A source-only answer that no longer surfaces the expected document must NOT score relevant — // otherwise a retrieval regression that stops returning MHSP.Discharge.pdf would hide here. + // A prose mention of the topic ("discharge") must NOT rescue an uncited answer — coverage is + // citation-based, not answer-text based (the doc-name alternatives include bare topic tokens). const withoutCite = { - answer: "No current indexed document directly supporting this request was found.", + answer: "No current source with discharge documentation guidance was found.", grounded: false, confidence: "unsupported", citations: [],