From 4f03aeffe599327c0ebf39a87da8293210bca87d Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:00:12 +0800 Subject: [PATCH 1/2] test(eval): add force-embedding flag + 10 vector-exercising golden cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The golden retrieval set was 100% lexical fast-path (embedding_skipped_rate=1.0), so it could not measure whether a re-index changes vector/embedding retrieval quality. - forceEmbedding option on searchChunksWithTelemetry (SearchChunksArgs): bypasses every lexical text-fast-path so retrieval always exercises the embedding/vector stage. Diagnostic/eval-only; folded into the search cache key; never set on production paths. - eval-retrieval.ts: per-case `forceEmbedding` field + a global `--force-embedding` flag. - 10 `vector-*` cases (psychiatric monographs: PTSD, OCD, panic, anorexia, GAD, Tourette, postnatal, bipolar, ADHD, opioid) with forceEmbedding=true. Each is a clinical query that must be answered by vector retrieval of the right monograph — verified live at document_recall@5=1.0, content_recall@5=1.0, all via strategy=hybrid (embedding used). Rationale: forcing embedding is the correct instrument for re-index measurement — you want to measure the vector index directly, not have a lexical shortcut mask a regression. Wording alone can't reliably force the vector path (the fast-path is driven by emergent lexical-match strength), so the flag makes these probes deterministic. Live golden eval: 34/34 pass (24 existing + 10 new), no regression. verify:cheap green (980). Co-Authored-By: Claude Fable 5 --- scripts/eval-retrieval.ts | 11 ++ scripts/fixtures/rag-retrieval-golden.json | 194 +++++++++++++++++++++ src/lib/rag.ts | 11 +- 3 files changed, 213 insertions(+), 3 deletions(-) diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 0a4332f2f..7218ae731 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -19,6 +19,10 @@ const goldenCaseSchema = z.object({ expectedContentTerms: z.array(contentExpectationSchema).default([]), topK: z.number().int().positive().default(8), expectTableEvidence: z.boolean().default(false), + // Bypass the lexical text-fast-path so this case always exercises the embedding/vector + // index. Use for "vector-*" probes that would otherwise be answered by a lexical shortcut, + // so a re-index's effect on vector retrieval is actually measured. + forceEmbedding: z.boolean().optional(), }); const goldenCasesSchema = z.array(goldenCaseSchema); @@ -37,6 +41,7 @@ type EvalArgs = { caseTimeoutMs: number; p90BudgetMs: number; p50BudgetMs: number; + forceEmbedding: boolean; }; export type GoldenRetrievalResult = { @@ -116,6 +121,7 @@ function parseArgs(argv: string[]): EvalArgs { caseTimeoutMs: inferredMode === "latency" ? 25_000 : 0, p90BudgetMs: 20_000, p50BudgetMs: 8_000, + forceEmbedding: false, }; for (let index = 0; index < argv.length; index += 1) { @@ -138,6 +144,10 @@ function parseArgs(argv: string[]): EvalArgs { if (args.caseTimeoutMs <= 0) args.caseTimeoutMs = 25_000; continue; } + if (token === "--force-embedding") { + args.forceEmbedding = true; + continue; + } const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); @@ -743,6 +753,7 @@ async function main() { topK: testCase.topK, minSimilarity: 0.12, skipCache: args.mode !== "latency", + forceEmbedding: testCase.forceEmbedding || args.forceEmbedding, }), ); const searchOutcome = await withCaseTimeout(searchPromise, args.caseTimeoutMs); diff --git a/scripts/fixtures/rag-retrieval-golden.json b/scripts/fixtures/rag-retrieval-golden.json index b31176b76..5f3cdabb4 100644 --- a/scripts/fixtures/rag-retrieval-golden.json +++ b/scripts/fixtures/rag-retrieval-golden.json @@ -224,5 +224,199 @@ ], "topK": 12, "expectTableEvidence": true + }, + { + "id": "vector-ptsd", + "query": "How is post-traumatic stress disorder managed in a person with intrusive flashbacks, nightmares, hyperarousal and avoidance after a traumatic event?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Traumatic Stress" + ], + "expectedContentTerms": [ + [ + "trauma", + "traumatic", + "ptsd", + "flashback" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-ocd", + "query": "How is obsessive-compulsive disorder managed in a person with distressing intrusive obsessions and repetitive compulsive rituals?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Obsessive" + ], + "expectedContentTerms": [ + [ + "obsess", + "compuls", + "intrusive", + "ocd" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-panic", + "query": "How is panic disorder managed in a person with recurrent unexpected panic attacks, palpitations and anticipatory anxiety?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Panic" + ], + "expectedContentTerms": [ + [ + "panic", + "attack", + "anxiety" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-anorexia", + "query": "How is anorexia nervosa managed in a person with severe dietary restriction, intense fear of weight gain and body-image disturbance?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Anorexia" + ], + "expectedContentTerms": [ + [ + "anorexia", + "eating", + "weight" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-gad-worry", + "query": "How is a patient managed who has persistent, excessive worry about many everyday things that they find very hard to control, along with restlessness and muscle tension?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Generalised Anxiety" + ], + "expectedContentTerms": [ + [ + "worry", + "anxiety", + "generalised" + ], + [ + "cbt", + "ssri", + "snri", + "antidepressant", + "psychotherapy" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-tourette", + "query": "How is Tourette syndrome managed in a child with chronic motor tics and vocal tics?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Tourette" + ], + "expectedContentTerms": [ + [ + "tic", + "tics", + "tourette" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-postnatal", + "query": "How is postnatal (postpartum) depression managed in a new mother who develops low mood and poor bonding with her infant in the first weeks postpartum?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Postnatal" + ], + "expectedContentTerms": [ + [ + "postnatal", + "postpartum", + "perinatal", + "depression" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-bipolar", + "query": "How is bipolar disorder managed in an adult with recurrent episodes of mania and depression?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Bipolar" + ], + "expectedContentTerms": [ + [ + "bipolar", + "mania", + "manic", + "mood" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-adhd", + "query": "How is attention deficit hyperactivity disorder managed in an adult with inattention, distractibility and impulsivity?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Attention Deficit Hyperactivity" + ], + "expectedContentTerms": [ + [ + "attention", + "adhd", + "hyperactiv", + "impuls" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true + }, + { + "id": "vector-opioid", + "query": "How is opioid use disorder managed in a person dependent on heroin?", + "expectedQueryClass": "broad_summary", + "expectedDocumentSubstrings": [ + "Opioid Use Disorder" + ], + "expectedContentTerms": [ + [ + "opioid", + "heroin", + "opiate", + "methadone", + "buprenorphine" + ] + ], + "topK": 8, + "expectTableEvidence": false, + "forceEmbedding": true } ] diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 6764accd2..9f8e6f059 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -307,6 +307,10 @@ export type SearchChunksArgs = { // Internal: set when this call is a re-run on a trigram-corrected query, to prevent the // unsupported-short-circuit typo-correction path from recursing more than once. typoCorrected?: boolean; + // Diagnostic/eval-only: bypass every lexical text-fast-path so retrieval always exercises + // the embedding/vector stage. Lets the golden eval measure the vector index directly for a + // re-index, instead of being masked by lexical shortcuts. Never set on production paths. + forceEmbedding?: boolean; }; export type AnswerProgressEvent = { @@ -1455,6 +1459,7 @@ function scopedSearchCacheKey(args: SearchChunksArgs, queryClass?: RagQueryClass args.ownerId ?? "anonymous", scopeKey(args), retrievalPlanCacheQuery(args, queryClass, queryVariants), + args.forceEmbedding ? "force-embedding" : "", ].join("|"); } @@ -5516,7 +5521,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { }); const baseTextFastPath = decideTextFastPath(args.query, baseTextResults, queryClassification.queryClass); - if (shouldReturnBeforeMemory(queryClassification.queryClass, baseTextFastPath)) { + if (!args.forceEmbedding && shouldReturnBeforeMemory(queryClassification.queryClass, baseTextFastPath)) { textFastResults = await attachPageVisualEvidence(supabase, baseTextResults); textFastResults = applySecondStageRerankIfNeeded({ queryClass: queryClassification.queryClass, @@ -5567,7 +5572,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { telemetry.rerank_latency_ms += Date.now() - rerankStartedAt; const boostedTextFastPath = decideTextFastPath(args.query, textFastResults, queryClassification.queryClass); - if (boostedTextFastPath.returnFastPath) { + if (!args.forceEmbedding && boostedTextFastPath.returnFastPath) { markEmbeddingSkippedByTextFastPath(telemetry, boostedTextFastPath.reason); telemetry.retrieval_strategy = "text_fast_path"; recordSearchScoreTelemetry(telemetry, textFastResults); @@ -5673,7 +5678,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { documentLookupResults, queryClassification.queryClass, ); - if (documentLookupFastPath.returnFastPath) { + if (!args.forceEmbedding && documentLookupFastPath.returnFastPath) { markEmbeddingSkippedByTextFastPath( telemetry, documentLookupFastPath.reason ? `document_lookup_fast_path:${documentLookupFastPath.reason}` : null, From a17fa3f2e967958d3faa2a72fbc4b7086843529a Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Sun, 5 Jul 2026 19:47:36 +0800 Subject: [PATCH 2/2] test(eval): harden force-embedding flag and vector-path eval guards Wire forceEmbedding through eval runners and retrieval cache keys, bypass coverage/lexical shortcuts when forced, and add golden-case failure metrics so vector regressions cannot hide behind text-fast-path or cache hits. --- scripts/eval-quality.ts | 15 ++++++++ scripts/eval-retrieval.ts | 24 ++++++++++++ src/lib/rag.ts | 50 ++++++++++++++++++------- tests/eval-quality.test.ts | 52 +++++++++++++++++++++++++- tests/retrieval-query-variants.test.ts | 12 ++++++ 5 files changed, 139 insertions(+), 14 deletions(-) diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 52cbc50f2..92c93703b 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -44,6 +44,7 @@ type EvalQualityArgs = { retrievalOnly: boolean; ragOnly: boolean; skipPreflight: boolean; + forceEmbedding: boolean; }; export type RagQualityResult = { @@ -149,6 +150,7 @@ function parseArgs(argv: string[]): EvalQualityArgs { retrievalOnly: false, ragOnly: false, skipPreflight: false, + forceEmbedding: false, }; for (let index = 0; index < argv.length; index += 1) { @@ -175,6 +177,10 @@ function parseArgs(argv: string[]): EvalQualityArgs { args.skipPreflight = true; continue; } + if (token === "--force-embedding") { + args.forceEmbedding = true; + continue; + } const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); @@ -475,6 +481,11 @@ export function buildEvalQualityReport(args: { `retrieval content_recall_at_5 ${retrievalSummary.content_recall_at_5} below ${qualityThresholds.retrievalContentRecallAt5}`, ); } + if (retrievalSummary.force_embedding_failure_count > 0) { + thresholdFailures.push( + `retrieval force_embedding_failure_count ${retrievalSummary.force_embedding_failure_count} above 0`, + ); + } if (governance.stale_rate > qualityThresholds.staleTopResultRate) { thresholdFailures.push( `top-result stale_rate ${governance.stale_rate} above ${qualityThresholds.staleTopResultRate}`, @@ -663,6 +674,8 @@ ${markdownTable([ ## Retrieval Decision Metrics ${markdownTable([ + ["Force-embedding cases", retrieval.force_embedding_case_count], + ["Force-embedding failures", retrieval.force_embedding_failure_count], ["Embedding skipped rate", retrieval.embedding_skipped_rate], ["Median text candidate budget", retrieval.median_text_candidate_budget], ["Second-stage rerank rate", retrieval.second_stage_rerank_rate], @@ -727,6 +740,7 @@ async function runRetrievalQualityCases(args: { ownerId?: string; limit?: number; query?: string; + forceEmbedding?: boolean; supabase: Awaited>; }) { const [{ searchChunksWithTelemetry }, capturedCases] = await Promise.all([ @@ -753,6 +767,7 @@ async function runRetrievalQualityCases(args: { topK: testCase.topK, minSimilarity: 0.12, skipCache: true, + forceEmbedding: testCase.forceEmbedding || args.forceEmbedding, }), ); const latencyMs = diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 7218ae731..23194974c 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -47,6 +47,7 @@ type EvalArgs = { export type GoldenRetrievalResult = { id: string; query: string; + forceEmbedding: boolean; expectedQueryClass: string; actualQueryClass: string | null; expectedDocumentSubstrings: string[]; @@ -468,6 +469,11 @@ export function evaluateGoldenRetrievalCase(args: { const tableEvidenceFoundAtK = hasTableEvidence(args.results, topK); const actualQueryClass = args.telemetry.query_class ?? null; const failures: string[] = []; + const vectorLayerCount = Object.entries(args.telemetry.retrieval_layer_counts ?? {}).reduce( + (sum, [layer, count]) => + ["embedding_fields", "index_units", "hybrid_vector", "vector_fallback"].includes(layer) ? sum + count : sum, + 0, + ); const hitAtK = documentHitsAtK.missing.length === 0 && contentHitsAtK.missing.length === 0 && @@ -485,10 +491,23 @@ export function evaluateGoldenRetrievalCase(args: { if (args.testCase.expectTableEvidence && !tableEvidenceFound) { failures.push("expected table evidence in top 5"); } + if (args.testCase.forceEmbedding) { + if (args.telemetry.embedding_skipped) failures.push("forceEmbedding expected embedding to run"); + if (args.telemetry.retrieval_strategy === "search_cache") failures.push("forceEmbedding served search cache"); + if ( + args.telemetry.retrieval_strategy === "text_fast_path" || + args.telemetry.retrieval_strategy === "document_lookup_fast_path" + ) { + failures.push(`forceEmbedding returned lexical strategy ${args.telemetry.retrieval_strategy}`); + } + if (args.telemetry.coverage_gate_decision === "accepted") failures.push("forceEmbedding returned coverage gate"); + if (vectorLayerCount <= 0) failures.push("forceEmbedding found no vector-layer candidates"); + } return { id: args.testCase.id, query: args.testCase.query, + forceEmbedding: args.testCase.forceEmbedding ?? false, expectedQueryClass: args.testCase.expectedQueryClass, actualQueryClass, expectedDocumentSubstrings: args.testCase.expectedDocumentSubstrings, @@ -560,6 +579,7 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[] } return counts; }, {}); + const forceEmbeddingResults = results.filter((result) => result.forceEmbedding); return { case_count: results.length, document_recall_at_5: Number( @@ -588,6 +608,8 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[] embedding_skip_reason_counts: embeddingSkipReasonCounts, text_fast_path_reason_counts: textFastPathReasonCounts, retrieval_layer_counts: layerCounts, + force_embedding_case_count: forceEmbeddingResults.length, + force_embedding_failure_count: forceEmbeddingResults.filter((result) => result.failures.length > 0).length, median_text_candidate_budget: percentile(textCandidateBudgets, 50), second_stage_rerank_rate: Number( (results.filter((result) => result.secondStageRerankUsed).length / Math.max(results.length, 1)).toFixed(4), @@ -661,6 +683,8 @@ function printHumanSummary(summary: ReturnType, queryClass?: RagQueryClass, queryVariants: string[] = [], @@ -1448,6 +1455,7 @@ export function retrievalPlanCacheQuery( `mode:${modeKey(args)}`, `topK:${args.topK ?? 8}`, `min:${args.minSimilarity ?? 0.15}`, + `forceEmbedding:${args.forceEmbedding ? "1" : "0"}`, `rag:${ragDeepMemoryVersion}`, ].join("|"); return queryCacheKeyForStorage(cacheKey); @@ -1459,7 +1467,6 @@ function scopedSearchCacheKey(args: SearchChunksArgs, queryClass?: RagQueryClass args.ownerId ?? "anonymous", scopeKey(args), retrievalPlanCacheQuery(args, queryClass, queryVariants), - args.forceEmbedding ? "force-embedding" : "", ].join("|"); } @@ -1546,7 +1553,7 @@ type SharedCacheMissReason = function sharedCacheSelector( supabase: ReturnType, kind: SharedCacheKind, - args: Pick, + args: Pick, indexingVersion: string, normalizedQuery: string = queryCacheKeyForStorage(normalizedCacheQuery(`${modeKey(args)} ${args.query}`)), ) { @@ -1711,7 +1718,10 @@ async function getSharedCachedSearch( } async function getSharedCachedAnswer( - args: Pick, + args: Pick< + SearchChunksArgs, + "query" | "documentId" | "documentIds" | "ownerId" | "skipCache" | "queryMode" | "forceEmbedding" + >, startedAt: number, ) { if (args.skipCache || env.RAG_ANSWER_CACHE_TTL_MS <= 0) return null; @@ -1744,7 +1754,7 @@ async function getSharedCachedAnswer( async function replaceSharedCacheRow( kind: SharedCacheKind, - args: Pick, + args: Pick, payload: unknown, ttlMs: number, normalizedQuery: string = queryCacheKeyForStorage(normalizedCacheQuery(`${modeKey(args)} ${args.query}`)), @@ -1796,7 +1806,10 @@ function setSharedCachedSearch( } function setSharedCachedAnswer( - args: Pick, + args: Pick< + SearchChunksArgs, + "query" | "documentId" | "documentIds" | "ownerId" | "skipCache" | "queryMode" | "forceEmbedding" + >, answer: RagAnswer, ) { if (args.skipCache || env.RAG_ANSWER_CACHE_TTL_MS <= 0) return; @@ -5370,6 +5383,9 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { // When the provider is source-only (offline mode, or auto mode without a usable key) we must // never call OpenAI for embeddings; retrieval falls back to the lexical text-fast-path only. const sourceOnlyRetrieval = isSourceOnlyMode(); + if (args.forceEmbedding && sourceOnlyRetrieval) { + throw new Error("forceEmbedding requires embedding-capable retrieval; source-only mode cannot exercise vectors."); + } // A3: shared across every withMemoryBoostedCandidates call in this request so the same // owner/query memory cards are fetched at most once per (query, embedding-present, count). const memoryCardCache: MemoryCardCache = new Map(); @@ -5446,7 +5462,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { telemetry.shared_cache_miss_reason = sharedCached.reason; } - if (shouldApplyUnsupportedSearchShortCircuit(retrievalQuery, queryAnalysis, ragAliasExpansions)) { + if (!args.forceEmbedding && shouldApplyUnsupportedSearchShortCircuit(retrievalQuery, queryAnalysis, ragAliasExpansions)) { // Item 10 follow-up (RC6): a typo can make an on-topic query ("schizophrenai management") look // unsupported and short-circuit before any layer runs. Before giving up, trigram-correct the // query against the known clinical-term vocabulary; if it changes, re-run the whole retrieval @@ -5704,8 +5720,8 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { telemetry, }); const coverageGate = evaluateEvidenceCoverageGate(args.query, coverageGateResults, queryClassification.queryClass); - applyCoverageGateTelemetry(telemetry, coverageGate, coverageGate.accepted); - if (coverageGate.accepted) { + applyCoverageGateTelemetry(telemetry, coverageGate, !args.forceEmbedding && coverageGate.accepted); + if (!args.forceEmbedding && coverageGate.accepted) { telemetry.retrieval_strategy = coverageGate.strategy; recordSearchScoreTelemetry(telemetry, coverageGateResults); setCachedSearch(args, coverageGateResults, telemetry, queryVariants); @@ -5733,7 +5749,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { } catch (error) { // In auto mode a failed embedding call (e.g. quota exhausted) degrades to the lexical // results already gathered rather than failing the whole search. "openai" mode rethrows. - if (!allowsAutoDegrade()) throw error; + if (args.forceEmbedding || !allowsAutoDegrade()) throw error; telemetry.embedding_skipped = true; telemetry.embedding_skip_reason = sourceOnlyReason(error); telemetry.vector_skipped_reason = classifyProviderFailure(error); @@ -5827,10 +5843,14 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { latencyMs: hybridResult.latencyMs, topScore: layerTopScore((hybridData ?? []) as SearchResult[]), }); + const vectorCandidates = mergeSearchResults( + mergeSearchResults((hybridData ?? []) as SearchResult[], embeddingFieldCandidates), + indexUnitCandidates, + ); if (!hybridError) { const rerankStartedAt = Date.now(); - const merged = mergeSearchResults((hybridData ?? []) as SearchResult[], textFastResults); + const merged = args.forceEmbedding ? vectorCandidates : mergeSearchResults(vectorCandidates, textFastResults); const mergedWithMetadata = await attachDocumentRankingMetadata(supabase, merged, args.ownerId); const memoryBoost = await withMemoryBoostedCandidates({ supabase, @@ -5888,7 +5908,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { return (data ?? []) as SearchResult[]; }), ).catch((error) => { - if (textFastResults.length > 0) return [] as SearchResult[][]; + if (!args.forceEmbedding && textFastResults.length > 0) return [] as SearchResult[][]; throw error; }); const fallbackLatencyMs = Date.now() - fallbackRpcStartedAt; @@ -5900,9 +5920,13 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { }); const rerankStartedAt = Date.now(); + const fallbackVectorCandidates = mergeSearchResults( + mergeSearchResults(resultSets.flat(), embeddingFieldCandidates), + indexUnitCandidates, + ); const mergedWithMetadata = await attachDocumentRankingMetadata( supabase, - mergeSearchResults(resultSets.flat(), textFastResults), + args.forceEmbedding ? fallbackVectorCandidates : mergeSearchResults(fallbackVectorCandidates, textFastResults), args.ownerId, ); const memoryBoost = await withMemoryBoostedCandidates({ diff --git a/tests/eval-quality.test.ts b/tests/eval-quality.test.ts index deffec1cd..b41fd45f2 100644 --- a/tests/eval-quality.test.ts +++ b/tests/eval-quality.test.ts @@ -9,12 +9,13 @@ import { sourceWarningsForRagQualityAnswer, type RagQualityResult, } from "../scripts/eval-quality"; -import type { GoldenRetrievalResult } from "../scripts/eval-retrieval"; +import { evaluateGoldenRetrievalCase, type GoldenRetrievalResult } from "../scripts/eval-retrieval"; function retrievalResult(overrides: Partial = {}): GoldenRetrievalResult { const base: GoldenRetrievalResult = { id: "retrieval-1", query: "What ANC threshold should withhold clozapine?", + forceEmbedding: false, expectedQueryClass: "table_threshold", actualQueryClass: "table_threshold", expectedDocumentSubstrings: ["clozapine.pdf"], @@ -165,6 +166,8 @@ describe("eval quality reporting", () => { structured_threshold_text_match: 1, }); expect(report.retrieval.summary.second_stage_rerank_rate).toBe(0.5); + expect(report.retrieval.summary.force_embedding_case_count).toBe(0); + expect(report.retrieval.summary.force_embedding_failure_count).toBe(0); expect(report.rag.summary.unsupported_correct_rate).toBe(0); expect(report.rag.summary.numeric_grounding_failure_rate).toBeCloseTo(0.3333, 4); expect(report.rag.summary.source_governance_danger_failure_rate).toBeCloseTo(0.3333, 4); @@ -179,6 +182,53 @@ describe("eval quality reporting", () => { ); }); + it("fails forced-embedding retrieval cases that return from cache, coverage, or lexical paths", () => { + const result = evaluateGoldenRetrievalCase({ + testCase: { + id: "vector-regression", + query: "How is panic disorder managed?", + expectedQueryClass: "broad_summary", + expectedDocumentSubstrings: [], + expectedContentTerms: [], + topK: 8, + expectTableEvidence: false, + forceEmbedding: true, + }, + results: [], + telemetry: { + query_class: "broad_summary", + retrieval_strategy: "text_fast_path", + embedding_skipped: true, + embedding_skip_reason: "strong_document_text_score", + text_fast_path_reason: "strong_document_text_score", + text_candidate_budget: 32, + text_candidate_count: 5, + vector_candidate_count: 0, + retrieval_layer_counts: { text_candidates: 5 }, + coverage_gate_decision: "accepted", + coverage_gate_reason: "document_title_evidence_gate", + second_stage_rerank_used: false, + }, + latencyMs: 10, + }); + + expect(result.forceEmbedding).toBe(true); + expect(result.failures).toEqual( + expect.arrayContaining([ + "forceEmbedding expected embedding to run", + "forceEmbedding returned lexical strategy text_fast_path", + "forceEmbedding returned coverage gate", + "forceEmbedding found no vector-layer candidates", + ]), + ); + const report = buildEvalQualityReport({ retrievalResults: [result], ragResults: [] }); + expect(report.retrieval.summary.force_embedding_case_count).toBe(1); + expect(report.retrieval.summary.force_embedding_failure_count).toBe(1); + expect(report.blocking_threshold_failures).toEqual( + expect.arrayContaining([expect.stringContaining("retrieval force_embedding_failure_count 1 above 0")]), + ); + }); + it("derives source governance warnings for direct RAG answers without precomputed warnings", () => { const warnings = sourceWarningsForRagQualityAnswer({ sourceGovernanceWarnings: undefined, diff --git a/tests/retrieval-query-variants.test.ts b/tests/retrieval-query-variants.test.ts index ddb62411e..a2f06474e 100644 --- a/tests/retrieval-query-variants.test.ts +++ b/tests/retrieval-query-variants.test.ts @@ -127,6 +127,18 @@ describe("retrieval query variants", () => { expect(textCandidateBudgetForQueryClass("unsupported_or_general", 12)).toBe(24); }); + it("keeps forced-embedding retrieval out of the ordinary search cache key", () => { + const baseArgs = { + query: "How is panic disorder managed?", + topK: 8, + minSimilarity: 0.12, + }; + + expect(retrievalPlanCacheQuery(baseArgs, "broad_summary")).not.toBe( + retrievalPlanCacheQuery({ ...baseArgs, forceEmbedding: true }, "broad_summary"), + ); + }); + it("allows direct document title hits to skip embedding retrieval", () => { expect( decideTextFastPath(