diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index 52cbc50f2..92c93703b 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -44,6 +44,7 @@ type EvalQualityArgs = { retrievalOnly: boolean; ragOnly: boolean; skipPreflight: boolean; + forceEmbedding: boolean; }; export type RagQualityResult = { @@ -149,6 +150,7 @@ function parseArgs(argv: string[]): EvalQualityArgs { retrievalOnly: false, ragOnly: false, skipPreflight: false, + forceEmbedding: false, }; for (let index = 0; index < argv.length; index += 1) { @@ -175,6 +177,10 @@ function parseArgs(argv: string[]): EvalQualityArgs { args.skipPreflight = true; continue; } + if (token === "--force-embedding") { + args.forceEmbedding = true; + continue; + } const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); @@ -475,6 +481,11 @@ export function buildEvalQualityReport(args: { `retrieval content_recall_at_5 ${retrievalSummary.content_recall_at_5} below ${qualityThresholds.retrievalContentRecallAt5}`, ); } + if (retrievalSummary.force_embedding_failure_count > 0) { + thresholdFailures.push( + `retrieval force_embedding_failure_count ${retrievalSummary.force_embedding_failure_count} above 0`, + ); + } if (governance.stale_rate > qualityThresholds.staleTopResultRate) { thresholdFailures.push( `top-result stale_rate ${governance.stale_rate} above ${qualityThresholds.staleTopResultRate}`, @@ -663,6 +674,8 @@ ${markdownTable([ ## Retrieval Decision Metrics ${markdownTable([ + ["Force-embedding cases", retrieval.force_embedding_case_count], + ["Force-embedding failures", retrieval.force_embedding_failure_count], ["Embedding skipped rate", retrieval.embedding_skipped_rate], ["Median text candidate budget", retrieval.median_text_candidate_budget], ["Second-stage rerank rate", retrieval.second_stage_rerank_rate], @@ -727,6 +740,7 @@ async function runRetrievalQualityCases(args: { ownerId?: string; limit?: number; query?: string; + forceEmbedding?: boolean; supabase: Awaited>; }) { const [{ searchChunksWithTelemetry }, capturedCases] = await Promise.all([ @@ -753,6 +767,7 @@ async function runRetrievalQualityCases(args: { topK: testCase.topK, minSimilarity: 0.12, skipCache: true, + forceEmbedding: testCase.forceEmbedding || args.forceEmbedding, }), ); const latencyMs = diff --git a/scripts/eval-retrieval.ts b/scripts/eval-retrieval.ts index 0a4332f2f..23194974c 100644 --- a/scripts/eval-retrieval.ts +++ b/scripts/eval-retrieval.ts @@ -19,6 +19,10 @@ const goldenCaseSchema = z.object({ expectedContentTerms: z.array(contentExpectationSchema).default([]), topK: z.number().int().positive().default(8), expectTableEvidence: z.boolean().default(false), + // Bypass the lexical text-fast-path so this case always exercises the embedding/vector + // index. Use for "vector-*" probes that would otherwise be answered by a lexical shortcut, + // so a re-index's effect on vector retrieval is actually measured. + forceEmbedding: z.boolean().optional(), }); const goldenCasesSchema = z.array(goldenCaseSchema); @@ -37,11 +41,13 @@ type EvalArgs = { caseTimeoutMs: number; p90BudgetMs: number; p50BudgetMs: number; + forceEmbedding: boolean; }; export type GoldenRetrievalResult = { id: string; query: string; + forceEmbedding: boolean; expectedQueryClass: string; actualQueryClass: string | null; expectedDocumentSubstrings: string[]; @@ -116,6 +122,7 @@ function parseArgs(argv: string[]): EvalArgs { caseTimeoutMs: inferredMode === "latency" ? 25_000 : 0, p90BudgetMs: 20_000, p50BudgetMs: 8_000, + forceEmbedding: false, }; for (let index = 0; index < argv.length; index += 1) { @@ -138,6 +145,10 @@ function parseArgs(argv: string[]): EvalArgs { if (args.caseTimeoutMs <= 0) args.caseTimeoutMs = 25_000; continue; } + if (token === "--force-embedding") { + args.forceEmbedding = true; + continue; + } const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`); @@ -458,6 +469,11 @@ export function evaluateGoldenRetrievalCase(args: { const tableEvidenceFoundAtK = hasTableEvidence(args.results, topK); const actualQueryClass = args.telemetry.query_class ?? null; const failures: string[] = []; + const vectorLayerCount = Object.entries(args.telemetry.retrieval_layer_counts ?? {}).reduce( + (sum, [layer, count]) => + ["embedding_fields", "index_units", "hybrid_vector", "vector_fallback"].includes(layer) ? sum + count : sum, + 0, + ); const hitAtK = documentHitsAtK.missing.length === 0 && contentHitsAtK.missing.length === 0 && @@ -475,10 +491,23 @@ export function evaluateGoldenRetrievalCase(args: { if (args.testCase.expectTableEvidence && !tableEvidenceFound) { failures.push("expected table evidence in top 5"); } + if (args.testCase.forceEmbedding) { + if (args.telemetry.embedding_skipped) failures.push("forceEmbedding expected embedding to run"); + if (args.telemetry.retrieval_strategy === "search_cache") failures.push("forceEmbedding served search cache"); + if ( + args.telemetry.retrieval_strategy === "text_fast_path" || + args.telemetry.retrieval_strategy === "document_lookup_fast_path" + ) { + failures.push(`forceEmbedding returned lexical strategy ${args.telemetry.retrieval_strategy}`); + } + if (args.telemetry.coverage_gate_decision === "accepted") failures.push("forceEmbedding returned coverage gate"); + if (vectorLayerCount <= 0) failures.push("forceEmbedding found no vector-layer candidates"); + } return { id: args.testCase.id, query: args.testCase.query, + forceEmbedding: args.testCase.forceEmbedding ?? false, expectedQueryClass: args.testCase.expectedQueryClass, actualQueryClass, expectedDocumentSubstrings: args.testCase.expectedDocumentSubstrings, @@ -550,6 +579,7 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[] } return counts; }, {}); + const forceEmbeddingResults = results.filter((result) => result.forceEmbedding); return { case_count: results.length, document_recall_at_5: Number( @@ -578,6 +608,8 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[] embedding_skip_reason_counts: embeddingSkipReasonCounts, text_fast_path_reason_counts: textFastPathReasonCounts, retrieval_layer_counts: layerCounts, + force_embedding_case_count: forceEmbeddingResults.length, + force_embedding_failure_count: forceEmbeddingResults.filter((result) => result.failures.length > 0).length, median_text_candidate_budget: percentile(textCandidateBudgets, 50), second_stage_rerank_rate: Number( (results.filter((result) => result.secondStageRerankUsed).length / Math.max(results.length, 1)).toFixed(4), @@ -651,6 +683,8 @@ function printHumanSummary(summary: ReturnType, queryClass?: RagQueryClass, queryVariants: string[] = [], @@ -1444,6 +1455,7 @@ export function retrievalPlanCacheQuery( `mode:${modeKey(args)}`, `topK:${args.topK ?? 8}`, `min:${args.minSimilarity ?? 0.15}`, + `forceEmbedding:${args.forceEmbedding ? "1" : "0"}`, `rag:${ragDeepMemoryVersion}`, ].join("|"); return queryCacheKeyForStorage(cacheKey); @@ -1541,7 +1553,7 @@ type SharedCacheMissReason = function sharedCacheSelector( supabase: ReturnType, kind: SharedCacheKind, - args: Pick, + args: Pick, indexingVersion: string, normalizedQuery: string = queryCacheKeyForStorage(normalizedCacheQuery(`${modeKey(args)} ${args.query}`)), ) { @@ -1706,7 +1718,10 @@ async function getSharedCachedSearch( } async function getSharedCachedAnswer( - args: Pick, + args: Pick< + SearchChunksArgs, + "query" | "documentId" | "documentIds" | "ownerId" | "skipCache" | "queryMode" | "forceEmbedding" + >, startedAt: number, ) { if (args.skipCache || env.RAG_ANSWER_CACHE_TTL_MS <= 0) return null; @@ -1739,7 +1754,7 @@ async function getSharedCachedAnswer( async function replaceSharedCacheRow( kind: SharedCacheKind, - args: Pick, + args: Pick, payload: unknown, ttlMs: number, normalizedQuery: string = queryCacheKeyForStorage(normalizedCacheQuery(`${modeKey(args)} ${args.query}`)), @@ -1791,7 +1806,10 @@ function setSharedCachedSearch( } function setSharedCachedAnswer( - args: Pick, + args: Pick< + SearchChunksArgs, + "query" | "documentId" | "documentIds" | "ownerId" | "skipCache" | "queryMode" | "forceEmbedding" + >, answer: RagAnswer, ) { if (args.skipCache || env.RAG_ANSWER_CACHE_TTL_MS <= 0) return; @@ -5365,6 +5383,9 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { // When the provider is source-only (offline mode, or auto mode without a usable key) we must // never call OpenAI for embeddings; retrieval falls back to the lexical text-fast-path only. const sourceOnlyRetrieval = isSourceOnlyMode(); + if (args.forceEmbedding && sourceOnlyRetrieval) { + throw new Error("forceEmbedding requires embedding-capable retrieval; source-only mode cannot exercise vectors."); + } // A3: shared across every withMemoryBoostedCandidates call in this request so the same // owner/query memory cards are fetched at most once per (query, embedding-present, count). const memoryCardCache: MemoryCardCache = new Map(); @@ -5441,7 +5462,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { telemetry.shared_cache_miss_reason = sharedCached.reason; } - if (shouldApplyUnsupportedSearchShortCircuit(retrievalQuery, queryAnalysis, ragAliasExpansions)) { + if (!args.forceEmbedding && shouldApplyUnsupportedSearchShortCircuit(retrievalQuery, queryAnalysis, ragAliasExpansions)) { // Item 10 follow-up (RC6): a typo can make an on-topic query ("schizophrenai management") look // unsupported and short-circuit before any layer runs. Before giving up, trigram-correct the // query against the known clinical-term vocabulary; if it changes, re-run the whole retrieval @@ -5516,7 +5537,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { }); const baseTextFastPath = decideTextFastPath(args.query, baseTextResults, queryClassification.queryClass); - if (shouldReturnBeforeMemory(queryClassification.queryClass, baseTextFastPath)) { + if (!args.forceEmbedding && shouldReturnBeforeMemory(queryClassification.queryClass, baseTextFastPath)) { textFastResults = await attachPageVisualEvidence(supabase, baseTextResults); textFastResults = applySecondStageRerankIfNeeded({ queryClass: queryClassification.queryClass, @@ -5567,7 +5588,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { telemetry.rerank_latency_ms += Date.now() - rerankStartedAt; const boostedTextFastPath = decideTextFastPath(args.query, textFastResults, queryClassification.queryClass); - if (boostedTextFastPath.returnFastPath) { + if (!args.forceEmbedding && boostedTextFastPath.returnFastPath) { markEmbeddingSkippedByTextFastPath(telemetry, boostedTextFastPath.reason); telemetry.retrieval_strategy = "text_fast_path"; recordSearchScoreTelemetry(telemetry, textFastResults); @@ -5673,7 +5694,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { documentLookupResults, queryClassification.queryClass, ); - if (documentLookupFastPath.returnFastPath) { + if (!args.forceEmbedding && documentLookupFastPath.returnFastPath) { markEmbeddingSkippedByTextFastPath( telemetry, documentLookupFastPath.reason ? `document_lookup_fast_path:${documentLookupFastPath.reason}` : null, @@ -5699,8 +5720,8 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { telemetry, }); const coverageGate = evaluateEvidenceCoverageGate(args.query, coverageGateResults, queryClassification.queryClass); - applyCoverageGateTelemetry(telemetry, coverageGate, coverageGate.accepted); - if (coverageGate.accepted) { + applyCoverageGateTelemetry(telemetry, coverageGate, !args.forceEmbedding && coverageGate.accepted); + if (!args.forceEmbedding && coverageGate.accepted) { telemetry.retrieval_strategy = coverageGate.strategy; recordSearchScoreTelemetry(telemetry, coverageGateResults); setCachedSearch(args, coverageGateResults, telemetry, queryVariants); @@ -5728,7 +5749,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { } catch (error) { // In auto mode a failed embedding call (e.g. quota exhausted) degrades to the lexical // results already gathered rather than failing the whole search. "openai" mode rethrows. - if (!allowsAutoDegrade()) throw error; + if (args.forceEmbedding || !allowsAutoDegrade()) throw error; telemetry.embedding_skipped = true; telemetry.embedding_skip_reason = sourceOnlyReason(error); telemetry.vector_skipped_reason = classifyProviderFailure(error); @@ -5822,10 +5843,14 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { latencyMs: hybridResult.latencyMs, topScore: layerTopScore((hybridData ?? []) as SearchResult[]), }); + const vectorCandidates = mergeSearchResults( + mergeSearchResults((hybridData ?? []) as SearchResult[], embeddingFieldCandidates), + indexUnitCandidates, + ); if (!hybridError) { const rerankStartedAt = Date.now(); - const merged = mergeSearchResults((hybridData ?? []) as SearchResult[], textFastResults); + const merged = args.forceEmbedding ? vectorCandidates : mergeSearchResults(vectorCandidates, textFastResults); const mergedWithMetadata = await attachDocumentRankingMetadata(supabase, merged, args.ownerId); const memoryBoost = await withMemoryBoostedCandidates({ supabase, @@ -5883,7 +5908,7 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { return (data ?? []) as SearchResult[]; }), ).catch((error) => { - if (textFastResults.length > 0) return [] as SearchResult[][]; + if (!args.forceEmbedding && textFastResults.length > 0) return [] as SearchResult[][]; throw error; }); const fallbackLatencyMs = Date.now() - fallbackRpcStartedAt; @@ -5895,9 +5920,13 @@ export async function searchChunksWithTelemetry(args: SearchChunksArgs) { }); const rerankStartedAt = Date.now(); + const fallbackVectorCandidates = mergeSearchResults( + mergeSearchResults(resultSets.flat(), embeddingFieldCandidates), + indexUnitCandidates, + ); const mergedWithMetadata = await attachDocumentRankingMetadata( supabase, - mergeSearchResults(resultSets.flat(), textFastResults), + args.forceEmbedding ? fallbackVectorCandidates : mergeSearchResults(fallbackVectorCandidates, textFastResults), args.ownerId, ); const memoryBoost = await withMemoryBoostedCandidates({ diff --git a/tests/eval-quality.test.ts b/tests/eval-quality.test.ts index deffec1cd..b41fd45f2 100644 --- a/tests/eval-quality.test.ts +++ b/tests/eval-quality.test.ts @@ -9,12 +9,13 @@ import { sourceWarningsForRagQualityAnswer, type RagQualityResult, } from "../scripts/eval-quality"; -import type { GoldenRetrievalResult } from "../scripts/eval-retrieval"; +import { evaluateGoldenRetrievalCase, type GoldenRetrievalResult } from "../scripts/eval-retrieval"; function retrievalResult(overrides: Partial = {}): GoldenRetrievalResult { const base: GoldenRetrievalResult = { id: "retrieval-1", query: "What ANC threshold should withhold clozapine?", + forceEmbedding: false, expectedQueryClass: "table_threshold", actualQueryClass: "table_threshold", expectedDocumentSubstrings: ["clozapine.pdf"], @@ -165,6 +166,8 @@ describe("eval quality reporting", () => { structured_threshold_text_match: 1, }); expect(report.retrieval.summary.second_stage_rerank_rate).toBe(0.5); + expect(report.retrieval.summary.force_embedding_case_count).toBe(0); + expect(report.retrieval.summary.force_embedding_failure_count).toBe(0); expect(report.rag.summary.unsupported_correct_rate).toBe(0); expect(report.rag.summary.numeric_grounding_failure_rate).toBeCloseTo(0.3333, 4); expect(report.rag.summary.source_governance_danger_failure_rate).toBeCloseTo(0.3333, 4); @@ -179,6 +182,53 @@ describe("eval quality reporting", () => { ); }); + it("fails forced-embedding retrieval cases that return from cache, coverage, or lexical paths", () => { + const result = evaluateGoldenRetrievalCase({ + testCase: { + id: "vector-regression", + query: "How is panic disorder managed?", + expectedQueryClass: "broad_summary", + expectedDocumentSubstrings: [], + expectedContentTerms: [], + topK: 8, + expectTableEvidence: false, + forceEmbedding: true, + }, + results: [], + telemetry: { + query_class: "broad_summary", + retrieval_strategy: "text_fast_path", + embedding_skipped: true, + embedding_skip_reason: "strong_document_text_score", + text_fast_path_reason: "strong_document_text_score", + text_candidate_budget: 32, + text_candidate_count: 5, + vector_candidate_count: 0, + retrieval_layer_counts: { text_candidates: 5 }, + coverage_gate_decision: "accepted", + coverage_gate_reason: "document_title_evidence_gate", + second_stage_rerank_used: false, + }, + latencyMs: 10, + }); + + expect(result.forceEmbedding).toBe(true); + expect(result.failures).toEqual( + expect.arrayContaining([ + "forceEmbedding expected embedding to run", + "forceEmbedding returned lexical strategy text_fast_path", + "forceEmbedding returned coverage gate", + "forceEmbedding found no vector-layer candidates", + ]), + ); + const report = buildEvalQualityReport({ retrievalResults: [result], ragResults: [] }); + expect(report.retrieval.summary.force_embedding_case_count).toBe(1); + expect(report.retrieval.summary.force_embedding_failure_count).toBe(1); + expect(report.blocking_threshold_failures).toEqual( + expect.arrayContaining([expect.stringContaining("retrieval force_embedding_failure_count 1 above 0")]), + ); + }); + it("derives source governance warnings for direct RAG answers without precomputed warnings", () => { const warnings = sourceWarningsForRagQualityAnswer({ sourceGovernanceWarnings: undefined, diff --git a/tests/retrieval-query-variants.test.ts b/tests/retrieval-query-variants.test.ts index ddb62411e..a2f06474e 100644 --- a/tests/retrieval-query-variants.test.ts +++ b/tests/retrieval-query-variants.test.ts @@ -127,6 +127,18 @@ describe("retrieval query variants", () => { expect(textCandidateBudgetForQueryClass("unsupported_or_general", 12)).toBe(24); }); + it("keeps forced-embedding retrieval out of the ordinary search cache key", () => { + const baseArgs = { + query: "How is panic disorder managed?", + topK: 8, + minSimilarity: 0.12, + }; + + expect(retrievalPlanCacheQuery(baseArgs, "broad_summary")).not.toBe( + retrievalPlanCacheQuery({ ...baseArgs, forceEmbedding: true }, "broad_summary"), + ); + }); + it("allows direct document title hits to skip embedding retrieval", () => { expect( decideTextFastPath(