test(eval): harden force-embedding flag and vector-path eval guards

scripts/eval-quality.ts

@@ -44,6 +44,7 @@ type EvalQualityArgs = {
   retrievalOnly: boolean;
   ragOnly: boolean;
   skipPreflight: boolean;
+  forceEmbedding: boolean;
 };
 
 export type RagQualityResult = {
@@ -149,6 +150,7 @@ function parseArgs(argv: string[]): EvalQualityArgs {
     retrievalOnly: false,
     ragOnly: false,
     skipPreflight: false,
+    forceEmbedding: false,
   };
 
   for (let index = 0; index < argv.length; index += 1) {
@@ -175,6 +177,10 @@ function parseArgs(argv: string[]): EvalQualityArgs {
       args.skipPreflight = true;
       continue;
     }
+    if (token === "--force-embedding") {
+      args.forceEmbedding = true;
+      continue;
+    }
 
     const value = argv[index + 1];
     if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`);
@@ -475,6 +481,11 @@ export function buildEvalQualityReport(args: {
         `retrieval content_recall_at_5 ${retrievalSummary.content_recall_at_5} below ${qualityThresholds.retrievalContentRecallAt5}`,
       );
     }
+    if (retrievalSummary.force_embedding_failure_count > 0) {
+      thresholdFailures.push(
+        `retrieval force_embedding_failure_count ${retrievalSummary.force_embedding_failure_count} above 0`,
+      );
+    }
     if (governance.stale_rate > qualityThresholds.staleTopResultRate) {
       thresholdFailures.push(
         `top-result stale_rate ${governance.stale_rate} above ${qualityThresholds.staleTopResultRate}`,
@@ -663,6 +674,8 @@ ${markdownTable([
 ## Retrieval Decision Metrics
 
 ${markdownTable([
+  ["Force-embedding cases", retrieval.force_embedding_case_count],
+  ["Force-embedding failures", retrieval.force_embedding_failure_count],
   ["Embedding skipped rate", retrieval.embedding_skipped_rate],
   ["Median text candidate budget", retrieval.median_text_candidate_budget],
   ["Second-stage rerank rate", retrieval.second_stage_rerank_rate],
@@ -727,6 +740,7 @@ async function runRetrievalQualityCases(args: {
   ownerId?: string;
   limit?: number;
   query?: string;
+  forceEmbedding?: boolean;
   supabase: Awaited<ReturnType<typeof loadAdminClient>>;
 }) {
   const [{ searchChunksWithTelemetry }, capturedCases] = await Promise.all([
@@ -753,6 +767,7 @@ async function runRetrievalQualityCases(args: {
         topK: testCase.topK,
         minSimilarity: 0.12,
         skipCache: true,
+        forceEmbedding: testCase.forceEmbedding || args.forceEmbedding,
       }),
     );
     const latencyMs =

scripts/eval-retrieval.ts

@@ -19,6 +19,10 @@ const goldenCaseSchema = z.object({
   expectedContentTerms: z.array(contentExpectationSchema).default([]),
   topK: z.number().int().positive().default(8),
   expectTableEvidence: z.boolean().default(false),
+  // Bypass the lexical text-fast-path so this case always exercises the embedding/vector
+  // index. Use for "vector-*" probes that would otherwise be answered by a lexical shortcut,
+  // so a re-index's effect on vector retrieval is actually measured.
+  forceEmbedding: z.boolean().optional(),
 });
 
 const goldenCasesSchema = z.array(goldenCaseSchema);
@@ -37,11 +41,13 @@ type EvalArgs = {
   caseTimeoutMs: number;
   p90BudgetMs: number;
   p50BudgetMs: number;
+  forceEmbedding: boolean;
 };
 
 export type GoldenRetrievalResult = {
   id: string;
   query: string;
+  forceEmbedding: boolean;
   expectedQueryClass: string;
   actualQueryClass: string | null;
   expectedDocumentSubstrings: string[];
@@ -116,6 +122,7 @@ function parseArgs(argv: string[]): EvalArgs {
     caseTimeoutMs: inferredMode === "latency" ? 25_000 : 0,
     p90BudgetMs: 20_000,
     p50BudgetMs: 8_000,
+    forceEmbedding: false,
   };
 
   for (let index = 0; index < argv.length; index += 1) {
@@ -138,6 +145,10 @@ function parseArgs(argv: string[]): EvalArgs {
       if (args.caseTimeoutMs <= 0) args.caseTimeoutMs = 25_000;
       continue;
     }
+    if (token === "--force-embedding") {
+      args.forceEmbedding = true;
+      continue;
+    }
 
     const value = argv[index + 1];
     if (!value || value.startsWith("--")) throw new Error(`Missing value for ${token}`);
@@ -458,6 +469,11 @@ export function evaluateGoldenRetrievalCase(args: {
   const tableEvidenceFoundAtK = hasTableEvidence(args.results, topK);
   const actualQueryClass = args.telemetry.query_class ?? null;
   const failures: string[] = [];
+  const vectorLayerCount = Object.entries(args.telemetry.retrieval_layer_counts ?? {}).reduce(
+    (sum, [layer, count]) =>
+      ["embedding_fields", "index_units", "hybrid_vector", "vector_fallback"].includes(layer) ? sum + count : sum,
+    0,
+  );
   const hitAtK =
     documentHitsAtK.missing.length === 0 &&
     contentHitsAtK.missing.length === 0 &&
@@ -475,10 +491,23 @@ export function evaluateGoldenRetrievalCase(args: {
   if (args.testCase.expectTableEvidence && !tableEvidenceFound) {
     failures.push("expected table evidence in top 5");
   }
+  if (args.testCase.forceEmbedding) {
+    if (args.telemetry.embedding_skipped) failures.push("forceEmbedding expected embedding to run");
+    if (args.telemetry.retrieval_strategy === "search_cache") failures.push("forceEmbedding served search cache");
+    if (
+      args.telemetry.retrieval_strategy === "text_fast_path" ||
+      args.telemetry.retrieval_strategy === "document_lookup_fast_path"
+    ) {
+      failures.push(`forceEmbedding returned lexical strategy ${args.telemetry.retrieval_strategy}`);
+    }
+    if (args.telemetry.coverage_gate_decision === "accepted") failures.push("forceEmbedding returned coverage gate");
+    if (vectorLayerCount <= 0) failures.push("forceEmbedding found no vector-layer candidates");
+  }
 
   return {
     id: args.testCase.id,
     query: args.testCase.query,
+    forceEmbedding: args.testCase.forceEmbedding ?? false,
     expectedQueryClass: args.testCase.expectedQueryClass,
     actualQueryClass,
     expectedDocumentSubstrings: args.testCase.expectedDocumentSubstrings,
@@ -550,6 +579,7 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[]
     }
     return counts;
   }, {});
+  const forceEmbeddingResults = results.filter((result) => result.forceEmbedding);
   return {
     case_count: results.length,
     document_recall_at_5: Number(
@@ -578,6 +608,8 @@ export function summarizeGoldenRetrievalResults(results: GoldenRetrievalResult[]
     embedding_skip_reason_counts: embeddingSkipReasonCounts,
     text_fast_path_reason_counts: textFastPathReasonCounts,
     retrieval_layer_counts: layerCounts,
+    force_embedding_case_count: forceEmbeddingResults.length,
+    force_embedding_failure_count: forceEmbeddingResults.filter((result) => result.failures.length > 0).length,
     median_text_candidate_budget: percentile(textCandidateBudgets, 50),
     second_stage_rerank_rate: Number(
       (results.filter((result) => result.secondStageRerankUsed).length / Math.max(results.length, 1)).toFixed(4),
@@ -651,6 +683,8 @@ function printHumanSummary(summary: ReturnType<typeof summarizeGoldenRetrievalRe
   console.log(`  retrieval_strategy_counts=${JSON.stringify(summary.retrieval_strategy_counts)}`);
   console.log(`  retrieval_plan_counts=${JSON.stringify(summary.retrieval_plan_counts)}`);
   console.log(`  retrieval_layer_counts=${JSON.stringify(summary.retrieval_layer_counts)}`);
+  console.log(`  force_embedding_case_count=${summary.force_embedding_case_count}`);
+  console.log(`  force_embedding_failure_count=${summary.force_embedding_failure_count}`);
   console.log(`  embedding_skipped_rate=${summary.embedding_skipped_rate}`);
   console.log(`  embedding_skip_reason_counts=${JSON.stringify(summary.embedding_skip_reason_counts)}`);
   console.log(`  text_fast_path_reason_counts=${JSON.stringify(summary.text_fast_path_reason_counts)}`);
@@ -743,6 +777,7 @@ async function main() {
         topK: testCase.topK,
         minSimilarity: 0.12,
         skipCache: args.mode !== "latency",
+        forceEmbedding: testCase.forceEmbedding || args.forceEmbedding,
       }),
     );
     const searchOutcome = await withCaseTimeout(searchPromise, args.caseTimeoutMs);

scripts/fixtures/rag-retrieval-golden.json

@@ -224,5 +224,199 @@
     ],
     "topK": 12,
     "expectTableEvidence": true
+  },
+  {
+    "id": "vector-ptsd",
+    "query": "How is post-traumatic stress disorder managed in a person with intrusive flashbacks, nightmares, hyperarousal and avoidance after a traumatic event?",
+    "expectedQueryClass": "broad_summary",
+    "expectedDocumentSubstrings": [
+      "Traumatic Stress"
+    ],
+    "expectedContentTerms": [
+      [
+        "trauma",
+        "traumatic",
+        "ptsd",
+        "flashback"
+      ]
+    ],
+    "topK": 8,
+    "expectTableEvidence": false,
+    "forceEmbedding": true
+  },
+  {
+    "id": "vector-ocd",
+    "query": "How is obsessive-compulsive disorder managed in a person with distressing intrusive obsessions and repetitive compulsive rituals?",
+    "expectedQueryClass": "broad_summary",
+    "expectedDocumentSubstrings": [
+      "Obsessive"
+    ],
+    "expectedContentTerms": [
+      [
+        "obsess",
+        "compuls",
+        "intrusive",
+        "ocd"
+      ]
+    ],
+    "topK": 8,
+    "expectTableEvidence": false,
+    "forceEmbedding": true
+  },
+  {
+    "id": "vector-panic",
+    "query": "How is panic disorder managed in a person with recurrent unexpected panic attacks, palpitations and anticipatory anxiety?",
+    "expectedQueryClass": "broad_summary",
+    "expectedDocumentSubstrings": [
+      "Panic"
+    ],
+    "expectedContentTerms": [
+      [
+        "panic",
+        "attack",
+        "anxiety"
+      ]
+    ],
+    "topK": 8,
+    "expectTableEvidence": false,
+    "forceEmbedding": true
+  },
+  {
+    "id": "vector-anorexia",
+    "query": "How is anorexia nervosa managed in a person with severe dietary restriction, intense fear of weight gain and body-image disturbance?",
+    "expectedQueryClass": "broad_summary",
+    "expectedDocumentSubstrings": [
+      "Anorexia"
+    ],
+    "expectedContentTerms": [
+      [
+        "anorexia",
+        "eating",
+        "weight"
+      ]
+    ],
+    "topK": 8,
+    "expectTableEvidence": false,
+    "forceEmbedding": true
+  },
+  {
+    "id": "vector-gad-worry",
+    "query": "How is a patient managed who has persistent, excessive worry about many everyday things that they find very hard to control, along with restlessness and muscle tension?",
+    "expectedQueryClass": "broad_summary",
+    "expectedDocumentSubstrings": [
+      "Generalised Anxiety"
+    ],
+    "expectedContentTerms": [
+      [
+        "worry",
+        "anxiety",
+        "generalised"
+      ],
+      [
+        "cbt",
+        "ssri",
+        "snri",
+        "antidepressant",
+        "psychotherapy"
+      ]
+    ],
+    "topK": 8,
+    "expectTableEvidence": false,
+    "forceEmbedding": true
+  },
+  {
+    "id": "vector-tourette",
+    "query": "How is Tourette syndrome managed in a child with chronic motor tics and vocal tics?",
+    "expectedQueryClass": "broad_summary",
+    "expectedDocumentSubstrings": [
+      "Tourette"
+    ],
+    "expectedContentTerms": [
+      [
+        "tic",
+        "tics",
+        "tourette"
+      ]
+    ],
+    "topK": 8,
+    "expectTableEvidence": false,
+    "forceEmbedding": true
+  },
+  {
+    "id": "vector-postnatal",
+    "query": "How is postnatal (postpartum) depression managed in a new mother who develops low mood and poor bonding with her infant in the first weeks postpartum?",
+    "expectedQueryClass": "broad_summary",
+    "expectedDocumentSubstrings": [
+      "Postnatal"
+    ],
+    "expectedContentTerms": [
+      [
+        "postnatal",
+        "postpartum",
+        "perinatal",
+        "depression"
+      ]
+    ],
+    "topK": 8,
+    "expectTableEvidence": false,
+    "forceEmbedding": true
+  },
+  {
+    "id": "vector-bipolar",
+    "query": "How is bipolar disorder managed in an adult with recurrent episodes of mania and depression?",
+    "expectedQueryClass": "broad_summary",
+    "expectedDocumentSubstrings": [
+      "Bipolar"
+    ],
+    "expectedContentTerms": [
+      [
+        "bipolar",
+        "mania",
+        "manic",
+        "mood"
+      ]
+    ],
+    "topK": 8,
+    "expectTableEvidence": false,
+    "forceEmbedding": true
+  },
+  {
+    "id": "vector-adhd",
+    "query": "How is attention deficit hyperactivity disorder managed in an adult with inattention, distractibility and impulsivity?",
+    "expectedQueryClass": "broad_summary",
+    "expectedDocumentSubstrings": [
+      "Attention Deficit Hyperactivity"
+    ],
+    "expectedContentTerms": [
+      [
+        "attention",
+        "adhd",
+        "hyperactiv",
+        "impuls"
+      ]
+    ],
+    "topK": 8,
+    "expectTableEvidence": false,
+    "forceEmbedding": true
+  },
+  {
+    "id": "vector-opioid",
+    "query": "How is opioid use disorder managed in a person dependent on heroin?",
+    "expectedQueryClass": "broad_summary",
+    "expectedDocumentSubstrings": [
+      "Opioid Use Disorder"
+    ],
+    "expectedContentTerms": [
+      [
+        "opioid",
+        "heroin",
+        "opiate",
+        "methadone",
+        "buprenorphine"
+      ]
+    ],
+    "topK": 8,
+    "expectTableEvidence": false,
+    "forceEmbedding": true
   }
 ]