fix(eval): recalibrate canary latency budgets to the post-#606 honest baseline

docs/retrieval-quality-runbook.md

@@ -142,7 +142,7 @@ Answer quality:
 - citation failure rate is above `0`
 - numeric grounding failure rate is above `0`
 - source-governance danger failure rate is above `0`
-- RAG p95 latency is above `25000ms`
+- RAG p95 latency is above `60000ms` (recalibrated 2026-07-13: honest post-#606 answer p95 from eval runners is ~48s — real generation, cross-region, and timeout→fallback chains; regression detection is baseline-relative via `latencyRegressionMs`)
 
 These thresholds are intended as a quality tripwire, not a substitute for clinical review.
 Warning-class source governance notes remain visible in reports and should be backfilled or explicitly baselined before release.

scripts/eval-quality.ts

@@ -142,18 +142,30 @@ const qualityThresholds = {
   // Latency gates default to the strict near-region ceilings that release and
   // local runs are held to. The Eval Canary measures from cross-region GitHub
   // runners → Sydney Supabase + OpenAI, where a real grounded answer pays full
-  // generation time and the provider-timeout→extractive-fallback chain costs
-  // 30s+ per affected case (issue #459 post-#606: quality metrics perfect, p95
+  // generation time (issue #459 post-#606: quality metrics perfect, p95
   // measured 48,256ms) — that workflow opts into the wider allowance by setting
   // EVAL_LATENCY_CONTEXT=cross-region-runner, so eval:quality:release keeps the
   // strict gate. User-facing latency is enforced separately by the answer SLO
   // deep probe, not this eval.
   ragP95LatencyMs: crossRegionRunnerLatencyContext ? 60_000 : 25_000,
   ragRouteP95LatencyMs: {
+    // Refusals must stay fast — a slow refusal means the pipeline burned generation time
+    // before giving up, which is exactly the waste mode #580 eliminated.
     unsupported: 4_000,
-    extractive: crossRegionRunnerLatencyContext ? 60_000 : 12_000,
+    // Direct source-stitching with no model generation. Generation-fallback chains no
+    // longer land in this bucket (see "fallback" below), so the runner allowance only
+    // needs cross-region RPC headroom — not the blanket 60s that pre-fallback-bucket
+    // calibration required — and a no-model extraction slowdown stays visible.
+    extractive: crossRegionRunnerLatencyContext ? 35_000 : 12_000,
     fast: 25_000,
-    strong: 35_000,
+    // Generation-fallback chains (latencyRouteForAnswer buckets any answer whose routing
+    // reason records a generation_fallback here): the failed generation spends up to
+    // OPENAI_ANSWER_TIMEOUT_MS (30s) before the source-backed fallback stitches an
+    // answer. Timeout-dominated, so the budget is region-insensitive.
+    fallback: 50_000,
+    // Strong may retry a truncated generation at a larger budget (self-heal) = up to two
+    // sequential generations; near-region runs keep the strict single-generation gate.
+    strong: crossRegionRunnerLatencyContext ? 60_000 : 35_000,
   } as Record<string, number>,
 };
 
@@ -643,6 +655,15 @@ function ragCaseDiagnosticsTable(results: RagQualityResult[]) {
 function latencyRouteForAnswer(answer: RagAnswer) {
   const route = answer.routingMode ?? "none";
   if ((answer.latencyTimings?.generation_latency_ms ?? 0) <= 0) return route;
+  // A generation ran and failed before a source-backed answer was assembled (e.g.
+  // provider_timeout -> extractive fallback). These chains structurally cost the failed
+  // generation's timeout PLUS the fallback work, so they get their own latency budget
+  // instead of inflating the plain fast budget, hiding inside the no-model extractive
+  // one, or — for strong-classified reasons (e.g. multi_document_comparison_synthesis) —
+  // blending into the strong budget. This check deliberately outranks the strong
+  // classifiers below: a successful strong generation (including quality retries) never
+  // records a generation_fallback, so genuine strong answers keep the strong budget.
+  if (/\bgeneration_fallback:/i.test(answer.routingReason ?? "")) return "fallback";
   if (route === "strong") return "strong";
   if (
     /^(?:broad_clinical_management_synthesis|clinical_risk_or_complex_query|limited_retrieval_strength|multi_document_comparison_synthesis|retrieval_gap_or_conflict)\b/i.test(

src/lib/reindex-eval-gate.ts

@@ -87,7 +87,14 @@ export const defaultReindexGateConfig: ReindexGateConfig = {
     expectedDangerWarningMissingCountCeiling: 0,
     staleReviewUnknownRateCeiling: 0.25,
     reviewRequiredRateCeiling: 0.25,
-    p95LatencyMsCeiling: 25000,
+    // Matches the eval-canary answer budget (scripts/eval-quality.ts ragP95LatencyMs).
+    // Recalibrated 2026-07-13: with grounded answering restored (#606), honest answer p95
+    // from eval runners is ~48s (real generation + cross-region + timeout->fallback
+    // chains); the old 25s ceiling was calibrated when fast confidence-gate refusals
+    // dominated the subset. Latency REGRESSION detection is unchanged — it stays with
+    // latencyRegressionMs against the baseline run, so a no-regression reindex at the
+    // honest baseline passes while a genuine slowdown still fails.
+    p95LatencyMsCeiling: 60000,
     rateRegressionTolerance: 0.02,
     latencyRegressionMs: 4000,
   },

tests/eval-quality.test.ts

@@ -234,26 +234,31 @@ describe("eval quality reporting", () => {
     );
   });
 
-  it("budgets a model-attempt extractive fallback against its attempted route", () => {
+  it("budgets a model-attempt extractive fallback against the fallback latency route", () => {
+    // A failed generation followed by a source-backed fallback structurally costs the
+    // generation timeout plus the fallback work, so it is budgeted in its own "fallback"
+    // bucket (50s) rather than the plain fast budget (25s) or the tight no-model
+    // extractive budget (12s). 35s here would fail both of those but must pass fallback.
     const report = buildEvalQualityReport({
       generatedAt: "2026-07-13T00:00:00.000Z",
       retrievalResults: [],
       ragResults: [
         ragResult({
           route: "extractive",
-          latencyRoute: "fast",
-          latencyMs: 15_000,
-          generationLatencyMs: 12_000,
+          latencyRoute: "fallback",
+          latencyMs: 35_000,
+          generationLatencyMs: 30_000,
           model: null,
           routingReason: "strong_routine_retrieval; generation_fallback:provider_timeout",
         }),
       ],
     });
 
-    expect(report.rag.summary.route_p95_latency_ms).toEqual({ fast: 15_000 });
+    expect(report.rag.summary.route_p95_latency_ms).toEqual({ fallback: 35_000 });
     expect(report.threshold_failures).not.toEqual(
       expect.arrayContaining([expect.stringContaining("route extractive")]),
     );
+    expect(report.threshold_failures).not.toEqual(expect.arrayContaining([expect.stringContaining("route fallback")]));
   });
 
   it("fails forced-embedding retrieval cases that return from cache, coverage, or lexical paths", () => {

tests/reindex-eval-gate.test.ts

@@ -304,10 +304,10 @@ describe("decideReindexGate — quality", () => {
       baselineRetrieval: retrieval(),
       candidateRetrieval: retrieval(),
       baselineQuality: quality(),
-      candidateQuality: quality({ p95_latency_ms: 26000 }),
+      candidateQuality: quality({ p95_latency_ms: 61000 }),
     });
     expect(decision.decision).toBe("NO_GO");
-    expect(decision.failures.join(" ")).toMatch(/p95_latency_ms 26000 above absolute ceiling 25000/);
+    expect(decision.failures.join(" ")).toMatch(/p95_latency_ms 61000 above absolute ceiling 60000/);
   });
 
   it("fails closed when only one side supplies a quality summary", () => {