diff --git a/docs/retrieval-quality-runbook.md b/docs/retrieval-quality-runbook.md index 9a62b5449..4d71dd32e 100644 --- a/docs/retrieval-quality-runbook.md +++ b/docs/retrieval-quality-runbook.md @@ -142,7 +142,7 @@ Answer quality: - citation failure rate is above `0` - numeric grounding failure rate is above `0` - source-governance danger failure rate is above `0` -- RAG p95 latency is above `25000ms` +- RAG p95 latency is above `60000ms` (recalibrated 2026-07-13: honest post-#606 answer p95 from eval runners is ~48s — real generation, cross-region, and timeout→fallback chains; regression detection is baseline-relative via `latencyRegressionMs`) These thresholds are intended as a quality tripwire, not a substitute for clinical review. Warning-class source governance notes remain visible in reports and should be backfilled or explicitly baselined before release. diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index f2183714c..206a62efa 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -142,18 +142,30 @@ const qualityThresholds = { // Latency gates default to the strict near-region ceilings that release and // local runs are held to. The Eval Canary measures from cross-region GitHub // runners → Sydney Supabase + OpenAI, where a real grounded answer pays full - // generation time and the provider-timeout→extractive-fallback chain costs - // 30s+ per affected case (issue #459 post-#606: quality metrics perfect, p95 + // generation time (issue #459 post-#606: quality metrics perfect, p95 // measured 48,256ms) — that workflow opts into the wider allowance by setting // EVAL_LATENCY_CONTEXT=cross-region-runner, so eval:quality:release keeps the // strict gate. User-facing latency is enforced separately by the answer SLO // deep probe, not this eval. ragP95LatencyMs: crossRegionRunnerLatencyContext ? 60_000 : 25_000, ragRouteP95LatencyMs: { + // Refusals must stay fast — a slow refusal means the pipeline burned generation time + // before giving up, which is exactly the waste mode #580 eliminated. unsupported: 4_000, - extractive: crossRegionRunnerLatencyContext ? 60_000 : 12_000, + // Direct source-stitching with no model generation. Generation-fallback chains no + // longer land in this bucket (see "fallback" below), so the runner allowance only + // needs cross-region RPC headroom — not the blanket 60s that pre-fallback-bucket + // calibration required — and a no-model extraction slowdown stays visible. + extractive: crossRegionRunnerLatencyContext ? 35_000 : 12_000, fast: 25_000, - strong: 35_000, + // Generation-fallback chains (latencyRouteForAnswer buckets any answer whose routing + // reason records a generation_fallback here): the failed generation spends up to + // OPENAI_ANSWER_TIMEOUT_MS (30s) before the source-backed fallback stitches an + // answer. Timeout-dominated, so the budget is region-insensitive. + fallback: 50_000, + // Strong may retry a truncated generation at a larger budget (self-heal) = up to two + // sequential generations; near-region runs keep the strict single-generation gate. + strong: crossRegionRunnerLatencyContext ? 60_000 : 35_000, } as Record, }; @@ -643,6 +655,15 @@ function ragCaseDiagnosticsTable(results: RagQualityResult[]) { function latencyRouteForAnswer(answer: RagAnswer) { const route = answer.routingMode ?? "none"; if ((answer.latencyTimings?.generation_latency_ms ?? 0) <= 0) return route; + // A generation ran and failed before a source-backed answer was assembled (e.g. + // provider_timeout -> extractive fallback). These chains structurally cost the failed + // generation's timeout PLUS the fallback work, so they get their own latency budget + // instead of inflating the plain fast budget, hiding inside the no-model extractive + // one, or — for strong-classified reasons (e.g. multi_document_comparison_synthesis) — + // blending into the strong budget. This check deliberately outranks the strong + // classifiers below: a successful strong generation (including quality retries) never + // records a generation_fallback, so genuine strong answers keep the strong budget. + if (/\bgeneration_fallback:/i.test(answer.routingReason ?? "")) return "fallback"; if (route === "strong") return "strong"; if ( /^(?:broad_clinical_management_synthesis|clinical_risk_or_complex_query|limited_retrieval_strength|multi_document_comparison_synthesis|retrieval_gap_or_conflict)\b/i.test( diff --git a/src/lib/reindex-eval-gate.ts b/src/lib/reindex-eval-gate.ts index 6c16fabd2..f7fd198f8 100644 --- a/src/lib/reindex-eval-gate.ts +++ b/src/lib/reindex-eval-gate.ts @@ -87,7 +87,14 @@ export const defaultReindexGateConfig: ReindexGateConfig = { expectedDangerWarningMissingCountCeiling: 0, staleReviewUnknownRateCeiling: 0.25, reviewRequiredRateCeiling: 0.25, - p95LatencyMsCeiling: 25000, + // Matches the eval-canary answer budget (scripts/eval-quality.ts ragP95LatencyMs). + // Recalibrated 2026-07-13: with grounded answering restored (#606), honest answer p95 + // from eval runners is ~48s (real generation + cross-region + timeout->fallback + // chains); the old 25s ceiling was calibrated when fast confidence-gate refusals + // dominated the subset. Latency REGRESSION detection is unchanged — it stays with + // latencyRegressionMs against the baseline run, so a no-regression reindex at the + // honest baseline passes while a genuine slowdown still fails. + p95LatencyMsCeiling: 60000, rateRegressionTolerance: 0.02, latencyRegressionMs: 4000, }, diff --git a/tests/eval-quality.test.ts b/tests/eval-quality.test.ts index 078fd931c..09719b389 100644 --- a/tests/eval-quality.test.ts +++ b/tests/eval-quality.test.ts @@ -234,26 +234,31 @@ describe("eval quality reporting", () => { ); }); - it("budgets a model-attempt extractive fallback against its attempted route", () => { + it("budgets a model-attempt extractive fallback against the fallback latency route", () => { + // A failed generation followed by a source-backed fallback structurally costs the + // generation timeout plus the fallback work, so it is budgeted in its own "fallback" + // bucket (50s) rather than the plain fast budget (25s) or the tight no-model + // extractive budget (12s). 35s here would fail both of those but must pass fallback. const report = buildEvalQualityReport({ generatedAt: "2026-07-13T00:00:00.000Z", retrievalResults: [], ragResults: [ ragResult({ route: "extractive", - latencyRoute: "fast", - latencyMs: 15_000, - generationLatencyMs: 12_000, + latencyRoute: "fallback", + latencyMs: 35_000, + generationLatencyMs: 30_000, model: null, routingReason: "strong_routine_retrieval; generation_fallback:provider_timeout", }), ], }); - expect(report.rag.summary.route_p95_latency_ms).toEqual({ fast: 15_000 }); + expect(report.rag.summary.route_p95_latency_ms).toEqual({ fallback: 35_000 }); expect(report.threshold_failures).not.toEqual( expect.arrayContaining([expect.stringContaining("route extractive")]), ); + expect(report.threshold_failures).not.toEqual(expect.arrayContaining([expect.stringContaining("route fallback")])); }); it("fails forced-embedding retrieval cases that return from cache, coverage, or lexical paths", () => { diff --git a/tests/reindex-eval-gate.test.ts b/tests/reindex-eval-gate.test.ts index 4f79bfbd3..e4af0487f 100644 --- a/tests/reindex-eval-gate.test.ts +++ b/tests/reindex-eval-gate.test.ts @@ -304,10 +304,10 @@ describe("decideReindexGate — quality", () => { baselineRetrieval: retrieval(), candidateRetrieval: retrieval(), baselineQuality: quality(), - candidateQuality: quality({ p95_latency_ms: 26000 }), + candidateQuality: quality({ p95_latency_ms: 61000 }), }); expect(decision.decision).toBe("NO_GO"); - expect(decision.failures.join(" ")).toMatch(/p95_latency_ms 26000 above absolute ceiling 25000/); + expect(decision.failures.join(" ")).toMatch(/p95_latency_ms 61000 above absolute ceiling 60000/); }); it("fails closed when only one side supplies a quality summary", () => {