From 87b996cf00f9eb9615bea7666797668b7ca7a5c5 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 14 Jul 2026 02:19:32 +0800 Subject: [PATCH 1/3] fix(eval): recalibrate canary latency budgets to the post-#606 honest baseline The pre-2026-07-13 budgets (p95 25s overall, 12s extractive) were calibrated while confidence_gate_blocked refusals dominated the answer subset - refusals return in ~1-2s, so the canary was timing fast wrong answers. With #606 restoring grounded answering, p95 reflects real work: the fast->extractive fallback chain spends the full 30s answer timeout before stitching sources, and #580's strong truncation self-heal can run two sequential generations. Post-fix observed subset p95 is 48.3s with every quality metric green (grounded 1.0, citation failure 0, numeric failure 0). New budgets catch regressions from that baseline instead of re-flagging the fix: overall 60s, extractive 50s, strong 60s. Fast (25s) and unsupported (4s) are unchanged - refusals must stay fast, which is exactly the waste mode #580 eliminated. These are eval-runner budgets (GitHub-hosted, cross-region to Sydney), not production UX targets; production SLOs stay in answer-slo.ts and docs/observability-slos.md. Closes the remaining red on #459. Co-Authored-By: Claude Fable 5 --- scripts/eval-quality.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index d7f481483..acda1f114 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -137,12 +137,27 @@ const qualityThresholds = { numericGroundingFailureRate: 0, staleTopResultRate: 0.25, reviewRequiredTopResultRate: 0.25, - ragP95LatencyMs: 25_000, + // Latency budgets are measured from the eval runner (GitHub-hosted, cross-region to the + // Sydney Supabase project + OpenAI), not from production. The pre-2026-07-13 budgets were + // calibrated when confidence_gate_blocked refusals dominated the subset — refusals return in + // ~1-2s, so the canary was timing fast wrong answers. After #606 restored grounded answering, + // p95 reflects real generation: the fast->extractive fallback chain spends the full 30s + // answer timeout before extracting, and #580's strong truncation self-heal can run two + // generations. Post-fix observed subset p95: 48.3s with every quality metric green. + // These budgets exist to catch latency REGRESSIONS from that honest baseline, not to encode + // a production UX target (production SLOs live in answer-slo.ts / docs/observability-slos.md). + ragP95LatencyMs: 60_000, ragRouteP95LatencyMs: { + // Refusals must stay fast — a slow refusal means the pipeline burned generation time + // before giving up, which is exactly the waste mode #580 eliminated. unsupported: 4_000, - extractive: 12_000, + // Includes timeout->extractive fallback cases, which spend OPENAI_ANSWER_TIMEOUT_MS (30s) + // on the failed generation before stitching sources. + extractive: 50_000, fast: 25_000, - strong: 35_000, + // Strong may retry a truncated generation at a larger budget (self-heal) = up to two + // sequential generations. + strong: 60_000, } as Record, }; From b6428227e5bec68618e17e1316595fe2c73953c6 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 14 Jul 2026 02:40:43 +0800 Subject: [PATCH 2/3] fix(eval): bucket generation-fallback chains separately; sync reindex gate ceiling Review follow-ups on the latency recalibration: - latencyRouteForAnswer now classifies answers whose routing reason records a generation_fallback into a dedicated "fallback" latency bucket (50s budget): a failed generation structurally costs its timeout plus the fallback work. This restores the tight budgets the first commit accidentally loosened or missed: extractive returns to 12s (no-model stitching must stay fast) and plain fast stays at 25s, so regressions on either path cannot hide inside the fallback allowance. - reindex-eval-gate quality p95 ceiling 25s -> 60s to match the canary budget (a no-regression reindex at the honest ~48s baseline must not be blocked by the absolute bar); baseline-relative regression detection via latencyRegressionMs is unchanged. Runbook updated to match. Co-Authored-By: Claude Fable 5 --- docs/retrieval-quality-runbook.md | 2 +- scripts/eval-quality.ts | 16 +++++++++++++--- src/lib/reindex-eval-gate.ts | 9 ++++++++- tests/eval-quality.test.ts | 15 ++++++++++----- tests/reindex-eval-gate.test.ts | 4 ++-- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/docs/retrieval-quality-runbook.md b/docs/retrieval-quality-runbook.md index 9a62b5449..4d71dd32e 100644 --- a/docs/retrieval-quality-runbook.md +++ b/docs/retrieval-quality-runbook.md @@ -142,7 +142,7 @@ Answer quality: - citation failure rate is above `0` - numeric grounding failure rate is above `0` - source-governance danger failure rate is above `0` -- RAG p95 latency is above `25000ms` +- RAG p95 latency is above `60000ms` (recalibrated 2026-07-13: honest post-#606 answer p95 from eval runners is ~48s — real generation, cross-region, and timeout→fallback chains; regression detection is baseline-relative via `latencyRegressionMs`) These thresholds are intended as a quality tripwire, not a substitute for clinical review. Warning-class source governance notes remain visible in reports and should be backfilled or explicitly baselined before release. diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index acda1f114..fbf5bd10b 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -151,10 +151,15 @@ const qualityThresholds = { // Refusals must stay fast — a slow refusal means the pipeline burned generation time // before giving up, which is exactly the waste mode #580 eliminated. unsupported: 4_000, - // Includes timeout->extractive fallback cases, which spend OPENAI_ANSWER_TIMEOUT_MS (30s) - // on the failed generation before stitching sources. - extractive: 50_000, + // Direct source-stitching with no model generation: keep the tight budget so a + // no-model extraction slowdown (search/stitching regression) cannot hide inside the + // fallback allowance below. + extractive: 12_000, fast: 25_000, + // Generation-fallback chains (latencyRouteForAnswer buckets any answer whose routing + // reason records a generation_fallback here): the failed generation spends up to + // OPENAI_ANSWER_TIMEOUT_MS (30s) before the source-backed fallback stitches an answer. + fallback: 50_000, // Strong may retry a truncated generation at a larger budget (self-heal) = up to two // sequential generations. strong: 60_000, @@ -655,6 +660,11 @@ function latencyRouteForAnswer(answer: RagAnswer) { ) { return "strong"; } + // A generation ran and failed before a source-backed answer was assembled (e.g. + // provider_timeout -> extractive fallback). These chains structurally cost the failed + // generation's timeout PLUS the fallback work, so they get their own latency budget + // instead of inflating the plain fast budget or hiding inside the no-model extractive one. + if (/\bgeneration_fallback:/i.test(answer.routingReason ?? "")) return "fallback"; return "fast"; } diff --git a/src/lib/reindex-eval-gate.ts b/src/lib/reindex-eval-gate.ts index 6c16fabd2..f7fd198f8 100644 --- a/src/lib/reindex-eval-gate.ts +++ b/src/lib/reindex-eval-gate.ts @@ -87,7 +87,14 @@ export const defaultReindexGateConfig: ReindexGateConfig = { expectedDangerWarningMissingCountCeiling: 0, staleReviewUnknownRateCeiling: 0.25, reviewRequiredRateCeiling: 0.25, - p95LatencyMsCeiling: 25000, + // Matches the eval-canary answer budget (scripts/eval-quality.ts ragP95LatencyMs). + // Recalibrated 2026-07-13: with grounded answering restored (#606), honest answer p95 + // from eval runners is ~48s (real generation + cross-region + timeout->fallback + // chains); the old 25s ceiling was calibrated when fast confidence-gate refusals + // dominated the subset. Latency REGRESSION detection is unchanged — it stays with + // latencyRegressionMs against the baseline run, so a no-regression reindex at the + // honest baseline passes while a genuine slowdown still fails. + p95LatencyMsCeiling: 60000, rateRegressionTolerance: 0.02, latencyRegressionMs: 4000, }, diff --git a/tests/eval-quality.test.ts b/tests/eval-quality.test.ts index 078fd931c..09719b389 100644 --- a/tests/eval-quality.test.ts +++ b/tests/eval-quality.test.ts @@ -234,26 +234,31 @@ describe("eval quality reporting", () => { ); }); - it("budgets a model-attempt extractive fallback against its attempted route", () => { + it("budgets a model-attempt extractive fallback against the fallback latency route", () => { + // A failed generation followed by a source-backed fallback structurally costs the + // generation timeout plus the fallback work, so it is budgeted in its own "fallback" + // bucket (50s) rather than the plain fast budget (25s) or the tight no-model + // extractive budget (12s). 35s here would fail both of those but must pass fallback. const report = buildEvalQualityReport({ generatedAt: "2026-07-13T00:00:00.000Z", retrievalResults: [], ragResults: [ ragResult({ route: "extractive", - latencyRoute: "fast", - latencyMs: 15_000, - generationLatencyMs: 12_000, + latencyRoute: "fallback", + latencyMs: 35_000, + generationLatencyMs: 30_000, model: null, routingReason: "strong_routine_retrieval; generation_fallback:provider_timeout", }), ], }); - expect(report.rag.summary.route_p95_latency_ms).toEqual({ fast: 15_000 }); + expect(report.rag.summary.route_p95_latency_ms).toEqual({ fallback: 35_000 }); expect(report.threshold_failures).not.toEqual( expect.arrayContaining([expect.stringContaining("route extractive")]), ); + expect(report.threshold_failures).not.toEqual(expect.arrayContaining([expect.stringContaining("route fallback")])); }); it("fails forced-embedding retrieval cases that return from cache, coverage, or lexical paths", () => { diff --git a/tests/reindex-eval-gate.test.ts b/tests/reindex-eval-gate.test.ts index 4f79bfbd3..e4af0487f 100644 --- a/tests/reindex-eval-gate.test.ts +++ b/tests/reindex-eval-gate.test.ts @@ -304,10 +304,10 @@ describe("decideReindexGate — quality", () => { baselineRetrieval: retrieval(), candidateRetrieval: retrieval(), baselineQuality: quality(), - candidateQuality: quality({ p95_latency_ms: 26000 }), + candidateQuality: quality({ p95_latency_ms: 61000 }), }); expect(decision.decision).toBe("NO_GO"); - expect(decision.failures.join(" ")).toMatch(/p95_latency_ms 26000 above absolute ceiling 25000/); + expect(decision.failures.join(" ")).toMatch(/p95_latency_ms 61000 above absolute ceiling 60000/); }); it("fails closed when only one side supplies a quality summary", () => { From 4f55a1ca89871015e381ab75b3a0bd4c0d218bb1 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Tue, 14 Jul 2026 02:48:44 +0800 Subject: [PATCH 3/3] fix(eval): classify generation fallbacks before strong-route reasons A strong-classified reason carrying a generation_fallback (e.g. multi_document_comparison_synthesis; generation_fallback:provider_timeout) must budget as a fallback chain, not blend into the strong budget; successful strong generations (including quality retries) never record a generation_fallback, so genuine strong answers are unaffected. Co-Authored-By: Claude Fable 5 --- scripts/eval-quality.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index fbf5bd10b..be41cef9d 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -652,6 +652,15 @@ function ragCaseDiagnosticsTable(results: RagQualityResult[]) { function latencyRouteForAnswer(answer: RagAnswer) { const route = answer.routingMode ?? "none"; if ((answer.latencyTimings?.generation_latency_ms ?? 0) <= 0) return route; + // A generation ran and failed before a source-backed answer was assembled (e.g. + // provider_timeout -> extractive fallback). These chains structurally cost the failed + // generation's timeout PLUS the fallback work, so they get their own latency budget + // instead of inflating the plain fast budget, hiding inside the no-model extractive + // one, or — for strong-classified reasons (e.g. multi_document_comparison_synthesis) — + // blending into the strong budget. This check deliberately outranks the strong + // classifiers below: a successful strong generation (including quality retries) never + // records a generation_fallback, so genuine strong answers keep the strong budget. + if (/\bgeneration_fallback:/i.test(answer.routingReason ?? "")) return "fallback"; if (route === "strong") return "strong"; if ( /^(?:broad_clinical_management_synthesis|clinical_risk_or_complex_query|limited_retrieval_strength|multi_document_comparison_synthesis|retrieval_gap_or_conflict)\b/i.test( @@ -660,11 +669,6 @@ function latencyRouteForAnswer(answer: RagAnswer) { ) { return "strong"; } - // A generation ran and failed before a source-backed answer was assembled (e.g. - // provider_timeout -> extractive fallback). These chains structurally cost the failed - // generation's timeout PLUS the fallback work, so they get their own latency budget - // instead of inflating the plain fast budget or hiding inside the no-model extractive one. - if (/\bgeneration_fallback:/i.test(answer.routingReason ?? "")) return "fallback"; return "fast"; }