From 107826475cdbd164ae1d69245bb737522cfddee5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 00:35:44 +0000 Subject: [PATCH 1/5] feat(rag): budget-aware generation deadlines + retrieval-exhausted telemetry (E-3b core) Implements the E-3b design: generation attempts are clamped to the route budget minus a measured 2s recovery reserve (single call-site covers all four attempt kinds); the truncation self-heal is gated on retry viability (reserve + 5s floor) with an observable truncation_retry_skipped_budget_reserve marker instead of a guaranteed-discard retry; the runtime records an additive route_budget_exhausted_by_retrieval flag; and the eval route-ceiling gate gains the triple-condition cross-region carve-out (context + runtime flag + zero generation) with a retrieval-exhausted audit cell - local/release gates stay strict, pinned by new unit tests. Unit coverage: route-budget 9/9 (constants + reserve/floor boundaries), eval-quality 27/27 (carve-out matrix). Integration tests for the fallback paths follow in the next commit. RAG impact: no retrieval behaviour change - answer-side timing/gating and telemetry only; retrieval, ranking, selection, comparators untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EXsJcLrbZUXwnBeG91cVo9 --- scripts/eval-quality.ts | 37 ++++++++++++++++++---- src/lib/rag/rag-route-budget.ts | 26 +++++++++++++++ src/lib/rag/rag.ts | 21 +++++++++++-- src/lib/types.ts | 3 ++ tests/eval-quality.test.ts | 56 +++++++++++++++++++++++++++++++++ tests/rag-route-budget.test.ts | 48 ++++++++++++++++++++++++++++ 6 files changed, 182 insertions(+), 9 deletions(-) diff --git a/scripts/eval-quality.ts b/scripts/eval-quality.ts index c516cd883..47dd71e86 100644 --- a/scripts/eval-quality.ts +++ b/scripts/eval-quality.ts @@ -89,6 +89,7 @@ export type RagQualityResult = { totalMs: number; routeBudgetMs: number; routeDeadlineExceeded: boolean; + budgetExhaustedByRetrieval?: boolean; }; routeCeilingExceeded?: boolean; estimatedCostUsd: number | null; @@ -150,7 +151,10 @@ export function deliveredGroundedAfterSourceGovernancePolicy( const crossRegionRunnerLatencyContext = process.env.EVAL_LATENCY_CONTEXT === "cross-region-runner"; -export function ragAnswerTimingDiagnostics(answer: Pick) { +export function ragAnswerTimingDiagnostics( + answer: Pick, + options?: { crossRegionRunner?: boolean }, +) { const latencyTimings = answer.latencyTimings; const answerRoute = answer.routingMode ?? "unsupported"; const defaultRouteBudgetMs = answerRouteBudgetMs[answerRoute as AnswerRouteMode] ?? 0; @@ -158,6 +162,14 @@ export function ragAnswerTimingDiagnostics(answer: Pick 0 : totalMs > routeBudgetMs), + routeCeilingExceeded: ceilingExcusedByCrossRegionRetrieval + ? false + : routeDeadlineExceeded || (routeBudgetMs === 0 ? generationMs > 0 : totalMs > routeBudgetMs), }; } @@ -744,7 +759,13 @@ function ragCaseDiagnosticsTable(results: RagQualityResult[]) { result.rpcLatencyMs, result.embeddingLatencyMs, result.timings?.routeBudgetMs, - result.timings?.routeDeadlineExceeded ? "yes" : "no", + // "retrieval-exhausted" keeps cross-region-suppressed ceilings auditable in + // run-over-run comparisons even though they no longer fail the gate. + result.timings?.routeDeadlineExceeded + ? result.timings?.budgetExhaustedByRetrieval + ? "retrieval-exhausted" + : "yes" + : "no", result.model, result.failures.length > 0 ? `failed (${result.failures.length})` : "passed", ] @@ -841,9 +862,13 @@ export function renderEvalQualityMarkdown(report: EvalQualityReport) { item.timings?.generationMs ?? "n/a" }ms verification=${item.timings?.verificationMs ?? "n/a"}ms total=${ item.timings?.totalMs ?? item.latencyMs - }ms budget=${ - item.timings?.routeBudgetMs ?? "n/a" - }ms deadline=${item.timings?.routeDeadlineExceeded ? "yes" : "no"}`, + }ms budget=${item.timings?.routeBudgetMs ?? "n/a"}ms deadline=${ + item.timings?.routeDeadlineExceeded + ? item.timings?.budgetExhaustedByRetrieval + ? "retrieval-exhausted" + : "yes" + : "no" + }`, ) .join("\n"); return `# Retrieval Quality Report diff --git a/src/lib/rag/rag-route-budget.ts b/src/lib/rag/rag-route-budget.ts index 0036f4cc4..9f412bd5a 100644 --- a/src/lib/rag/rag-route-budget.ts +++ b/src/lib/rag/rag-route-budget.ts @@ -7,6 +7,16 @@ export const answerRouteBudgetMs = { strong: 35_000, } as const satisfies Record; +// A generation attempt must never spend the recovery path's share of the route budget. +// Fixed (not a fraction): recovery cost is O(1) — measured 26ms post-retrieval extractive +// (eval run #57 clozapine case) and ~100ms-1s for fallback + verification + logging +// cross-region — so 2s is ~2x the worst observed while costing only 8% of the fast budget. +export const generationRecoveryReserveMs = 2_000; +// A truncation self-heal retry needs at least this much wall time to be worth attempting: +// truncated attempts measure ~20s+ before hitting max_output_tokens, and the strong retry +// spends MORE reasoning under a boosted cap, so anything shorter is a guaranteed-discard. +export const minimumGenerationRetryMs = 5_000; + export class AnswerRouteDeadlineExceededError extends Error { readonly routeMode: AnswerRouteMode; readonly budgetMs: number; @@ -25,10 +35,20 @@ export type AnswerRouteDeadline = { readonly deadlineExceeded: boolean; remainingMs(): number; requestTimeoutMs(maximumMs: number): number; + /** Like requestTimeoutMs, but holds back generationRecoveryReserveMs so the + * source-backed recovery path always fits inside the route budget. */ + generationRequestTimeoutMs(maximumMs: number): number; race(promise: Promise): Promise; dispose(): void; }; +/** True when enough budget remains for a generation retry to plausibly complete + * AND still leave the recovery reserve. Zero-budget routes always return false. */ +export function deadlineAllowsGenerationRetry(deadline: Pick) { + if (deadline.budgetMs <= 0) return false; + return deadline.remainingMs() >= generationRecoveryReserveMs + minimumGenerationRetryMs; +} + export function answerRouteResultCanBeCached(deadline: Pick) { return !deadline.deadlineExceeded; } @@ -87,6 +107,12 @@ export function createAnswerRouteDeadline(args: { throwIfAborted(); return Math.max(1, Math.min(maximumMs, remainingMs())); }, + generationRequestTimeoutMs(maximumMs) { + throwIfAborted(); + // Structurally identical to requestTimeoutMs (same 1ms degenerate floor, same + // zero-budget behavior) minus the recovery reserve. + return Math.max(1, Math.min(maximumMs, remainingMs() - generationRecoveryReserveMs)); + }, race(promise: Promise) { try { throwIfAborted(); diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index 6425d2439..c0ced5bbd 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -167,6 +167,7 @@ import { import { answerRouteResultCanBeCached, createAnswerRouteDeadline, + deadlineAllowsGenerationRetry, isAnswerRouteDeadlineExceeded, } from "@/lib/rag/rag-route-budget"; import { fetchRelatedDocumentMetadata, fetchRelatedDocuments } from "@/lib/document-enrichment"; @@ -3570,11 +3571,16 @@ async function answerQuestionWithScopeUncoalesced( callerSignal: args.signal, startedAt, }); + // True exactly when pre-answer work (dominated by retrieval) consumed the whole route + // budget before any generation could start — e.g. slow cross-region RPC on an extractive + // route. Additive telemetry only; deadlineExceeded semantics are unchanged. + const routeBudgetExhaustedByRetrieval = routeDeadline.budgetMs > 0 && routeDeadline.remainingMs() <= 0; const routeTimingDiagnostics = () => ({ retrieval_latency_ms: searchLatencyMs, routing_latency_ms: routingLatencyMs, route_budget_ms: routeDeadline.budgetMs, route_deadline_exceeded: routeDeadline.deadlineExceeded, + route_budget_exhausted_by_retrieval: routeBudgetExhaustedByRetrieval, }); const finalizeAnswer = (answer: RagAnswer, numericVerificationSources?: SearchResult[]) => { const verificationStartedAt = Date.now(); @@ -4059,7 +4065,10 @@ ${qualityRetryInstruction}` schemaName: "clinical_rag_answer", instructions: answerInstructions, promptCacheKey: ragAnswerPromptVersion, - timeoutMs: routeDeadline.requestTimeoutMs(env.OPENAI_ANSWER_TIMEOUT_MS), + // Reserve-aware: an attempt may never spend the recovery path's share of the + // route budget (eval run #57: a provider timeout consumed the whole fallback + // budget and the successful recovery still missed the route ceiling by 54ms). + timeoutMs: routeDeadline.generationRequestTimeoutMs(env.OPENAI_ANSWER_TIMEOUT_MS), maxRetries: 0, reasoningEffort: useStrongReasoning ? strongReasoningEffortForQueryClass(queryClass, env.OPENAI_STRONG_REASONING_EFFORT) @@ -4261,8 +4270,14 @@ ${qualityRetryInstruction}` }); // Adopted from main: retry truncation once for BOTH fast- and strong-routed first attempts // (previously fast-only), keyed on route.mode rather than model identity so it stays correct - // when the tiers share a model. - if (generated.truncated && !retriedWithStrong) { + // when the tiers share a model. Budget-gated (E-3b): truncated attempts measure ~20s+ + // before hitting max_output_tokens, so a retry into a nearly-spent budget is a + // guaranteed-discard — skip it and let the existing source-backed recovery deliver. + if (generated.truncated && !retriedWithStrong && !deadlineAllowsGenerationRetry(routeDeadline)) { + answerRetryReasons.push( + `truncation_retry_skipped_budget_reserve:${generationRetryReason(route.mode === "fast" ? "fast" : "strong", generated)}`, + ); + } else if (generated.truncated && !retriedWithStrong) { const retryPrefix = route.mode === "fast" ? "fast" : "strong"; const retryReason = `${generationRetryReason(retryPrefix, generated)}_retry_strong`; answerRetryCount += 1; diff --git a/src/lib/types.ts b/src/lib/types.ts index 353720173..913f12f52 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -1009,6 +1009,9 @@ export type RagAnswer = { verification_latency_ms?: number; route_budget_ms?: number; route_deadline_exceeded?: boolean; + /** Pre-answer work (dominated by retrieval) consumed the entire route budget before + * generation could start. Additive; optional for mixed-version telemetry compat. */ + route_budget_exhausted_by_retrieval?: boolean; total_latency_ms?: number; }; openAIRequestIds?: string[]; diff --git a/tests/eval-quality.test.ts b/tests/eval-quality.test.ts index 7c4ece94f..2161ede72 100644 --- a/tests/eval-quality.test.ts +++ b/tests/eval-quality.test.ts @@ -740,3 +740,59 @@ describe("eval quality reporting", () => { expect(retrievalCasesForProviderMode(cases, "openai")).toEqual(cases); }); }); + +describe("cross-region retrieval-exhausted carve-out (E-3b)", () => { + const exhaustedTimings = { + total_latency_ms: 13_352, + generation_latency_ms: 0, + route_budget_ms: 12_000, + route_deadline_exceeded: true, + route_budget_exhausted_by_retrieval: true, + }; + + it("keeps the strict gate in local/release contexts even when the flag is set", () => { + expect( + ragAnswerTimingDiagnostics({ routingMode: "extractive", latencyTimings: exhaustedTimings }).routeCeilingExceeded, + ).toBe(true); + }); + + it("suppresses the ceiling only for cross-region + runtime flag + zero generation", () => { + expect( + ragAnswerTimingDiagnostics( + { routingMode: "extractive", latencyTimings: exhaustedTimings }, + { crossRegionRunner: true }, + ).routeCeilingExceeded, + ).toBe(false); + }); + + it("still fails cross-region when the runtime flag is absent", () => { + expect( + ragAnswerTimingDiagnostics( + { + routingMode: "extractive", + latencyTimings: { ...exhaustedTimings, route_budget_exhausted_by_retrieval: false }, + }, + { crossRegionRunner: true }, + ).routeCeilingExceeded, + ).toBe(true); + }); + + it("still fails cross-region when generation consumed time", () => { + expect( + ragAnswerTimingDiagnostics( + { + routingMode: "extractive", + latencyTimings: { ...exhaustedTimings, generation_latency_ms: 1 }, + }, + { crossRegionRunner: true }, + ).routeCeilingExceeded, + ).toBe(true); + }); + + it("carries the flag through timings for report auditing", () => { + expect( + ragAnswerTimingDiagnostics({ routingMode: "extractive", latencyTimings: exhaustedTimings }).timings + .budgetExhaustedByRetrieval, + ).toBe(true); + }); +}); diff --git a/tests/rag-route-budget.test.ts b/tests/rag-route-budget.test.ts index 5d27714bc..739be844b 100644 --- a/tests/rag-route-budget.test.ts +++ b/tests/rag-route-budget.test.ts @@ -5,6 +5,9 @@ import { answerRouteResultCanBeCached, answerRouteBudgetMs, createAnswerRouteDeadline, + deadlineAllowsGenerationRetry, + generationRecoveryReserveMs, + minimumGenerationRetryMs, } from "../src/lib/rag/rag-route-budget"; afterEach(() => { @@ -64,3 +67,48 @@ describe("RAG route deadlines", () => { deadline.dispose(); }); }); + +describe("budget-aware generation deadlines (E-3b)", () => { + it("pins the reserve and retry-floor constants", () => { + expect(generationRecoveryReserveMs).toBe(2_000); + expect(minimumGenerationRetryMs).toBe(5_000); + }); + + it("holds back the recovery reserve from generation timeouts", async () => { + vi.useFakeTimers(); + const deadline = createAnswerRouteDeadline({ routeMode: "fast", startedAt: Date.now() }); + await vi.advanceTimersByTimeAsync(10_000); + // 15_000 remaining: plain requestTimeoutMs grants it all, the generation + // variant holds back the 2_000ms recovery reserve. + expect(deadline.requestTimeoutMs(30_000)).toBe(15_000); + expect(deadline.generationRequestTimeoutMs(30_000)).toBe(13_000); + deadline.dispose(); + }); + + it("floors at 1ms when the reserve exceeds remaining budget", async () => { + vi.useFakeTimers(); + const deadline = createAnswerRouteDeadline({ routeMode: "extractive", startedAt: Date.now() }); + await vi.advanceTimersByTimeAsync(answerRouteBudgetMs.extractive - 1_500); + expect(deadline.generationRequestTimeoutMs(30_000)).toBe(1); + deadline.dispose(); + }); + + it("gates generation retries on reserve + viability floor", async () => { + vi.useFakeTimers(); + const deadline = createAnswerRouteDeadline({ routeMode: "fast", startedAt: Date.now() }); + // 25_000 budget: allowed until remaining < 7_000 (2_000 reserve + 5_000 floor). + expect(deadlineAllowsGenerationRetry(deadline)).toBe(true); + await vi.advanceTimersByTimeAsync(18_000); + expect(deadline.remainingMs()).toBe(7_000); + expect(deadlineAllowsGenerationRetry(deadline)).toBe(true); + await vi.advanceTimersByTimeAsync(1); + expect(deadlineAllowsGenerationRetry(deadline)).toBe(false); + deadline.dispose(); + }); + + it("never allows generation retries on zero-budget routes", () => { + const deadline = createAnswerRouteDeadline({ routeMode: "unsupported", startedAt: Date.now() }); + expect(deadlineAllowsGenerationRetry(deadline)).toBe(false); + deadline.dispose(); + }); +}); From 314d03f0ec22cbae78d07df1925f887a4381400a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 00:47:32 +0000 Subject: [PATCH 2/5] =?UTF-8?q?test(rag):=20E-3b=20integration=20coverage?= =?UTF-8?q?=20=E2=80=94=20budget=20cap=20red-proof,=20self-heal=20gate,=20?= =?UTF-8?q?retrieval-exhausted=20flag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five fake-timer integration tests: the reserve-aware timeout is pinned three independent ways (exact 23000ms grant, deadline flag stays clear, total under budget - a revert to requestTimeoutMs fails all three); the truncation self-heal skip pins the exact answer_retry_reasons marker with no retry-count increment and provider called once; the offline harness pins route_budget_exhausted_by_retrieval both true (13s retrieval on a 12s extractive budget, provider never called, grounded extractive answer) and false (fast retrieval). 52/52 across both files. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EXsJcLrbZUXwnBeG91cVo9 --- tests/rag-answer-fallback.test.ts | 264 ++++++++++++++++++++++++++++++ tests/rag-offline-answer.test.ts | 53 ++++++ 2 files changed, 317 insertions(+) diff --git a/tests/rag-answer-fallback.test.ts b/tests/rag-answer-fallback.test.ts index be474b5ab..ccb49dbbe 100644 --- a/tests/rag-answer-fallback.test.ts +++ b/tests/rag-answer-fallback.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { answerRouteBudgetMs, generationRecoveryReserveMs } from "../src/lib/rag/rag-route-budget"; import type { RagAnswer, SearchResult } from "../src/lib/types"; function retrievalRpcBaseName(name: string) { @@ -2969,3 +2970,266 @@ describe("RAG structured-output fallback", () => { expect(search.telemetry.retrieval_strategy).toBe("hybrid"); }); }); + +describe("budget-aware generation deadlines", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + /** Mirrors the "recovers lithium dosing" fixture above: a fast-routed dose query whose + * every generation attempt resolves truncated (max_output_tokens). The first attempt + * optionally burns fake wall-clock before resolving, so the remaining route budget can + * be pushed below the recovery reserve + retry viability floor. */ + async function lithiumTruncatedGenerationAnswer(consumeFirstAttemptMs: number) { + vi.stubEnv("OPENAI_API_KEY", "test-key"); + vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0"); + vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0"); + + const waSource = (overrides: Partial, publisherCode: "FSH" | "EMHS") => + source({ + ...overrides, + source_metadata: { + source_title: overrides.title ?? "Lithium guideline", + publisher: + publisherCode === "FSH" ? "Fiona Stanley Fremantle Hospitals Group" : "East Metropolitan Health Service", + publisher_code: publisherCode, + jurisdiction: "Australia/WA", + version: "1", + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "current", + clinical_validation_status: "locally_reviewed", + extraction_quality: "good", + }, + }); + const sources = [ + waSource( + { + id: "fsh-lithium-1", + document_id: "fsh-lithium", + title: "Lithium Therapy - Initiation and Continuation Guideline", + section_heading: "Initiation", + content: + "For lithium initiation in adults, start lithium carbonate at 250 mg at night and titrate according to the serum lithium concentration.", + }, + "FSH", + ), + waSource( + { + id: "emhs-lithium-1", + document_id: "emhs-lithium", + title: "Lithium Clinical Guideline", + section_heading: "Target range", + content: + "The usual target serum lithium concentration is 0.6 to 0.8 mmol/L for maintenance treatment in adults.", + }, + "EMHS", + ), + waSource( + { + id: "fsh-lithium-2", + document_id: "fsh-lithium", + title: "Lithium Therapy - Initiation and Continuation Guideline", + section_heading: "Monitoring after dose changes", + content: + "Measure the serum lithium concentration 12 hours after the previous dose and repeat it 5 to 7 days after a dose change.", + }, + "FSH", + ), + waSource( + { + id: "emhs-lithium-2", + document_id: "emhs-lithium", + title: "Lithium Clinical Guideline", + section_heading: "Dose adjustment", + content: + "Use a lower lithium starting dose in older adults and people with impaired renal function, with closer serum monitoring.", + }, + "EMHS", + ), + source({ + id: "bmj-paediatric-depression", + document_id: "bmj-paediatric-depression", + title: "Depression in children", + content: "Psychological therapy is considered for depression in children and young people.", + source_metadata: { + source_title: "Depression in children", + publisher: "BMJ Best Practice", + publisher_code: "BMJ", + jurisdiction: "International", + version: null, + publication_date: null, + review_date: null, + uploaded_at: null, + indexed_at: null, + uploaded_by: null, + document_status: "current", + clinical_validation_status: "unverified", + extraction_quality: "good", + }, + }), + ]; + const rpc = vi.fn(async (name: string) => { + if (retrievalRpcBaseName(name) === "match_document_chunks_text") return { data: sources, error: null }; + if (retrievalRpcBaseName(name) === "get_related_document_metadata") return { data: [], error: null }; + return { data: [], error: null }; + }); + let requestIndex = 0; + const generateStructuredTextResult = vi.fn(async () => { + requestIndex += 1; + if (requestIndex === 1 && consumeFirstAttemptMs > 0) { + vi.setSystemTime(new Date(Date.now() + consumeFirstAttemptMs)); + } + return { + text: "", + model: "gpt-5.4-mini", + operation: "answer", + latencyMs: 12, + requestId: `req_truncated_${requestIndex}`, + usage: { input_tokens: 100, output_tokens: 650, total_tokens: 750 }, + status: "incomplete", + truncated: true, + incompleteReason: "max_output_tokens", + }; + }); + + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ + rpc, + from: vi.fn(() => new EmptyQuery()), + }), + })); + vi.doMock("@/lib/openai", () => ({ + embedTextWithTelemetry: vi.fn(async () => ({ embedding: [0.1, 0.2, 0.3], cacheHit: false })), + generateStructuredTextResult, + })); + + const { answerQuestionWithScope } = await import("../src/lib/rag/rag"); + const answer = await answerQuestionWithScope({ + query: "Lithium dosing", + ownerId: undefined, + logQuery: false, + skipCache: true, + }); + return { answer, generateStructuredTextResult }; + } + + it("caps a generation attempt so source-backed recovery fits inside the route budget", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-14T00:00:00.000Z")); + vi.stubEnv("OPENAI_API_KEY", "test-key"); + // Stubbed far above the 25_000ms fast-route budget so the granted timeout can only + // come from the deadline: budget - 2_000ms recovery reserve = 23_000ms. Reverting the + // call site to the reserve-free requestTimeoutMs would grant the full 25_000ms. + vi.stubEnv("OPENAI_ANSWER_TIMEOUT_MS", "60000"); + vi.stubEnv("RAG_SEARCH_CACHE_TTL_MS", "0"); + vi.stubEnv("RAG_ANSWER_CACHE_TTL_MS", "0"); + + // Same retrieval fixture as the model-synthesis test above, which routes fast. + const clozapineSource = source({ + id: "clozapine-monitoring-1", + document_id: "clozapine-doc", + title: "Medication guideline", + file_name: "medication-guideline.pdf", + page_number: 11, + section_heading: "Monitoring", + content: + "Medication point: • Copy of the Consent to Clozapine Treatment Form EMR0270. Medication point: • Prescribe initiation of Clozapine on the WA Adult Clozapine Initiation and Titration form. Medication point: • Ensure consumers complete the Clozapine Monitoring Form on initiation.", + similarity: 0.94, + hybrid_score: 0.94, + text_rank: 0, + }); + const rpc = vi.fn(async (name: string) => { + if (retrievalRpcBaseName(name) === "match_document_chunks_text") return { data: [clozapineSource], error: null }; + if (retrievalRpcBaseName(name) === "get_related_document_metadata") return { data: [], error: null }; + return { data: [], error: null }; + }); + vi.doMock("@/lib/supabase/admin", () => ({ + createAdminClient: () => ({ + rpc, + from: vi.fn(() => new EmptyQuery()), + }), + })); + const grantedTimeoutsMs: number[] = []; + const generateStructuredTextResult = vi.fn( + async (_input: string, _schema: unknown, options?: { timeoutMs?: number }) => { + grantedTimeoutsMs.push(options?.timeoutMs ?? Number.NaN); + // Consume the entire granted window, then fail like a provider timeout. With the + // reserve subtracted this leaves 2_000ms of route budget for the recovery path; + // without it, recovery would start with the budget already fully spent. + vi.setSystemTime(new Date(Date.now() + (options?.timeoutMs ?? 0))); + throw new Error("OpenAI timed out. Trying source-only fallback response."); + }, + ); + vi.doMock("@/lib/openai", () => ({ + embedTextWithTelemetry: vi.fn(async () => ({ embedding: [0.1, 0.2, 0.3], cacheHit: false })), + generateStructuredTextResult, + })); + + const { answerQuestionWithScope } = await import("../src/lib/rag/rag"); + const answer = await answerQuestionWithScope({ + query: "what monitoring is required for clozapine", + ownerId: undefined, + logQuery: false, + skipCache: true, + }); + + // Fake timers pin elapsed-at-call to exactly 0ms, so the received timeout must equal + // budget - reserve. This is the assertion that fails if generationRequestTimeoutMs is + // reverted to requestTimeoutMs at the generation call site. + expect(generateStructuredTextResult).toHaveBeenCalledTimes(1); + expect(grantedTimeoutsMs).toEqual([answerRouteBudgetMs.fast - generationRecoveryReserveMs]); + expect(answer.latencyTimings?.route_budget_ms).toBe(answerRouteBudgetMs.fast); + // The attempt used its whole window, yet the reserve kept the source-backed recovery + // inside the route budget. + expect(answer.latencyTimings?.route_deadline_exceeded).toBe(false); + expect(answer.latencyTimings?.total_latency_ms).toBeLessThan(answerRouteBudgetMs.fast); + expect(answer.routingReason).toContain("generation_fallback:provider_timeout"); + expect(answer.sources.length).toBeGreaterThan(0); + }); + + it("skips the truncation self-heal when the budget reserve would be breached", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-14T00:00:00.000Z")); + // Burn 20_000ms of the 25_000ms fast budget inside the first attempt before it + // resolves truncated: the 5_000ms left is below generationRecoveryReserveMs + + // minimumGenerationRetryMs (7_000ms), so the strong self-heal must be skipped + // instead of spending the recovery reserve on a guaranteed-discard retry. + const { answer, generateStructuredTextResult } = await lithiumTruncatedGenerationAnswer(20_000); + + expect(generateStructuredTextResult).toHaveBeenCalledTimes(1); + // The skip is recorded without counting as a retry; the terminal truncation throw + // then lands on the existing source-backed recovery. + expect(answer.latencyTimings?.answer_retry_reasons).toEqual([ + "truncation_retry_skipped_budget_reserve:fast_max_output_tokens", + "generation_max_output_tokens", + ]); + expect(answer.latencyTimings?.answer_retry_count).toBe(1); + expect(answer.routingReason).toContain("generation_fallback:provider_incomplete_max_output_tokens"); + expect(answer.routingReason).toContain("source_backed_extractive_fallback"); + expect(answer.routingMode).toBe("extractive"); + expect(answer.grounded).toBe(true); + expect(answer.confidence).not.toBe("unsupported"); + expect(answer.citations.length).toBeGreaterThan(0); + expect(answer.answer.replace(/\*\*/g, "")).toMatch(/lithium|250 mg/i); + }); + + it("keeps the truncation self-heal when the budget reserve still fits", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-14T00:00:00.000Z")); + // Identical truncated first attempt with no wall-clock consumed: the full budget + // remains, so the strong self-heal retry must still run. + const { answer, generateStructuredTextResult } = await lithiumTruncatedGenerationAnswer(0); + + expect(generateStructuredTextResult).toHaveBeenCalledTimes(2); + expect(answer.latencyTimings?.answer_retry_reasons).toEqual([ + "fast_max_output_tokens_retry_strong", + "strong_max_output_tokens", + ]); + expect(answer.latencyTimings?.answer_retry_count).toBe(2); + expect(answer.routingReason).toContain("source_backed_extractive_fallback"); + }); +}); diff --git a/tests/rag-offline-answer.test.ts b/tests/rag-offline-answer.test.ts index e56d74a0e..f1c0355a4 100644 --- a/tests/rag-offline-answer.test.ts +++ b/tests/rag-offline-answer.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { answerRouteBudgetMs } from "../src/lib/rag/rag-route-budget"; import type { SearchResult } from "../src/lib/types"; function source(overrides: Partial = {}): SearchResult { @@ -240,3 +241,55 @@ describe("source-only / offline answers", () => { expect(answer.citations.map((citation) => citation.chunk_id)).toEqual(["clozapine-chunk-1", "clozapine-chunk-2"]); }); }); + +describe("route budget consumed by retrieval", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("flags route budget exhausted by retrieval", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-14T00:00:00.000Z")); + // Retrieval burns 13_000ms of fake wall-clock before returning sources — more than + // the whole 12_000ms extractive route budget — so the route deadline is created with + // its budget already fully consumed by pre-generation work. + let retrievalDelayApplied = false; + const { answer, generateStructuredTextResult } = await answerOffline( + "What ANC threshold should withhold clozapine?", + [], + (name) => { + if (name === "match_document_chunks_text_v2" || name === "match_document_chunks_text") { + if (!retrievalDelayApplied) { + retrievalDelayApplied = true; + vi.setSystemTime(new Date(Date.now() + answerRouteBudgetMs.extractive + 1_000)); + } + return { data: [source()], error: null }; + } + return { data: [], error: null }; + }, + ); + + expect(generateStructuredTextResult).not.toHaveBeenCalled(); + expect(answer.latencyTimings?.route_budget_ms).toBe(answerRouteBudgetMs.extractive); + expect(answer.latencyTimings?.route_budget_exhausted_by_retrieval).toBe(true); + expect(answer.latencyTimings?.route_deadline_exceeded).toBe(true); + // Budget exhaustion is additive telemetry, not a failure: the answer must still be a + // grounded extractive answer built without any provider call. + expect(answer.routingMode).toBe("extractive"); + expect(answer.grounded).toBe(true); + expect(answer.confidence).not.toBe("unsupported"); + expect(answer.citations.map((citation) => citation.chunk_id)).toContain("clozapine-chunk-1"); + }); + + it("keeps the exhaustion flag false for fast retrieval", async () => { + const { answer, generateStructuredTextResult } = await answerOffline( + "What ANC threshold should withhold clozapine?", + [source()], + ); + + expect(generateStructuredTextResult).not.toHaveBeenCalled(); + expect(answer.latencyTimings?.route_budget_exhausted_by_retrieval).toBe(false); + expect(answer.latencyTimings?.route_deadline_exceeded).toBe(false); + expect(answer.grounded).toBe(true); + }); +}); From eb08ec5476989ba268f9bd2290be82f4884fcd0c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 00:56:45 +0000 Subject: [PATCH 3/5] docs(ledger): clinical-governance review row for the E-3b diff (APPROVE-WITH-NITS) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EXsJcLrbZUXwnBeG91cVo9 --- docs/branch-review-ledger.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index b369ea87a..117896f32 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -666,3 +666,4 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-20 | claude/clinical-kb-pwa-review-asi3wb (restarted from 5b4098d; PR: E-2 baseline record + instrument fix) | see PR head | ADDENDUM Phase E-2 BASELINE BANKED (canary dispatch #57, run 29786560936, main 5b4098d, answer_case_limit=44): golden retrieval 36/36 green in-run (no-regression net held); eval:quality full-44 RED on exactly two gates — citation_failure_rate 0.0227 (1/44: neuroleptic-side-effect-escalation — expected doc never retrieved, generation quality-failed, extractive fallback with 1 citation) and route_ceiling_failure_count 2 (clozapine-anc-withhold-threshold: 13.3s pure retrieval vs 12s extractive budget, RPC 9.3s, zero generation; agitation-arousal-typo-dosing: 25054ms vs 25000ms after provider_timeout ate 22.6s pre-recovery). Green gates: grounded_supported 1.0, unsupported_correct 1.0 (all 14 refusals incl. both prompt-injection probes at 2ms), numeric grounding failures 0, governance danger 0, p95 17.4s. Non-blocking signals: expected_source_hit 0.6136 (both admission-discharge cases miss MHSP.AdmissionCommunityPts.pdf to sibling NMHS/RKPG policies — labeling-vs-ranking question, §3.1 class), source governance warning rate 0.8182 (metadata debt, waivable class). SYSTEMIC E-3 TARGET: ~9 of 19 generation attempts discarded (fast output fails quality gate → extractive fallback wins; ~7 cases carry fast_quality_retry_strong→extractive reasons) = ~half of generation latency+spend wasted. Instrument defect found+fixed this PR: targeting step was skipped after the red gate (GitHub failure-skip semantics) → if: gains !cancelled() so baselines observe red gates. Spend: est ~$1-2 actual (19 OpenAI-request cases; cost rates unset in CI so report shows n/a) of ≤$20 Phase E envelope. Next: cheap re-dispatch (default limit 8 + answer_quality_eval=true) to bank the skipped 30-case targeting baseline. | Gates this PR: check:github-actions PASS; check:ci-scope PASS; check:gate-manifest PASS; prettier clean. Baseline evidence: job log run 29786560936 (5 failing-case diagnostics + Answer Metrics table read in full) | | 2026-07-21 | claude/clinical-kb-pwa-review-asi3wb (restarted from 87815f4; PR: E-3a cost self-reporting) | see PR head + follow-up pin | Phase E-3a wave (all $0 diagnostics complete, recorded here) + I8 fix: eval-canary env gains the three RAG_EVAL_*_USD_PER_MILLION rates (gpt-5.6-terra standard tier, verified from the live OpenAI pricing page 2026-07-21; documented as a LOWER BOUND since gpt-5.6-sol strong retries cost 2x and the estimator applies one rate set) so estimated_cost_usd stops reading n/a in CI. E-3a findings: (1) discarded-generation histogram from run #57 — fast_quality_retry_strong→extractive x6, generation_quality_failed x4, provider_timeout x2; gate sub-reasons fragment_like x2, bad_final_answer_quality x1, ungrounded_extractive_fallback x1 → E-3c targets the fast-attempt-fails-extractive-wins shape first. (2) I2 resolved: routeCeilingExceeded (eval-quality.ts:157/171) keys on the RUNTIME's own route_budget_ms + deadline flag, NOT the cross-region-widened eval gates — clozapine case = 13.3s retrieval vs the runtime's 12s extractive budget, geography-amplified; design → E-3b. (3) FIXTURE-CORPUS DRIFT (major): the live corpus contains ZERO MHSP*-named files; the 44-case fixture's expectedFiles are all MHSP.*/CG.MHSP.* and pass only via the eval-document-matching alias tier. All three expected-doc failures triaged: neuroleptic-side-effect-escalation = REAL retrieval-coverage gap (alias exists; Neuroleptic Side Effects (AKG).pdf indexed 11 chunks, 3 contain escalat%, yet answer retrieval returned a single off-point source) → protected-path item awaiting user go-ahead; admission-discharge x2 = top-5 preference for sibling NMHS/RKPG policies over the existing aliased AKG doc → labeling-vs-ranking decision presented to user. No fixture/alias edits made (ground truth requires sign-off). | check:github-actions PASS; check:ci-scope PASS; check:gate-manifest PASS; prettier clean; Supabase access read-only (documents/document_chunks SELECTs); OpenAI pricing page read via WebFetch | | 2026-07-21 | claude/clinical-kb-pwa-review-asi3wb (restarted from c308fdc; PR: labeling widen) | see PR head | USER-APPROVED ground-truth widening (AskUserQuestion 2026-07-21: "Widen to accept siblings") for the two admission-discharge cases: the WIDER alias tier's AdmissionCommunityPts entry gains the two NMHS "Admission to Discharge" titles (Community Mental Health + Mental Health Inpatients). Deliberate exclusions documented in-code: discharge-only docs must not satisfy the admission slot; the MHHITH programme policy is too narrow. STRICT golden tier untouched (bulk-merge prohibition respected). Verified by replaying run #57's actual top-5 lists through expectedFileCoverage: admission-discharge-coverage-paraphrase now passes; admission-discharge-comparison STILL FAILS honestly (its top-5 carries no admission-side doc at all — dup discharge-planning + Falls Prevention) and is retained as a genuine comparison-class retrieval-coverage signal, folded into the same investigation as the neuroleptic case. | typecheck PASS; prettier clean; probe script replay recorded (paraphrase allHit true / comparison allHit false, missing admission slot); no retrieval code touched; protected-file edit (eval-document-matching.ts) per explicit user authorization | +| 2026-07-21 | claude/clinical-kb-pwa-review-asi3wb (PR: E-3b budget-aware generation deadlines) | 314d03f | Clinical-governance review of E-3b diff `git diff origin/main...HEAD` (commits 1078264 + 314d03f): answer-side generation timing/gating + telemetry — reserve-aware generation timeout (`generationRequestTimeoutMs`), truncation self-heal budget gate (`deadlineAllowsGenerationRetry`), `route_budget_exhausted_by_retrieval` telemetry, and the cross-region eval carve-out in scripts/eval-quality.ts. | APPROVE-WITH-NITS. No P0/P1/P2. (a) Conservative failure preserved: reserve-aware timeout fires ~2s early producing the SAME error type — an internal SDK timeout is mapped by mapOpenAIError to PublicApiError(openai_timeout) (openai.ts:525-529), NOT a bare DOMException, so the rag.ts:4636 re-throw guard is not tripped and the existing source-backed fallback/extractive recovery is reached; truncation-skip falls through to the terminal throw (rag.ts:4309-4313) into the same catch. No new answer-producing path. (b) Safety gates intact: all recovery answers finalize through finalizeAnswer→finalizeRagAnswerQuality and isSafeExtractiveFallbackCandidate (grounded/confidence/quality/numeric) — none touched. (c) Eval carve-out env-gated: crossRegionRunner && budgetExhaustedByRetrieval && generationMs===0; EVAL_LATENCY_CONTEXT set only in eval-canary.yml:164, prod caller (eval-quality.ts:1095) passes no options → inert in release/local; generationMs===0 requirement means a generation-side failure (generationMs>0) is never suppressed. (d)/(e) Telemetry additions non-PHI (boolean + mechanical retry-reason strings); no privacy/query-privacy/cross-border/verification source files touched; no new provider call. Nits (P3, non-blocking): carve-out also excuses fast/strong routes when generationMs===0 (sound — provably no generation ran); `requestTimeoutMs` now prod-dead (test-only); report label "retrieval-exhausted" (routeDeadlineExceeded && flag) is broader than actual gate suppression (audit label only, gate stays strict). | 88 offline unit tests PASS (tests/rag-route-budget.test.ts, tests/eval-quality.test.ts, tests/rag-offline-answer.test.ts, tests/rag-answer-fallback.test.ts); no provider/Supabase/OpenAI calls; no files mutated except this ledger row | From 56c5d2cb27c63d39c360137f1181dda4edf05830 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 00:58:06 +0000 Subject: [PATCH 4/5] docs(ledger): E-3b implementation row + run #58 targeting baseline bank Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EXsJcLrbZUXwnBeG91cVo9 --- docs/branch-review-ledger.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index 117896f32..a5612feed 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -667,3 +667,4 @@ Use this ledger to prevent repeated branch and PR reviews when the reviewed HEAD | 2026-07-21 | claude/clinical-kb-pwa-review-asi3wb (restarted from 87815f4; PR: E-3a cost self-reporting) | see PR head + follow-up pin | Phase E-3a wave (all $0 diagnostics complete, recorded here) + I8 fix: eval-canary env gains the three RAG_EVAL_*_USD_PER_MILLION rates (gpt-5.6-terra standard tier, verified from the live OpenAI pricing page 2026-07-21; documented as a LOWER BOUND since gpt-5.6-sol strong retries cost 2x and the estimator applies one rate set) so estimated_cost_usd stops reading n/a in CI. E-3a findings: (1) discarded-generation histogram from run #57 — fast_quality_retry_strong→extractive x6, generation_quality_failed x4, provider_timeout x2; gate sub-reasons fragment_like x2, bad_final_answer_quality x1, ungrounded_extractive_fallback x1 → E-3c targets the fast-attempt-fails-extractive-wins shape first. (2) I2 resolved: routeCeilingExceeded (eval-quality.ts:157/171) keys on the RUNTIME's own route_budget_ms + deadline flag, NOT the cross-region-widened eval gates — clozapine case = 13.3s retrieval vs the runtime's 12s extractive budget, geography-amplified; design → E-3b. (3) FIXTURE-CORPUS DRIFT (major): the live corpus contains ZERO MHSP*-named files; the 44-case fixture's expectedFiles are all MHSP.*/CG.MHSP.* and pass only via the eval-document-matching alias tier. All three expected-doc failures triaged: neuroleptic-side-effect-escalation = REAL retrieval-coverage gap (alias exists; Neuroleptic Side Effects (AKG).pdf indexed 11 chunks, 3 contain escalat%, yet answer retrieval returned a single off-point source) → protected-path item awaiting user go-ahead; admission-discharge x2 = top-5 preference for sibling NMHS/RKPG policies over the existing aliased AKG doc → labeling-vs-ranking decision presented to user. No fixture/alias edits made (ground truth requires sign-off). | check:github-actions PASS; check:ci-scope PASS; check:gate-manifest PASS; prettier clean; Supabase access read-only (documents/document_chunks SELECTs); OpenAI pricing page read via WebFetch | | 2026-07-21 | claude/clinical-kb-pwa-review-asi3wb (restarted from c308fdc; PR: labeling widen) | see PR head | USER-APPROVED ground-truth widening (AskUserQuestion 2026-07-21: "Widen to accept siblings") for the two admission-discharge cases: the WIDER alias tier's AdmissionCommunityPts entry gains the two NMHS "Admission to Discharge" titles (Community Mental Health + Mental Health Inpatients). Deliberate exclusions documented in-code: discharge-only docs must not satisfy the admission slot; the MHHITH programme policy is too narrow. STRICT golden tier untouched (bulk-merge prohibition respected). Verified by replaying run #57's actual top-5 lists through expectedFileCoverage: admission-discharge-coverage-paraphrase now passes; admission-discharge-comparison STILL FAILS honestly (its top-5 carries no admission-side doc at all — dup discharge-planning + Falls Prevention) and is retained as a genuine comparison-class retrieval-coverage signal, folded into the same investigation as the neuroleptic case. | typecheck PASS; prettier clean; probe script replay recorded (paraphrase allHit true / comparison allHit false, missing admission slot); no retrieval code touched; protected-file edit (eval-document-matching.ts) per explicit user authorization | | 2026-07-21 | claude/clinical-kb-pwa-review-asi3wb (PR: E-3b budget-aware generation deadlines) | 314d03f | Clinical-governance review of E-3b diff `git diff origin/main...HEAD` (commits 1078264 + 314d03f): answer-side generation timing/gating + telemetry — reserve-aware generation timeout (`generationRequestTimeoutMs`), truncation self-heal budget gate (`deadlineAllowsGenerationRetry`), `route_budget_exhausted_by_retrieval` telemetry, and the cross-region eval carve-out in scripts/eval-quality.ts. | APPROVE-WITH-NITS. No P0/P1/P2. (a) Conservative failure preserved: reserve-aware timeout fires ~2s early producing the SAME error type — an internal SDK timeout is mapped by mapOpenAIError to PublicApiError(openai_timeout) (openai.ts:525-529), NOT a bare DOMException, so the rag.ts:4636 re-throw guard is not tripped and the existing source-backed fallback/extractive recovery is reached; truncation-skip falls through to the terminal throw (rag.ts:4309-4313) into the same catch. No new answer-producing path. (b) Safety gates intact: all recovery answers finalize through finalizeAnswer→finalizeRagAnswerQuality and isSafeExtractiveFallbackCandidate (grounded/confidence/quality/numeric) — none touched. (c) Eval carve-out env-gated: crossRegionRunner && budgetExhaustedByRetrieval && generationMs===0; EVAL_LATENCY_CONTEXT set only in eval-canary.yml:164, prod caller (eval-quality.ts:1095) passes no options → inert in release/local; generationMs===0 requirement means a generation-side failure (generationMs>0) is never suppressed. (d)/(e) Telemetry additions non-PHI (boolean + mechanical retry-reason strings); no privacy/query-privacy/cross-border/verification source files touched; no new provider call. Nits (P3, non-blocking): carve-out also excuses fast/strong routes when generationMs===0 (sound — provably no generation ran); `requestTimeoutMs` now prod-dead (test-only); report label "retrieval-exhausted" (routeDeadlineExceeded && flag) is broader than actual gate suppression (audit label only, gate stays strict). | 88 offline unit tests PASS (tests/rag-route-budget.test.ts, tests/eval-quality.test.ts, tests/rag-offline-answer.test.ts, tests/rag-answer-fallback.test.ts); no provider/Supabase/OpenAI calls; no files mutated except this ledger row | +| 2026-07-21 | claude/clinical-kb-pwa-review-asi3wb (PR: E-3b budget-aware generation deadlines) | 314d03f (+eb08ec5 review row) | ADDENDUM 5 wave E-3b implemented per the design-agent plan: generation attempts clamped to route budget minus a measured 2s recovery reserve (single call-site, all four attempt kinds); truncation self-heal gated on retry viability (reserve+5s floor) with observable truncation_retry_skipped_budget_reserve marker; additive route_budget_exhausted_by_retrieval runtime flag; eval route-ceiling gains the triple-condition cross-region carve-out (context + runtime flag + zero generation) with retrieval-exhausted audit cells — local/release gates provably strict. Fixes I3 (54ms budget overrun after 22.6s provider timeout), I5 (82s truncation waste class), resolves I2 (clozapine 13.3s retrieval vs 12s runtime budget = geography, now suppressed ONLY in the sanctioned cross-region context with full auditability). Reviewer verdicts: rag-retrieval-reviewer APPROVE-WITH-NITS (2 P3: prod-dead requestTimeoutMs retained for symmetry; report-cell coupling cosmetic; cached-replay invariant PROVEN — budget-exhausted answers never cached, carve-out unreachable via replay; marker isolation proven — SLO counters key on fallback_reason not answer_retry_reasons); clinical-governance-reviewer APPROVE-WITH-NITS (prior row) — internal-timeout→PublicApiError→existing-fallback path verified, all safety gates still applied to recovery answers. ALSO BANKED — E-2 targeting baseline (canary run #58, 29788404357, all-green incl. first execution of the !cancelled()-fixed instrument, ~$1-2): metric_rates relevance 0.6 / readability 1.0 / artifact_leaks 1.0 / intent_coverage 0.9333 / fail_closed 0.9; targeting_rate 0.5909 (13/22); by intent: document_lookup 5/5, red_result_action 3/3, contraindication 2/2, dose 1/5, monitoring_schedule 1/5, pathway_referral 1/2; all 9 misses = missing dose figure/schedule-interval (answer lengths 73-232 chars) → E-3c co-primary target alongside the wasted-generation class. Phase E spend ≈$3-6 of ≤$20. | Red-proofs: reserve pinned 3 independent ways (exact 23000ms grant, deadline flag clear, total under budget); self-heal skip pins exact marker + single provider call; offline flag pinned true/false. Focused: route-budget 9/9, eval-quality 27/27, fallback+offline 52/52, parser/abort regressions 15/15. Full suite 3043 passed / 1 known container pdf artifact. typecheck+lint+prettier clean. No provider calls; live proof = E-4 paired run | From 500a66ea0553f97b7a600620b6e999774679f12e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 21 Jul 2026 01:00:17 +0000 Subject: [PATCH 5/5] fix(rag): trim E-3b comments to fit the rag.ts no-growth maintainability budget Static-pr caught rag.ts at 5033 lines vs the 5030 no-growth budget - the E-3b additions tipped the tracked monolith over its cap. Rationale comments live in rag-route-budget.ts and the design record; the rag.ts copies shrink to one-liners (comments only, zero logic change - suites re-verified). The real extraction debt stays tracked in the maturity audit. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EXsJcLrbZUXwnBeG91cVo9 --- src/lib/rag/rag.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/lib/rag/rag.ts b/src/lib/rag/rag.ts index c0ced5bbd..75d1043a2 100644 --- a/src/lib/rag/rag.ts +++ b/src/lib/rag/rag.ts @@ -3572,8 +3572,7 @@ async function answerQuestionWithScopeUncoalesced( startedAt, }); // True exactly when pre-answer work (dominated by retrieval) consumed the whole route - // budget before any generation could start — e.g. slow cross-region RPC on an extractive - // route. Additive telemetry only; deadlineExceeded semantics are unchanged. + // budget before generation could start. Additive telemetry; deadlineExceeded unchanged. const routeBudgetExhaustedByRetrieval = routeDeadline.budgetMs > 0 && routeDeadline.remainingMs() <= 0; const routeTimingDiagnostics = () => ({ retrieval_latency_ms: searchLatencyMs, @@ -4065,9 +4064,7 @@ ${qualityRetryInstruction}` schemaName: "clinical_rag_answer", instructions: answerInstructions, promptCacheKey: ragAnswerPromptVersion, - // Reserve-aware: an attempt may never spend the recovery path's share of the - // route budget (eval run #57: a provider timeout consumed the whole fallback - // budget and the successful recovery still missed the route ceiling by 54ms). + // Reserve-aware: never spend the recovery path's share of the route budget. timeoutMs: routeDeadline.generationRequestTimeoutMs(env.OPENAI_ANSWER_TIMEOUT_MS), maxRetries: 0, reasoningEffort: useStrongReasoning @@ -4270,8 +4267,7 @@ ${qualityRetryInstruction}` }); // Adopted from main: retry truncation once for BOTH fast- and strong-routed first attempts // (previously fast-only), keyed on route.mode rather than model identity so it stays correct - // when the tiers share a model. Budget-gated (E-3b): truncated attempts measure ~20s+ - // before hitting max_output_tokens, so a retry into a nearly-spent budget is a + // when the tiers share a model. Budget-gated: a retry into a nearly-spent budget is a // guaranteed-discard — skip it and let the existing source-backed recovery deliver. if (generated.truncated && !retriedWithStrong && !deadlineAllowsGenerationRetry(routeDeadline)) { answerRetryReasons.push(