From 5874814cd3e448dfa358a6e500115094cb3124cf Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:17:56 +0800 Subject: [PATCH] fix(rag): stop reasoning-token starvation that discarded answers as "unsupported" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gpt-5.5 draws reasoning tokens from the SAME max_output_tokens budget as the visible answer. The strong route ran high reasoning effort against a 4000-token cap, so reasoning exhausted the budget and returned `incomplete: max_output_tokens` before writing the answer — the answer was then discarded and the request degraded to "unsupported" after 60-90s (confirmed via live rag_queries telemetry). The same high effort also overran the 30s answer timeout (provider_timeout). It is a class of bug: every reasoning-model call sized max_output_tokens without reserving reasoning headroom, including the ingestion enrichment/extraction paths that ran silently over the corpus. - env: OPENAI_MAX_OUTPUT_TOKENS 4000->16000; OPENAI_STRONG_REASONING_EFFORT high->medium. medium resolves every query class, including the safety-critical medication_dose_risk and table_threshold that strongReasoningEffortForQueryClass kept at the full configured effort and that starved first. - openai: reasoningHeadroomFloor(effort) floors max_output_tokens by effort in responseBody so no call site can under-provision reasoning headroom. Math.max only ever raises a budget, and the budget is a ceiling (billed per token used), so it is free. - rag: truncation self-heal — strong escalations retry at a boosted cap (2x, >=24000) instead of re-truncating on the same budget; a cumulative generation wall-clock budget skips the third (polish) generation to bound tail latency, keeping the valid strong answer rather than risking a truncation -> unsupported tail. - ingestion: document-enrichment and model-index-extraction now observe truncation and warn loudly (with document identity) instead of silently cutting off enrichment. - observability: answer-slo adds truncation/timeout fallback counters to the /api/health deep probe so this regression class is visible without hand-running SQL. Verified: verify:cheap (1673 tests) + typecheck + prettier offline; live golden retrieval 36/36 (content_mrr@10 0.9414); live eval:quality --rag-only eliminated the max_output_tokens and provider_timeout fallbacks. An A/B baseline (pre-fix vs fix on the same live corpus) is byte-identical on every grounding/citation/failure metric (zero quality regression) and cuts p95 generation latency ~14s (75.9s -> 61.5s). Co-Authored-By: Claude Opus 4.8 --- src/lib/document-enrichment.ts | 18 +++++-- src/lib/env.ts | 28 ++++++++--- src/lib/model-index-extraction.ts | 17 +++++-- src/lib/observability/answer-slo.ts | 24 +++++++++- src/lib/openai.ts | 26 ++++++++++- src/lib/rag.ts | 43 ++++++++++++++--- tests/answer-slo.test.ts | 34 ++++++++++---- tests/document-enrichment.test.ts | 10 +++- tests/model-index-extraction.test.ts | 10 +++- tests/openai-cache.test.ts | 70 +++++++++++++++++++++++++++- 10 files changed, 247 insertions(+), 33 deletions(-) diff --git a/src/lib/document-enrichment.ts b/src/lib/document-enrichment.ts index 7cc7dc633..0332cddb1 100644 --- a/src/lib/document-enrichment.ts +++ b/src/lib/document-enrichment.ts @@ -15,7 +15,7 @@ import { compactPromptChunk, selectCoverageAwarePromptChunks, } from "@/lib/indexing-coverage"; -import { generateStructuredTextResponse } from "@/lib/openai"; +import { generateStructuredTextResult } from "@/lib/openai"; import { ragDeepMemoryVersion } from "@/lib/deep-memory"; import { normalizeDocumentLabelForStorage } from "@/lib/document-tags"; import { cleanClinicalSummaryText, fenceSourceEvidence, sourceTextForModelEvidence } from "@/lib/source-text-sanitizer"; @@ -555,11 +555,13 @@ export async function generateDocumentEnrichment(args: { chunks: EnrichmentChunk[]; images?: EnrichmentImage[]; }) { - const raw = await generateStructuredTextResponse( + const result = await generateStructuredTextResult( buildEnrichmentPrompt({ ...args, images: args.images ?? [] }), summarySchema, { model: env.OPENAI_STRONG_ANSWER_MODEL || env.OPENAI_FAST_ANSWER_MODEL, + // Answer-size budget; responseBody() floors the effective cap by reasoning effort so + // medium-effort reasoning cannot starve the enrichment JSON (reasoningHeadroomFloor). maxOutputTokens: 2400, operation: "summary", schemaName: "clinical_document_enrichment", @@ -568,7 +570,17 @@ export async function generateDocumentEnrichment(args: { textVerbosity: "medium", }, ); - const parsed = parseGeneratedSummary(raw, args.document, args.chunks, args.images ?? []); + // Ingestion runs unattended over the whole corpus, so a truncated enrichment must be loud, + // not silent — a silently cut-off summary degrades retrieval for that document and would be + // re-wasted on every re-index. Warn with the document identity so it is greppable and + // re-processable; parsing still proceeds on the partial text. + if (result.truncated) { + console.warn("document enrichment truncated", { + document: args.document.file_name ?? args.document.title, + reason: result.incompleteReason ?? result.status ?? "unknown", + }); + } + const parsed = parseGeneratedSummary(result.text, args.document, args.chunks, args.images ?? []); const inferred = inferLabels(args.document); const organization = classifyDocumentOrganization({ ...args.document, diff --git a/src/lib/env.ts b/src/lib/env.ts index 4861497f7..e2d849e4d 100644 --- a/src/lib/env.ts +++ b/src/lib/env.ts @@ -33,11 +33,19 @@ const envSchema = z.object({ // Strong tier intentionally stays on the standard (non-"pro") model. Fast vs strong // is differentiated by reasoning effort (OPENAI_*_REASONING_EFFORT), not model tier. OPENAI_STRONG_ANSWER_MODEL: z.string().default("gpt-5.5"), - // Reasoning models (gpt-5*) draw reasoning tokens from this same budget, so a - // low cap can starve the JSON answer payload and silently truncate clinical - // content (doses/thresholds cut mid-sentence). Raised default to 4000 for headroom; - // if output is still cut off, createTextResult now flags it as truncated (GEN-C1). - OPENAI_MAX_OUTPUT_TOKENS: z.coerce.number().int().positive().default(4000), + // Reasoning models (gpt-5*) draw reasoning tokens from this SAME budget as the + // visible answer, so a low cap makes medium/high-effort reasoning consume the whole + // budget *thinking* and return `incomplete: max_output_tokens` BEFORE it writes the + // JSON answer — the answer is then discarded and the request degrades to + // "unsupported" after 60-90s of wasted generation (GEN-C1). The old 4000 default did + // exactly this on the strong route in production (confirmed via rag_queries telemetry). + // 16000 gives ample headroom for reasoning + a full clinical answer; it is a CEILING + // (billed per token actually used), not a spend commitment, so raising it costs + // nothing when answers finish early. responseBody() additionally floors the effective + // budget by reasoning effort so no call site can under-provision (reasoningHeadroomFloor + // in openai.ts), and the answer path self-heals a truncation by retrying with a larger + // cap before falling back. + OPENAI_MAX_OUTPUT_TOKENS: z.coerce.number().int().positive().default(16000), OPENAI_QUERY_CACHE_SIZE: z.coerce.number().int().nonnegative().default(200), // Max inputs per embeddings request. The OpenAI embeddings endpoint caps a single // request at 2048 inputs / ~300k tokens; a full-corpus re-embed of ~400k texts in one @@ -65,7 +73,15 @@ const envSchema = z.object({ .default("false") .transform((value) => value === "true"), OPENAI_FAST_REASONING_EFFORT: z.enum(["none", "low", "medium", "high", "xhigh"]).default("low"), - OPENAI_STRONG_REASONING_EFFORT: z.enum(["none", "low", "medium", "high", "xhigh"]).default("high"), + // "high" reasoning on gpt-5.5 is both slow (overruns OPENAI_ANSWER_TIMEOUT_MS -> + // provider_timeout) and token-hungry (exhausts OPENAI_MAX_OUTPUT_TOKENS -> truncation) — + // the two dominant answer-generation failure modes seen in production. "medium" is ample + // for answers grounded in retrieved sources and roughly halves generation time. Note + // strongReasoningEffortForQueryClass() previously kept the FULL configured effort for the + // safety-critical medication_dose_risk/table_threshold classes, so with the old "high" + // default those exact classes ran high-effort and starved first; medium fixes them too. + // That function never raises effort above this configured value. + OPENAI_STRONG_REASONING_EFFORT: z.enum(["none", "low", "medium", "high", "xhigh"]).default("medium"), OPENAI_SUMMARY_REASONING_EFFORT: z.enum(["none", "low", "medium", "high", "xhigh"]).default("medium"), OPENAI_VISION_REASONING_EFFORT: z.enum(["none", "low", "medium", "high", "xhigh"]).default("low"), OPENAI_TEXT_VERBOSITY: z.enum(["low", "medium", "high"]).default("low"), diff --git a/src/lib/model-index-extraction.ts b/src/lib/model-index-extraction.ts index 4fd5439b2..f2b70f8df 100644 --- a/src/lib/model-index-extraction.ts +++ b/src/lib/model-index-extraction.ts @@ -5,7 +5,7 @@ import { buildIndexingCoverageProfile, selectCoverageAwarePromptChunks, } from "@/lib/indexing-coverage"; -import { generateStructuredTextResponse } from "@/lib/openai"; +import { generateStructuredTextResult } from "@/lib/openai"; import { cleanClinicalSummaryText, fenceSourceEvidence, sourceTextForModelEvidence } from "@/lib/source-text-sanitizer"; export const modelIndexExtractionVersion = "model-heavy-index-v1" as const; @@ -368,8 +368,10 @@ export async function generateModelIndexProfile(args: { }) { if (args.chunks.length === 0) return emptyProfile(); const model = env.OPENAI_STRONG_ANSWER_MODEL || env.OPENAI_ANSWER_MODEL; - const raw = await generateStructuredTextResponse(buildPrompt({ ...args, images: args.images ?? [] }), schema, { + const result = await generateStructuredTextResult(buildPrompt({ ...args, images: args.images ?? [] }), schema, { model, + // Answer-size budget; responseBody() floors the effective cap by reasoning effort so + // medium-effort reasoning cannot starve the model-index JSON (reasoningHeadroomFloor). maxOutputTokens: 3200, operation: "summary", schemaName: "clinical_model_index_profile", @@ -377,7 +379,16 @@ export async function generateModelIndexProfile(args: { reasoningEffort: "medium", textVerbosity: "medium", }); - return parseProfile(raw, args.chunks, args.images ?? [], model); + // Ingestion runs unattended over the whole corpus; a truncated extraction silently drops + // model-index coverage for the document. Warn loudly (greppable, with document identity) + // instead of failing silent; parsing still proceeds on the partial text. + if (result.truncated) { + console.warn("model-index extraction truncated", { + document: args.document.file_name ?? args.document.title, + reason: result.incompleteReason ?? result.status ?? "unknown", + }); + } + return parseProfile(result.text, args.chunks, args.images ?? [], model); } export function fallbackModelIndexProfile() { diff --git a/src/lib/observability/answer-slo.ts b/src/lib/observability/answer-slo.ts index ba32d6a80..a4b69c286 100644 --- a/src/lib/observability/answer-slo.ts +++ b/src/lib/observability/answer-slo.ts @@ -16,9 +16,18 @@ export type AnswerSloSnapshot = { totalQueries: number; hybridRpcErrorQueries: number; degradedQueries: number; + // Subsets of degradedQueries broken out by the two dominant answer-generation waste + // classes: reasoning/answer token starvation (fallback_reason contains + // "max_output_tokens") and generation timeouts (contains "timeout"). These are exactly + // the failure modes the OPENAI_MAX_OUTPUT_TOKENS raise + reasoning-effort drop target, + // surfaced as scrapeable counters so a regression is visible without hand-running SQL. + truncationFallbackQueries: number; + timeoutFallbackQueries: number; // 0..1; 0 when there were no queries in the window (avoid divide-by-zero noise). hybridRpcErrorRate: number; degradedRate: number; + truncationFallbackRate: number; + timeoutFallbackRate: number; }; type CountResult = { count: number | null; error: unknown }; @@ -30,6 +39,7 @@ type SloCountBuilder = PromiseLike & { gt(column: string, value: string): SloCountBuilder; is(column: string, value: null): SloCountBuilder; not(column: string, operator: string, value: null): SloCountBuilder; + ilike(column: string, pattern: string): SloCountBuilder; }; export type SloProbeClient = { @@ -60,13 +70,17 @@ export async function answerSloSnapshot(client: SloProbeClient, windowMinutes = // discriminator without hiding normal production answers from the SLO. .is("metadata->>event_type", null); - const [total, hybrid, degraded] = await Promise.all([ + const [total, hybrid, degraded, truncation, timeout] = await Promise.all([ base(), base().not("metadata->hybrid_rpc_errors", "is", null), base().not("metadata->>fallback_reason", "is", null), + // fallback_reason values look like "...generation_fallback:provider_incomplete_max_output_tokens" + // and "...generation_fallback:provider_timeout" (confirmed in live rag_queries). + base().ilike("metadata->>fallback_reason", "%max_output_tokens%"), + base().ilike("metadata->>fallback_reason", "%timeout%"), ]); - for (const result of [total, hybrid, degraded]) { + for (const result of [total, hybrid, degraded, truncation, timeout]) { if (result.error) { if (result.error instanceof Error) throw result.error; // Supabase surfaces a plain PostgrestError object ({ message, code, ... }), @@ -82,13 +96,19 @@ export async function answerSloSnapshot(client: SloProbeClient, windowMinutes = const totalQueries = total.count ?? 0; const hybridRpcErrorQueries = hybrid.count ?? 0; const degradedQueries = degraded.count ?? 0; + const truncationFallbackQueries = truncation.count ?? 0; + const timeoutFallbackQueries = timeout.count ?? 0; return { windowMinutes, totalQueries, hybridRpcErrorQueries, degradedQueries, + truncationFallbackQueries, + timeoutFallbackQueries, hybridRpcErrorRate: rate(hybridRpcErrorQueries, totalQueries), degradedRate: rate(degradedQueries, totalQueries), + truncationFallbackRate: rate(truncationFallbackQueries, totalQueries), + timeoutFallbackRate: rate(timeoutFallbackQueries, totalQueries), }; } diff --git a/src/lib/openai.ts b/src/lib/openai.ts index 07a9188bc..0c736d6cd 100644 --- a/src/lib/openai.ts +++ b/src/lib/openai.ts @@ -199,6 +199,30 @@ function resolveReasoningEffort(model: string, effort: OpenAIReasoningEffort) { return capabilities.allowedReasoningEfforts.has(effort) ? effort : "low"; } +// Reasoning models spend reasoning tokens from the SAME budget as the visible answer +// (max_output_tokens). A budget sized only for the answer lets reasoning starve it -> +// `incomplete: max_output_tokens` (the answer is discarded and the request degrades). +// This floor guarantees a minimum TOTAL budget per effort level so no call site can +// under-provision reasoning headroom — a site that declares its answer size gets that +// answer size plus guaranteed reasoning room, and a site asking for MORE than the floor +// keeps its own value (Math.max). The floor only ever RAISES a budget, and the budget is +// a ceiling on token spend (billed per token actually used), so lifting small budgets to +// it is free when the response finishes early. +function reasoningHeadroomFloor(effort: OpenAIReasoningEffort): number { + switch (effort) { + case "xhigh": + return 16000; + case "high": + return 12000; + case "medium": + return 8000; + case "low": + return 2000; + default: + return 0; + } +} + function responseBody( input: OpenAIResponseInput, resolved: ResolvedTextGenerationOptions, @@ -222,7 +246,7 @@ function responseBody( model: resolved.model, input, ...(resolved.instructions ? { instructions: resolved.instructions } : {}), - max_output_tokens: resolved.maxOutputTokens, + max_output_tokens: Math.max(resolved.maxOutputTokens, reasoningHeadroomFloor(resolvedReasoningEffort)), store: env.OPENAI_STORE_RESPONSES, prompt_cache_key: resolved.promptCacheKey ?? promptCacheKeyFor(operation), ...(promptCacheRetention ? { prompt_cache_retention: promptCacheRetention } : {}), diff --git a/src/lib/rag.ts b/src/lib/rag.ts index 9c50cd178..7db8736a1 100644 --- a/src/lib/rag.ts +++ b/src/lib/rag.ts @@ -4598,7 +4598,7 @@ ${buildRagSourceBlock(contextResults, { query: answerFocusQuery, queryClass })}` async function generateWithModel( model: string, contextResults: SearchResult[], - options?: { strong?: boolean; qualityRetryInstruction?: string }, + options?: { strong?: boolean; qualityRetryInstruction?: string; maxOutputTokensOverride?: number }, ): Promise { const qualityRetryInstruction = options?.qualityRetryInstruction; // Fast vs strong is differentiated by reasoning effort, not model identity, so the @@ -4619,7 +4619,7 @@ ${qualityRetryInstruction}` try { const result = await generateStructuredTextResult(input, answerJsonOutputSchemaForResults(contextResults), { model, - maxOutputTokens: env.OPENAI_MAX_OUTPUT_TOKENS, + maxOutputTokens: options?.maxOutputTokensOverride ?? env.OPENAI_MAX_OUTPUT_TOKENS, operation: "answer", schemaName: "clinical_rag_answer", instructions: answerInstructions, @@ -4647,6 +4647,18 @@ ${qualityRetryInstruction}` } } + // Truncation self-heal budget: a max_output_tokens truncation means reasoning+answer + // exhausted the cap, not that the model failed. The strong retries below spend MORE + // reasoning than the first attempt, so they get a boosted cap — escalating to strong on + // the SAME budget is what previously burned a second full generation and still fell + // through to "unsupported". Billed per token actually used, so this is free unless hit. + const strongRetryMaxOutputTokens = Math.max(env.OPENAI_MAX_OUTPUT_TOKENS * 2, 24000); + // Cap cumulative generation wall-clock so a fast -> strong -> quality-repair chain can't + // stack three ~timeout-length calls into a ~90s tail. The quality-repair is a polish pass + // over an already-valid, cited strong answer, so once this budget is spent we keep the + // strong answer rather than risk a third generation (and a truncation -> unsupported tail). + const generationTotalBudgetMs = env.OPENAI_ANSWER_TIMEOUT_MS * 2; + /** Generation incomplete reason. */ function generationIncompleteReason(result: OpenAITextResult) { return result.incompleteReason ?? (result.status === "incomplete" ? "incomplete" : "unknown"); @@ -4808,7 +4820,10 @@ ${qualityRetryInstruction}` retriedWithStrong = true; await args.onProgress?.({ stage: "retrying", - message: "Fast answer hit the output limit, retrying with the strong model.", + message: + route.mode === "fast" + ? "Fast answer hit the output limit, retrying with the strong model and a larger output budget." + : "Answer hit the output limit, retrying with a larger output budget.", mode: "strong", model: env.OPENAI_STRONG_ANSWER_MODEL, reason: routingReason, @@ -4819,7 +4834,12 @@ ${qualityRetryInstruction}` // Widen the retry context from the trimmed fast set to the full result set, but keep the P9 // per-document crowding cap — the strong-initial route is capped, so the retry must be too. packedContextResults = await packContextForGeneration(capPerDocumentCrowding(answerInputResults)); - generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults, { strong: true }); + // Boost the cap: a max_output_tokens truncation retried on the SAME budget with MORE + // reasoning (strong) just re-truncates. This is the truncation self-heal. + generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults, { + strong: true, + maxOutputTokensOverride: strongRetryMaxOutputTokens, + }); retrievalDiagnostics.routeMode = "strong"; } if (generated.truncated) { @@ -4896,7 +4916,12 @@ ${qualityRetryInstruction}` args.onRevising?.(retryReason); // Same as the truncation retry above: widen but keep the P9 per-document crowding cap. packedContextResults = await packContextForGeneration(capPerDocumentCrowding(answerInputResults)); - generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults, { strong: true }); + // Strong spends more reasoning tokens than the fast attempt it is replacing, so it needs + // the boosted cap to avoid truncating (and degrading to unsupported) on the escalation. + generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults, { + strong: true, + maxOutputTokensOverride: strongRetryMaxOutputTokens, + }); retrievalDiagnostics.routeMode = "strong"; if (generated.truncated) { const truncatedReason = generationRetryReason("strong", generated); @@ -4917,7 +4942,12 @@ ${qualityRetryInstruction}` ? generatedAnswerQualityFailureReason(answer, args.query, queryClass) : null; const answerNeedsStrongQualityRepair = usedStrongModel && Boolean(strongQualityFailureReason); - if (answerNeedsStrongQualityRepair) { + if (answerNeedsStrongQualityRepair && generationLatencyMs >= generationTotalBudgetMs) { + // A4 tail-latency guard: out of the cumulative generation time budget, so keep the + // valid (if imperfect) cited strong answer instead of spending a third generation + // and risking a truncation -> unsupported tail. Recorded for observability. + answerRetryReasons.push(`strong_quality_repair_skipped_time_budget:${strongQualityFailureReason}`); + } else if (answerNeedsStrongQualityRepair) { routingReason = `${routingReason}; strong_quality_retry`; answerRetryCount += 1; answerRetryReasons.push("strong_quality_retry"); @@ -4931,6 +4961,7 @@ ${qualityRetryInstruction}` args.onRevising?.("strong_quality_retry"); generated = await generateWithModel(env.OPENAI_STRONG_ANSWER_MODEL, packedContextResults, { strong: true, + maxOutputTokensOverride: strongRetryMaxOutputTokens, qualityRetryInstruction: `The previous answer failed deterministic validation (${strongQualityFailureReason}). Return schema-valid output only, with a complete natural clinical synthesis in the answer field. The first sentence must directly answer the question as a full sentence. Every clinical claim must be supported by valid retrieved citation_chunk_id values; do not invent citation IDs. Avoid template/source-inventory wording and do not include JSON fragments inside text fields. If the evidence cannot support the requested clinical answer, return a concise source-gap answer instead. If the question is a simple definition or direct fact question, answer only that question and return answerSections as an empty array unless a source-gap or safety caveat is essential.`, }); retrievalDiagnostics.routeMode = "strong"; diff --git a/tests/answer-slo.test.ts b/tests/answer-slo.test.ts index 2e7fea7fc..59cb766f7 100644 --- a/tests/answer-slo.test.ts +++ b/tests/answer-slo.test.ts @@ -3,14 +3,18 @@ import { describe, expect, it } from "vitest"; import { answerSloSnapshot, type SloProbeClient } from "@/lib/observability/answer-slo"; // Fake PostgREST count builder: from().select().gt() is the "total" query; adding -// .not(column,...) narrows it to the hybrid-error or degraded count based on the -// filtered column. Awaiting resolves to { count, error }. +// .not(column,...) narrows it to the hybrid-error or degraded count, and .ilike(col, +// pattern) narrows it to the truncation or timeout fallback subset by pattern; the third +// arg records the base .is(...) filters so tests can assert event-type scoping. Awaiting +// resolves to { count, error }. +type SloFilterKey = "total" | "hybrid" | "degraded" | "truncation" | "timeout"; + function fakeClient( - counts: { total: number; hybrid: number; degraded: number }, + counts: { total: number; hybrid: number; degraded: number; truncation?: number; timeout?: number }, error?: unknown, observedBaseFilters: Array<{ column: string; value: null }> = [], ): SloProbeClient { - const build = (filter: "total" | "hybrid" | "degraded") => { + const build = (filter: SloFilterKey) => { const builder = { gt: () => builder, is: (column: string, value: null) => { @@ -18,8 +22,10 @@ function fakeClient( return builder; }, not: (column: string) => build(column.includes("hybrid_rpc_errors") ? "hybrid" : "degraded"), + ilike: (_column: string, pattern: string) => + build(pattern.includes("max_output_tokens") ? "truncation" : "timeout"), then: (resolve: (value: { count: number | null; error: unknown }) => unknown) => - resolve({ count: error ? null : counts[filter], error: error ?? null }), + resolve({ count: error ? null : (counts[filter] ?? 0), error: error ?? null }), }; return builder; }; @@ -28,15 +34,22 @@ function fakeClient( describe("answerSloSnapshot", () => { it("computes counts and rates over the window", async () => { - const snapshot = await answerSloSnapshot(fakeClient({ total: 20, hybrid: 3, degraded: 2 }), 60); + const snapshot = await answerSloSnapshot( + fakeClient({ total: 20, hybrid: 3, degraded: 5, truncation: 1, timeout: 4 }), + 60, + ); expect(snapshot).toMatchObject({ windowMinutes: 60, totalQueries: 20, hybridRpcErrorQueries: 3, - degradedQueries: 2, + degradedQueries: 5, + truncationFallbackQueries: 1, + timeoutFallbackQueries: 4, }); expect(snapshot.hybridRpcErrorRate).toBeCloseTo(0.15, 5); - expect(snapshot.degradedRate).toBeCloseTo(0.1, 5); + expect(snapshot.degradedRate).toBeCloseTo(0.25, 5); + expect(snapshot.truncationFallbackRate).toBeCloseTo(0.05, 5); + expect(snapshot.timeoutFallbackRate).toBeCloseTo(0.2, 5); }); it("counts privacy-redacted answer rows while excluding search observations by event type", async () => { @@ -46,8 +59,9 @@ describe("answerSloSnapshot", () => { ); expect(snapshot.totalQueries).toBe(7); + // Five base() queries now scope by event_type: total, hybrid, degraded, truncation, timeout. expect(observedBaseFilters).toEqual( - Array.from({ length: 3 }, () => ({ column: "metadata->>event_type", value: null })), + Array.from({ length: 5 }, () => ({ column: "metadata->>event_type", value: null })), ); }); @@ -56,6 +70,8 @@ describe("answerSloSnapshot", () => { expect(snapshot.totalQueries).toBe(0); expect(snapshot.hybridRpcErrorRate).toBe(0); expect(snapshot.degradedRate).toBe(0); + expect(snapshot.truncationFallbackRate).toBe(0); + expect(snapshot.timeoutFallbackRate).toBe(0); }); it("throws when a count query errors so the probe is not falsely healthy", async () => { diff --git a/tests/document-enrichment.test.ts b/tests/document-enrichment.test.ts index dabfc797d..216047c2d 100644 --- a/tests/document-enrichment.test.ts +++ b/tests/document-enrichment.test.ts @@ -10,8 +10,16 @@ vi.mock("@/lib/env", () => ({ }, })); +// The source now calls generateStructuredTextResult (object result) so it can observe +// truncation; wrap the string-returning inner mock so the existing +// .mockResolvedValue(JSON.stringify(...)) setups and .mock.calls assertions keep working. vi.mock("@/lib/openai", () => ({ - generateStructuredTextResponse: mocks.generateStructuredTextResponse, + generateStructuredTextResult: vi.fn(async (...args: unknown[]) => ({ + text: await mocks.generateStructuredTextResponse(...args), + truncated: false, + status: "completed" as const, + incompleteReason: undefined, + })), })); import { generateDocumentEnrichment, ragEnrichmentVersion, upsertDocumentEnrichment } from "@/lib/document-enrichment"; diff --git a/tests/model-index-extraction.test.ts b/tests/model-index-extraction.test.ts index b23c24721..566c60cea 100644 --- a/tests/model-index-extraction.test.ts +++ b/tests/model-index-extraction.test.ts @@ -11,8 +11,16 @@ vi.mock("@/lib/env", () => ({ }, })); +// The source now calls generateStructuredTextResult (object result) so it can observe +// truncation; wrap the string-returning inner mock so the existing +// .mockResolvedValue(JSON.stringify(...)) setups and .mock.calls assertions keep working. vi.mock("@/lib/openai", () => ({ - generateStructuredTextResponse: mocks.generateStructuredTextResponse, + generateStructuredTextResult: vi.fn(async (...args: unknown[]) => ({ + text: await mocks.generateStructuredTextResponse(...args), + truncated: false, + status: "completed" as const, + incompleteReason: undefined, + })), })); import { generateModelIndexProfile } from "@/lib/model-index-extraction"; diff --git a/tests/openai-cache.test.ts b/tests/openai-cache.test.ts index 2ffbac77a..3e4d09dc0 100644 --- a/tests/openai-cache.test.ts +++ b/tests/openai-cache.test.ts @@ -147,7 +147,10 @@ describe("OpenAI query embedding cache", () => { expect(capturedOptions).toMatchObject({ timeout: 1234, maxRetries: 1, signal: controller.signal }); expect(capturedBody).toMatchObject({ model: "gpt-5.5", - max_output_tokens: 200, + // The caller's maxOutputTokens (200) is floored to the "high"-effort reasoning + // headroom (12000) by responseBody so reasoning tokens cannot starve the answer + // (reasoningHeadroomFloor). The floor only ever raises a budget. + max_output_tokens: 12000, store: false, prompt_cache_key: "clinical-rag-answer-v18", prompt_cache_retention: "24h", @@ -246,6 +249,71 @@ describe("OpenAI query embedding cache", () => { expect(JSON.stringify(capturedBodies)).not.toContain("minimal"); }); + it("floors max_output_tokens by reasoning effort but never lowers a larger budget", async () => { + const capturedBodies: Record[] = []; + + vi.stubEnv("OPENAI_API_KEY", "test-key"); + vi.stubEnv("OPENAI_STRONG_ANSWER_MODEL", "gpt-5.5"); + + vi.doMock("openai", () => ({ + default: class MockOpenAI { + embeddings = { create: vi.fn() }; + responses = { + create: vi.fn((body: Record) => { + capturedBodies.push(body); + return { + withResponse: async () => ({ + data: { status: "completed", output_text: '{"answer":"ok"}' }, + request_id: `req_${capturedBodies.length}`, + }), + }; + }), + }; + }, + })); + + const { generateStructuredTextResult } = await import("../src/lib/openai"); + const schema = { type: "object", properties: {}, required: [] }; + + // Tiny declared budget + medium effort -> floored to the medium reasoning headroom (8000). + await generateStructuredTextResult("Q", schema, { + model: "gpt-5.5", + operation: "answer", + schemaName: "clinical_test", + maxOutputTokens: 300, + reasoningEffort: "medium", + }); + // Tiny budget + low effort -> floored to the low headroom (2000). + await generateStructuredTextResult("Q", schema, { + model: "gpt-5.5", + operation: "answer", + schemaName: "clinical_test", + maxOutputTokens: 300, + reasoningEffort: "low", + }); + // A budget larger than the floor passes through unchanged (Math.max only raises). + await generateStructuredTextResult("Q", schema, { + model: "gpt-5.5", + operation: "answer", + schemaName: "clinical_test", + maxOutputTokens: 16000, + reasoningEffort: "medium", + }); + // Non-reasoning model gets no floor (no reasoning tokens to reserve). + await generateStructuredTextResult("Q", schema, { + model: "gpt-4.1-mini", + operation: "answer", + schemaName: "clinical_test", + maxOutputTokens: 300, + reasoningEffort: "high", + }); + + expect(capturedBodies[0]).toMatchObject({ max_output_tokens: 8000 }); + expect(capturedBodies[1]).toMatchObject({ max_output_tokens: 2000 }); + expect(capturedBodies[2]).toMatchObject({ max_output_tokens: 16000 }); + expect(capturedBodies[3]).toMatchObject({ max_output_tokens: 300 }); + }); + it("flags truncated (incomplete) responses (GEN-C1)", async () => { vi.stubEnv("OPENAI_API_KEY", "test-key");