diff --git a/crates/buzz-agent/src/handoff.rs b/crates/buzz-agent/src/handoff.rs index 5fdbc3079d..4748678059 100644 --- a/crates/buzz-agent/src/handoff.rs +++ b/crates/buzz-agent/src/handoff.rs @@ -3,6 +3,7 @@ use crate::config::{ HANDOFF_MAX_OUTPUT_TOKENS, HANDOFF_MAX_TOOL_NAMES, HANDOFF_MIN_PROMPT_BUDGET_BYTES, HANDOFF_ORIGINAL_TASK_MAX_BYTES, MAX_CONTEXT_RECOVERIES_PER_RUN, }; +use crate::llm::summary_completion_cap; use crate::types::HistoryItem; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -35,10 +36,21 @@ pub(crate) enum ContextRecovery { Exhausted, } -const HANDOFF_SYSTEM_PROMPT: &str = "You are generating a context handoff summary for the next \ -turn of an autonomous agent. Be concise but thorough. Cover: what the original task was, what \ -you accomplished, key decisions made, what remains, and one concrete next step. Output plain \ -text only — no tool calls, no JSON. Stay under 8192 tokens."; +/// System prompt for the handoff summarizer. `LazyLock` + `format!` so the +/// token figure is derived from [`HANDOFF_MAX_OUTPUT_TOKENS`] instead of a +/// duplicated literal, and "visible plain-text summary" makes explicit that +/// the limit is on summary text, not on any hidden reasoning the model does +/// first (which is budgeted separately on the wire — see +/// `openrouter_summary_body`). +static HANDOFF_SYSTEM_PROMPT: std::sync::LazyLock = std::sync::LazyLock::new(|| { + format!( + "You are generating a context handoff summary for the next turn of an autonomous agent. \ + Be concise but thorough. Cover: what the original task was, what you accomplished, key \ + decisions made, what remains, and one concrete next step. Output plain text only — no \ + tool calls, no JSON. Keep the visible plain-text summary under \ + {HANDOFF_MAX_OUTPUT_TOKENS} tokens." + ) +}); impl RunCtx<'_> { pub(crate) async fn maybe_handoff(&mut self, handoff_attempts: &mut usize) -> HandoffOutcome { @@ -170,7 +182,7 @@ impl RunCtx<'_> { _ = self.cancel.changed() => return HandoffOutcome::Cancelled, r = self.llm.summarize( self.cfg, - HANDOFF_SYSTEM_PROMPT, + &HANDOFF_SYSTEM_PROMPT, &prompt, HANDOFF_MAX_OUTPUT_TOKENS, self.effective_model, @@ -331,7 +343,7 @@ impl RunCtx<'_> { Some(explicit) => explicit.saturating_sub(fixed_bytes), None => handoff_prompt_budget_bytes( self.cfg.max_context_tokens, - HANDOFF_MAX_OUTPUT_TOKENS, + summary_completion_cap(self.cfg.provider, HANDOFF_MAX_OUTPUT_TOKENS), fixed_bytes, ), }; @@ -513,8 +525,9 @@ fn byte_fallback_threshold( mod tests { use super::{ byte_fallback_threshold, estimate_tokens_from_bytes, handoff_prompt_budget_bytes, - token_threshold, + summary_completion_cap, token_threshold, HANDOFF_SYSTEM_PROMPT, }; + use crate::config::{Provider, HANDOFF_MAX_OUTPUT_TOKENS}; #[test] fn handoff_prompt_budget_reserves_summary_output_and_fixed_prompt() { @@ -526,6 +539,71 @@ mod tests { assert_eq!(handoff_prompt_budget_bytes(1_000, 2_000, 10_000), 0); } + /// OpenRouter's summary request grants reasoning an equal budget on top of + /// the visible-text budget, so its completion cap is 2× the handoff text + /// budget; the input budget must reserve that doubled cap. At the + /// 1-byte/token upper bound, prompt bytes bound prompt tokens, so the join + /// to pin is: (budget + fixed prompt) + actual completion cap ≤ window. + /// Reserving only `HANDOFF_MAX_OUTPUT_TOKENS` would break this by exactly + /// one extra reasoning budget at the maximum constructed prompt. + #[test] + fn openrouter_prompt_budget_reserves_doubled_completion_cap() { + let cap = summary_completion_cap(Provider::OpenRouter, HANDOFF_MAX_OUTPUT_TOKENS); + assert_eq!( + cap, + 2 * HANDOFF_MAX_OUTPUT_TOKENS, + "OpenRouter doubles: text + reasoning" + ); + let window = 200_000u64; + let fixed = 1_000usize; + let budget = handoff_prompt_budget_bytes(window, cap, fixed); + assert_eq!(budget, 182_616); // 200_000 - 16_384 - 1_000 + let max_prompt_tokens = estimate_tokens_from_bytes(budget + fixed); + assert!( + max_prompt_tokens + u64::from(cap) <= window, + "input + completion allowance must fit the configured window" + ); + // The old single reservation violates the same join — the regression + // this guards against. + let stale_budget = handoff_prompt_budget_bytes(window, HANDOFF_MAX_OUTPUT_TOKENS, fixed); + assert!( + estimate_tokens_from_bytes(stale_budget + fixed) + u64::from(cap) > window, + "reserving only the text budget must be observable as an overflow here" + ); + } + + /// Anthropic/OpenAI/Databricks summary bodies request exactly the caller's + /// budget, so their input reservation is unchanged. + #[test] + fn non_openrouter_completion_cap_is_the_callers_budget() { + for provider in [ + Provider::Anthropic, + Provider::OpenAi, + Provider::Databricks, + Provider::DatabricksV2, + ] { + assert_eq!( + summary_completion_cap(provider, HANDOFF_MAX_OUTPUT_TOKENS), + HANDOFF_MAX_OUTPUT_TOKENS + ); + } + } + + /// The prompt's token figure is derived from `HANDOFF_MAX_OUTPUT_TOKENS` + /// and names the *visible plain-text summary* as its target, so hidden + /// reasoning (budgeted separately on the wire) is not the referent. + #[test] + fn handoff_system_prompt_derives_limit_and_targets_visible_text() { + let expected = format!( + "Keep the visible plain-text summary under {HANDOFF_MAX_OUTPUT_TOKENS} tokens." + ); + assert!( + HANDOFF_SYSTEM_PROMPT.contains(&expected), + "prompt must derive its token figure from HANDOFF_MAX_OUTPUT_TOKENS: {}", + *HANDOFF_SYSTEM_PROMPT + ); + } + #[test] fn token_threshold_uses_fraction_when_output_is_small() { // 200k window, 1k output. fractional = 0.9*200000 = 180000; diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index a091790425..289adbd1ad 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -2237,22 +2237,56 @@ pub(crate) fn build_token_source(cfg: &Config) -> Result, A } } +/// Completion-token cap that [`Llm::summarize`] actually requests from +/// `provider`, given the caller's visible-text budget. OpenRouter grants +/// reasoning a separate, equal budget on top of the text budget (see +/// [`openrouter_summary_body`]), so its top-level cap is double the caller's +/// budget; every other provider requests the caller's budget unchanged. +/// Callers that reserve output headroom in an input budget +/// (`handoff_prompt_budget_bytes`) must reserve THIS value, not the text +/// budget — otherwise input plus the actual completion allowance can exceed +/// the configured context window. +pub(crate) fn summary_completion_cap(provider: Provider, max_output_tokens: u32) -> u32 { + match provider { + Provider::OpenRouter => max_output_tokens.saturating_mul(2), + Provider::Anthropic | Provider::OpenAi | Provider::Databricks | Provider::DatabricksV2 => { + max_output_tokens + } + } +} + /// Build the request body for `Llm::summarize` on `Provider::OpenRouter`. /// Extracted so tests can assert on the actual wire shape instead of a -/// hand-rolled literal — summaries never carry `reasoning` (see -/// `apply_openrouter_mutations`, which the summary path never calls). -/// It spells the token limit `max_tokens` directly for the same reason: the -/// mutation that renames it is never applied here. +/// hand-rolled literal — summaries never carry config-driven reasoning +/// *effort* (see `apply_openrouter_mutations`, which the summary path never +/// calls). It spells the token limit `max_tokens` directly for the same +/// reason: the mutation that renames it is never applied here. fn openrouter_summary_body( effective_model: &str, system_prompt: &str, user_prompt: &str, max_output_tokens: u32, ) -> Value { + // Reasoning models spend output tokens thinking before emitting any + // visible text, and that spend counts against `max_tokens`. Left + // unseparated, a model can burn the entire cap mid-reasoning and return an + // empty `content` — observed with deepseek-v4-flash, where 13 consecutive + // handoff attempts length-stopped inside the reasoning channel and every + // one degraded to lossy history truncation. Give reasoning its own + // equal-sized budget on top of the text budget so `max_output_tokens` + // remains what the caller means: visible summary text. `exclude` keeps the + // reasoning out of the response body; `summarize()` only reads `content`. + // Non-reasoning endpoints ignore the `reasoning` object (see + // `apply_openrouter_mutations` on why it is never paired with + // `provider.require_parameters`). json!({ "model": effective_model, "stream": false, - "max_tokens": max_output_tokens, + "max_tokens": summary_completion_cap(Provider::OpenRouter, max_output_tokens), + "reasoning": { + "max_tokens": max_output_tokens, + "exclude": true, + }, "messages": [ { "role": "system", "content": system_prompt }, { "role": "user", "content": user_prompt }, @@ -6263,8 +6297,14 @@ mod tests { assert!(body.get("max_completion_tokens").is_none()); } + /// The summary body reserves `max_output_tokens` for visible text by + /// granting reasoning a separate, equal budget on top and excluding it + /// from the response. Without the separation, a reasoning model can spend + /// the entire cap thinking and length-stop with empty `content`, which + /// `summarize()` reports as an empty summary and the handoff degrades to + /// lossy truncation. #[test] - fn openrouter_summary_carries_neither_reasoning_nor_provider() { + fn openrouter_summary_budgets_reasoning_separately_and_carries_no_provider() { let body = openrouter_summary_body( "anthropic/claude-opus-4-7", "summarize", @@ -6274,14 +6314,25 @@ mod tests { assert_eq!(body["model"], "anthropic/claude-opus-4-7"); assert_eq!(body["messages"][0]["role"], "system"); assert_eq!(body["messages"][1]["content"], "text to summarize"); - assert_eq!(body["max_tokens"], 1024); + assert_eq!( + body["max_tokens"], 2048, + "total cap must cover the text budget plus the reasoning budget" + ); + assert_eq!( + body["reasoning"]["max_tokens"], 1024, + "reasoning gets its own budget so it cannot starve the summary text" + ); + assert_eq!( + body["reasoning"]["exclude"], true, + "reasoning must not be included in the response; summarize() reads only content" + ); assert!( - body.get("max_completion_tokens").is_none(), - "summary body must use OpenRouter's token-limit spelling" + body["reasoning"].get("effort").is_none(), + "budget-based cap only; effort stays unset for the summary call" ); assert!( - body.get("reasoning").is_none(), - "summary body must not carry reasoning" + body.get("max_completion_tokens").is_none(), + "summary body must use OpenRouter's token-limit spelling" ); assert!( body.get("provider").is_none(),