From 3b373b26e0a81f13da5d5c8faf4c14441652ad70 Mon Sep 17 00:00:00 2001 From: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz> Date: Wed, 5 Aug 2026 10:05:18 -0400 Subject: [PATCH 1/3] fix(agent): recover from unsupported image input Co-authored-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz> Signed-off-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz> --- crates/buzz-agent/src/agent.rs | 89 +++++++++++++++++++++++++++++++++- crates/buzz-agent/src/llm.rs | 47 +++++++++++++++++- crates/buzz-agent/src/types.rs | 5 ++ 3 files changed, 137 insertions(+), 4 deletions(-) diff --git a/crates/buzz-agent/src/agent.rs b/crates/buzz-agent/src/agent.rs index ff87a33a1a..b3dfdd3aa6 100644 --- a/crates/buzz-agent/src/agent.rs +++ b/crates/buzz-agent/src/agent.rs @@ -21,6 +21,34 @@ use crate::wire::{self, WireSender}; const ERROR_REFLECTION_SUFFIX: &str = "\n\n[Reflect] Before retrying, identify the cause and change your approach."; +const UNSUPPORTED_IMAGE_TOOL_MESSAGE: &str = "The current model does not support image input. The image was removed from conversation history so this turn can continue. Use a text-based inspection tool or ask the user for a textual description instead."; + +/// Remove image blocks that the provider has explicitly rejected while keeping +/// their surrounding tool result (and therefore the tool-call/result pairing) +/// intact. Returns the number of images removed; zero means the provider error +/// cannot be safely recovered by mutating history. +fn replace_unsupported_images(history: &mut [HistoryItem]) -> usize { + let mut replaced = 0; + for item in history { + let HistoryItem::ToolResult(result) = item else { + continue; + }; + let before = result.content.len(); + result + .content + .retain(|content| !matches!(content, ToolResultContent::Image { .. })); + let removed = before - result.content.len(); + if removed > 0 { + replaced += removed; + result.is_error = true; + result.content.push(ToolResultContent::Text( + UNSUPPORTED_IMAGE_TOOL_MESSAGE.to_string(), + )); + } + } + replaced +} + /// Maximum reply reminders emitted per prompt when `require_reply` is on. /// /// After this many, the turn is allowed to end whether or not anything was @@ -249,10 +277,10 @@ impl RunCtx<'_> { tools.push(builtin::load_skill_def()); } round = round.saturating_add(1); - let response = tokio::select! { + let response_result = tokio::select! { biased; _ = self.cancel.changed() => return Ok(StopReason::Cancelled), - r = self.llm.complete(self.cfg, self.system_prompt, self.history, &tools, self.effective_model) => r?, + r = self.llm.complete(self.cfg, self.system_prompt, self.history, &tools, self.effective_model) => r, _ = async { // Keepalive ticker: emit a lightweight session update every 30s // while waiting on the LLM provider. This resets the ACP harness @@ -275,6 +303,22 @@ impl RunCtx<'_> { } } => unreachable!(), }; + let response = match response_result { + Ok(response) => response, + Err(AgentError::UnsupportedImageInput(detail)) => { + let removed = replace_unsupported_images(self.history); + if removed == 0 { + return Err(AgentError::UnsupportedImageInput(detail)); + } + tracing::warn!( + model = self.effective_model, + removed_images = removed, + "provider rejected image input; removed images from history and continuing turn" + ); + continue; + } + Err(error) => return Err(error), + }; // Record provider-reported input usage so the next loop iteration's // handoff gate can compare it against the token budget. We capture @@ -1075,6 +1119,47 @@ mod tests { assert!(total_after <= max_bytes); } + #[test] + fn unsupported_images_become_recoverable_tool_errors() { + let mut history = vec![ + HistoryItem::Assistant { + text: String::new(), + tool_calls: vec![ToolCall { + provider_id: "call-image".into(), + name: "dev__view_image".into(), + arguments: json!({ "source": "spec.png" }), + provider_extra: Default::default(), + }], + reasoning_details: None, + }, + HistoryItem::ToolResult(ToolResult { + provider_id: "call-image".into(), + content: vec![ + ToolResultContent::Text("10x10 image from spec.png".into()), + ToolResultContent::Image { + data: "aW1n".into(), + mime_type: "image/png".into(), + }, + ], + is_error: false, + }), + ]; + + assert_eq!(replace_unsupported_images(&mut history), 1); + let HistoryItem::ToolResult(result) = &history[1] else { + panic!("tool result must stay paired with the assistant tool call"); + }; + assert_eq!(result.provider_id, "call-image"); + assert!(result.is_error); + assert!(result + .content + .iter() + .all(|content| !matches!(content, ToolResultContent::Image { .. }))); + assert!(result.text().contains("does not support image input")); + assert!(result.text().contains("10x10 image from spec.png")); + assert_eq!(replace_unsupported_images(&mut history), 0); + } + #[test] fn truncate_history_noop_when_under_budget() { let mut history = vec![ diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index 220d99f9a4..3511b86587 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -1706,6 +1706,15 @@ fn is_retryable_transport_error(e: &reqwest::Error) -> bool { e.is_timeout() || e.is_connect() || e.is_request() } +fn is_unsupported_image_input_error(body: &str) -> bool { + body.to_ascii_lowercase() + .contains("no endpoints found that support image input") +} + +fn unsupported_image_input_error(error_body: String) -> AgentError { + AgentError::UnsupportedImageInput(error_body) +} + /// Build the terminal `AgentError::Llm` for a `post()` exit that has given up /// retrying — persistent retryable status, transport failure, or a body-read /// break. `detail` carries the specific cause (status/body, or the transport @@ -1864,9 +1873,12 @@ where // upstream capacity — no retry was attempted, so cumulative duration // would be misleading. if status == 404 { + let error_body = read_error_body(resp).await; + if is_unsupported_image_input_error(&error_body) { + return Err(PostError::Agent(unsupported_image_input_error(error_body))); + } return Err(PostError::Agent(AgentError::LlmModelNotFound(format!( - "{status}: {}", - read_error_body(resp).await + "{status}: {error_body}" )))); } if !status.is_success() { @@ -2117,6 +2129,9 @@ async fn openrouter_post( // about the model, and reporting a parameter problem as // `LlmModelNotFound` (or vice versa) sends the user to the wrong fix. let error_body = read_error_body(resp).await; + if is_unsupported_image_input_error(&error_body) { + return Err(unsupported_image_input_error(error_body)); + } if error_body.contains("No endpoints found that can handle the requested parameters") { return Err(openrouter_parameter_routing_error(&error_body)); } @@ -6217,6 +6232,34 @@ mod tests { ); } + /// A provider's explicit image-capability rejection is a recoverable typed + /// error, not a missing model. The agent loop uses this signal to remove the + /// image from history before retrying the next LLM round. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn openrouter_post_404_unsupported_image_is_typed_and_not_retried() { + let (url, _captured, attempts) = spawn_openrouter_stub(vec![CannedResponse::new( + 404, + r#"{"error":{"message":"No endpoints found that support image input"}}"#, + )]) + .await; + let http = Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); + let err = openrouter_post(&http, &format!("{url}/x"), &json!({}), "key") + .await + .unwrap_err(); + assert!( + matches!(&err, AgentError::UnsupportedImageInput(s) if s.contains("support image input")), + "image rejection must reach the history-recovery path: got {err:?}" + ); + assert_eq!( + attempts.load(std::sync::atomic::Ordering::SeqCst), + 1, + "a deterministic capability rejection must not be retried" + ); + } + /// Every other 404 still maps to `LlmModelNotFound`, including one that /// shares the `No endpoints found` prefix but is about the model rather than /// the parameters — the discriminator is narrow enough that a genuinely diff --git a/crates/buzz-agent/src/types.rs b/crates/buzz-agent/src/types.rs index e386421981..fce705d69d 100644 --- a/crates/buzz-agent/src/types.rs +++ b/crates/buzz-agent/src/types.rs @@ -388,6 +388,10 @@ pub enum AgentError { Llm(String), LlmAuth(String), LlmModelNotFound(String), + /// The provider explicitly rejected image content for the selected model. + /// Kept distinct so the agent loop can remove the unsupported image from + /// replayed history and give the model a recoverable tool error. + UnsupportedImageInput(String), Mcp(String), Cancelled, } @@ -399,6 +403,7 @@ impl std::fmt::Display for AgentError { Self::Llm(s) => write!(f, "llm: {s}"), Self::LlmAuth(s) => write!(f, "llm auth: {s}"), Self::LlmModelNotFound(s) => write!(f, "llm model not found: {s}"), + Self::UnsupportedImageInput(s) => write!(f, "llm image input unsupported: {s}"), Self::Mcp(s) => write!(f, "mcp: {s}"), Self::Cancelled => write!(f, "cancelled"), } From a210305019b33d5f56677b4c82bab79e4ac52d24 Mon Sep 17 00:00:00 2001 From: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz> Date: Wed, 5 Aug 2026 10:15:24 -0400 Subject: [PATCH 2/3] test(agent): cover unsupported image recovery Co-authored-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz> Signed-off-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz> --- crates/buzz-agent/src/llm.rs | 10 +- crates/buzz-agent/tests/bin/fake_mcp.rs | 11 +- crates/buzz-agent/tests/fake_llm.rs | 137 ++++++++++++++++++++++-- 3 files changed, 143 insertions(+), 15 deletions(-) diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index 3511b86587..99e74c6d67 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -1711,10 +1711,6 @@ fn is_unsupported_image_input_error(body: &str) -> bool { .contains("no endpoints found that support image input") } -fn unsupported_image_input_error(error_body: String) -> AgentError { - AgentError::UnsupportedImageInput(error_body) -} - /// Build the terminal `AgentError::Llm` for a `post()` exit that has given up /// retrying — persistent retryable status, transport failure, or a body-read /// break. `detail` carries the specific cause (status/body, or the transport @@ -1875,7 +1871,9 @@ where if status == 404 { let error_body = read_error_body(resp).await; if is_unsupported_image_input_error(&error_body) { - return Err(PostError::Agent(unsupported_image_input_error(error_body))); + return Err(PostError::Agent(AgentError::UnsupportedImageInput( + error_body, + ))); } return Err(PostError::Agent(AgentError::LlmModelNotFound(format!( "{status}: {error_body}" @@ -2130,7 +2128,7 @@ async fn openrouter_post( // `LlmModelNotFound` (or vice versa) sends the user to the wrong fix. let error_body = read_error_body(resp).await; if is_unsupported_image_input_error(&error_body) { - return Err(unsupported_image_input_error(error_body)); + return Err(AgentError::UnsupportedImageInput(error_body)); } if error_body.contains("No endpoints found that can handle the requested parameters") { return Err(openrouter_parameter_routing_error(&error_body)); diff --git a/crates/buzz-agent/tests/bin/fake_mcp.rs b/crates/buzz-agent/tests/bin/fake_mcp.rs index 5b660da48c..1b7f346162 100644 --- a/crates/buzz-agent/tests/bin/fake_mcp.rs +++ b/crates/buzz-agent/tests/bin/fake_mcp.rs @@ -12,6 +12,7 @@ //! (use a large value, e.g. 999, to simulate hang) //! FAKE_MCP_RESULT_SIZE=N — `tools/call` returns an N-byte text result //! (default: the literal "ok"); grows history +//! FAKE_MCP_IMAGE_RESULT=1 — `tools/call` returns text plus a PNG image block //! FAKE_MCP_PID_FILE=path — write the child PID to `path` on startup //! (for tests that want to verify the child died) //! FAKE_MCP_SPAWN_GRANDCHILD=1 @@ -300,10 +301,18 @@ fn main() { } else { "ok".to_owned() }; + let content = if env_flag("FAKE_MCP_IMAGE_RESULT") { + json!([ + { "type": "text", "text": result_text }, + { "type": "image", "data": "aW1n", "mimeType": "image/png" }, + ]) + } else { + json!([{ "type": "text", "text": result_text }]) + }; write_response( id, json!({ - "content": [{ "type": "text", "text": result_text }], + "content": content, "isError": false, }), ); diff --git a/crates/buzz-agent/tests/fake_llm.rs b/crates/buzz-agent/tests/fake_llm.rs index ef6f9d2d80..a2d5ac4039 100644 --- a/crates/buzz-agent/tests/fake_llm.rs +++ b/crates/buzz-agent/tests/fake_llm.rs @@ -57,9 +57,26 @@ async fn spawn_fake_llm(responses: Vec) -> String { url } +struct CannedResponse { + status: u16, + body: Value, +} + /// Like `spawn_fake_llm` but also captures the full JSON request body from each /// incoming HTTP request. Returns (url, captured_requests). async fn spawn_capturing_fake_llm(responses: Vec) -> (String, Arc>>) { + spawn_capturing_fake_llm_with_statuses( + responses + .into_iter() + .map(|body| CannedResponse { status: 200, body }) + .collect(), + ) + .await +} + +async fn spawn_capturing_fake_llm_with_statuses( + responses: Vec, +) -> (String, Arc>>) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let url = format!("http://{}", listener.local_addr().unwrap()); let queue = Arc::new(Mutex::new(VecDeque::from(responses))); @@ -122,15 +139,22 @@ async fn spawn_capturing_fake_llm(responses: Vec) -> (String, Arc Date: Wed, 5 Aug 2026 10:42:55 -0400 Subject: [PATCH 3/3] test(agent): cover the unsupported-image loop guard The `removed == 0` branch in the recovery path is the only thing stopping a phrase-matched 404 with no image in history from re-requesting forever: `max_rounds` defaults to 0 (unlimited) in production, and buzz-acp's queue retry never fires because the turn never returns. Deleting that guard left the whole package suite green, so the branch had no coverage. This test drives the typed error with an empty history and asserts the turn fails with the typed error after exactly one request. Co-authored-by: Sami Signed-off-by: Sami --- crates/buzz-agent/tests/fake_llm.rs | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/crates/buzz-agent/tests/fake_llm.rs b/crates/buzz-agent/tests/fake_llm.rs index a2d5ac4039..4253ef329c 100644 --- a/crates/buzz-agent/tests/fake_llm.rs +++ b/crates/buzz-agent/tests/fake_llm.rs @@ -437,6 +437,67 @@ async fn unsupported_image_response_recovers_without_replaying_image() { h.shutdown().await; } +/// The recovery path must only fire when it actually removed an image. If the +/// provider emits the unsupported-image phrase while history holds no image +/// (a misclassification, or a provider that returns the phrase for an +/// unrelated reason), mutating nothing and continuing would spin the turn loop +/// forever — `max_rounds` defaults to 0 (unlimited) in production, so nothing +/// downstream bounds it. The turn must fail with the typed error instead. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unsupported_image_without_image_in_history_fails_instead_of_looping() { + // Five rejections but MAX_ROUNDS=4: if the guard is removed the loop + // re-requests without ever mutating history and drains the queue. + let responses = (0..5) + .map(|_| CannedResponse { + status: 404, + body: json!({ + "error": { "message": "No endpoints found that support image input" } + }), + }) + .collect(); + let (url, captures) = spawn_capturing_fake_llm_with_statuses(responses).await; + let mut h = Harness::spawn(&url).await; + + h.send( + "initialize", + json!({"protocolVersion":2,"clientCapabilities":{}}), + ) + .await; + let _ = h.recv().await; + let session_id = h + .send("session/new", json!({ "cwd": "/tmp", "mcpServers": [] })) + .await; + let session = h.recv_until(|v| v["id"] == json!(session_id)).await; + let sid = session["result"]["sessionId"].as_str().unwrap(); + + let prompt_id = h + .send( + "session/prompt", + json!({ + "sessionId": sid, + "prompt": [{"type":"text","text":"no image here"}], + }), + ) + .await; + let reply = h.recv_until(|v| v["id"] == json!(prompt_id)).await; + + assert!( + reply.get("result").is_none(), + "an unrecoverable image rejection must not complete the turn: {reply}" + ); + let message = reply["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("image input unsupported"), + "the typed error must surface to the caller: {reply}" + ); + assert_eq!( + captures.lock().await.len(), + 1, + "the loop must not re-request after a rejection it could not repair" + ); + h.shutdown().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn rejects_concurrent_prompts() { // Slow first response so the second prompt arrives mid-flight.