diff --git a/.changeset/refuse-unsupported-image-formats.md b/.changeset/refuse-unsupported-image-formats.md new file mode 100644 index 0000000000..5a8c43f76f --- /dev/null +++ b/.changeset/refuse-unsupported-image-formats.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Stop unsupported image formats (AVIF, BMP, TIFF, ICO, …) from breaking sessions at every entry point — including remote image URLs and images mislabeled by a tool — and recover an already-stuck session by dropping the offending image and retrying, so one such image can no longer make every later request fail. diff --git a/packages/acp-adapter/src/convert.ts b/packages/acp-adapter/src/convert.ts index 4e28a1c62c..3d388f7b7c 100644 --- a/packages/acp-adapter/src/convert.ts +++ b/packages/acp-adapter/src/convert.ts @@ -3,6 +3,8 @@ import { log, buildImageCompressionCaption, compressBase64ForModel, + gateImageFormatParts, + parseImageDataUrl, persistOriginalImage, type PromptPart, type TelemetryClient, @@ -16,6 +18,9 @@ import { isHideOutputMarker } from './marker'; * Convert an array of ACP {@link ContentBlock}s into the SDK's * {@link PromptPart} array. * + * Image parts are built from the client-declared MIME verbatim; run the + * result through {@link compressPromptImageParts} before submitting so + * unsupported formats are dropped and MIME aliases canonicalized. */ export function acpBlocksToPromptParts( blocks: readonly ContentBlock[], @@ -81,6 +86,13 @@ export function acpBlocksToPromptParts( * server's upload-time step. Best effort: a part that cannot be compressed is * passed through unchanged. * + * The format gate (`gateImageFormatParts`) runs first: parts whose MIME is + * outside the provider-accepted set are never forwarded — the part is + * dropped and a text notice stands in, so one unsupported image cannot + * poison the session history; accepted MIME aliases (`image/jpg`, + * case/whitespace variants) are rewritten to the canonical form strict + * provider whitelists require. + * * Compression is never silent: a re-encoded image gains a caption text part * immediately before it stating what the original was, and the original bytes * are persisted (into `originalsDir` — typically the session's @@ -102,7 +114,7 @@ export async function compressPromptImageParts( } = {}, ): Promise { const out: PromptPart[] = []; - for (const part of parts) { + for (const part of gateImageFormatParts(parts) as PromptPart[]) { if (part.type === 'image_url') { const parsed = parseImageDataUrl(part.imageUrl.url); if (parsed !== null) { @@ -150,12 +162,6 @@ export async function compressPromptImageParts( return out; } -function parseImageDataUrl(url: string): { mimeType: string; base64: string } | null { - const match = /^data:([^;,]+);base64,(.*)$/s.exec(url); - if (match === null) return null; - return { mimeType: match[1]!, base64: match[2]! }; -} - /** * Minimum-viable XML-attribute escaping for prompt-embedded resource * wrappers. The output is consumed by an LLM, not parsed by a canonical diff --git a/packages/acp-adapter/test/convert.test.ts b/packages/acp-adapter/test/convert.test.ts index 0a280ab6b4..c9f3aa9c9e 100644 --- a/packages/acp-adapter/test/convert.test.ts +++ b/packages/acp-adapter/test/convert.test.ts @@ -426,4 +426,32 @@ describe('compressPromptImageParts', () => { const compressed = await compressPromptImageParts(parts); expect(compressed).toEqual(parts); }); + + it('replaces an image the provider cannot accept with a text notice', async () => { + // An AVIF image must never reach the session history — the provider + // rejects it and every later request would fail. A notice stands in. + const parts = acpBlocksToPromptParts([ + textBlock('look at this'), + imageBlock(Buffer.from([1, 2, 3]).toString('base64'), 'image/avif'), + ]); + const compressed = await compressPromptImageParts(parts); + + expect(compressed).toHaveLength(2); + expect(compressed[0]).toEqual({ type: 'text', text: 'look at this' }); + const notice = compressed[1]; + if (notice?.type !== 'text') throw new Error('expected a text notice'); + expect(notice.text).toContain('image/avif'); + }); + + it('forwards accepted MIME aliases in canonical form', async () => { + // Strict provider whitelists reject the raw `image/jpg` alias — the part + // must land in the session with the canonical MIME. + const base64 = Buffer.from([1, 2, 3]).toString('base64'); + const parts = acpBlocksToPromptParts([imageBlock(base64, 'image/jpg')]); + const compressed = await compressPromptImageParts(parts); + + expect(compressed).toEqual([ + { type: 'image_url', imageUrl: { url: `data:image/jpeg;base64,${base64}` } }, + ]); + }); }); diff --git a/packages/agent-core/src/agent/compaction/full.ts b/packages/agent-core/src/agent/compaction/full.ts index a982ee568e..b3a17b752f 100644 --- a/packages/agent-core/src/agent/compaction/full.ts +++ b/packages/agent-core/src/agent/compaction/full.ts @@ -16,6 +16,7 @@ import { APIRequestTooLargeError, APIStatusError, createUserMessage, + isImageFormatError, } from '@moonshot-ai/kosong'; import type { Agent } from '..'; @@ -468,16 +469,20 @@ export class FullCompaction { summary = extractCompactionSummary(response); break; } catch (error) { - // A request-body-size rejection (HTTP 413) is first retried with - // media parts replaced by text markers: accumulated base64 payloads - // are the usual culprit, and a text summary does not need them — - // the conversation already narrates what was seen, and the - // ReadMediaFile `` text wrapper survives. Only - // the summarizer input copy is rewritten; the real history keeps - // its media. A 413 after the strip (or with no media to strip) - // falls through to the overflow shrink below — dropping oldest - // messages shrinks the body too. - if (error instanceof APIRequestTooLargeError && !mediaStripAttempted) { + // A request-body-size rejection (HTTP 413) or an image-format + // rejection is first retried with media parts replaced by text + // markers: accumulated base64 payloads are the usual 413 culprit, + // a poisoned image the format-rejection culprit, and a text summary + // needs neither — the conversation already narrates what was seen, + // and the ReadMediaFile `` text wrapper survives. + // Only the summarizer input copy is rewritten; the real history + // keeps its media. A rejection after the strip (or with no media to + // strip) falls through to the overflow shrink below for a 413, and + // propagates for a format error — dropping oldest messages cannot + // fix a poisoned image's format. + const mediaRejected = + error instanceof APIRequestTooLargeError || isImageFormatError(error); + if (mediaRejected && !mediaStripAttempted) { mediaStripAttempted = true; const stripped = replaceMediaPartsWithMarkers(historyForModel); if (stripped !== historyForModel) { diff --git a/packages/agent-core/src/agent/context/index.ts b/packages/agent-core/src/agent/context/index.ts index 286c936556..7967b188a6 100644 --- a/packages/agent-core/src/agent/context/index.ts +++ b/packages/agent-core/src/agent/context/index.ts @@ -20,6 +20,7 @@ import { import { degradeOlderMediaParts, MEDIA_DEGRADE_KEEP_RECENT, + MEDIA_STRIPPED_PLACEHOLDERS, project, type ProjectionAnomaly, type ProjectOptions, @@ -501,6 +502,16 @@ export class ContextMemory { return degradeOlderMediaParts(this.messages, MEDIA_DEGRADE_KEEP_RECENT); } + // Fallback projection for the image-format resend: EVERY media part + // replaced by a text marker. Unlike the 413 case (too MUCH media), a + // format rejection means at least one image is poison and the error never + // says which — only a full strip guarantees the resend carries none. + // Purely read-side, and only used after the provider already rejected an + // image; see the image-format fallback in `turn-step`. + get mediaStrippedMessages(): Message[] { + return degradeOlderMediaParts(this.messages, 0, MEDIA_STRIPPED_PLACEHOLDERS); + } + useProjectedHistoryFrom(source: ContextMemory): void { this.clear(); this.pushHistory(...trimTrailingOpenToolExchange(source.project(source.history))); diff --git a/packages/agent-core/src/agent/context/projector.ts b/packages/agent-core/src/agent/context/projector.ts index bace0331a4..b19877f40a 100644 --- a/packages/agent-core/src/agent/context/projector.ts +++ b/packages/agent-core/src/agent/context/projector.ts @@ -482,26 +482,46 @@ const MEDIA_DEGRADED_PLACEHOLDERS = { '[video omitted: dropped to fit the provider request size limit; re-read the file to view it]', } as const; +/** + * Markers for the media-stripped resend after the provider rejected an + * image's FORMAT (not its size): the image marker points the model at + * re-reading the file, whose refusal carries per-OS conversion instructions; + * audio/video are collateral of the full strip and say so. + */ +export const MEDIA_STRIPPED_PLACEHOLDERS = { + image_url: + '[image omitted: the provider rejected this image; re-read the file for conversion instructions]', + audio_url: + '[audio omitted: dropped along with a rejected image; re-read the file to hear it]', + video_url: + '[video omitted: dropped along with a rejected image; re-read the file to view it]', +} as const; + +type MediaPlaceholderSet = typeof MEDIA_DEGRADED_PLACEHOLDERS | typeof MEDIA_STRIPPED_PLACEHOLDERS; + function isDegradableMediaPart( part: ContentPart, -): part is ContentPart & { type: keyof typeof MEDIA_DEGRADED_PLACEHOLDERS } { +): part is ContentPart & { type: keyof MediaPlaceholderSet } { return part.type in MEDIA_DEGRADED_PLACEHOLDERS; } /** * Replace all but the `keepRecent` most recent media parts with deterministic * text markers. This is the media-degraded projection used to resend a request - * the provider rejected as too large (HTTP 413 on accumulated base64 media): - * a purely read-side transform — the underlying history is left untouched — - * that trades old pixels for bytes while the surrounding text (including - * ReadMediaFile's `` wrapper) survives, so the model can - * re-read any file it still needs. Untouched messages are returned by - * reference, and when nothing needs degrading the input array itself is - * returned. + * the provider rejected as too large (HTTP 413 on accumulated base64 media) + * and — with `keepRecent = 0` and `MEDIA_STRIPPED_PLACEHOLDERS` — the resend + * after an image-format rejection, where the poisoned image could be anywhere + * and only a full strip guarantees a clean request. A purely read-side + * transform — the underlying history is left untouched — that trades pixels + * for deliverability while the surrounding text (including ReadMediaFile's + * `` wrapper) survives, so the model can re-read any file + * it still needs. Untouched messages are returned by reference, and when + * nothing needs degrading the input array itself is returned. */ export function degradeOlderMediaParts( messages: readonly Message[], keepRecent: number, + placeholders: MediaPlaceholderSet = MEDIA_DEGRADED_PLACEHOLDERS, ): Message[] { const mediaCount = messages.reduce( (count, message) => count + message.content.filter(isDegradableMediaPart).length, @@ -515,7 +535,7 @@ export function degradeOlderMediaParts( const content = message.content.map((part): ContentPart => { if (toDegrade === 0 || !isDegradableMediaPart(part)) return part; toDegrade -= 1; - return { type: 'text', text: MEDIA_DEGRADED_PLACEHOLDERS[part.type] }; + return { type: 'text', text: placeholders[part.type] }; }); return { ...message, content }; }); diff --git a/packages/agent-core/src/agent/records/types.ts b/packages/agent-core/src/agent/records/types.ts index 53cac76f07..fed30c675b 100644 --- a/packages/agent-core/src/agent/records/types.ts +++ b/packages/agent-core/src/agent/records/types.ts @@ -177,9 +177,9 @@ export interface AgentRecordEvents { messageCount: number; turnStep?: string; attempt?: string; - /** Set when this request is a fallback resend (strict rebuild or - * media-degraded rebuild). */ - projection?: 'strict' | 'media-degraded'; + /** Set when this request is a fallback resend (strict rebuild, + * media-degraded rebuild, or media-stripped rebuild). */ + projection?: 'strict' | 'media-degraded' | 'media-stripped'; /** Compaction only: messages dropped so far by overflow/empty shrinking. */ droppedCount?: number; }; diff --git a/packages/agent-core/src/agent/turn/index.ts b/packages/agent-core/src/agent/turn/index.ts index 2ee83c5054..b392b41723 100644 --- a/packages/agent-core/src/agent/turn/index.ts +++ b/packages/agent-core/src/agent/turn/index.ts @@ -34,6 +34,7 @@ import { } from '../../loop/index'; import type { AgentEvent, TurnEndedEvent, TurnEndReason } from '../../rpc'; import type { TelemetryPropertyValue } from '../../telemetry'; +import { gateImageFormatParts } from '../../tools/support/image-compress'; import { abortable, isUserCancellation, userCancellationReason } from '../../utils/abort'; import { USER_PROMPT_ORIGIN, type PromptOrigin } from '../context'; import { renderUserPromptHookBlockResult, renderUserPromptHookResult } from '../../session/hooks'; @@ -134,20 +135,27 @@ export class TurnFlow { // Returns the new turnId, or null if the turn was marked as resuming. prompt(input: readonly ContentPart[], origin: PromptOrigin = USER_PROMPT_ORIGIN): number | null { + // The last funnel before a prompt lands in the session history: images + // in formats providers reject (AVIF, HEIC, …) become text notices here, + // so no caller — the SDK/RPC prompt path included — can poison the + // session. Upstream ingestion points already gate; this is the backstop. + const gated = gateImageFormatParts(input); this.agent.records.logRecord({ type: 'turn.prompt', - input, + input: gated, origin, }); - return this.launch(input, origin); + return this.launch(gated, origin); } // Returns the new turnId, or null if the input was buffered as a steer // message or the turn was marked as resuming. steer(input: readonly ContentPart[], origin: PromptOrigin = USER_PROMPT_ORIGIN): number | null { + // Same format gate as prompt() — steer input enters the history too. + const gated = gateImageFormatParts(input); this.agent.records.logRecord({ type: 'turn.steer', - input, + input: gated, origin, }); // Buffer while a turn is active OR a manual compaction holds the context; @@ -155,10 +163,10 @@ export class TurnFlow { // (summary + reinjection) is done. Returning null means "buffered" — which is // exactly what fire-and-forget callers (background notifications, cron) assume. if (this.activeTurn || this.agent.fullCompaction.isCompacting) { - this.steerBuffer.push({ input, origin }); + this.steerBuffer.push({ input: gated, origin }); return null; } - return this.launch(input, origin); + return this.launch(gated, origin); } retry(trigger?: string): number | null { @@ -725,6 +733,7 @@ export class TurnFlow { buildMessages: () => this.agent.context.messages, buildMessagesStrict: () => this.agent.context.strictMessages, buildMessagesMediaDegraded: () => this.agent.context.mediaDegradedMessages, + buildMessagesMediaStripped: () => this.agent.context.mediaStrippedMessages, dispatchEvent: this.buildDispatchEvent(turnId), // Re-read per step (not snapshotted per turn) so a select_tools load // is dispatchable on the very next step of the same turn. diff --git a/packages/agent-core/src/index.ts b/packages/agent-core/src/index.ts index b4ce829e27..afc53fccd2 100644 --- a/packages/agent-core/src/index.ts +++ b/packages/agent-core/src/index.ts @@ -64,12 +64,24 @@ export { compressImageContentParts, cropImageForModel, formatByteSize, + gateImageFormatParts, resolveMaxImageEdgePx, resolveReadImageByteBudget, IMAGE_BYTE_BUDGET, MAX_IMAGE_EDGE_PX, READ_IMAGE_BYTE_BUDGET, } from './tools/support/image-compress'; +export { + MODEL_ACCEPTED_IMAGE_MIMES, + buildImageConversionGuidance, + buildUnsupportedImageNotice, + decodeBase64Prefix, + isModelAcceptedImageMime, + normalizeImageMime, + parseImageDataUrl, + resolveEffectiveImageMime, + unsupportedImageMimeFromUrl, +} from './tools/support/image-format-policy'; export { ImageLimits } from './tools/support/image-limits'; export type { CompressAnnotateOptions, diff --git a/packages/agent-core/src/loop/llm.ts b/packages/agent-core/src/loop/llm.ts index 0e25ce6dfa..00ca36ded3 100644 --- a/packages/agent-core/src/loop/llm.ts +++ b/packages/agent-core/src/loop/llm.ts @@ -34,9 +34,10 @@ export interface LLMRequestLogFields { /** Request purpose; absent means a regular loop step. */ readonly kind?: 'loop' | 'compaction'; /** Set when the messages are a fallback resend projection: the strict - * wire-compliant rebuild, or the media-degraded rebuild after a - * request-too-large rejection. */ - readonly projection?: 'strict' | 'media-degraded'; + * wire-compliant rebuild, the media-degraded rebuild after a + * request-too-large rejection, or the media-stripped rebuild after an + * image-format rejection. */ + readonly projection?: 'strict' | 'media-degraded' | 'media-stripped'; /** Compaction only: messages dropped so far by overflow/empty shrinking. */ readonly droppedCount?: number; } diff --git a/packages/agent-core/src/loop/run-turn.ts b/packages/agent-core/src/loop/run-turn.ts index 767da02b78..b05f8cf7d7 100644 --- a/packages/agent-core/src/loop/run-turn.ts +++ b/packages/agent-core/src/loop/run-turn.ts @@ -50,6 +50,15 @@ export interface RunTurnInput { * so each step does not pay a fresh rejection. */ readonly buildMessagesMediaDegraded?: LoopMessageBuilder | undefined; + /** + * Optional media-stripped rebuild of the request messages: EVERY media + * part replaced by a text marker. Used to resend once after the provider + * rejects an image's format (see `executeLoopStep`); the poisoned image + * could be anywhere in the history, so only a full strip guarantees a + * clean request. After a successful stripped resend, later steps of the + * same turn build from this projection directly. + */ + readonly buildMessagesMediaStripped?: LoopMessageBuilder | undefined; readonly dispatchEvent: LoopEventDispatcher; readonly tools?: readonly ExecutableTool[] | undefined; /** @@ -84,6 +93,7 @@ export async function runTurn(input: RunTurnInput): Promise { buildMessages, buildMessagesStrict, buildMessagesMediaDegraded, + buildMessagesMediaStripped, dispatchEvent, tools, buildTools, @@ -104,6 +114,9 @@ export async function runTurn(input: RunTurnInput): Promise { // history is deterministically over the provider's body-size limit, so // rebuilding it would pay a fresh rejection on every step. let mediaDegradedActive = false; + // Same for the media-stripped resend after an image-format rejection: the + // poison is still in the full history, so later steps stay stripped. + let mediaStrippedActive = false; const recordStepUsage = async ( stepUsage: TokenUsage, ): Promise => { @@ -125,11 +138,14 @@ export async function runTurn(input: RunTurnInput): Promise { turnId, signal, buildMessages: - mediaDegradedActive && buildMessagesMediaDegraded !== undefined - ? buildMessagesMediaDegraded - : buildMessages, + mediaStrippedActive && buildMessagesMediaStripped !== undefined + ? buildMessagesMediaStripped + : mediaDegradedActive && buildMessagesMediaDegraded !== undefined + ? buildMessagesMediaDegraded + : buildMessages, buildMessagesStrict, buildMessagesMediaDegraded, + buildMessagesMediaStripped, dispatchEvent, llm, tools, @@ -147,6 +163,7 @@ export async function runTurn(input: RunTurnInput): Promise { }); activeStep = undefined; mediaDegradedActive = mediaDegradedActive || stepResult.mediaDegradedResendUsed === true; + mediaStrippedActive = mediaStrippedActive || stepResult.mediaStrippedResendUsed === true; if (stepResult.stopReason === 'tool_use') { continue; diff --git a/packages/agent-core/src/loop/turn-step.ts b/packages/agent-core/src/loop/turn-step.ts index 09e83c5ee9..1a5a16650b 100644 --- a/packages/agent-core/src/loop/turn-step.ts +++ b/packages/agent-core/src/loop/turn-step.ts @@ -11,6 +11,7 @@ import { randomUUID } from 'node:crypto'; import { APIRequestTooLargeError, + isImageFormatError, isRecoverableRequestStructureError, type TokenUsage, } from '@moonshot-ai/kosong'; @@ -41,6 +42,8 @@ export interface ExecuteLoopStepDeps { readonly buildMessagesStrict?: LoopMessageBuilder | undefined; /** See RunTurnInput.buildMessagesMediaDegraded. */ readonly buildMessagesMediaDegraded?: LoopMessageBuilder | undefined; + /** See RunTurnInput.buildMessagesMediaStripped. */ + readonly buildMessagesMediaStripped?: LoopMessageBuilder | undefined; readonly dispatchEvent: LoopEventDispatcher; readonly llm: LLM; readonly tools?: readonly ExecutableTool[] | undefined; @@ -70,6 +73,12 @@ export async function executeLoopStep(deps: ExecuteLoopStepDeps): Promise<{ * rejection on every step of the turn. */ readonly mediaDegradedResendUsed?: boolean; + /** + * True when this step only succeeded after resending with every media + * part stripped (image-format rejection). The turn loop keeps later steps + * on the stripped projection for the same reason as above. + */ + readonly mediaStrippedResendUsed?: boolean; }> { const { turnId, @@ -77,6 +86,7 @@ export async function executeLoopStep(deps: ExecuteLoopStepDeps): Promise<{ buildMessages, buildMessagesStrict, buildMessagesMediaDegraded, + buildMessagesMediaStripped, dispatchEvent, llm, tools, @@ -155,6 +165,7 @@ export async function executeLoopStep(deps: ExecuteLoopStepDeps): Promise<{ } as const; let response: LLMChatResponse; let mediaDegradedResendUsed = false; + let mediaStrippedResendUsed = false; try { response = await chatWithRetry({ ...retryInput, params: chatParams }); } catch (error) { @@ -195,6 +206,45 @@ export async function executeLoopStep(deps: ExecuteLoopStepDeps): Promise<{ log?.info('recovered after media-degraded resend', { turnStep: `${turnId}.${String(currentStep)}`, }); + } else if (buildMessagesMediaStripped !== undefined && isImageFormatError(error)) { + // The provider rejected an IMAGE in the request (unsupported format or + // undecodable data). Unlike the 413 case — too MUCH media — the error + // never says WHICH image is poison, and the same history is re-sent + // every turn, so the session would stay stuck. Resend ONCE with every + // media part replaced by a text marker: the only projection guaranteed + // to carry no poison. Read-side only — the history keeps its media, + // and the `` wrappers survive so the model can + // re-read files (getting conversion guidance for refused formats). A + // rejection of that rebuild propagates unchanged. + signal.throwIfAborted(); + log?.warn('provider rejected an image in the request; resending with all media stripped', { + turnStep: `${turnId}.${String(currentStep)}`, + model: llm.modelName, + }); + const strippedMessages = await buildMessagesMediaStripped(); + signal.throwIfAborted(); + try { + response = await chatWithRetry({ + ...retryInput, + params: { + ...chatParams, + messages: strippedMessages, + requestLogFields: { projection: 'media-stripped' }, + }, + }); + } catch (strippedError) { + log?.error('media-stripped resend still rejected by provider', { + turnStep: `${turnId}.${String(currentStep)}`, + model: llm.modelName, + originalError: errorMessage(error), + strippedError: errorMessage(strippedError), + }); + throw strippedError; + } + mediaStrippedResendUsed = true; + log?.info('recovered after media-stripped resend', { + turnStep: `${turnId}.${String(currentStep)}`, + }); } else if (buildMessagesStrict !== undefined && isRecoverableRequestStructureError(error)) { // A structural request rejection (tool_use/tool_result pairing, empty or // whitespace-only text, non-user first message, non-alternating roles) means @@ -300,6 +350,7 @@ export async function executeLoopStep(deps: ExecuteLoopStepDeps): Promise<{ stopReason: stopTurnAfterStep && effectiveStopReason === 'tool_use' ? 'end_turn' : effectiveStopReason, mediaDegradedResendUsed, + mediaStrippedResendUsed, }; } diff --git a/packages/agent-core/src/mcp/output.ts b/packages/agent-core/src/mcp/output.ts index 84e246a9d9..08fe82e9a1 100644 --- a/packages/agent-core/src/mcp/output.ts +++ b/packages/agent-core/src/mcp/output.ts @@ -32,6 +32,10 @@ import type { ContentPart } from '@moonshot-ai/kosong'; import type { TelemetryClient } from '#/telemetry'; import { compressImageContentParts } from '../tools/support/image-compress'; +import { + buildUnsupportedImageNotice, + isModelAcceptedImageMime, +} from '../tools/support/image-format-policy'; import { persistOriginalImage } from '../tools/support/image-originals'; import type { MCPContentBlock, MCPToolResult } from './types'; @@ -133,6 +137,15 @@ export function convertMCPContentBlock(block: MCPContentBlock): ContentPart | nu if (block.type === 'resource_link' && typeof block.uri === 'string') { const mimeType = block.mimeType ?? 'application/octet-stream'; if (mimeType.startsWith('image/')) { + // The declared MIME is the only format signal for a remote image: an + // extensionless or signed URL gives the extension gate nothing to work + // with, and the provider fetches it server-side. When the server + // honestly declares a format providers reject (e.g. an image search + // tool returning AVIF links), drop the image for a notice that keeps + // the URL — the model can still fetch and convert it. + if (!isModelAcceptedImageMime(mimeType)) { + return { type: 'text', text: buildUnsupportedImageNotice(mimeType, block.uri) }; + } return { type: 'image_url', imageUrl: { url: block.uri } }; } if (mimeType.startsWith('audio/')) { diff --git a/packages/agent-core/src/tools/builtin/file/read-media.ts b/packages/agent-core/src/tools/builtin/file/read-media.ts index 4543c4e33c..05ce837ccd 100644 --- a/packages/agent-core/src/tools/builtin/file/read-media.ts +++ b/packages/agent-core/src/tools/builtin/file/read-media.ts @@ -48,6 +48,10 @@ import { type ImageCompressionTelemetry, type ImageCropRegion, } from '../../support/image-compress'; +import { + buildImageConversionGuidance, + isModelAcceptedImageMime, +} from '../../support/image-format-policy'; import { ImageLimits } from '../../support/image-limits'; import { toInputJsonSchema } from '../../support/input-schema'; import { literalRulePattern, matchesPathRuleSubject } from '../../support/rule-match'; @@ -214,45 +218,6 @@ function buildMediaNote(input: { // ── Implementation ─────────────────────────────────────────────────── -/** - * Refusal message for HEIC/HEIF with a conversion command matching the - * execution environment (`kaos.osEnv.osKind` — where Bash actually runs, so - * SSH/container sessions get the right command too). macOS converts with the - * built-in `sips`; Linux and Windows have no built-in HEIC decoder, so the - * guidance names the common tools and how to get them. - */ -function buildHeicConversionGuidance(path: string, mimeType: string, osKind: string): string { - const converted = path.replace(/\.[^./\\]+$/, '') + '.jpg'; - return ( - `"${path}" is a ${mimeType} image, which the provider does not accept. ` + - 'Convert it to JPEG first, then read the converted file. ' + - heicConversionCommand(path, converted, osKind) - ); -} - -function heicConversionCommand(path: string, converted: string, osKind: string): string { - switch (osKind) { - case 'macOS': - return `On macOS: sips -s format jpeg "${path}" --out "${converted}"`; - case 'Linux': - return ( - `On Linux: heif-convert "${path}" "${converted}" (package libheif-examples), ` + - `or with ImageMagick: magick "${path}" "${converted}"` - ); - case 'Windows': - return ( - `On Windows, with ImageMagick: magick "${path}" "${converted}" ` + - '(install it first if missing: winget install ImageMagick.ImageMagick)' - ); - default: - return ( - `Options: sips -s format jpeg "${path}" --out "${converted}" (macOS), ` + - `heif-convert "${path}" "${converted}" (Linux, package libheif-examples), ` + - `or magick "${path}" "${converted}" (ImageMagick)` - ); - } -} - export class ReadMediaFileTool implements BuiltinTool { readonly name = 'ReadMediaFile' as const; readonly description: string; @@ -336,15 +301,22 @@ export class ReadMediaFileTool implements BuiltinTool { 'Tell the user to use a model with image input capability.', }; } - // HEIC/HEIF must never reach the provider: no provider accepts them, - // and once the image_url lands in the history every subsequent request - // in the session is rejected. Refuse with a conversion command for the - // execution environment instead — the model can run it through Bash - // (under the normal permission flow) and read the converted file. - if (fileType.mimeType === 'image/heic' || fileType.mimeType === 'image/heif') { + // Formats outside the provider-accepted set (AVIF, HEIC, BMP, TIFF, + // ICO, …) must never reach the model: once the image_url lands in the + // history every subsequent request in the session is rejected. Refuse + // with a conversion command for the execution environment instead — + // the model can run it through Bash (under the normal permission flow) + // and read the converted file. The accepted set and guidance live in + // support/image-format-policy, the single source of truth every + // ingestion point shares. + if (fileType.kind === 'image' && !isModelAcceptedImageMime(fileType.mimeType)) { return { isError: true, - output: buildHeicConversionGuidance(args.path, fileType.mimeType, this.kaos.osEnv.osKind), + output: buildImageConversionGuidance( + args.path, + fileType.mimeType, + this.kaos.osEnv.osKind, + ), }; } if (fileType.kind === 'video' && !this.capabilities.video_in) { diff --git a/packages/agent-core/src/tools/support/image-compress.ts b/packages/agent-core/src/tools/support/image-compress.ts index 75f9d46bdd..ecc8cdf9e7 100644 --- a/packages/agent-core/src/tools/support/image-compress.ts +++ b/packages/agent-core/src/tools/support/image-compress.ts @@ -16,8 +16,12 @@ * prompt. Callers simply send the original instead. * - PNG, JPEG, and (non-animated) WebP are re-encoded; WebP re-encodes * through the PNG/JPEG ladder, so only its decoder wasm ships. GIF and - * animated WebP are passed through to preserve animation. Unknown formats - * are passed through. + * animated WebP are passed through to preserve animation. Formats outside + * the provider-accepted set (see ./image-format-policy) are never + * forwarded by the part-level helpers — they are replaced with a text + * notice; the byte-level helpers still pass anything they cannot + * re-encode through unchanged, so callers must gate on + * `isModelAcceptedImageMime` first. * - Compression must never be silent to the model: results carry the * original dimensions, {@link buildImageCompressionCaption} renders the * shared "what was compressed, where is the original" note every ingestion @@ -33,6 +37,17 @@ import type { ContentPart } from '@moonshot-ai/kosong'; import type { TelemetryClient } from '#/telemetry'; import { sniffImageDimensions } from './file-type'; +import { + buildMalformedImageNotice, + buildUnsupportedImageNotice, + decodeBase64Prefix, + isDataUrl, + isModelAcceptedImageMime, + normalizeImageMime, + parseImageDataUrl, + resolveEffectiveImageMime, + unsupportedImageMimeFromUrl, +} from './image-format-policy'; import { decodeWebp, isAnimatedWebp } from './webp-decode'; /** @@ -252,7 +267,7 @@ export async function compressImageForModel( const maxEdge = options.maxEdge ?? resolveMaxImageEdgePx(); const byteBudget = options.byteBudget ?? IMAGE_BYTE_BUDGET; const maxDecodeBytes = options.maxDecodeBytes ?? MAX_DECODE_BYTES; - const normalizedMime = normalizeMime(mimeType); + const normalizedMime = normalizeImageMime(mimeType); const dims = sniffImageDimensions(bytes); const passthrough = (): CompressImageResult => ({ @@ -376,7 +391,10 @@ export interface CompressBase64Result { /** * Convenience wrapper for call sites that already hold base64 (MCP results, * data URLs). Decodes, compresses, and re-encodes to base64. Best effort: - * returns the original base64 unchanged on any failure. + * returns the original base64 unchanged on any failure — including formats it + * cannot re-encode, so callers must refuse MIME types outside the + * provider-accepted set (`isModelAcceptedImageMime`) before building an + * image part from the result. */ export async function compressBase64ForModel( base64: string, @@ -404,7 +422,7 @@ export async function compressBase64ForModel( reportCompressEvent(options.telemetry, { outcome: 'passthrough_guard', startedAt, - inputMime: normalizeMime(mimeType), + inputMime: normalizeImageMime(mimeType), exifTransposed: false, result, }); @@ -428,7 +446,7 @@ export async function compressBase64ForModel( reportCompressEvent(options.telemetry, { outcome: 'passthrough_error', startedAt, - inputMime: normalizeMime(mimeType), + inputMime: normalizeImageMime(mimeType), exifTransposed: false, result, }); @@ -475,6 +493,78 @@ export interface CompressedContentParts { readonly captions: readonly string[]; } +/** + * Enforce the provider-accepted image format set (see ./image-format-policy) + * on a content-part list. Inline `data:` image parts whose MIME is outside + * the accepted set are dropped and replaced with a text notice, so one + * unsupported image cannot poison the session history. Accepted images are + * forwarded only as the byte-exact canonical data URL: an alias + * (`image/jpg`), case/whitespace variants, or MIME parameters + * (`image/jpeg;charset=utf-8`) all rebuild to the bare canonical form, + * because strict provider whitelists exact-match the full header. Remote + * (non-data) image URLs and non-image parts pass through — a URL carries no + * bytes to inspect. + * + * The BYTES are authoritative, not the declared MIME: the header of each + * inline image is sniffed, and a mismatch (e.g. AVIF bytes an MCP image + * search tool labels `image/png`) is gated on what the image IS — the + * provider decodes bytes, not labels. When the sniffer doesn't recognize + * the bytes (corrupt image, exotic container), the declared MIME stands + * and the 400-recovery path remains the backstop. + * + * This is the format gate shared by every ingestion point; run it BEFORE + * compression so unsupported bytes are never decoded. + */ +export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart[] { + const out: ContentPart[] = []; + for (const part of parts) { + if (part.type === 'image_url') { + const parsed = parseImageDataUrl(part.imageUrl.url); + if (parsed === null) { + // A `data:` URL that failed to parse (missing `;base64,` separator, + // empty MIME, …) is guaranteed to fail at the provider — Anthropic + // throws on it, OpenAI-compat servers 400. Drop it for a notice at + // ingestion instead of leaving it to poison the session and trigger + // the media-stripped resend on every later turn. + if (isDataUrl(part.imageUrl.url)) { + out.push({ type: 'text', text: buildMalformedImageNotice(part.imageUrl.url) }); + continue; + } + // Remote image URL (no bytes to sniff): reject when its path + // extension names a format providers reject (e.g. a search-tool + // link ending in `.avif`) — the notice keeps the URL so the model + // can still fetch and convert the image. Extensionless / unknown + // URLs pass through to the provider — and to the 400 recovery. + const extMime = unsupportedImageMimeFromUrl(part.imageUrl.url); + if (extMime !== null) { + out.push({ + type: 'text', + text: buildUnsupportedImageNotice(extMime, part.imageUrl.url), + }); + continue; + } + out.push(part); + continue; + } + const effectiveMime = resolveEffectiveImageMime( + parsed.mimeType, + decodeBase64Prefix(parsed.base64), + ); + if (!isModelAcceptedImageMime(effectiveMime)) { + out.push({ type: 'text', text: buildUnsupportedImageNotice(effectiveMime) }); + continue; + } + const canonicalUrl = `data:${normalizeImageMime(effectiveMime)};base64,${parsed.base64}`; + if (part.imageUrl.url !== canonicalUrl) { + out.push({ type: 'image_url', imageUrl: { ...part.imageUrl, url: canonicalUrl } }); + continue; + } + } + out.push(part); + } + return out; +} + /** * Compress any inline base64 image parts in a content-part list — used by * the MCP tool-result path (prompt ingestion compresses per image with @@ -483,6 +573,13 @@ export interface CompressedContentParts { * through, as are non-image parts. Best effort: a part that fails to * compress is left unchanged. * + * The format gate ({@link gateImageFormatParts}) runs first: parts whose + * MIME is outside the provider-accepted set are never forwarded — the part + * is dropped and a text notice stands in, so one unsupported image cannot + * poison the session history. This is the MCP funnel's enforcement point — + * MCP servers can return any `image/*` MIME (e.g. AVIF from an image search + * tool). + * * With `annotate` set, every image that was actually re-encoded gets a * caption in {@link CompressedContentParts.captions} so the model knows it * is looking at a downsampled copy. `annotate.persistOriginal` additionally @@ -497,7 +594,7 @@ export async function compressImageContentParts( const { annotate, ...compressOptions } = options; const out: ContentPart[] = []; const captions: string[] = []; - for (const part of parts) { + for (const part of gateImageFormatParts(parts)) { if (part.type === 'image_url') { const parsed = parseImageDataUrl(part.imageUrl.url); if (parsed !== null) { @@ -624,7 +721,7 @@ export async function cropImageForModel( const maxEdge = options.maxEdge ?? resolveMaxImageEdgePx(); const byteBudget = options.byteBudget ?? IMAGE_BYTE_BUDGET; const maxDecodeBytes = options.maxDecodeBytes ?? MAX_DECODE_BYTES; - const normalizedMime = normalizeMime(mimeType); + const normalizedMime = normalizeImageMime(mimeType); const fail = (errorKind: CropErrorKind, error: string): CropImageFailure => { reportCropEvent(options.telemetry, { startedAt, ok: false, errorKind }); @@ -857,12 +954,6 @@ export function formatByteSize(bytes: number): string { return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; } -function parseImageDataUrl(url: string): { mimeType: string; base64: string } | null { - const match = /^data:([^;,]+);base64,(.*)$/s.exec(url); - if (match === null) return null; - return { mimeType: match[1]!, base64: match[2]! }; -} - // ── internals ──────────────────────────────────────────────────────── /** The concrete jimp image instance type, derived from the lazily-loaded module. */ @@ -1006,11 +1097,6 @@ function fitWithinEdge(image: JimpImage, edge: number): boolean { return true; } -function normalizeMime(mimeType: string): string { - const lower = mimeType.trim().toLowerCase(); - return lower === 'image/jpg' ? 'image/jpeg' : lower; -} - // ── telemetry ──────────────────────────────────────────────────────── /** Failure classification carried by the `image_crop` event. */ @@ -1055,7 +1141,7 @@ function reportCompressEvent( source: telemetry.source, outcome: input.outcome, input_mime: input.inputMime, - output_mime: normalizeMime(input.result.mimeType), + output_mime: normalizeImageMime(input.result.mimeType), original_bytes: input.result.originalByteLength, final_bytes: input.result.finalByteLength, original_width: input.result.originalWidth, diff --git a/packages/agent-core/src/tools/support/image-format-policy.ts b/packages/agent-core/src/tools/support/image-format-policy.ts new file mode 100644 index 0000000000..db72e4bb18 --- /dev/null +++ b/packages/agent-core/src/tools/support/image-format-policy.ts @@ -0,0 +1,270 @@ +/** + * Provider-accepted image formats — the single source of truth. + * + * Model providers accept only PNG, JPEG, GIF, and WebP image blocks. An + * `image_url` part carrying any other MIME (AVIF, HEIC, BMP, TIFF, ICO, …) + * is rejected by the API — and because prompts and tool results persist in + * the session history, that one part makes every subsequent request fail + * too ("session poisoning"). Every ingestion point therefore refuses + * unsupported formats instead of passing the bytes through: ReadMediaFile + * refuses with a conversion command the model can run, and prompt/MCP + * ingestion replaces the image with a text notice. + * + * The policy is deliberately a closed set, not a denylist: a format is only + * ever sent when it is known to be accepted. Supporting a new format means + * adding it to {@link MODEL_ACCEPTED_IMAGE_MIMES}; tailoring the refusal + * guidance for a newly-seen unsupported format means adding one row to + * {@link UNSUPPORTED_IMAGE_FORMATS}. + * + * Inbound MIME strings are normalized for the DECISION + * ({@link normalizeImageMime}: case, whitespace, `image/jpg`), but every + * call site must forward the CANONICAL MIME into the session — strict + * provider whitelists (e.g. Anthropic's) reject the raw alias, which would + * re-create the very session poisoning this module exists to prevent. + * + * Scope: only inline `data:` images can be gated. A remote http(s) image URL + * (an MCP `resource_link`, a REST `source.kind: 'url'` part) carries no + * bytes to inspect, and providers that support URL images fetch them + * server-side; those pass through unchanged. + */ + +import { IMAGE_MIME_BY_SUFFIX, sniffMediaFromMagic } from './file-type'; + +/** Image MIME types every provider accepts. The closed set. */ +export const MODEL_ACCEPTED_IMAGE_MIMES: ReadonlySet = new Set([ + 'image/png', + 'image/jpeg', + 'image/gif', + 'image/webp', +]); + +/** Human-readable list of the accepted formats, for notices. */ +const ACCEPTED_FORMATS_TEXT = 'PNG, JPEG, GIF, and WebP'; + +interface UnsupportedImageFormatInfo { + /** + * A format-specific Linux decoder named in the conversion guidance (e.g. + * heif-convert for HEIC/HEIF). Other OSes, and formats without a dedicated + * decoder, are guided to sips (macOS) or ImageMagick. + */ + readonly linuxDecoder?: { readonly command: string; readonly packageName: string }; +} + +/** + * Unsupported formats worth tailoring the guidance for, by normalized MIME. + * A missing entry still means "refuse" — the entry only adds a + * format-specific conversion hint. + */ +const UNSUPPORTED_IMAGE_FORMATS: Readonly> = + Object.freeze({ + 'image/avif': {}, + 'image/heic': { linuxDecoder: { command: 'heif-convert', packageName: 'libheif-examples' } }, + 'image/heif': { linuxDecoder: { command: 'heif-convert', packageName: 'libheif-examples' } }, + 'image/bmp': {}, + 'image/tiff': {}, + 'image/x-icon': {}, + }); + +/** + * Lowercase, drop MIME parameters, and apply the `image/jpg` alias. Parameter + * stripping keeps a declared media type like `image/jpeg; charset=utf-8` + * consistent with a data-URL MIME token (which the parser already clips at + * the first `;`), so an accepted image with parameters is treated exactly + * like the bare form instead of being misread as unsupported. + */ +export function normalizeImageMime(mimeType: string): string { + const lower = mimeType.trim().toLowerCase(); + const semi = lower.indexOf(';'); + const base = (semi === -1 ? lower : lower.slice(0, semi)).trim(); + return base === 'image/jpg' ? 'image/jpeg' : base; +} + +/** + * Base64 characters decoded for magic sniffing: 48 chars ≈ 36 bytes, which + * covers every signature the sniffer reads (RIFF chunk at offset 8, ftyp + * brand at offset 8-12, ASF at 16). + */ +const BASE64_SNIFF_CHARS = 48; + +/** + * Decode just the prefix of a base64 payload needed for magic-byte + * sniffing, without allocating the full image. `Buffer.from` never throws + * on malformed base64 — it decodes what it can. + */ +export function decodeBase64Prefix(base64: string): Buffer { + return Buffer.from(base64.slice(0, BASE64_SNIFF_CHARS), 'base64'); +} + +/** + * The MIME an image should be judged by: the sniffed bytes when the magic + * header is recognized (bytes are authoritative — a mislabeled image, e.g. + * AVIF bytes an MCP image search tool labels `image/png`, is gated on what + * it IS, because the provider decodes bytes not labels), else the declared + * MIME. A header recognized as a non-image container also wins, so a video + * file hiding in an image part is refused instead of trusted. + */ +export function resolveEffectiveImageMime(declaredMime: string, header: Uint8Array): string { + const sniffed = sniffMediaFromMagic(header); + return sniffed !== null ? sniffed.mimeType : declaredMime; +} + +/** + * Whether a non-data image URL points at a format providers reject, judged + * by its path extension. A best-effort heuristic for the case where there + * are no bytes to sniff (a remote http(s) image): catches the common + * search-tool direct link ending in `.avif`. Query string and fragment are + * ignored; the match is case-insensitive. URLs without an extension, or + * whose extension lies, fall through to the provider — and to the 400 + * recovery — unchanged. + * + * Returns the unsupported MIME when the extension is known and not in the + * accepted set (so the notice names the right format), else null. + */ +export function unsupportedImageMimeFromUrl(url: string): string | null { + let path = url; + const query = path.indexOf('?'); + if (query !== -1) path = path.slice(0, query); + const hash = path.indexOf('#'); + if (hash !== -1) path = path.slice(0, hash); + const dot = path.lastIndexOf('.'); + if (dot === -1) return null; + const ext = path.slice(dot).toLowerCase(); + // `.svg` is deliberately absent from IMAGE_MIME_BY_SUFFIX — SVG files are + // text for the file tools — but as a remote image URL it is accepted by no + // provider, so flag it here without touching the shared suffix map. + const mime = ext === '.svg' ? 'image/svg+xml' : IMAGE_MIME_BY_SUFFIX[ext]; + if (mime === undefined || isModelAcceptedImageMime(mime)) return null; + return mime; +} + +/** + * Parse an image `data:` URL into its MIME and base64 payload. The MIME is + * returned raw — callers decide via {@link isModelAcceptedImageMime} and + * forward {@link normalizeImageMime}. MIME parameters are tolerated and + * ignored (`data:image/avif;charset=utf-8;base64,…`), so a parameter-bearing + * URL cannot slip past the format gate. The scheme and `base64` marker are + * matched case-insensitively (RFC 2045 encoding names are case-insensitive), + * so an uppercase `;BASE64,` cannot slip past either — and since callers + * rebuild to the canonical URL, the marker comes back out lowercase. + * Returns null for non-data URLs (e.g. a remote http(s) image — see the + * scope note in the module header). + */ +export function parseImageDataUrl(url: string): { mimeType: string; base64: string } | null { + const match = /^data:([^;,]+)(?:;[^;,]+)*?;base64,(.*)$/si.exec(url); + if (match === null) return null; + return { mimeType: match[1]!, base64: match[2]! }; +} + +/** + * Whether a URL claims to be a `data:` URL (the scheme is case-insensitive). + * Used to distinguish "failed to parse a data URL" (malformed — guaranteed + * to fail at the provider) from "not a data URL" (a remote http(s) image + * the provider fetches). + */ +export function isDataUrl(url: string): boolean { + return url.toLowerCase().startsWith('data:'); +} + +/** + * Whether an image with this MIME may be sent to the model. Only the closed + * accepted set passes; everything else must be refused at the entry point — + * once an unsupported `image_url` lands in the session history, every later + * request in the session is rejected by the provider. + */ +export function isModelAcceptedImageMime(mimeType: string): boolean { + return MODEL_ACCEPTED_IMAGE_MIMES.has(normalizeImageMime(mimeType)); +} + +/** + * Refusal for an unsupported image that has a readable file path, with a + * conversion command matching the execution environment (`kaos.osEnv.osKind` + * — where Bash actually runs, so SSH/container sessions get the right command + * too). The model can run the command through Bash (under the normal + * permission flow) and read the converted file. + * + * macOS converts with the built-in `sips`; Linux and Windows have no built-in + * decoder for these formats, so the guidance names ImageMagick (plus the + * format's dedicated Linux decoder when one exists, e.g. heif-convert). + */ +export function buildImageConversionGuidance( + path: string, + mimeType: string, + osKind: string, +): string { + const converted = path.replace(/\.[^./\\]+$/, '') + '.jpg'; + return ( + `"${path}" is an ${mimeType} image, which the provider does not accept. ` + + 'Convert it to JPEG first, then read the converted file. ' + + imageConversionCommand( + path, + converted, + osKind, + UNSUPPORTED_IMAGE_FORMATS[normalizeImageMime(mimeType)], + ) + ); +} + +function imageConversionCommand( + path: string, + converted: string, + osKind: string, + format: UnsupportedImageFormatInfo | undefined, +): string { + const magick = `magick "${path}" "${converted}"`; + const linuxDecoder = format?.linuxDecoder; + switch (osKind) { + case 'macOS': + return `On macOS: sips -s format jpeg "${path}" --out "${converted}"`; + case 'Linux': + return linuxDecoder === undefined + ? `On Linux, with ImageMagick: ${magick}` + : `On Linux: ${linuxDecoder.command} "${path}" "${converted}" ` + + `(package ${linuxDecoder.packageName}), or with ImageMagick: ${magick}`; + case 'Windows': + return ( + `On Windows, with ImageMagick: ${magick} ` + + '(install it first if missing: winget install ImageMagick.ImageMagick)' + ); + default: + return ( + `Options: sips -s format jpeg "${path}" --out "${converted}" (macOS)` + + (linuxDecoder === undefined + ? '' + : `, ${linuxDecoder.command} "${path}" "${converted}" ` + + `(Linux, package ${linuxDecoder.packageName})`) + + `, or ${magick} (ImageMagick)` + ); + } +} + +/** + * Short notice standing in for an unsupported image where there is no file + * path to point at (MCP tool results, prompt uploads): the image part is + * dropped and this text replaces it, so the model knows what happened and + * the session history stays free of formats the provider rejects. + */ +export function buildUnsupportedImageNotice(mimeType: string, name?: string): string { + const what = + name === undefined || name.length === 0 + ? `unsupported image format ${mimeType}` + : `"${name}" uses unsupported image format ${mimeType}`; + return ( + `[Image omitted: ${what}. Model providers accept only ${ACCEPTED_FORMATS_TEXT} — ` + + 'convert it to PNG or JPEG and try again.]' + ); +} + +/** + * Notice standing in for an image part whose `data:` URL could not be parsed + * at all (missing `;base64,` separator, empty MIME, …): the provider is + * guaranteed to reject it, so it is dropped at ingestion instead of being + * left to poison the session and trigger the media-stripped resend on every + * later turn. The URL is truncated — a malformed payload can be huge. + */ +export function buildMalformedImageNotice(url: string): string { + const shown = url.length > 80 ? `${url.slice(0, 80)}…` : url; + return ( + `[Image omitted: "${shown}" is not a valid data URL (its header or payload ` + + 'could not be parsed). Re-encode the image as PNG or JPEG and try again.]' + ); +} diff --git a/packages/agent-core/test/agent/turn.test.ts b/packages/agent-core/test/agent/turn.test.ts index 47335196b1..a0b0ae43e7 100644 --- a/packages/agent-core/test/agent/turn.test.ts +++ b/packages/agent-core/test/agent/turn.test.ts @@ -12,6 +12,7 @@ import { APIRequestTooLargeError, APIStatusError, APITimeoutError, + ChatProviderError, type ChatProvider, type Message, type ModelCapability, @@ -33,7 +34,12 @@ import type { } from '../../src/session/subagent-host'; import { recordingTelemetry, type TelemetryRecord } from '../fixtures/telemetry'; import { createFakeKaos } from '../tools/fixtures/fake-kaos'; -import { createCommandKaos, testAgent, type TestAgentOptions } from './harness/agent'; +import { + createCommandKaos, + testAgent, + type TestAgentContext, + type TestAgentOptions, +} from './harness/agent'; import { executeTool } from '../tools/fixtures/execute-tool'; import { agentTask } from './background/helpers'; @@ -132,6 +138,268 @@ describe('Agent turn flow', () => { ).toHaveLength(3); }); + it('gates unsupported image formats at the prompt and steer entry so the session cannot be poisoned', async () => { + const histories: Message[][] = []; + const generate: GenerateFn = async (_provider, _system, _tools, history) => { + histories.push(structuredClone(history)); + return { + id: 'mock-format-gate', + message: { role: 'assistant', content: [{ type: 'text', text: 'done' }], toolCalls: [] }, + usage: { inputOther: 1, output: 1, inputCacheRead: 0, inputCacheCreation: 0 }, + finishReason: 'completed', + rawFinishReason: 'stop', + }; + }; + const ctx = testAgent({ generate }); + ctx.configure({ + provider: { type: 'kimi', apiKey: 'test-key', model: 'kimi-code' }, + modelCapabilities: { + image_in: true, + video_in: false, + audio_in: false, + thinking: false, + tool_use: true, + max_context_tokens: 256_000, + }, + }); + + // The SDK/RPC prompt path carries no upstream gate: the turn entry is + // the last funnel before parts land in the session history. + await ctx.rpc.prompt({ + input: [ + { type: 'text', text: 'what is in these images?' }, + { type: 'image_url', imageUrl: { url: 'data:image/avif;base64,QUJD' } }, + { type: 'image_url', imageUrl: { url: 'data:image/jpg;base64,REVG' } }, + ], + }); + await ctx.untilTurnEnd(); + + // The AVIF image never reaches the model: a notice stands in, and the + // accepted image/jpg alias is forwarded as canonical image/jpeg. + const sentParts = histories[0]!.flatMap((message) => message.content); + const sentImages = sentParts.filter((part) => part.type === 'image_url'); + expect(sentImages).toEqual([ + { type: 'image_url', imageUrl: { url: 'data:image/jpeg;base64,REVG' } }, + ]); + const sentText = sentParts + .filter((part) => part.type === 'text') + .map((part) => part.text) + .join('\n'); + expect(sentText).toContain('image/avif'); + + // The history itself is clean — no image/avif part can re-poison later turns. + const historyParts = ctx.agent.context.history.flatMap((message) => message.content); + expect( + historyParts.some( + (part) => part.type === 'image_url' && part.imageUrl.url.includes('image/avif'), + ), + ).toBe(false); + + // Steer input enters the history the same way and gets the same gate. + await ctx.rpc.steer({ + input: [{ type: 'image_url', imageUrl: { url: 'data:image/heic;base64,QUJD' } }], + }); + await ctx.untilTurnEnd(); + + // The steer turn's history also carries the first turn's (canonical) + // image; what must be gone is the HEIC one. + const steerParts = histories[1]!.flatMap((message) => message.content); + expect( + steerParts.some( + (part) => part.type === 'image_url' && part.imageUrl.url.includes('image/heic'), + ), + ).toBe(false); + expect( + steerParts + .filter((part) => part.type === 'text') + .map((part) => part.text) + .join('\n'), + ).toContain('image/heic'); + + // A mislabeled image is gated on its real bytes: AVIF bytes labeled + // image/png never reach the model. + const avif = Buffer.alloc(16); + avif.writeUInt32BE(16, 0); + avif.write('ftyp', 4, 'latin1'); + avif.write('avif', 8, 'latin1'); + await ctx.rpc.prompt({ + input: [ + { + type: 'image_url', + imageUrl: { url: `data:image/png;base64,${avif.toString('base64')}` }, + }, + ], + }); + await ctx.untilTurnEnd(); + + // The third turn's history also carries the first turn's canonical + // image; what must be gone is the mislabeled AVIF payload. + const mislabeledParts = histories[2]!.flatMap((message) => message.content); + expect( + mislabeledParts.some( + (part) => + part.type === 'image_url' && part.imageUrl.url.includes(avif.toString('base64')), + ), + ).toBe(false); + expect( + mislabeledParts + .filter((part) => part.type === 'text') + .map((part) => part.text) + .join('\n'), + ).toContain('image/avif'); + }); + + describe('image-format recovery', () => { + const IMAGE_CAPABLE: ModelCapability = { + image_in: true, + video_in: false, + audio_in: false, + thinking: false, + tool_use: true, + max_context_tokens: 256_000, + }; + + // Simulate a legacy/pre-gate history that already carries a poisoned + // image. The turn.prompt gate only sanitizes NEW prompt input, not the + // pre-existing context, so this reaches the provider unmodified. + function plantPoisonedImage(ctx: TestAgentContext): void { + ctx.agent.context.appendUserMessage( + [ + { type: 'text', text: '' }, + { type: 'image_url', imageUrl: { url: 'data:image/avif;base64,QUJD' } }, + { type: 'text', text: '' }, + ], + { kind: 'user' }, + ); + } + + function okResponse() { + return { + id: 'mock-recovery', + message: { + role: 'assistant' as const, + content: [{ type: 'text' as const, text: 'ok' }], + toolCalls: [], + }, + usage: { inputOther: 1, output: 1, inputCacheRead: 0, inputCacheCreation: 0 }, + finishReason: 'completed' as const, + rawFinishReason: 'stop', + }; + } + + it('strips all media and retries once on a server image-format 400', async () => { + let attempts = 0; + const histories: Message[][] = []; + const generate: GenerateFn = async (_p, _s, _t, history) => { + attempts += 1; + histories.push(structuredClone(history)); + if (attempts === 1) throw new APIStatusError(400, 'unsupported image format'); + return okResponse(); + }; + const ctx = testAgent({ generate }); + ctx.configure({ + provider: { type: 'kimi', apiKey: 'test-key', model: 'kimi-code' }, + modelCapabilities: IMAGE_CAPABLE, + }); + plantPoisonedImage(ctx); + + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'continue' }] }); + await ctx.untilTurnEnd(); + + expect(attempts).toBe(2); + expect(histories[0]!.flatMap((m) => m.content).some((p) => p.type === 'image_url')).toBe(true); + expect(histories[1]!.flatMap((m) => m.content).some((p) => p.type === 'image_url')).toBe(false); + // Read-side only: the real history keeps the poisoned image. + expect( + ctx.agent.context.history.flatMap((m) => m.content).some((p) => p.type === 'image_url'), + ).toBe(true); + }); + + it('strips all media and retries once on kosong client-side image error', async () => { + let attempts = 0; + const generate: GenerateFn = async () => { + attempts += 1; + if (attempts === 1) { + throw new ChatProviderError('Unsupported media type for base64 image: image/avif'); + } + return okResponse(); + }; + const ctx = testAgent({ generate }); + ctx.configure({ + provider: { type: 'kimi', apiKey: 'test-key', model: 'kimi-code' }, + modelCapabilities: IMAGE_CAPABLE, + }); + plantPoisonedImage(ctx); + + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'continue' }] }); + await ctx.untilTurnEnd(); + + // isRetryableGenerateError excludes image-format errors, so no transient + // retries burn first — exactly one throw then one recovered resend. + expect(attempts).toBe(2); + }); + + it('does NOT recover a non-image 400 (no wasted resend)', async () => { + let attempts = 0; + const generate: GenerateFn = async () => { + attempts += 1; + throw new APIStatusError(400, 'max_tokens must be positive'); + }; + const ctx = testAgent({ generate }); + ctx.configure({ + provider: { type: 'kimi', apiKey: 'test-key', model: 'kimi-code' }, + modelCapabilities: IMAGE_CAPABLE, + }); + plantPoisonedImage(ctx); + + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'continue' }] }); + await ctx.untilTurnEnd(); + + expect(attempts).toBe(1); + }); + + it('does NOT recover image count/size/support errors (no silent blind resend)', async () => { + // "too many images" mentions "image" but is not a format/data error: + // stripping media would let the turn complete with the model blind to + // the user's images, hiding the real problem. Surface it instead. + let attempts = 0; + const generate: GenerateFn = async () => { + attempts += 1; + throw new APIStatusError(400, 'too many images in request'); + }; + const ctx = testAgent({ generate }); + ctx.configure({ + provider: { type: 'kimi', apiKey: 'test-key', model: 'kimi-code' }, + modelCapabilities: IMAGE_CAPABLE, + }); + plantPoisonedImage(ctx); + + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'continue' }] }); + await ctx.untilTurnEnd(); + + expect(attempts).toBe(1); + }); + + it('surfaces the error when the strip resend also fails (no infinite loop)', async () => { + let attempts = 0; + const generate: GenerateFn = async () => { + attempts += 1; + throw new APIStatusError(400, 'unsupported image format'); + }; + const ctx = testAgent({ generate }); + ctx.configure({ + provider: { type: 'kimi', apiKey: 'test-key', model: 'kimi-code' }, + modelCapabilities: IMAGE_CAPABLE, + }); + plantPoisonedImage(ctx); + + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'continue' }] }); + await ctx.untilTurnEnd(); + + expect(attempts).toBe(2); + }); + }); + it('tracks turn_started and turn_interrupted telemetry', async () => { const records: TelemetryRecord[] = []; const ctx = testAgent({ telemetry: recordingTelemetry(records) }); diff --git a/packages/agent-core/test/mcp/output.test.ts b/packages/agent-core/test/mcp/output.test.ts index 24d2f9fd35..19023b0b54 100644 --- a/packages/agent-core/test/mcp/output.test.ts +++ b/packages/agent-core/test/mcp/output.test.ts @@ -180,6 +180,22 @@ describe('convertMCPContentBlock', () => { }); }); + test('replaces resource_link with an unsupported image mimeType with a notice', () => { + // A signed/extensionless URL gives the extension gate nothing to work + // with; the declared MIME is the only signal — an honestly-declared + // unsupported format must not become an image_url. + const block = assertValidMcpBlock({ + type: 'resource_link', + name: 'photo', + uri: 'https://cdn.example.com/v2/image?id=123', + mimeType: 'image/avif', + }); + const part = convertMCPContentBlock(block); + if (part?.type !== 'text') throw new Error('expected a text notice'); + expect(part.text).toContain('image/avif'); + expect(part.text).toContain('https://cdn.example.com/v2/image?id=123'); + }); + test('returns null for resource_link with unsupported mimeType', () => { const block = assertValidMcpBlock({ type: 'resource_link', @@ -280,6 +296,124 @@ describe('mcpResultToExecutableOutput', () => { ]); }); + test('replaces an image the provider cannot accept with a text notice (media-only)', async () => { + // An image search tool returning AVIF: forwarding the image_url would + // make every later request in the session fail. The media-only wrap is + // kept, with a notice standing in for the dropped image. + const out = await mcpResultToExecutableOutput( + result([{ type: 'image', data: 'QUJD', mimeType: 'image/avif' }]), + 'mcp__search__image', + ); + expect(out.isError).toBe(false); + const parts = out.output as ContentPart[]; + expect(parts[0]).toEqual({ type: 'text', text: '' }); + expect(parts.some((p) => p.type === 'image_url')).toBe(false); + const notice = parts.find( + (p) => p.type === 'text' && p.text.includes('image/avif'), + ); + expect(notice).toBeDefined(); + expect(parts.at(-1)).toEqual({ type: 'text', text: '' }); + }); + + test('drops an unsupported image but keeps the accompanying text', async () => { + const out = await mcpResultToExecutableOutput( + result([ + { type: 'text', text: 'found an image' }, + { type: 'image', data: 'QUJD', mimeType: 'image/heic' }, + ]), + 'mcp__search__image', + ); + const parts = out.output as ContentPart[]; + expect(parts.some((p) => p.type === 'image_url')).toBe(false); + expect(parts[0]).toEqual({ type: 'text', text: 'found an image' }); + expect(parts.some((p) => p.type === 'text' && p.text.includes('image/heic'))).toBe(true); + }); + + test('gates a mislabeled image on its real bytes (image search tool)', async () => { + // An image search tool returning AVIF bytes labeled image/png: the + // bytes are authoritative — the image must not reach the session + // history, and the notice names the real format. + const avif = Buffer.alloc(16); + avif.writeUInt32BE(16, 0); + avif.write('ftyp', 4, 'latin1'); + avif.write('avif', 8, 'latin1'); + const out = await mcpResultToExecutableOutput( + result([{ type: 'image', data: avif.toString('base64'), mimeType: 'image/png' }]), + 'mcp__search__image', + ); + const parts = out.output as ContentPart[]; + expect(parts.some((p) => p.type === 'image_url')).toBe(false); + expect(parts.some((p) => p.type === 'text' && p.text.includes('image/avif'))).toBe(true); + }); + + test('forwards the image/jpg alias as canonical image/jpeg', async () => { + // Strict provider whitelists reject the raw `image/jpg` alias — the part + // must land in the session with the canonical MIME. + const out = await mcpResultToExecutableOutput( + result([{ type: 'image', data: 'QUJD', mimeType: 'image/jpg' }]), + 'mcp__s__t', + ); + const parts = out.output as ContentPart[]; + const image = parts.find((p) => p.type === 'image_url'); + expect(image).toEqual({ + type: 'image_url', + imageUrl: { url: 'data:image/jpeg;base64,QUJD' }, + }); + }); + + test('drops remote images by declared MIME and by URL extension, passes accepted links through', async () => { + // An MCP resource_link carries no bytes, so the only format signals are + // the declared MIME and the URL extension: an honestly-declared AVIF + // becomes a notice even with an extensionless (signed) URL — the + // extension gate alone cannot see it; a server that declares PNG but + // links an `.avif` URL is caught by the extension; an accepted link + // passes through. + const out = await mcpResultToExecutableOutput( + result([ + assertValidMcpBlock({ + type: 'resource_link', + name: 'photo', + uri: 'https://cdn.example.com/v2/image?id=123', + mimeType: 'image/avif', + }), + assertValidMcpBlock({ + type: 'resource_link', + name: 'pic.avif', + uri: 'https://example.com/pic.avif', + mimeType: 'image/png', + }), + assertValidMcpBlock({ + type: 'resource_link', + name: 'ok.png', + uri: 'https://example.com/ok.png?size=full#frame', + mimeType: 'image/png', + }), + ]), + 'mcp__s__t', + ); + const parts = out.output as ContentPart[]; + expect(parts).toContainEqual({ + type: 'image_url', + imageUrl: { url: 'https://example.com/ok.png?size=full#frame' }, + }); + // Neither AVIF link survives as an image_url. + expect( + parts.some( + (p) => + p.type === 'image_url' && + (p.imageUrl.url.includes('cdn.example.com') || p.imageUrl.url.includes('pic.avif')), + ), + ).toBe(false); + const notices = parts + .filter((p) => p.type === 'text') + .map((p) => (p as { text: string }).text) + .join('\n'); + expect(notices).toContain('image/avif'); + // Both notices keep their URL so the model can fetch and convert. + expect(notices).toContain('https://cdn.example.com/v2/image?id=123'); + expect(notices).toContain('https://example.com/pic.avif'); + }); + test('does NOT wrap when a non-empty text part accompanies the media', async () => { const out = await mcpResultToExecutableOutput( result([ diff --git a/packages/agent-core/test/tools/image-compress.test.ts b/packages/agent-core/test/tools/image-compress.test.ts index d60f3820a7..4f0cb5be33 100644 --- a/packages/agent-core/test/tools/image-compress.test.ts +++ b/packages/agent-core/test/tools/image-compress.test.ts @@ -47,6 +47,7 @@ import { compressImageForModel, cropImageForModel, extractImageCompressionCaptions, + gateImageFormatParts, IMAGE_BYTE_BUDGET, MAX_IMAGE_EDGE_ENV, MAX_IMAGE_EDGE_PX, @@ -60,6 +61,8 @@ import { ImageLimits } from '../../src/tools/support/image-limits'; // eslint-disable-next-line import/no-unresolved import { sniffImageDimensions } from '../../src/tools/support/file-type'; // eslint-disable-next-line import/no-unresolved +import { normalizeImageMime, unsupportedImageMimeFromUrl } from '../../src/tools/support/image-format-policy'; +// eslint-disable-next-line import/no-unresolved import type { TelemetryClient, TelemetryProperties } from '../../src/telemetry'; // ── fixtures ───────────────────────────────────────────────────────── @@ -836,6 +839,278 @@ describe('compressImageContentParts', () => { expect(imagePart.imageUrl.id).toBe('att-1'); expect(imagePart.imageUrl.url).not.toBe(dataUrl('image/png', big)); }); + + it('drops image parts the provider cannot accept, replacing each with a notice', async () => { + // MCP servers can return any image/* MIME (e.g. an AVIF from an image + // search tool). Forwarding it would poison the session history, so the + // part is dropped and a text notice stands in. + const parts = [ + { type: 'text' as const, text: 'search results' }, + { type: 'image_url' as const, imageUrl: { url: dataUrl('image/avif', new Uint8Array([1, 2, 3])) } }, + { type: 'image_url' as const, imageUrl: { url: dataUrl('image/heic', new Uint8Array([4, 5, 6])) } }, + ]; + const { parts: out, captions } = await compressImageContentParts(parts); + + expect(out[0]).toEqual({ type: 'text', text: 'search results' }); + expect(out.some((p) => p.type === 'image_url')).toBe(false); + const notices = out.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text); + expect(notices.some((t) => t.includes('image/avif'))).toBe(true); + expect(notices.some((t) => t.includes('image/heic'))).toBe(true); + // Dropping is not compression: no captions are produced. + expect(captions).toEqual([]); + }); + + it('passes the accepted formats through the format gate untouched', async () => { + for (const mime of ['image/png', 'image/jpeg', 'image/gif', 'image/webp']) { + const url = dataUrl(mime, new Uint8Array([1, 2, 3])); + const parts = [{ type: 'image_url' as const, imageUrl: { url } }]; + const { parts: out } = await compressImageContentParts(parts); + expect(out[0]).toEqual({ type: 'image_url', imageUrl: { url } }); + } + }); + + it('forwards accepted MIME aliases in canonical form', async () => { + // `image/jpg` (and case/whitespace variants) pass the gate, but the raw + // alias must not land in the session: strict provider whitelists (e.g. + // Anthropic's) reject it and every later request would fail. + const bytes = new Uint8Array([1, 2, 3]); + const base64 = Buffer.from(bytes).toString('base64'); + for (const alias of ['image/jpg', 'Image/JPEG', ' image/jpeg ']) { + const parts = [ + { type: 'image_url' as const, imageUrl: { url: `data:${alias};base64,${base64}` } }, + ]; + const { parts: out, captions } = await compressImageContentParts(parts); + expect(out[0]).toEqual({ + type: 'image_url', + imageUrl: { url: `data:image/jpeg;base64,${base64}` }, + }); + // Rewriting the MIME is not compression: no caption. + expect(captions).toEqual([]); + } + }); + + it('drops an unsupported image even when its data URL carries MIME parameters', async () => { + const parts = [ + { + type: 'image_url' as const, + imageUrl: { url: dataUrl('image/avif;charset=utf-8', new Uint8Array([1, 2, 3])) }, + }, + ]; + const { parts: out } = await compressImageContentParts(parts); + expect(out.some((p) => p.type === 'image_url')).toBe(false); + expect(out[0]).toMatchObject({ type: 'text' }); + expect((out[0] as { text: string }).text).toContain('image/avif'); + }); +}); + +// ── format gate (shared by every ingestion point) ──────────────────── + +describe('gateImageFormatParts', () => { + function dataUrl(mime: string, bytes: Uint8Array): string { + return `data:${mime};base64,${Buffer.from(bytes).toString('base64')}`; + } + + it('replaces every unsupported inline image with a notice and keeps the rest', () => { + const parts = [ + { type: 'text' as const, text: 'results' }, + { type: 'image_url' as const, imageUrl: { url: dataUrl('image/avif', new Uint8Array([1])) } }, + { type: 'image_url' as const, imageUrl: { url: dataUrl('image/bmp', new Uint8Array([2])) } }, + { type: 'video_url' as const, videoUrl: { url: dataUrl('video/mp4', new Uint8Array([3])) } }, + { type: 'image_url' as const, imageUrl: { url: dataUrl('image/png', new Uint8Array([4])) } }, + ]; + const out = gateImageFormatParts(parts); + + expect(out[0]).toEqual({ type: 'text', text: 'results' }); + // Both unsupported images became notices naming their MIME. + const notices = out.filter((p) => p.type === 'text').map((p) => (p as { text: string }).text); + expect(notices.some((t) => t.includes('image/avif'))).toBe(true); + expect(notices.some((t) => t.includes('image/bmp'))).toBe(true); + // Video parts and the accepted image pass through untouched. + expect(out).toContainEqual(parts[3]); + expect(out).toContainEqual(parts[4]); + expect( + out.some( + (p) => p.type === 'image_url' && !p.imageUrl.url.startsWith('data:image/png'), + ), + ).toBe(false); + }); + + it('rewrites accepted MIME aliases to canonical form', () => { + const base64 = Buffer.from([1, 2, 3]).toString('base64'); + const out = gateImageFormatParts([ + { type: 'image_url', imageUrl: { url: `data:image/jpg;base64,${base64}` } }, + ]); + expect(out[0]).toEqual({ + type: 'image_url', + imageUrl: { url: `data:image/jpeg;base64,${base64}` }, + }); + }); + + it('rewrites an accepted MIME carrying parameters to the bare canonical form', () => { + // Strict provider whitelists exact-match the full data-URL header, so + // `image/jpeg;charset=utf-8` would be rejected just like an alias. + const base64 = Buffer.from([1, 2, 3]).toString('base64'); + const out = gateImageFormatParts([ + { type: 'image_url', imageUrl: { url: `data:image/jpeg;charset=utf-8;base64,${base64}` } }, + ]); + expect(out[0]).toEqual({ + type: 'image_url', + imageUrl: { url: `data:image/jpeg;base64,${base64}` }, + }); + }); + + it('gates on the sniffed bytes, not the declared MIME', () => { + const pngBytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3, 4]); + const ftyp = (brand: string): Buffer => { + const buf = Buffer.alloc(16); + buf.writeUInt32BE(16, 0); + buf.write('ftyp', 4, 'latin1'); + buf.write(brand, 8, 'latin1'); + return buf; + }; + + // AVIF bytes labeled image/png (a mislabeling MCP image search tool): + // dropped as the AVIF it is — the provider decodes bytes, not labels. + const mislabeled = gateImageFormatParts([ + { type: 'image_url', imageUrl: { url: `data:image/png;base64,${ftyp('avif').toString('base64')}` } }, + ]); + expect(mislabeled.some((p) => p.type === 'image_url')).toBe(false); + expect((mislabeled[0] as { text: string }).text).toContain('image/avif'); + + // A video container hiding in an image part is refused too. + const video = gateImageFormatParts([ + { type: 'image_url', imageUrl: { url: `data:image/png;base64,${ftyp('isom').toString('base64')}` } }, + ]); + expect(video.some((p) => p.type === 'image_url')).toBe(false); + expect((video[0] as { text: string }).text).toContain('video/mp4'); + + // PNG bytes labeled image/avif: rescued — forwarded as the PNG it is. + const rescued = gateImageFormatParts([ + { type: 'image_url', imageUrl: { url: `data:image/avif;base64,${pngBytes.toString('base64')}` } }, + ]); + expect(rescued[0]).toEqual({ + type: 'image_url', + imageUrl: { url: `data:image/png;base64,${pngBytes.toString('base64')}` }, + }); + + // Unrecognized bytes (corrupt image): the declared MIME stands; the + // 400-recovery path is the backstop for this case. + const garbage = gateImageFormatParts([ + { type: 'image_url', imageUrl: { url: `data:image/png;base64,${Buffer.from([1, 2, 3]).toString('base64')}` } }, + ]); + expect(garbage[0]).toMatchObject({ type: 'image_url' }); + }); + + it('parses the base64 marker case-insensitively', () => { + // `;BASE64,` is a legal data URL (RFC 2045 encoding names are + // case-insensitive): an uppercase marker must not slip past the gate as + // if it were a remote URL, and the canonical rebuild lowercases it. + const base64 = Buffer.from([1, 2, 3]).toString('base64'); + + const accepted = gateImageFormatParts([ + { type: 'image_url', imageUrl: { url: `data:image/jpeg;BASE64,${base64}` } }, + ]); + expect(accepted[0]).toEqual({ + type: 'image_url', + imageUrl: { url: `data:image/jpeg;base64,${base64}` }, + }); + + const unsupported = gateImageFormatParts([ + { type: 'image_url', imageUrl: { url: `data:image/avif;BASE64,${base64}` } }, + ]); + expect(unsupported.some((p) => p.type === 'image_url')).toBe(false); + expect((unsupported[0] as { text: string }).text).toContain('image/avif'); + }); + + it('drops remote image URLs whose extension is unsupported, passes others through', () => { + // No bytes to inspect, so the gate uses the path extension: a known-bad + // extension becomes a notice; extensionless / unknown / accepted + // extensions pass through to the provider (and the 400 recovery). + for (const bad of [ + 'https://example.com/pic.avif', + 'https://example.com/pic.AVIF', + 'https://example.com/pic.heic?size=full', + 'https://example.com/scan.tiff#frame', + 'https://example.com/icon.ico', + 'https://example.com/logo.svg', + ]) { + const out = gateImageFormatParts([{ type: 'image_url', imageUrl: { url: bad } }]); + expect(out[0]).toMatchObject({ type: 'text' }); + // The notice keeps the URL so the model can fetch and convert the image. + expect((out[0] as { text: string }).text).toContain(bad); + } + for (const ok of [ + 'https://example.com/pic.png', + 'https://example.com/pic.jpg?size=full#frame', + 'https://example.com/avatar', + 'https://cdn.example.com/v2/image?id=123', + ]) { + const part = { type: 'image_url' as const, imageUrl: { url: ok } }; + expect(gateImageFormatParts([part])).toEqual([part]); + } + }); + + it('drops a malformed data URL instead of letting it poison the session', () => { + // A `data:` URL parseImageDataUrl cannot parse is guaranteed to fail at + // the provider (Anthropic throws, OpenAI-compat 400s): dropping it at + // ingestion beats paying a rejected request + media strip every turn. + const cases = [ + 'data:image/avif', + 'data:image/png;notbase64,QUJD', + 'data:;base64,QUJD', + 'data:image/png;base64', + 'DATA:image/avif', + ]; + for (const url of cases) { + const out = gateImageFormatParts([{ type: 'image_url', imageUrl: { url } }]); + expect(out.some((p) => p.type === 'image_url')).toBe(false); + expect(out[0]).toMatchObject({ type: 'text' }); + expect((out[0] as { text: string }).text).toContain('not a valid data URL'); + } + }); + + it('truncates a long malformed data URL in the notice', () => { + const url = `data:image/png${'x'.repeat(500)}`; + const out = gateImageFormatParts([{ type: 'image_url', imageUrl: { url } }]); + const notice = (out[0] as { text: string }).text; + expect(notice.length).toBeLessThan(250); + expect(notice).not.toContain(url); + }); +}); + +describe('normalizeImageMime', () => { + it('lowercases, strips MIME parameters, and applies the jpg alias', () => { + expect(normalizeImageMime('image/png')).toBe('image/png'); + expect(normalizeImageMime('Image/JPEG')).toBe('image/jpeg'); + expect(normalizeImageMime('image/jpg')).toBe('image/jpeg'); + expect(normalizeImageMime(' image/webp ')).toBe('image/webp'); + // Parameters (e.g. charset) are dropped so a declared media type stays + // consistent with a data-URL MIME token. + expect(normalizeImageMime('image/jpeg; charset=utf-8')).toBe('image/jpeg'); + expect(normalizeImageMime('IMAGE/PNG;foo=bar')).toBe('image/png'); + }); +}); + +describe('unsupportedImageMimeFromUrl', () => { + it('flags known-unsupported extensions and ignores query/fragment/case', () => { + expect(unsupportedImageMimeFromUrl('https://example.com/pic.avif')).toBe('image/avif'); + expect(unsupportedImageMimeFromUrl('https://example.com/pic.AVIF?x=1')).toBe('image/avif'); + expect(unsupportedImageMimeFromUrl('https://example.com/photo.HEIC#frame')).toBe('image/heic'); + expect(unsupportedImageMimeFromUrl('https://example.com/scan.tiff')).toBe('image/tiff'); + expect(unsupportedImageMimeFromUrl('https://example.com/icon.ico')).toBe('image/x-icon'); + // .svg is not in the shared suffix map (SVG is text for the file tools), + // but remote SVG images are accepted by no provider. + expect(unsupportedImageMimeFromUrl('https://example.com/logo.svg')).toBe('image/svg+xml'); + expect(unsupportedImageMimeFromUrl('https://example.com/logo.svgz')).toBe('image/svg+xml'); + }); + + it('returns null for accepted, extensionless, or unknown URLs', () => { + expect(unsupportedImageMimeFromUrl('https://example.com/pic.png')).toBeNull(); + expect(unsupportedImageMimeFromUrl('https://example.com/pic.jpg')).toBeNull(); + expect(unsupportedImageMimeFromUrl('https://example.com/avatar')).toBeNull(); + expect(unsupportedImageMimeFromUrl('https://cdn.example.com/v2/image?id=123')).toBeNull(); + expect(unsupportedImageMimeFromUrl('https://example.com/readme.json')).toBeNull(); + }); }); // ── original-dimension metadata ────────────────────────────────────── diff --git a/packages/agent-core/test/tools/read-media.test.ts b/packages/agent-core/test/tools/read-media.test.ts index 37825ce0c6..78b61c819b 100644 --- a/packages/agent-core/test/tools/read-media.test.ts +++ b/packages/agent-core/test/tools/read-media.test.ts @@ -643,10 +643,11 @@ describe('ReadMediaFileTool', () => { ); }); - it('ships sniffed image formats to the provider without gating', async () => { - // A `.png` file that is actually a BMP is reported as `image/bmp`. The - // tool does not gate on image format — it ships the real bytes with the - // sniffed MIME, and the provider decides which formats it accepts. + it('refuses a sniffed-but-unsupported image format instead of shipping it to the provider', async () => { + // A `.png` file that is actually a BMP is sniffed as `image/bmp`. The tool + // must not ship the bytes: the provider rejects BMP, and once the + // image_url lands in the history every later request in the session fails. + // It refuses with conversion guidance instead. const data = Buffer.concat([Buffer.from('BM'), Buffer.from('bmpdata')]); const tool = makeReadMediaTool({ stat: vi.fn().mockResolvedValue({ ...DEFAULT_STAT, stSize: data.length }), @@ -660,10 +661,10 @@ describe('ReadMediaFileTool', () => { signal, }); - const parts = outputParts(result); - expect((parts[1] as { imageUrl: { url: string } }).imageUrl.url).toBe( - `data:image/bmp;base64,${data.toString('base64')}`, - ); + expect(result.isError).toBe(true); + expect(result.output).toContain('image/bmp'); + expect(result.output).toContain('Convert it to JPEG'); + expect(result.output).toContain('/workspace/photo.jpg'); }); it('rejects a media-extension file whose bytes are not a supported image', async () => { @@ -1019,7 +1020,7 @@ describe('ReadMediaFileTool', () => { }); }); - describe('provider-unsupported formats (HEIC/HEIF)', () => { + describe('provider-unsupported formats', () => { /** Minimal ISO-BMFF header: size + 'ftyp' + the given brand. */ function ftypHeader(brand: string): Buffer { const bytes = Buffer.alloc(16); @@ -1029,8 +1030,7 @@ describe('ReadMediaFileTool', () => { return bytes; } - function heicTool(osKind: string, brand = 'heic'): ReadMediaFileTool { - const data = ftypHeader(brand); + function unsupportedTool(osKind: string, data: Buffer): ReadMediaFileTool { const kaos = createFakeKaos({ stat: vi.fn().mockResolvedValue({ ...DEFAULT_STAT, stSize: data.length }), readBytes: vi.fn().mockResolvedValue(data), @@ -1039,6 +1039,10 @@ describe('ReadMediaFileTool', () => { return new ReadMediaFileTool(kaos, PERMISSIVE_WORKSPACE, capabilities()); } + function heicTool(osKind: string, brand = 'heic'): ReadMediaFileTool { + return unsupportedTool(osKind, ftypHeader(brand)); + } + it('refuses HEIC with sips guidance on macOS instead of sending it to the provider', async () => { const result = await executeTool(heicTool('macOS'), { turnId: 't1', @@ -1095,6 +1099,52 @@ describe('ReadMediaFileTool', () => { expect(region.isError).toBe(true); expect(region.output).toContain('sips'); }); + + it('refuses AVIF (still and animated brands) with conversion guidance', async () => { + for (const brand of ['avif', 'avis']) { + const result = await executeTool(unsupportedTool('macOS', ftypHeader(brand)), { + turnId: 't1', + toolCallId: `c_avif_${brand}`, + args: { path: '/workspace/photo.avif' }, + signal, + }); + expect(result.isError).toBe(true); + expect(result.output).toContain('image/avif'); + expect(result.output).toContain('sips -s format jpeg'); + expect(result.output).toContain('/workspace/photo.jpg'); + } + }); + + it('refuses AVIF on Linux with ImageMagick guidance (no heif-convert)', async () => { + const result = await executeTool(unsupportedTool('Linux', ftypHeader('avif')), { + turnId: 't1', + toolCallId: 'c_avif_linux', + args: { path: '/workspace/photo.avif' }, + signal, + }); + expect(result.isError).toBe(true); + expect(result.output).toContain('magick'); + expect(result.output).not.toContain('heif-convert'); + }); + + it('refuses BMP, TIFF, and ICO with per-OS conversion guidance', async () => { + const cases: readonly { data: Buffer; mime: string; path: string }[] = [ + { data: Buffer.concat([Buffer.from('BM'), Buffer.from('bmpdata')]), mime: 'image/bmp', path: '/workspace/photo.bmp' }, + { data: Buffer.from([0x49, 0x49, 0x2a, 0x00, 0x00]), mime: 'image/tiff', path: '/workspace/scan.tiff' }, + { data: Buffer.from([0x00, 0x00, 0x01, 0x00, 0x00]), mime: 'image/x-icon', path: '/workspace/favicon.ico' }, + ]; + for (const c of cases) { + const result = await executeTool(unsupportedTool('macOS', c.data), { + turnId: 't1', + toolCallId: `c_${c.mime.replace('/', '_')}`, + args: { path: c.path }, + signal, + }); + expect(result.isError).toBe(true); + expect(result.output).toContain(c.mime); + expect(result.output).toContain('sips'); + } + }); }); describe('read byte budget', () => { diff --git a/packages/kosong/src/errors.ts b/packages/kosong/src/errors.ts index 818b903df0..770228b789 100644 --- a/packages/kosong/src/errors.ts +++ b/packages/kosong/src/errors.ts @@ -145,8 +145,85 @@ export function isRetryableGenerateError(error: unknown): boolean { // instances are deliberately excluded above: deterministic 4xx // (400/401/403/404/422) and the recovery-owned context-overflow / // request-too-large subclasses keep their dedicated handling instead of - // burning retries first. - return error instanceof ChatProviderError; + // burning retries first. Image-format rejections are likewise excluded: + // they are deterministic per history and recovered by the media-stripped + // resend (see isImageFormatError), so retrying the identical request first + // would only burn the retry budget. + return error instanceof ChatProviderError && !isImageFormatError(error); +} + +// Client-side image rejections thrown before the request is sent (kosong's +// own media whitelist in the Anthropic adapter). +const IMAGE_FORMAT_PROVIDER_MESSAGE_PATTERNS = [ + /unsupported media type for base64 image/, + /invalid data url for image/, +] as const; + +// Server-side image rejections that are safe to recover by stripping media: +// an unsupported/invalid media type or undecodable image data. These are +// deliberately narrow and grounded in the documented messages of the major +// providers (Anthropic, OpenAI, Moonshot/Kimi, Gemini) — image COUNT/SIZE +// limits or image-input-disabled errors also mention "image", but stripping +// media either over-recovers or hides a real configuration problem the user +// should see; only format/data rejections are guaranteed to be fixed by +// removing the offending image. +// +// Matching on provider message text is inherently best-effort: these strings +// are not a stable contract, so a novel phrasing is missed and the error +// propagates (the pre-recovery behavior). The entry-point format gate is the +// structural defense; this recovery only backstops the residue. +// Every pattern mentions "image" literally, and MEDIA_TYPE_FIELD_PATTERN is +// separately gated on an "image" anchor — so audio/video media rejections +// ("unsupported media type", "invalid media type") can never be classified +// as image errors here. All documented provider image rejections mention +// "image", so the restriction costs no known match. +const IMAGE_FORMAT_STATUS_MESSAGE_PATTERNS = [ + // Unsupported format — OpenAI / Moonshot "unsupported image …". + /unsupported image (?:url|format|type)/, + // Undecodable / corrupt image data. + /does not represent a valid image/, + /could not (?:process|decode) (?:the |input )?image/, + /unable to process (?:the |input )?image/, + /failed to decode (?:the )?image/, + /invalid image(?: data| type| format)?/, +] as const; + +// Anthropic `media_type` & Gemini `mime_type` enum violations name the field +// — recoverable only when the message is about an IMAGE. A video/audio +// `media_type` rejection must surface instead of being blindly +// media-stripped: unlike images there is no conversion-guidance path for +// video today, so dropping the user's video silently would hide the real +// error. Every documented image media_type message also mentions "image", +// so the anchor costs nothing on the known cases. +const MEDIA_TYPE_FIELD_PATTERN = /(?:media|mime)_?type/; + +/** + * Whether the provider rejected an IMAGE in the request because of its + * FORMAT or DATA — an unsupported media type or undecodable image bytes. + * The rejection is deterministic for a given history (the same image is + * re-sent on every request, so the session would fail every turn), and the + * only recovery is to resend once with all media stripped (see the + * media-stripped resend in the agent loop). Body-size (413), context + * overflow, image count/size limits, image-input-disabled rejections, and + * non-image (audio/video) media rejections are excluded — the first two + * have their own recoveries, and the rest are not fixed by stripping media. + */ +export function isImageFormatError(error: unknown): boolean { + if (error instanceof APIStatusError) { + if (error instanceof APIContextOverflowError) return false; + if (error instanceof APIRequestTooLargeError) return false; + if (error.statusCode !== 400) return false; + const lowerMessage = error.message.toLowerCase(); + return ( + IMAGE_FORMAT_STATUS_MESSAGE_PATTERNS.some((pattern) => pattern.test(lowerMessage)) || + (MEDIA_TYPE_FIELD_PATTERN.test(lowerMessage) && lowerMessage.includes('image')) + ); + } + if (error instanceof ChatProviderError) { + const lowerMessage = error.message.toLowerCase(); + return IMAGE_FORMAT_PROVIDER_MESSAGE_PATTERNS.some((pattern) => pattern.test(lowerMessage)); + } + return false; } // `terminated` is the undici signature for an SSE/HTTP body stream that is diff --git a/packages/kosong/src/index.ts b/packages/kosong/src/index.ts index 97909a16a3..63e07c49c8 100644 --- a/packages/kosong/src/index.ts +++ b/packages/kosong/src/index.ts @@ -68,6 +68,7 @@ export { APITimeoutError, ChatProviderError, isContextOverflowStatusError, + isImageFormatError, isProviderRateLimitError, isRecoverableRequestStructureError, isRequestTooLargeStatusError, diff --git a/packages/kosong/test/errors.test.ts b/packages/kosong/test/errors.test.ts index 5fa195f882..1eb62efdbf 100644 --- a/packages/kosong/test/errors.test.ts +++ b/packages/kosong/test/errors.test.ts @@ -7,6 +7,7 @@ import { APIStatusError, APITimeoutError, ChatProviderError, + isImageFormatError, isProviderRateLimitError, isRecoverableRequestStructureError, isRetryableGenerateError, @@ -545,3 +546,124 @@ describe('isProviderRateLimitError', () => { expect(isProviderRateLimitError(new Error('context length exceeded'))).toBe(false); }); }); + +describe('isImageFormatError', () => { + it('matches documented provider image format/data rejections', () => { + // OpenAI + expect( + isImageFormatError( + new APIStatusError(400, 'The image data you provided does not represent a valid image'), + ), + ).toBe(true); + // Anthropic media_type enum violation + expect( + isImageFormatError( + new APIStatusError( + 400, + "messages.0.content.1.image.source.base64.media_type: Input should be 'image/jpeg'", + ), + ), + ).toBe(true); + // Anthropic decode failure + expect(isImageFormatError(new APIStatusError(400, 'Could not process image'))).toBe(true); + // Moonshot/Kimi (from the Kimi Code error reference) + expect( + isImageFormatError( + new APIStatusError(400, 'Invalid request: unsupported image url: /tmp/photo.avif'), + ), + ).toBe(true); + expect(isImageFormatError(new APIStatusError(400, 'unsupported image format'))).toBe(true); + // Gemini + expect(isImageFormatError(new APIStatusError(400, 'Unable to process input image'))).toBe(true); + expect( + isImageFormatError( + new APIStatusError(400, 'The mime_type must accurately match the actual image format'), + ), + ).toBe(true); + }); + + it('matches kosong client-side image whitelist throws', () => { + expect( + isImageFormatError(new ChatProviderError('Unsupported media type for base64 image: image/avif')), + ).toBe(true); + expect( + isImageFormatError( + new ChatProviderError('Invalid data URL for image: data:image/avif;BASE64,AAA'), + ), + ).toBe(true); + }); + + it('does not match a non-image 400, an unrelated status, or overflow/413 subclasses', () => { + expect(isImageFormatError(new APIStatusError(400, 'max_tokens must be positive'))).toBe(false); + expect(isImageFormatError(new APIStatusError(422, 'image is bad'))).toBe(false); + expect(isImageFormatError(new APIStatusError(401, 'invalid api key'))).toBe(false); + expect( + isImageFormatError(new APIContextOverflowError(400, 'context length exceeded for image model')), + ).toBe(false); + expect( + isImageFormatError(new APIRequestTooLargeError(413, 'image request too large')), + ).toBe(false); + expect(isImageFormatError(new ChatProviderError('connection reset'))).toBe(false); + expect(isImageFormatError(new Error('image is bad'))).toBe(false); + }); + + it('does not match image count/size/support errors that stripping media cannot fix', () => { + // Stripping media to zero would let these requests "succeed" with the + // model blind to the user's images — hiding the real error. They must + // surface instead of triggering a media-stripped resend. + expect(isImageFormatError(new APIStatusError(400, 'too many images in request'))).toBe(false); + expect( + isImageFormatError(new APIStatusError(400, 'image dimension 5000 exceeds maximum 2048')), + ).toBe(false); + expect( + isImageFormatError(new APIStatusError(400, 'image input is disabled for this model')), + ).toBe(false); + expect(isImageFormatError(new APIStatusError(400, 'image_url is not allowed'))).toBe(false); + // Documented provider messages that are image-shaped but not + // format/data errors: Anthropic's per-image size cap, Moonshot's + // capability code, Gemini's unsupported-inlineData rejection. + expect( + isImageFormatError( + new APIStatusError( + 400, + 'messages.44.content.1.image.source.base64: image exceeds 5 MB maximum: 11641928 bytes > 5242880 bytes', + ), + ), + ).toBe(false); + expect(isImageFormatError(new APIStatusError(400, 'Image Input Not Supported'))).toBe(false); + expect( + isImageFormatError(new APIStatusError(400, "`inlineData` isn't supported by this model.")), + ).toBe(false); + // Video/audio media_type errors are NOT image errors: they must surface + // (no conversion-guidance path exists for video) instead of triggering a + // blind media-stripped resend. + expect( + isImageFormatError( + new APIStatusError( + 400, + "messages.0.content.1.video.source.base64.media_type: Input should be 'video/mp4'", + ), + ), + ).toBe(false); + // Bare "media type" phrasings for audio/video inputs likewise surface. + expect( + isImageFormatError(new APIStatusError(400, 'unsupported media type for audio input')), + ).toBe(false); + expect(isImageFormatError(new APIStatusError(400, 'invalid media type'))).toBe(false); + }); + + it('is excluded from the transient-retry fallback so dedicated recovery fires first', () => { + // A base ChatProviderError is normally retried as an unclassified + // transient; image-format errors must not be, or the run would burn the + // retry budget on an identical request before reaching the media strip. + expect(isRetryableGenerateError(new ChatProviderError('transient blip'))).toBe(true); + expect( + isRetryableGenerateError( + new ChatProviderError('Unsupported media type for base64 image: image/avif'), + ), + ).toBe(false); + expect( + isRetryableGenerateError(new APIStatusError(400, 'unsupported image format')), + ).toBe(false); + }); +}); diff --git a/packages/node-sdk/src/index.ts b/packages/node-sdk/src/index.ts index c6e1dc6d2c..7f5806c97e 100644 --- a/packages/node-sdk/src/index.ts +++ b/packages/node-sdk/src/index.ts @@ -75,8 +75,13 @@ export { installGlobalProxyDispatcher } from '@moonshot-ai/agent-core'; // pre-compression bytes readable (ReadMediaFile + region) for detail. export { buildImageCompressionCaption, + buildUnsupportedImageNotice, compressImageForModel, compressBase64ForModel, + gateImageFormatParts, + isModelAcceptedImageMime, + normalizeImageMime, + parseImageDataUrl, persistOriginalImage, sessionMediaOriginalsDir, IMAGE_BYTE_BUDGET, diff --git a/packages/server/src/routes/prompts.ts b/packages/server/src/routes/prompts.ts index ac0d1cd738..5104bd8a8b 100644 --- a/packages/server/src/routes/prompts.ts +++ b/packages/server/src/routes/prompts.ts @@ -13,7 +13,7 @@ import { promptSteerResultSchema, type PromptSubmission, } from '@moonshot-ai/protocol'; -import { IPromptService, AuthModelNotResolvedError, AuthProvisioningRequiredError, AuthTokenMissingError, AuthTokenUnauthorizedError, PromptAlreadyCompletedError, PromptNotFoundError, SessionBusyError, SessionNotFoundError, FileNotFoundError, ICoreProcessService, IEnvironmentService, IFileStore, buildImageCompressionCaption, compressImageForModel, compressBase64ForModel, persistOriginalImage, sessionMediaOriginalsDir, withTelemetryContext, type IInstantiationService, type GetResult, type ImageCompressionTelemetry, type TelemetryClient } from '@moonshot-ai/agent-core'; +import { IPromptService, AuthModelNotResolvedError, AuthProvisioningRequiredError, AuthTokenMissingError, AuthTokenUnauthorizedError, PromptAlreadyCompletedError, PromptNotFoundError, SessionBusyError, SessionNotFoundError, FileNotFoundError, ICoreProcessService, IEnvironmentService, IFileStore, buildImageCompressionCaption, buildUnsupportedImageNotice, compressImageForModel, compressBase64ForModel, decodeBase64Prefix, isModelAcceptedImageMime, normalizeImageMime, persistOriginalImage, resolveEffectiveImageMime, sessionMediaOriginalsDir, unsupportedImageMimeFromUrl, withTelemetryContext, type IInstantiationService, type GetResult, type ImageCompressionTelemetry, type TelemetryClient } from '@moonshot-ai/agent-core'; import { z } from 'zod'; @@ -308,7 +308,22 @@ async function resolvePromptMediaFiles( // input-stage step as the file path below, for REST clients that submit an // image as `{ source: { kind: 'base64' } }` instead of uploading a file. if (part.type === 'image' && part.source.kind === 'base64') { - const compressed = await compressBase64ForModel(part.source.data, part.source.media_type, { + // Formats the provider cannot accept must never enter the session + // history — one unsupported image_url makes every later request fail. + // The bytes are authoritative: an image labeled image/png that is + // actually AVIF is gated on the sniffed format, not the label. Drop + // the image; a notice stands in so the model knows what happened. + const effectiveMime = resolveEffectiveImageMime( + part.source.media_type, + decodeBase64Prefix(part.source.data), + ); + if (!isModelAcceptedImageMime(effectiveMime)) { + content.push({ type: 'text', text: buildUnsupportedImageNotice(effectiveMime) }); + changed = true; + continue; + } + const canonicalMime = normalizeImageMime(effectiveMime); + const compressed = await compressBase64ForModel(part.source.data, canonicalMime, { maxEdge: options.maxImageEdgePx, telemetry: telemetryFor('prompt_inline'), }); @@ -346,11 +361,32 @@ async function resolvePromptMediaFiles( source: { kind: 'base64', media_type: compressed.mimeType, data: compressed.base64 }, }); changed = true; + } else if (canonicalMime !== part.source.media_type) { + // Accepted but aliased (image/jpg, case/whitespace) or mislabeled + // (jpeg bytes declared png): forward the canonical MIME — strict + // provider whitelists reject anything else. + content.push({ ...part, source: { ...part.source, media_type: canonicalMime } }); + changed = true; } else { content.push(part); } continue; } + // Remote image URL: no bytes to sniff, so reject when its path extension + // names a format providers reject (e.g. a link ending in `.avif`) — the + // notice keeps the URL so the model can still fetch and convert the + // image. Extensionless / unknown URLs pass through to the provider and + // the 400 recovery. Image+URL parts that pass are re-emitted unchanged. + if (part.type === 'image' && part.source.kind === 'url') { + const extMime = unsupportedImageMimeFromUrl(part.source.url); + if (extMime !== null) { + content.push({ type: 'text', text: buildUnsupportedImageNotice(extMime, part.source.url) }); + changed = true; + continue; + } + content.push(part); + continue; + } if ((part.type !== 'image' && part.type !== 'video') || part.source.kind !== 'file') { content.push(part); continue; @@ -374,6 +410,18 @@ async function resolvePromptMediaFiles( let mediaType = file.meta.media_type; let bytes: Uint8Array = data; if (part.type === 'image') { + // Same format gate as the inline path above, and again the bytes are + // authoritative: an upload whose Content-Type lies (AVIF bytes sent + // as image/png) becomes a notice instead of an image part. + mediaType = resolveEffectiveImageMime(mediaType, data); + if (!isModelAcceptedImageMime(mediaType)) { + content.push({ type: 'text', text: buildUnsupportedImageNotice(mediaType, file.meta.name) }); + changed = true; + continue; + } + // Forward the canonical MIME (image/jpg → image/jpeg, case/whitespace) + // — strict provider whitelists reject the raw alias. + mediaType = normalizeImageMime(mediaType); const compressed = await compressImageForModel(data, mediaType, { maxEdge: options.maxImageEdgePx, telemetry: telemetryFor('prompt_file'), diff --git a/packages/server/test/prompt.e2e.test.ts b/packages/server/test/prompt.e2e.test.ts index 1df6c6e480..fee7ebe22b 100644 --- a/packages/server/test/prompt.e2e.test.ts +++ b/packages/server/test/prompt.e2e.test.ts @@ -358,6 +358,50 @@ describe('POST /api/v1/sessions/{sid}/prompts — submit validation (W7.2 / Chai ]); }); + it('replaces a remote image URL whose extension is unsupported with a notice', async () => { + // No bytes to sniff, so the gate uses the path extension: a `.avif` + // link becomes a notice (the provider would fetch it server-side and + // 400); an extensionless or accepted-extension URL passes through. + let submitted: PromptSubmission | undefined; + const r = await bootDaemon([ + [ + IPromptService, + createPromptServiceOverride({ + submit: async (_sid, body) => { + submitted = body; + return { + prompt_id: 'prompt_from_stub', + user_message_id: 'msg_from_stub', + status: 'running', + content: body.content, + created_at: '2026-06-09T00:00:00.000Z', + }; + }, + }), + ], + ]); + const sid = await createSession(r); + + const res = await appOf(r).inject({ + method: 'POST', + url: `/api/v1/sessions/${sid}/prompts`, + payload: { + content: [ + { type: 'text', text: 'describe this' }, + { type: 'image', source: { kind: 'url', url: 'https://example.com/pic.avif' } }, + ], + }, + }); + expect(envelopeOf(res.json()).code).toBe(0); + + expect(submitted?.content[0]).toEqual({ type: 'text', text: 'describe this' }); + const notice = submitted?.content[1]; + if (notice?.type !== 'text') throw new Error('expected a text notice'); + expect(notice.text).toContain('image/avif'); + expect(notice.text).toContain('https://example.com/pic.avif'); + expect(submitted?.content.some((p) => p.type === 'image')).toBe(false); + }); + it('uploads a real PNG image file and resolves it before submitting the prompt', async () => { let submitted: PromptSubmission | undefined; const r = await bootDaemon([ @@ -565,6 +609,301 @@ describe('POST /api/v1/sessions/{sid}/prompts — submit validation (W7.2 / Chai expect(persisted.equals(Buffer.from(base64, 'base64'))).toBe(true); }); + it('replaces an inline base64 image the provider cannot accept with a notice', async () => { + // An AVIF inline image must never reach the session history — the + // provider rejects it and every later request would fail. + let submitted: PromptSubmission | undefined; + const r = await bootDaemon([ + [ + IPromptService, + createPromptServiceOverride({ + submit: async (_sid, body) => { + submitted = body; + return { + prompt_id: 'prompt_from_stub', + user_message_id: 'msg_from_stub', + status: 'running', + content: body.content, + created_at: '2026-06-09T00:00:00.000Z', + }; + }, + }), + ], + ]); + const sid = await createSession(r); + + const res = await appOf(r).inject({ + method: 'POST', + url: `/api/v1/sessions/${sid}/prompts`, + payload: { + content: [ + { type: 'text', text: 'what is this?' }, + { + type: 'image', + source: { + kind: 'base64', + media_type: 'image/avif', + data: Buffer.from([1, 2, 3]).toString('base64'), + }, + }, + ], + }, + }); + expect(envelopeOf(res.json()).code).toBe(0); + + expect(submitted?.content[0]).toEqual({ type: 'text', text: 'what is this?' }); + const notice = submitted?.content[1]; + if (notice?.type !== 'text') throw new Error('expected a text notice'); + expect(notice.text).toContain('image/avif'); + expect(submitted?.content.some((p) => p.type === 'image')).toBe(false); + }); + + it('gates an inline base64 image on its real bytes when the declared MIME lies', async () => { + // AVIF bytes declared image/png: the provider decodes bytes, not labels, + // so the sniffed format decides — the image must not reach the session. + let submitted: PromptSubmission | undefined; + const r = await bootDaemon([ + [ + IPromptService, + createPromptServiceOverride({ + submit: async (_sid, body) => { + submitted = body; + return { + prompt_id: 'prompt_from_stub', + user_message_id: 'msg_from_stub', + status: 'running', + content: body.content, + created_at: '2026-06-09T00:00:00.000Z', + }; + }, + }), + ], + ]); + const sid = await createSession(r); + + const avif = Buffer.alloc(16); + avif.writeUInt32BE(16, 0); + avif.write('ftyp', 4, 'latin1'); + avif.write('avif', 8, 'latin1'); + + const res = await appOf(r).inject({ + method: 'POST', + url: `/api/v1/sessions/${sid}/prompts`, + payload: { + content: [ + { + type: 'image', + source: { kind: 'base64', media_type: 'image/png', data: avif.toString('base64') }, + }, + ], + }, + }); + expect(envelopeOf(res.json()).code).toBe(0); + + const notice = submitted?.content[0]; + if (notice?.type !== 'text') throw new Error('expected a text notice'); + expect(notice.text).toContain('image/avif'); + expect(submitted?.content.some((p) => p.type === 'image')).toBe(false); + }); + + it('gates an uploaded file on its real bytes when the Content-Type lies', async () => { + let submitted: PromptSubmission | undefined; + const r = await bootDaemon([ + [ + IPromptService, + createPromptServiceOverride({ + submit: async (_sid, body) => { + submitted = body; + return { + prompt_id: 'prompt_from_stub', + user_message_id: 'msg_from_stub', + status: 'running', + content: body.content, + created_at: '2026-06-09T00:00:00.000Z', + }; + }, + }), + ], + ]); + const sid = await createSession(r); + + const avif = Buffer.alloc(16); + avif.writeUInt32BE(16, 0); + avif.write('ftyp', 4, 'latin1'); + avif.write('avif', 8, 'latin1'); + + const upload = buildMultipart({ + file: { + fieldName: 'file', + filename: 'photo.png', + contentType: 'image/png', + data: avif, + }, + }); + const uploadRes = await appOf(r).inject({ + method: 'POST', + url: '/api/v1/files', + payload: upload.body, + headers: { 'content-type': upload.contentType }, + }); + const uploadEnv = envelopeOf<{ id: string; media_type: string }>(uploadRes.json()); + expect(uploadEnv.code).toBe(0); + + const res = await appOf(r).inject({ + method: 'POST', + url: `/api/v1/sessions/${sid}/prompts`, + payload: { + content: [{ type: 'image', source: { kind: 'file', file_id: uploadEnv.data!.id } }], + }, + }); + expect(envelopeOf(res.json()).code).toBe(0); + + const notice = submitted?.content[0]; + if (notice?.type !== 'text') throw new Error('expected a text notice'); + expect(notice.text).toContain('image/avif'); + expect(submitted?.content.some((p) => p.type === 'image')).toBe(false); + }); + + it('forwards an inline base64 image with an aliased MIME in canonical form', async () => { + // Strict provider whitelists reject the raw `image/jpg` alias — the part + // must land in the session with the canonical MIME. + let submitted: PromptSubmission | undefined; + const r = await bootDaemon([ + [ + IPromptService, + createPromptServiceOverride({ + submit: async (_sid, body) => { + submitted = body; + return { + prompt_id: 'prompt_from_stub', + user_message_id: 'msg_from_stub', + status: 'running', + content: body.content, + created_at: '2026-06-09T00:00:00.000Z', + }; + }, + }), + ], + ]); + const sid = await createSession(r); + + const base64 = Buffer.from([1, 2, 3]).toString('base64'); + const res = await appOf(r).inject({ + method: 'POST', + url: `/api/v1/sessions/${sid}/prompts`, + payload: { + content: [ + { type: 'image', source: { kind: 'base64', media_type: 'image/jpg', data: base64 } }, + ], + }, + }); + expect(envelopeOf(res.json()).code).toBe(0); + + expect(submitted?.content).toEqual([ + { type: 'image', source: { kind: 'base64', media_type: 'image/jpeg', data: base64 } }, + ]); + }); + + it('treats an inline base64 image with a parameterized media type like the bare form', async () => { + // `image/jpeg; charset=utf-8` is the same accepted image as `image/jpeg`: + // MIME parameters are stripped before the acceptance check, so it is + // forwarded (canonicalized), not dropped as if unsupported. Garbage + // bytes force the declared-MIME fallback path that exercises the strip. + let submitted: PromptSubmission | undefined; + const r = await bootDaemon([ + [ + IPromptService, + createPromptServiceOverride({ + submit: async (_sid, body) => { + submitted = body; + return { + prompt_id: 'prompt_from_stub', + user_message_id: 'msg_from_stub', + status: 'running', + content: body.content, + created_at: '2026-06-09T00:00:00.000Z', + }; + }, + }), + ], + ]); + const sid = await createSession(r); + + const base64 = Buffer.from([1, 2, 3]).toString('base64'); + const res = await appOf(r).inject({ + method: 'POST', + url: `/api/v1/sessions/${sid}/prompts`, + payload: { + content: [ + { + type: 'image', + source: { kind: 'base64', media_type: 'image/jpeg; charset=utf-8', data: base64 }, + }, + ], + }, + }); + expect(envelopeOf(res.json()).code).toBe(0); + + expect(submitted?.content).toEqual([ + { type: 'image', source: { kind: 'base64', media_type: 'image/jpeg', data: base64 } }, + ]); + }); + + it('replaces an uploaded image the provider cannot accept with a notice naming the file', async () => { + let submitted: PromptSubmission | undefined; + const r = await bootDaemon([ + [ + IPromptService, + createPromptServiceOverride({ + submit: async (_sid, body) => { + submitted = body; + return { + prompt_id: 'prompt_from_stub', + user_message_id: 'msg_from_stub', + status: 'running', + content: body.content, + created_at: '2026-06-09T00:00:00.000Z', + }; + }, + }), + ], + ]); + const sid = await createSession(r); + + const upload = buildMultipart({ + file: { + fieldName: 'file', + filename: 'photo.avif', + contentType: 'image/avif', + data: Buffer.from([1, 2, 3]), + }, + }); + const uploadRes = await appOf(r).inject({ + method: 'POST', + url: '/api/v1/files', + payload: upload.body, + headers: { 'content-type': upload.contentType }, + }); + const uploadEnv = envelopeOf<{ id: string; media_type: string }>(uploadRes.json()); + expect(uploadEnv.code).toBe(0); + expect(uploadEnv.data?.media_type).toBe('image/avif'); + + const res = await appOf(r).inject({ + method: 'POST', + url: `/api/v1/sessions/${sid}/prompts`, + payload: { + content: [{ type: 'image', source: { kind: 'file', file_id: uploadEnv.data!.id } }], + }, + }); + expect(envelopeOf(res.json()).code).toBe(0); + + const notice = submitted?.content[0]; + if (notice?.type !== 'text') throw new Error('expected a text notice'); + expect(notice.text).toContain('image/avif'); + expect(notice.text).toContain('photo.avif'); + expect(submitted?.content.some((p) => p.type === 'image')).toBe(false); + }); + it('scopes prompt image compression telemetry to the session', async () => { // The agent-side image_compress sources inherit the session context from // their per-session telemetry client; the prompt-ingestion sources must