Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/hide-image-compression-caption.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@moonshot-ai/kimi-code": patch
---

Fix compressed-image prompts leaking an internal `<system>` compression note into the visible message and the session title.
46 changes: 45 additions & 1 deletion packages/agent-core/src/agent/context/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { createToolMessage, type ContentPart, type Message } from '@moonshot-ai/
import type { Agent } from '..';
import { ErrorCodes, KimiError } from '../../errors';
import type { ExecutableToolResult, LoopRecordedEvent } from '../../loop';
import { extractImageCompressionCaptions } from '../../tools/support/image-compress';
import { estimateTokens, estimateTokensForMessages } from '../../utils/tokens';
import { escapeXml } from '../../utils/xml-escape';
import {
Expand Down Expand Up @@ -66,9 +67,23 @@ export class ContextMemory {
origin: PromptOrigin = USER_PROMPT_ORIGIN,
): void {
if (content.length === 0) return;
// Prompt ingestion (server upload/base64 route, TUI paste, ACP) annotates
// a compressed image with an inline `<system>` caption next to the image.
// Left inside the user message, that raw markup is user-visible in every
// history projection (TUI replay, vis, export). Reroute each caption
// through the built-in system-reminder injection — hidden by its
// `injection` origin — and keep only the real user content here.
const { captions, parts } =
origin.kind === 'user'
? splitImageCompressionCaptions(content)
: { captions: [], parts: [...content] };
for (const caption of captions) {
this.appendSystemReminder(caption, { kind: 'injection', variant: 'image_compression' });

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve image-compression reminders through compaction

When a compressed-image prompt later gets compacted, this new reminder has origin.kind === 'injection'; compactionUserMessageDisposition drops all injections, while collectCompactableUserMessages keeps the real user message (including the compressed image_url) verbatim. After compaction the model can still see the downsampled image but no longer receives the original dimensions/path or the ReadMediaFile guidance that used to stay inline with the kept user message, so follow-up turns lose the full-fidelity readback path. Please keep/reinject this image_compression variant during compaction or attach the caption to the compactable image message.

Useful? React with 👍 / 👎.

}
if (parts.length === 0) return;
this.appendMessage({
role: 'user',
content: [...content],
content: parts,
toolCalls: [],
origin,
});
Expand Down Expand Up @@ -702,6 +717,35 @@ function isEmptyEquivalentContentArray(output: readonly ContentPart[]): boolean
return output.every((part) => part.type === 'text' && part.text.trim().length === 0);
}

// Split inline image-compression captions (see buildImageCompressionCaption)
// out of user prompt content. A caption may be a standalone text part (server
// route, ACP) or merged into an adjacent text segment (TUI paste), so each
// text part is scanned rather than matched whole. Text left empty once its
// captions are removed is dropped entirely.
function splitImageCompressionCaptions(content: readonly ContentPart[]): {
captions: readonly string[];
parts: ContentPart[];
} {
const captions: string[] = [];
const parts: ContentPart[] = [];
for (const part of content) {
if (part.type !== 'text') {
parts.push(part);
continue;
}
const extracted = extractImageCompressionCaptions(part.text);
if (extracted.captions.length === 0) {
parts.push(part);
continue;
}
captions.push(...extracted.captions);
if (extracted.text.trim().length > 0) {
parts.push({ type: 'text', text: extracted.text });
}
}
return { captions, parts };
}

function isEmptyOutputText(output: string): boolean {
return output.trim().length === 0 || output.trim() === TOOL_OUTPUT_EMPTY_TEXT;
}
Expand Down
10 changes: 8 additions & 2 deletions packages/agent-core/src/session/prompt-metadata.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { ActivatePluginCommandPayload, ActivateSkillPayload, PromptPayload } from '#/rpc';
import { extractImageCompressionCaptions } from '#/tools/support/image-compress';
import type { ContentPart } from '@moonshot-ai/kosong';

const MAX_TITLE_LENGTH = 200;
Expand Down Expand Up @@ -38,8 +39,13 @@ export function promptMetadataTextFromPluginCommand(

function promptPartText(part: ContentPart): string | undefined {
switch (part.type) {
case 'text':
return part.text;
case 'text': {
// Prompt ingestion may have annotated a compressed image with an inline
// caption (see buildImageCompressionCaption). It is harness metadata,
// not something the user typed, so keep it out of titles/lastPrompt.
const { text } = extractImageCompressionCaptions(part.text);
return text.trim().length === 0 ? undefined : text;
}
case 'image_url':
return '[image]';
case 'audio_url':
Expand Down
53 changes: 52 additions & 1 deletion packages/agent-core/src/tools/support/image-compress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@
* original dimensions, {@link buildImageCompressionCaption} renders the
* shared "what was compressed, where is the original" note every ingestion
* point can place next to the image, and {@link cropImageForModel} lets a
* caller read a region of the original back at full fidelity.
* caller read a region of the original back at full fidelity. In user
* prompts the context layer later reroutes that note through the hidden
* system-reminder injection via {@link extractImageCompressionCaptions},
* so its raw `<system>` markup never renders in the UI.
*/

import type { ContentPart } from '@moonshot-ai/kosong';
Expand Down Expand Up @@ -553,6 +556,14 @@ export interface ImageCompressionCaptionInput {
* model knows it is looking at a downsampled copy: what the original was, what
* was actually sent, and — when the original is on disk — where to read it
* back (via ReadMediaFile `region`) for full-fidelity detail.
*
* Two channels consume this note differently:
* - Tool results (MCP images) keep it inline — `<system>` status text inside
* tool output is the established convention there.
* - User prompts must not render raw `<system>` markup in the UI, so the
* context layer detects the caption via
* {@link extractImageCompressionCaptions} and reroutes it through the
* built-in system-reminder injection (hidden by its `injection` origin).
*/
export function buildImageCompressionCaption(input: ImageCompressionCaptionInput): string {
const sentences = [
Expand All @@ -572,6 +583,46 @@ export function buildImageCompressionCaption(input: ImageCompressionCaptionInput
return `<system>${sentences.join(' ')}</system>`;
}

/**
* Fixed opening every {@link buildImageCompressionCaption} note starts with —
* the anchor {@link extractImageCompressionCaptions} matches on. Keep the two
* in sync.
*/
const CAPTION_OPENING = '<system>Image compressed to fit model limits:';

/**
* A full caption embedded in arbitrary text. The body is sentences plus a
* quoted file path and never contains `</system>`, so the non-greedy scan to
* the closing tag is exact.
*/
const CAPTION_PATTERN = /<system>(Image compressed to fit model limits:[\s\S]*?)<\/system>/g;

export interface ImageCompressionCaptionExtraction {
/** Caption bodies found, in order, without the `<system>` wrapper. */
readonly captions: readonly string[];
/** The input text with every caption removed. */
readonly text: string;
}

/**
* Find every {@link buildImageCompressionCaption} note embedded in `text` and
* return the unwrapped caption bodies plus the text without them. Prompt
* ingestion (server upload/base64 route, TUI paste, ACP) places the caption
* inline next to the image — sometimes merged into an adjacent text segment —
* and the context layer uses this to reroute the note through the built-in
* system-reminder injection instead of leaving raw `<system>` markup in the
* user-visible message.
*/
export function extractImageCompressionCaptions(text: string): ImageCompressionCaptionExtraction {
if (!text.includes(CAPTION_OPENING)) return { captions: [], text };
const captions: string[] = [];
const remainder = text.replace(CAPTION_PATTERN, (_match, body: string) => {
captions.push(body);
return '';
});
return { captions, text: remainder };
}

function describeImageVariant(variant: ImageVariantDescription): string {
const size = `${variant.mimeType} (${formatByteSize(variant.byteLength)})`;
if (variant.width > 0 && variant.height > 0) {
Expand Down
74 changes: 74 additions & 0 deletions packages/agent-core/test/agent/context.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { describe, expect, it, vi } from 'vitest';
import { renderNotificationXml } from '../../src/agent/context/notification-xml';
import { project } from '../../src/agent/context/projector';
import type { ContextMessage } from '../../src/agent/context/types';
import { buildImageCompressionCaption } from '../../src/tools/support/image-compress';
import { estimateTokensForMessages } from '../../src/utils/tokens';
import { createFakeKaos } from '../tools/fixtures/fake-kaos';
import { recordingTelemetry, type TelemetryRecord } from '../fixtures/telemetry';
Expand Down Expand Up @@ -59,6 +60,79 @@ describe('Agent context', () => {
expect(ctx.agent.context.messages.some((message) => 'origin' in message)).toBe(false);
});

it('reroutes an inline image-compression caption into a hidden system reminder', () => {
const ctx = testAgent();
ctx.configure();

const caption = buildImageCompressionCaption({
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
originalPath: '/tmp/originals/shot.png',
});
// The TUI merges the caption into the preceding text segment; the server
// route emits it as a standalone part. Cover the merged (harder) shape.
ctx.agent.context.appendUserMessage([
{ type: 'text', text: `能展示但是没有快捷键提示${caption}` },
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
]);

const textOf = (message: ContextMessage): string =>
message.content.map((part) => (part.type === 'text' ? part.text : '')).join('');

expect(ctx.agent.context.history.map(({ role, origin }) => ({ role, origin }))).toEqual([
{ role: 'user', origin: { kind: 'injection', variant: 'image_compression' } },
{ role: 'user', origin: { kind: 'user' } },
]);
const [reminder, userMessage] = ctx.agent.context.history;
expect(textOf(reminder!)).toContain('<system-reminder>');
expect(textOf(reminder!)).toContain('Image compressed to fit model limits');
expect(textOf(reminder!)).toContain('/tmp/originals/shot.png');
expect(textOf(reminder!)).not.toContain('<system>');
expect(textOf(userMessage!)).toBe('能展示但是没有快捷键提示');
expect(userMessage!.content.some((part) => part.type === 'image_url')).toBe(true);
});

it('drops a caption-only text part instead of leaving an empty user text part', () => {
const ctx = testAgent();
ctx.configure();

const caption = buildImageCompressionCaption({
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
originalPath: '/tmp/originals/shot.png',
});
ctx.agent.context.appendUserMessage([
{ type: 'text', text: caption },
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
]);

const [, userMessage] = ctx.agent.context.history;
expect(userMessage!.content).toEqual([
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
]);
});

it('leaves caption-shaped text alone on non-user origins', () => {
const ctx = testAgent();
ctx.configure();

const caption = buildImageCompressionCaption({
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
originalPath: '/tmp/originals/shot.png',
});
ctx.agent.context.appendUserMessage([{ type: 'text', text: caption }], {
kind: 'hook_result',
event: 'PostToolUse',
});

expect(ctx.agent.context.history).toHaveLength(1);
expect(ctx.agent.context.history[0]!.origin).toEqual({
kind: 'hook_result',
event: 'PostToolUse',
});
});

it('tracks conversation_undo when undoHistory reverts a user message', async () => {
const records: TelemetryRecord[] = [];
const ctx = testAgent({ telemetry: recordingTelemetry(records) });
Expand Down
Loading
Loading