diff --git a/.changeset/lan-remote-control.md b/.changeset/lan-remote-control.md new file mode 100644 index 0000000000..83ff9d633e --- /dev/null +++ b/.changeset/lan-remote-control.md @@ -0,0 +1,11 @@ +--- +"@moonshot-ai/kimi-code": patch +"@moonshot-ai/vis-server": patch +--- + +feat(vis): show LAN URLs when binding to 0.0.0.0 for remote control + +When vis-server binds to 0.0.0.0 or :: (all interfaces), the startup +banner and CLI output now display the actual LAN IP addresses that +other devices on the same network can use to connect. This enables +lan-range remote control from phones, tablets, or other machines. diff --git a/.changeset/prefer-readmediafile-video.md b/.changeset/prefer-readmediafile-video.md new file mode 100644 index 0000000000..4bf1e5cdaa --- /dev/null +++ b/.changeset/prefer-readmediafile-video.md @@ -0,0 +1,10 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +feat(agent): prefer ReadMediaFile for video analysis in system prompt + +Updated the system prompt (`system.md`) to explicitly guide the AI to +use `ReadMediaFile` directly for video analysis, rather than writing +Python scripts to extract frames. This leverages the built-in video +input capability of supported models and avoids unnecessary overhead. diff --git a/apps/kimi-code/src/cli/sub/vis.ts b/apps/kimi-code/src/cli/sub/vis.ts index 7e6f2e1f25..6fea80b985 100644 --- a/apps/kimi-code/src/cli/sub/vis.ts +++ b/apps/kimi-code/src/cli/sub/vis.ts @@ -21,6 +21,7 @@ export interface StartedVisServer { readonly port: number; readonly host: string; readonly url: string; + readonly lanUrls?: string[]; readonly close: () => Promise; } @@ -86,6 +87,12 @@ export async function handleVis(deps: VisDeps, opts: VisOptions): Promise : `${server.url}sessions/${encodeURIComponent(opts.sessionId)}`; deps.stdout.write(`kimi vis is running at ${server.url}\n`); + if (server.lanUrls !== undefined && server.lanUrls.length > 0) { + deps.stdout.write(`LAN access:\n`); + for (const lanUrl of server.lanUrls) { + deps.stdout.write(` ${lanUrl}\n`); + } + } deps.stdout.write('Press Ctrl-C to stop.\n'); if (opts.open) { diff --git a/apps/vis/server/src/config.ts b/apps/vis/server/src/config.ts index 0af5b9f32d..ed258c2195 100644 --- a/apps/vis/server/src/config.ts +++ b/apps/vis/server/src/config.ts @@ -1,5 +1,22 @@ import { homedir } from 'node:os'; import { join } from 'node:path'; +import os from 'node:os'; + +export function isAllInterfaces(host: string): boolean { + return host === '0.0.0.0' || host === '::'; +} + +export function getLocalNetworkAddresses(port: number): string[] { + const addresses: string[] = []; + for (const [, ifaceList] of Object.entries(os.networkInterfaces())) { + for (const iface of ifaceList ?? []) { + if (!iface.internal && iface.family === 'IPv4') { + addresses.push(`http://${iface.address}:${port}/`); + } + } + } + return addresses; +} /** Resolve KIMI_CODE_HOME (env > ~/.kimi-code). */ export function resolveKimiCodeHome(): string { diff --git a/apps/vis/server/src/index.ts b/apps/vis/server/src/index.ts index 0047337367..80ef2b47b5 100644 --- a/apps/vis/server/src/index.ts +++ b/apps/vis/server/src/index.ts @@ -5,9 +5,9 @@ import { formatStartupBanner } from './startup-banner'; async function main(): Promise { const host = resolveHost(); const authToken = resolveVisAuthToken(host); - const { port } = await startVisServer({ host, authToken }); + const { port, lanUrls } = await startVisServer({ host, authToken }); process.stdout.write( - formatStartupBanner({ authToken, host, kimiCodeHome: KIMI_CODE_HOME, port }), + formatStartupBanner({ authToken, host, kimiCodeHome: KIMI_CODE_HOME, port, lanUrls }), ); } diff --git a/apps/vis/server/src/start.ts b/apps/vis/server/src/start.ts index 1a33cf2ae0..f305f02b81 100644 --- a/apps/vis/server/src/start.ts +++ b/apps/vis/server/src/start.ts @@ -1,7 +1,7 @@ import { serve } from '@hono/node-server'; import { createApp } from './app'; -import { hostForUrl, resolveHost, resolveKimiCodeHome, resolvePort, resolveVisAuthToken } from './config'; +import { getLocalNetworkAddresses, hostForUrl, isAllInterfaces, resolveHost, resolveKimiCodeHome, resolvePort, resolveVisAuthToken } from './config'; import type { WebAsset } from './lib/web-asset'; export interface StartVisServerOptions { @@ -18,6 +18,7 @@ export interface StartedVisServer { readonly port: number; readonly host: string; readonly url: string; + readonly lanUrls?: string[]; readonly close: () => Promise; } @@ -36,6 +37,7 @@ export async function startVisServer( port: info.port, host, url: `http://${hostForUrl(host)}:${info.port}/`, + lanUrls: isAllInterfaces(host) ? getLocalNetworkAddresses(info.port) : undefined, close: () => new Promise((done, fail) => { server.close((err?: Error) => (err ? fail(err) : done())); diff --git a/apps/vis/server/src/startup-banner.ts b/apps/vis/server/src/startup-banner.ts index a32589e89b..d2c2c457f2 100644 --- a/apps/vis/server/src/startup-banner.ts +++ b/apps/vis/server/src/startup-banner.ts @@ -5,12 +5,19 @@ export interface StartupBannerOptions { readonly host: string; readonly kimiCodeHome: string; readonly port: number; + readonly lanUrls?: string[]; } export function formatStartupBanner(options: StartupBannerOptions): string { const authStatus = options.authToken === undefined ? 'auth=disabled' : 'auth=required'; - return ( + let banner = `[vis-server] listening on http://${hostForUrl(options.host)}:${String(options.port)} ` + - `(${authStatus}, KIMI_CODE_HOME=${options.kimiCodeHome})\n` - ); + `(${authStatus}, KIMI_CODE_HOME=${options.kimiCodeHome})\n`; + if (options.lanUrls !== undefined && options.lanUrls.length > 0) { + banner += + `[vis-server] LAN access:\n` + + options.lanUrls.map((url) => ` - ${url}`).join('\n') + + '\n'; + } + return banner; } diff --git a/apps/vis/server/test/lib/config.test.ts b/apps/vis/server/test/lib/config.test.ts index e0f064fe59..c938c82053 100644 --- a/apps/vis/server/test/lib/config.test.ts +++ b/apps/vis/server/test/lib/config.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { hostForUrl } from '../../src/config'; +import { hostForUrl, isAllInterfaces, getLocalNetworkAddresses } from '../../src/config'; describe('hostForUrl', () => { it('brackets a bare IPv6 literal for use in a URL', () => { @@ -18,3 +18,39 @@ describe('hostForUrl', () => { expect(hostForUrl('[::1]')).toBe('[::1]'); }); }); + +describe('isAllInterfaces', () => { + it('returns true for 0.0.0.0', () => { + expect(isAllInterfaces('0.0.0.0')).toBe(true); + }); + + it('returns true for ::', () => { + expect(isAllInterfaces('::')).toBe(true); + }); + + it('returns false for loopback hosts', () => { + expect(isAllInterfaces('127.0.0.1')).toBe(false); + expect(isAllInterfaces('localhost')).toBe(false); + expect(isAllInterfaces('::1')).toBe(false); + }); +}); + +describe('getLocalNetworkAddresses', () => { + it('returns non-empty array of IPv4 URLs for a valid port', () => { + const addresses = getLocalNetworkAddresses(3001); + expect(addresses.length).toBeGreaterThan(0); + for (const addr of addresses) { + expect(addr).toMatch(/^http:\/\/\d+\.\d+\.\d+\.\d+:3001\/$/); + } + }); + + it('returns different URLs for different ports', () => { + const addrs3001 = getLocalNetworkAddresses(3001); + const addrs8080 = getLocalNetworkAddresses(8080); + expect(addrs3001.length).toBe(addrs8080.length); + for (let i = 0; i < addrs3001.length; i++) { + expect(addrs3001[i]).toContain(':3001/'); + expect(addrs8080[i]).toContain(':8080/'); + } + }); +}); diff --git a/packages/agent-core-v2/src/app/agentProfileCatalog/system.md b/packages/agent-core-v2/src/app/agentProfileCatalog/system.md index fe939693a8..af909cff8a 100644 --- a/packages/agent-core-v2/src/app/agentProfileCatalog/system.md +++ b/packages/agent-core-v2/src/app/agentProfileCatalog/system.md @@ -58,7 +58,7 @@ The user may ask you to research on certain topics, process or generate certain - Understand the user's requirements thoroughly, ask for clarification before you start if needed. - Make plans before doing deep or wide research, to ensure you are always on track. - Search on the Internet if possible, with carefully-designed search queries to improve efficiency and accuracy. -- Use proper tools or shell commands or Python packages to process or generate images, videos, PDFs, docs, spreadsheets, presentations, or other multimedia files. Detect if there are already such tools in the environment. If you have to install third-party tools/packages, you MUST ensure that they are installed in a virtual/isolated environment. +- Use proper tools or shell commands or Python packages to process or generate images, videos, PDFs, docs, spreadsheets, presentations, or other multimedia files. For video analysis, prefer `ReadMediaFile` directly over writing Python scripts to extract frames. Detect if there are already such tools in the environment. If you have to install third-party tools/packages, you MUST ensure that they are installed in a virtual/isolated environment. - Once you generate or edit any images, videos or other media files, try to read it again before proceed, to ensure that the content is as expected. - Avoid installing or deleting anything to/from outside of the current working directory. If you have to do so, ask the user for confirmation. diff --git a/packages/agent-core-v2/src/app/web/providers/local-fetch-url.ts b/packages/agent-core-v2/src/app/web/providers/local-fetch-url.ts index bd785f0d21..388f8d6bf2 100644 --- a/packages/agent-core-v2/src/app/web/providers/local-fetch-url.ts +++ b/packages/agent-core-v2/src/app/web/providers/local-fetch-url.ts @@ -21,6 +21,10 @@ import { parseHTML as rawParseHTML } from 'linkedom'; import { Agent, type Dispatcher } from 'undici'; import { isProxyConfigured, makeNoProxyMatcher, resolveNoProxy } from '#/_base/utils/proxy'; +import { + MODEL_ACCEPTED_IMAGE_MIMES, + normalizeImageMime, +} from '#/agent/media/image-format-policy'; import { HttpFetchError, type UrlFetcher, type UrlFetchResult } from '../tools/fetch-url-types'; @@ -109,6 +113,27 @@ export class LocalFetchURLProvider implements UrlFetcher { } } + const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); + + // Handle accepted image formats directly — convert to base64 data URL + // so the model can view them inline. + const normalizedMime = normalizeImageMime(contentType); + if (MODEL_ACCEPTED_IMAGE_MIMES.has(normalizedMime)) { + const buffer = await response.arrayBuffer(); + if (buffer.byteLength > this.maxBytes) { + throw new Error( + `Response body too large: ${String(buffer.byteLength)} bytes exceeds maxBytes (${String(this.maxBytes)}).`, + ); + } + const base64 = Buffer.from(buffer).toString('base64'); + const dataUrl = `data:${normalizedMime};base64,${base64}`; + return { + content: `Fetched image (${normalizedMime}).`, + kind: 'image', + imageUrl: dataUrl, + }; + } + const body = await response.text(); const actualBytes = Buffer.byteLength(body, 'utf8'); @@ -118,7 +143,6 @@ export class LocalFetchURLProvider implements UrlFetcher { ); } - const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); if (contentType.startsWith('text/plain') || contentType.startsWith('text/markdown')) { return { content: body, kind: 'passthrough' }; } diff --git a/packages/agent-core-v2/src/app/web/tools/fetch-url-types.ts b/packages/agent-core-v2/src/app/web/tools/fetch-url-types.ts index de43ab052c..cf33464a8d 100644 --- a/packages/agent-core-v2/src/app/web/tools/fetch-url-types.ts +++ b/packages/agent-core-v2/src/app/web/tools/fetch-url-types.ts @@ -10,11 +10,13 @@ * - `extracted` — the body was an HTML page; only the main article text * was extracted and returned. */ -export type UrlFetchKind = 'passthrough' | 'extracted'; +export type UrlFetchKind = 'passthrough' | 'extracted' | 'image'; export interface UrlFetchResult { readonly content: string; readonly kind: UrlFetchKind; + /** When the fetched URL is an image, the image data as a base64 data URL. */ + readonly imageUrl?: string; } export interface UrlFetcher { diff --git a/packages/agent-core-v2/src/app/web/tools/fetch-url.ts b/packages/agent-core-v2/src/app/web/tools/fetch-url.ts index 6c847b0d04..c0127cedaa 100644 --- a/packages/agent-core-v2/src/app/web/tools/fetch-url.ts +++ b/packages/agent-core-v2/src/app/web/tools/fetch-url.ts @@ -58,7 +58,19 @@ export class FetchURLTool implements BuiltinTool { { toolCallId, signal }: ExecutableToolContext, ): Promise { try { - const { content, kind } = await this.fetcher.fetch(args.url, { toolCallId, signal }); + const { content, kind, imageUrl } = await this.fetcher.fetch(args.url, { toolCallId, signal }); + + // When the fetcher returns an image, emit it as an image_url content part + // so the model can see it directly. + if (kind === 'image' && imageUrl !== undefined) { + return { + isError: false, + output: [ + { type: 'text', text: content }, + { type: 'image_url', image_url: { url: imageUrl } }, + ], + }; + } if (!content) { return { diff --git a/packages/agent-core-v2/test/app/web/tools/fetch-url.test.ts b/packages/agent-core-v2/test/app/web/tools/fetch-url.test.ts index 233833062d..f7c33d92bb 100644 --- a/packages/agent-core-v2/test/app/web/tools/fetch-url.test.ts +++ b/packages/agent-core-v2/test/app/web/tools/fetch-url.test.ts @@ -90,32 +90,43 @@ describe('FetchURLTool abort signal', () => { }); describe('FetchURLTool output note', () => { - async function runKind(kind: UrlFetchResult['kind']): Promise { + async function runKind(kind: UrlFetchResult['kind'], imageUrl?: string): Promise { const fetch = vi .fn() - .mockResolvedValue({ content: 'BODY', kind } satisfies UrlFetchResult); + .mockResolvedValue({ content: 'BODY', kind, imageUrl } satisfies UrlFetchResult); const tool = new FetchURLTool({ fetch }); - const result = await execute(tool, 'https://example.com', new AbortController().signal); - expect(result.isError).toBe(false); - if (typeof result.output !== 'string') throw new Error('expected string output'); - return result.output; + return execute(tool, 'https://example.com', new AbortController().signal); } it('puts the passthrough note and citation reminder at the front of output', async () => { - const output = await runKind('passthrough'); - expect(output).toBe( + const result = await runKind('passthrough'); + expect(result.isError).toBe(false); + if (typeof result.output !== 'string') throw new Error('expected string output'); + expect(result.output).toBe( 'The returned content is the full response body, returned verbatim. ' + 'If you use it in your answer, cite this page as a markdown link, e.g. [title](url).\n\nBODY', ); }); it('puts the extracted note and citation reminder at the front of output', async () => { - const output = await runKind('extracted'); - expect(output).toBe( + const result = await runKind('extracted'); + expect(result.isError).toBe(false); + if (typeof result.output !== 'string') throw new Error('expected string output'); + expect(result.output).toBe( 'The returned content is the main text extracted from the page. ' + 'If you use it in your answer, cite this page as a markdown link, e.g. [title](url).\n\nBODY', ); }); + + it('returns image as ContentPart[] when kind is image', async () => { + const result = await runKind('image', 'data:image/png;base64,abc123'); + expect(result.isError).toBe(false); + if (typeof result.output === 'string') throw new Error('expected ContentPart[] output'); + expect(result.output).toEqual([ + { type: 'text', text: 'BODY' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,abc123' } }, + ]); + }); }); describe('LocalFetchURLProvider abort signal', () => { diff --git a/packages/agent-core/src/tools/builtin/web/fetch-url.ts b/packages/agent-core/src/tools/builtin/web/fetch-url.ts index d130e3977a..1d28c47879 100644 --- a/packages/agent-core/src/tools/builtin/web/fetch-url.ts +++ b/packages/agent-core/src/tools/builtin/web/fetch-url.ts @@ -26,13 +26,15 @@ import DESCRIPTION from './fetch-url.md?raw'; * - `extracted` — the body was an HTML page; only the main article text * was extracted and returned. */ -export type UrlFetchKind = 'passthrough' | 'extracted'; +export type UrlFetchKind = 'passthrough' | 'extracted' | 'image'; export interface UrlFetchResult { - /** The text handed to the LLM. */ + /** The text handed to the LLM, or a description when the result is an image. */ content: string; - /** Whether `content` is a verbatim passthrough or extracted main text. */ + /** Whether `content` is a verbatim passthrough, extracted main text, or an image. */ kind: UrlFetchKind; + /** When the fetched URL is an image, the image data as a base64 data URL. */ + imageUrl?: string; } export interface UrlFetcher { @@ -89,7 +91,19 @@ export class FetchURLTool implements BuiltinTool { }: ExecutableToolContext, ): Promise { try { - const { content, kind } = await this.fetcher.fetch(args.url, { toolCallId }); + const { content, kind, imageUrl } = await this.fetcher.fetch(args.url, { toolCallId }); + + // When the fetcher returns an image, emit it as an image_url content part + // so the model can see it directly. + if (kind === 'image' && imageUrl !== undefined) { + return { + isError: false, + output: [ + { type: 'text', text: content }, + { type: 'image_url', image_url: { url: imageUrl } }, + ], + }; + } if (!content) { return { diff --git a/packages/agent-core/src/tools/providers/local-fetch-url.ts b/packages/agent-core/src/tools/providers/local-fetch-url.ts index 1b8e50844e..28765f1913 100644 --- a/packages/agent-core/src/tools/providers/local-fetch-url.ts +++ b/packages/agent-core/src/tools/providers/local-fetch-url.ts @@ -27,6 +27,10 @@ import { Agent, type Dispatcher } from 'undici'; import { isProxyConfigured, makeNoProxyMatcher, resolveNoProxy } from '../../utils/proxy'; import { HttpFetchError, type UrlFetcher, type UrlFetchResult } from '../builtin'; +import { + MODEL_ACCEPTED_IMAGE_MIMES, + normalizeImageMime, +} from '../../tools/support/image-format-policy'; // Readability's .d.ts references the global `Document` type, but this // package compiles with `lib: ES2023` (no DOM). Extracting the @@ -253,6 +257,27 @@ export class LocalFetchURLProvider implements UrlFetcher { } } + const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); + + // Handle accepted image formats directly — convert to base64 data URL + // so the model can view them inline. + const normalizedMime = normalizeImageMime(contentType); + if (MODEL_ACCEPTED_IMAGE_MIMES.has(normalizedMime)) { + const buffer = await response.arrayBuffer(); + if (buffer.byteLength > this.maxBytes) { + throw new Error( + `Response body too large: ${String(buffer.byteLength)} bytes exceeds maxBytes (${String(this.maxBytes)}).`, + ); + } + const base64 = Buffer.from(buffer).toString('base64'); + const dataUrl = `data:${normalizedMime};base64,${base64}`; + return { + content: `Fetched image (${normalizedMime}).`, + kind: 'image', + imageUrl: dataUrl, + }; + } + const body = await response.text(); // Servers may omit content-length — measure again defensively. @@ -263,7 +288,6 @@ export class LocalFetchURLProvider implements UrlFetcher { ); } - const contentType = (response.headers.get('content-type') ?? '').toLowerCase(); if (contentType.startsWith('text/plain') || contentType.startsWith('text/markdown')) { return { content: body, kind: 'passthrough' }; } diff --git a/packages/agent-core/test/tools/providers/local-fetch-url.test.ts b/packages/agent-core/test/tools/providers/local-fetch-url.test.ts index ff23ea1d8d..4487e3ebac 100644 --- a/packages/agent-core/test/tools/providers/local-fetch-url.test.ts +++ b/packages/agent-core/test/tools/providers/local-fetch-url.test.ts @@ -88,6 +88,70 @@ describe('LocalFetchURLProvider content kind', () => { expect(result.kind).toBe('extracted'); expect(result.content).toContain('quick brown fox'); }); + + it('converts supported image responses to base64 data URLs', async () => { + const pngBytes = Buffer.from([0x89, 0x50, 0x4e, 0x47]); // PNG magic + const fetchImpl = vi.fn().mockResolvedValue( + new Response(pngBytes, { + status: 200, + headers: { 'content-type': 'image/png' }, + }), + ); + const provider = new LocalFetchURLProvider({ fetchImpl }); + + const result = await provider.fetch('https://example.com/image.png'); + + expect(result.kind).toBe('image'); + expect(result.content).toContain('image/png'); + expect(result.imageUrl).toMatch(/^data:image\/png;base64,/); + }); + + it('normalizes image/jpg to image/jpeg when converting', async () => { + const jpegBytes = Buffer.from([0xff, 0xd8, 0xff]); // JPEG magic + const fetchImpl = vi.fn().mockResolvedValue( + new Response(jpegBytes, { + status: 200, + headers: { 'content-type': 'image/jpg' }, + }), + ); + const provider = new LocalFetchURLProvider({ fetchImpl }); + + const result = await provider.fetch('https://example.com/image.jpg'); + + expect(result.kind).toBe('image'); + expect(result.imageUrl).toMatch(/^data:image\/jpeg;base64,/); + }); + + it('rejects oversized images by content-length', async () => { + const fetchImpl = vi.fn().mockResolvedValue( + new Response(Buffer.alloc(100), { + status: 200, + headers: { + 'content-type': 'image/png', + 'content-length': String(11 * 1024 * 1024), + }, + }), + ); + const provider = new LocalFetchURLProvider({ fetchImpl }); + + await expect(provider.fetch('https://example.com/big.png')).rejects.toThrow( + 'Response body too large', + ); + }); + + it('rejects oversized images by actual body size', async () => { + const fetchImpl = vi.fn().mockResolvedValue( + new Response(Buffer.alloc(11 * 1024 * 1024), { + status: 200, + headers: { 'content-type': 'image/png' }, + }), + ); + const provider = new LocalFetchURLProvider({ fetchImpl }); + + await expect(provider.fetch('https://example.com/big.png')).rejects.toThrow( + 'Response body too large', + ); + }); }); describe('LocalFetchURLProvider SSRF guard', () => {