diff --git a/.changeset/upload-no-size-cap.md b/.changeset/upload-no-size-cap.md new file mode 100644 index 0000000000..1afa1ea34f --- /dev/null +++ b/.changeset/upload-no-size-cap.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": minor +--- + +Remove the 50 MB size limit on file uploads to the built-in server, so large attachments (for example in the web UI) no longer fail with an upload-too-large error. Uploads now stream to disk instead of being buffered in memory. diff --git a/packages/agent-core-v2/src/_base/utils/fs.ts b/packages/agent-core-v2/src/_base/utils/fs.ts index a93e9699a2..b9222b01ab 100644 --- a/packages/agent-core-v2/src/_base/utils/fs.ts +++ b/packages/agent-core-v2/src/_base/utils/fs.ts @@ -112,3 +112,48 @@ export async function atomicWrite( } } } + +/** + * Streamed variant of `atomicWrite`: same tmp + fsync + rename discipline, but + * the content arrives as an `AsyncIterable` so arbitrarily large values never + * sit in memory at once. + */ +export async function atomicWriteStream( + filePath: string, + source: AsyncIterable, + mode?: number, +): Promise { + const hex = randomBytes(4).toString('hex'); + const tmpPath = `${filePath}.tmp.${process.pid}.${hex}`; + let renamed = false; + try { + const fh = await open(tmpPath, 'w', mode); + try { + for await (const chunk of source) { + if (chunk.byteLength > 0) { + await fh.writeFile(chunk); + } + } + await fh.sync(); + } finally { + await fh.close(); + } + if (process.platform === 'win32') { + try { + await unlink(filePath); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== 'ENOENT') throw error; + } + } + await rename(tmpPath, filePath); + renamed = true; + } finally { + if (!renamed) { + try { + await unlink(tmpPath); + } catch { + } + } + } +} diff --git a/packages/agent-core-v2/src/app/file/fileService.ts b/packages/agent-core-v2/src/app/file/fileService.ts index 476b615e63..0115d59f8c 100644 --- a/packages/agent-core-v2/src/app/file/fileService.ts +++ b/packages/agent-core-v2/src/app/file/fileService.ts @@ -25,8 +25,6 @@ export const fileMetaSchema = z.object({ }); export type FileMeta = z.infer; -export const DEFAULT_MAX_UPLOAD_BYTES = 50 * 1024 * 1024; - export interface SaveOptions { readonly name?: string; readonly mimeType?: string; @@ -57,7 +55,6 @@ export const IFileService: ServiceIdentifier = createDecorator DEFAULT_MAX_UPLOAD_BYTES) { - throw fileTooLargeError(bytes, DEFAULT_MAX_UPLOAD_BYTES); + let size = 0; + const counting = async function* (): AsyncIterable { + for await (const chunk of source) { + const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as string); + size += buf.length; + yield buf; } - chunks.push(buf); + }; + try { + await this.blobs.putStream(BLOB_SCOPE, id, counting()); + } catch (error) { + // best-effort cleanup of a partially written blob + await this.blobs.delete(BLOB_SCOPE, id).catch(() => undefined); + throw error; } - const data = Buffer.concat(chunks); - - await this.blobs.put(BLOB_SCOPE, id, data); const now = Date.now(); const meta: FileMeta = { id, name: options.name ?? filename, media_type: options.mimeType ?? 'application/octet-stream', - size: data.length, + size, created_at: new Date(now).toISOString(), ...(options.expiresInSec !== undefined ? { expires_at: new Date(now + options.expiresInSec * 1000).toISOString() } diff --git a/packages/agent-core-v2/src/persistence/backends/memory/inMemoryStorageService.ts b/packages/agent-core-v2/src/persistence/backends/memory/inMemoryStorageService.ts index cf5f716c0e..29d252aa18 100644 --- a/packages/agent-core-v2/src/persistence/backends/memory/inMemoryStorageService.ts +++ b/packages/agent-core-v2/src/persistence/backends/memory/inMemoryStorageService.ts @@ -69,6 +69,28 @@ export class InMemoryStorageService implements IFileSystemStorageService { this.notifyWatchers(scope, key); } + async writeStream( + scope: string, + key: string, + source: AsyncIterable, + _options: StorageWriteOptions = {}, + ): Promise { + const chunks: Uint8Array[] = []; + let total = 0; + for await (const chunk of source) { + chunks.push(chunk); + total += chunk.byteLength; + } + const merged = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + merged.set(chunk, offset); + offset += chunk.byteLength; + } + this.bucket(scope).set(key, merged); + this.notifyWatchers(scope, key); + } + async append( scope: string, key: string, diff --git a/packages/agent-core-v2/src/persistence/backends/node-fs/blobStoreService.ts b/packages/agent-core-v2/src/persistence/backends/node-fs/blobStoreService.ts index caf7443703..e783e27e77 100644 --- a/packages/agent-core-v2/src/persistence/backends/node-fs/blobStoreService.ts +++ b/packages/agent-core-v2/src/persistence/backends/node-fs/blobStoreService.ts @@ -19,6 +19,10 @@ export class BlobStoreService implements IBlobStore { await this.storage.write(scope, key, data, { atomic: true }); } + async putStream(scope: string, key: string, source: AsyncIterable): Promise { + await this.storage.writeStream(scope, key, source, { atomic: true }); + } + async get(scope: string, key: string): Promise { return this.storage.read(scope, key); } diff --git a/packages/agent-core-v2/src/persistence/backends/node-fs/fileStorageService.ts b/packages/agent-core-v2/src/persistence/backends/node-fs/fileStorageService.ts index 4bb02fcbf0..d16312c3e4 100644 --- a/packages/agent-core-v2/src/persistence/backends/node-fs/fileStorageService.ts +++ b/packages/agent-core-v2/src/persistence/backends/node-fs/fileStorageService.ts @@ -8,6 +8,8 @@ * Primitives: * - `write` → `atomicWrite` (tmp + fsync + rename) followed by a directory * fsync, so the replacement is both atomic and durable. + * - `writeStream` → the streamed form of `write` (`atomicWriteStream`), for + * values too large to buffer in memory. * - `append` → `open('a')` + write + `fh.sync()` (when `durable`), plus a * one-time directory fsync per scope. * - `watch` → chokidar on the parent directory, filtered to the exact key and @@ -29,7 +31,7 @@ import { dirname, join, normalize } from 'pathe'; import { DisposableStore, combinedDisposable, toDisposable, type IDisposable } from '#/_base/di/lifecycle'; import { Emitter, type Event } from '#/_base/event'; import { onUnexpectedError } from '#/_base/errors/unexpectedError'; -import { atomicWrite, syncDir } from '#/_base/utils/fs'; +import { atomicWrite, atomicWriteStream, syncDir } from '#/_base/utils/fs'; import type { IFileSystemStorageService, @@ -102,6 +104,22 @@ export class FileStorageService implements IFileSystemStorageService { } } + async writeStream( + scope: string, + key: string, + source: AsyncIterable, + _options: StorageWriteOptions = {}, + ): Promise { + const filePath = this.path(scope, key); + try { + await mkdir(dirname(filePath), { recursive: true, mode: this.dirMode }); + await atomicWriteStream(filePath, source, this.fileMode); + await this.syncDirOnce(dirname(filePath)); + } catch (error) { + throw toStorageIoError(error, { path: filePath, op: 'write' }); + } + } + async append( scope: string, key: string, diff --git a/packages/agent-core-v2/src/persistence/interface/blobStore.ts b/packages/agent-core-v2/src/persistence/interface/blobStore.ts index eeb8c99b9c..a4d8c32025 100644 --- a/packages/agent-core-v2/src/persistence/interface/blobStore.ts +++ b/packages/agent-core-v2/src/persistence/interface/blobStore.ts @@ -15,6 +15,7 @@ export interface IBlobStore { readonly _serviceBrand: undefined; put(scope: string, key: string, data: Uint8Array): Promise; + putStream(scope: string, key: string, source: AsyncIterable): Promise; get(scope: string, key: string): Promise; getStream(scope: string, key: string, range?: BlobReadRange): AsyncIterable; has(scope: string, key: string): Promise; diff --git a/packages/agent-core-v2/src/persistence/interface/storage.ts b/packages/agent-core-v2/src/persistence/interface/storage.ts index 83c57ca17d..fa552ba437 100644 --- a/packages/agent-core-v2/src/persistence/interface/storage.ts +++ b/packages/agent-core-v2/src/persistence/interface/storage.ts @@ -13,6 +13,10 @@ * the last value" semantics. Keeping both as first-class primitives lets each * implementation implement them optimally (file: `open('a')` vs tmp+rename). * + * `writeStream` is the streamed form of `write` for values too large to hold + * in memory: same whole-value replacement semantics (tmp + rename on the file + * backend), but the bytes arrive as an `AsyncIterable`. + * * The service is byte-oriented and scope/key-addressed: `scope` maps to a * directory, `key` maps to a filename. It knows nothing about JSON, records, * configs, versions or framing. Those concerns live in the typed Store facades @@ -124,6 +128,12 @@ export interface IFileSystemStorageService { read(scope: string, key: string): Promise; readStream(scope: string, key: string, range?: StorageReadRange): AsyncIterable; write(scope: string, key: string, data: Uint8Array, options?: StorageWriteOptions): Promise; + writeStream( + scope: string, + key: string, + source: AsyncIterable, + options?: StorageWriteOptions, + ): Promise; append(scope: string, key: string, data: Uint8Array, options?: StorageAppendOptions): Promise; list(scope: string, prefix?: string): Promise; delete(scope: string, key: string): Promise; diff --git a/packages/agent-core-v2/test/agent/media/videoResolver.test.ts b/packages/agent-core-v2/test/agent/media/videoResolver.test.ts index 9f2b76fdca..9d66d2d807 100644 --- a/packages/agent-core-v2/test/agent/media/videoResolver.test.ts +++ b/packages/agent-core-v2/test/agent/media/videoResolver.test.ts @@ -57,6 +57,11 @@ function blobStore(): IBlobStore { put: async (scope, key, bytes) => { data.set(`${scope}/${key}`, bytes); }, + putStream: async (scope, key, source) => { + const chunks: Uint8Array[] = []; + for await (const chunk of source) chunks.push(chunk); + data.set(`${scope}/${key}`, Buffer.concat(chunks)); + }, get: async (scope, key) => data.get(`${scope}/${key}`), getStream: async function* () {}, has: async (scope, key) => data.has(`${scope}/${key}`), diff --git a/packages/agent-core-v2/test/agent/task/taskService.test.ts b/packages/agent-core-v2/test/agent/task/taskService.test.ts index d7e3f49319..58e62d786d 100644 --- a/packages/agent-core-v2/test/agent/task/taskService.test.ts +++ b/packages/agent-core-v2/test/agent/task/taskService.test.ts @@ -152,6 +152,7 @@ describe('AgentTaskService', () => { read: async () => undefined, readStream: async function* () {}, write: async () => {}, + writeStream: async () => {}, append: async () => {}, list: async () => [], delete: async () => {}, @@ -802,6 +803,7 @@ describe('AgentTaskService', () => { read: async () => undefined, readStream: async function* () {}, write: async () => {}, + writeStream: async () => {}, append: async (_scope: string, _key: string, chunk: Uint8Array) => { persistedChars += chunk.byteLength; }, diff --git a/packages/agent-core-v2/test/app/file/fileService.test.ts b/packages/agent-core-v2/test/app/file/fileService.test.ts index 253b1c4865..b112e09b74 100644 --- a/packages/agent-core-v2/test/app/file/fileService.test.ts +++ b/packages/agent-core-v2/test/app/file/fileService.test.ts @@ -13,7 +13,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { SyncDescriptor } from '#/_base/di/descriptors'; import { DisposableStore } from '#/_base/di/lifecycle'; import { createServices, type TestInstantiationService } from '#/_base/di/test'; -import { DEFAULT_MAX_UPLOAD_BYTES, FileErrors, IFileService } from '#/app/file/fileService'; +import { FileErrors, IFileService } from '#/app/file/fileService'; import { FileServiceImpl } from '#/app/file/fileServiceImpl'; import { IFileSystemStorageService } from '#/persistence/interface/storage'; import { InMemoryStorageService } from '#/persistence/backends/memory/inMemoryStorageService'; @@ -115,11 +115,22 @@ describe('FileServiceImpl', () => { }); }); - it('rejects an upload that exceeds the cap', async () => { - const big = Buffer.alloc(DEFAULT_MAX_UPLOAD_BYTES + 1, 0); - await expect(store().save(readable(big), 'big.bin')).rejects.toMatchObject({ - code: FileErrors.codes.FILE_TOO_LARGE, - }); + it('streams a multi-chunk upload and records the total size', async () => { + const chunks = [Buffer.from('aaa'), Buffer.from('bbbb'), Buffer.from('cc')]; + const meta = await store().save(Readable.from(chunks), 'chunked.bin'); + + expect(meta.size).toBe(9); + const { stream } = await store().get(meta.id); + expect((await readAll(stream())).toString()).toBe('aaabbbbcc'); + }); + + it('cleans up the blob when the source stream fails mid-upload', async () => { + const failing = Readable.from((async function* () { + yield Buffer.from('partial'); + throw new Error('source exploded'); + })()); + + await expect(store().save(failing, 'broken.bin')).rejects.toThrow('source exploded'); expect(await backend.list('files')).toHaveLength(0); }); diff --git a/packages/agent-core-v2/test/persistence/backends/node-fs/appendLogStore.test.ts b/packages/agent-core-v2/test/persistence/backends/node-fs/appendLogStore.test.ts index a6b24eb762..52d7eaadaa 100644 --- a/packages/agent-core-v2/test/persistence/backends/node-fs/appendLogStore.test.ts +++ b/packages/agent-core-v2/test/persistence/backends/node-fs/appendLogStore.test.ts @@ -34,6 +34,7 @@ function chunkedStorage(chunks: Uint8Array[]): IFileSystemStorageService { for (const c of chunks) yield c; }, write: async () => {}, + writeStream: async () => {}, append: async () => {}, list: async () => [], delete: async () => {}, diff --git a/packages/agent-core-v2/test/persistence/backends/node-fs/fileStorageService.test.ts b/packages/agent-core-v2/test/persistence/backends/node-fs/fileStorageService.test.ts index c173164776..4721e32794 100644 --- a/packages/agent-core-v2/test/persistence/backends/node-fs/fileStorageService.test.ts +++ b/packages/agent-core-v2/test/persistence/backends/node-fs/fileStorageService.test.ts @@ -87,3 +87,41 @@ describe('FileStorageService — error translation', () => { }); }); }); + +describe('FileStorageService — writeStream', () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'fss-stream-')); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('writes a chunked source and replaces the whole value', async () => { + const svc = new FileStorageService(dir); + await svc.write('scope', 'k.bin', encoder.encode('old')); + await svc.writeStream('scope', 'k.bin', (async function* () { + yield encoder.encode('aa'); + yield encoder.encode('bbb'); + })()); + + const chunks: Uint8Array[] = []; + for await (const chunk of svc.readStream('scope', 'k.bin')) chunks.push(chunk); + expect(Buffer.concat(chunks).toString()).toBe('aabbb'); + }); + + it('leaves no target file behind when the source fails mid-stream', async () => { + const svc = new FileStorageService(dir); + await expect( + svc.writeStream('scope', 'k.bin', (async function* () { + yield encoder.encode('partial'); + throw new Error('boom'); + })()), + ).rejects.toThrow(); + + expect(await svc.read('scope', 'k.bin')).toBeUndefined(); + expect(await svc.list('scope')).toEqual([]); + }); +}); diff --git a/packages/kap-server/src/protocol/error-codes.ts b/packages/kap-server/src/protocol/error-codes.ts index b5a4ff0711..e8f033aa78 100644 --- a/packages/kap-server/src/protocol/error-codes.ts +++ b/packages/kap-server/src/protocol/error-codes.ts @@ -124,7 +124,7 @@ export const ErrorCode = { /** 临时文件已过期 */ FILE_EXPIRED: 41003, - /** 上传超 50MB */ + /** 文件过大(如 session 导出超限;/files 上传不设上限) */ FILE_TOO_LARGE: 41301, /** fs.read 超 10MB */ FS_TOO_LARGE: 41302, diff --git a/packages/kap-server/src/routes/files.ts b/packages/kap-server/src/routes/files.ts index ff74e99ffc..6d5e8af732 100644 --- a/packages/kap-server/src/routes/files.ts +++ b/packages/kap-server/src/routes/files.ts @@ -6,15 +6,15 @@ * DELETE /files/{file_id} delete a file → { deleted: true } * * Backed by the v2 `IFileService` (Core scope), which stores bytes in - * `IBlobStore` and the metadata index alongside them. Mirrors the v1 server's - * wire behavior (envelope codes 40407 / 41301, 50 MiB cap, content-disposition) - * but resolves the store through `core.accessor.get`. + * `IBlobStore` and the metadata index alongside them. Uploads stream straight + * to the store with no size cap (local single-user deployment), and the route + * mirrors the v1 server's wire behavior (envelope codes 40407 / 41301, + * content-disposition) while resolving the store through `core.accessor.get`. */ import multipart from '@fastify/multipart'; import { - DEFAULT_MAX_UPLOAD_BYTES, ErrorCodes, IFileService, Error2, @@ -76,7 +76,11 @@ interface FilesReply { export function registerFilesRoutes(app: FilesRouteHost, core: Scope): void { app.register(multipart, { limits: { - fileSize: DEFAULT_MAX_UPLOAD_BYTES, + // No upload size cap — local single-user deployment. The limit must be + // set explicitly: @fastify/multipart defaults `fileSize` to Fastify's + // `bodyLimit` (1 MiB) when it is left undefined, and silently truncates + // the file stream at that size. + fileSize: Number.MAX_SAFE_INTEGER, files: 1, }, }); @@ -108,14 +112,9 @@ export function registerFilesRoutes(app: FilesRouteHost, core: Scope): void { const store = core.accessor.get(IFileService); - const partFile = part.file as NodeJS.ReadableStream & { truncated?: boolean }; - let busboyTruncated = false; - partFile.on('limit', () => { - busboyTruncated = true; - }); try { const meta = await store.save( - partFile as unknown as import('node:stream').Readable, + part.file as unknown as import('node:stream').Readable, part.filename, { name: nameOverride ?? part.filename, @@ -123,18 +122,6 @@ export function registerFilesRoutes(app: FilesRouteHost, core: Scope): void { expiresInSec, }, ); - if (busboyTruncated || partFile.truncated === true) { - try { - await store.delete(meta.id); - } catch { - // best-effort cleanup of the truncated blob - } - sendMappedError(reply as unknown as FilesReply, req, new Error2( - ErrorCodes.FILE_TOO_LARGE, - `upload size exceeds limit ${DEFAULT_MAX_UPLOAD_BYTES} bytes`, - )); - return; - } reply.send(okEnvelope(meta, req.id)); } catch (error) { sendMappedError(reply as unknown as FilesReply, req, error); @@ -239,19 +226,6 @@ function sendMappedError(reply: FilesReply, req: { id: string }, err: unknown): reply.code(404).send(errEnvelope(ErrorCode.FILE_NOT_FOUND, 'file not found', requestId)); return; } - if (err instanceof Error2 && err.code === ErrorCodes.FILE_TOO_LARGE) { - reply.code(413).send(errEnvelope(ErrorCode.FILE_TOO_LARGE, 'upload too large (>50MB)', requestId)); - return; - } - if ( - typeof err === 'object' && - err !== null && - 'name' in err && - (err as { name: string }).name === 'FST_REQ_FILE_TOO_LARGE' - ) { - reply.code(413).send(errEnvelope(ErrorCode.FILE_TOO_LARGE, 'upload too large (>50MB)', requestId)); - return; - } requestLog(req)?.error({ err }, 'file request failed'); reply .code(500) diff --git a/packages/kap-server/test/files.test.ts b/packages/kap-server/test/files.test.ts index 6a9060d1d2..4a94dacdf0 100644 --- a/packages/kap-server/test/files.test.ts +++ b/packages/kap-server/test/files.test.ts @@ -1,11 +1,11 @@ /** * `/api/v1/files` end-to-end for the v2 server. * - * Mirrors the v1 server's files e2e (upload → download → delete → 404, the - * 50 MiB cap, unknown ids, index persistence across restart, the `name` - * override, and the missing-file validation) but boots `startServer` from - * server-v2 and drives it through Fastify `app.inject` with hand-built - * multipart bodies. + * Mirrors the v1 server's files e2e (upload → download → delete → 404, + * large uploads with no size cap, unknown ids, index persistence across + * restart, the `name` override, and the missing-file validation) but boots + * `startServer` from server-v2 and drives it through Fastify `app.inject` + * with hand-built multipart bodies. */ import { mkdtempSync, rmSync } from 'node:fs'; @@ -168,7 +168,7 @@ describe('POST /api/v1/files (server-v2)', () => { expect((get2Res.json() as Envelope).code).toBe(40407); }); - it('upload > 50MB → 41301', async () => { + it('uploads a file larger than the former 50 MiB cap', async () => { const r = await boot(); const big = Buffer.alloc(51 * 1024 * 1024, 0); const mp = buildMultipart({ @@ -185,8 +185,19 @@ describe('POST /api/v1/files (server-v2)', () => { payload: mp.body, headers: { 'content-type': mp.contentType }, }); - expect(res.statusCode).toBe(413); - expect((res.json() as Envelope).code).toBe(41301); + expect(res.statusCode).toBe(200); + const env = res.json() as Envelope<{ id: string; size: number }>; + expect(env.code).toBe(0); + expect(env.data!.size).toBe(big.length); + + const getRes = await appOf(r).inject({ + method: 'GET', + url: `/api/v1/files/${env.data!.id}`, + }); + expect(getRes.statusCode).toBe(200); + // `expect(buffer).toEqual(buffer)` deep-compares 51M elements and blows + // the heap — use the native memcmp instead. + expect(getRes.rawPayload.equals(big)).toBe(true); }); it('GET / DELETE unknown file_id → 40407', async () => { diff --git a/packages/protocol/src/error-codes.ts b/packages/protocol/src/error-codes.ts index 27cce69d39..ee1ebc344f 100644 --- a/packages/protocol/src/error-codes.ts +++ b/packages/protocol/src/error-codes.ts @@ -109,7 +109,7 @@ export const ErrorCode = { /** 临时文件已过期 */ FILE_EXPIRED: 41003, - /** 上传超 50MB */ + /** 文件过大(如 session 导出超限;/files 上传不设上限) */ FILE_TOO_LARGE: 41301, /** fs.read 超 10MB */ FS_TOO_LARGE: 41302, diff --git a/packages/protocol/src/rest/file.ts b/packages/protocol/src/rest/file.ts index 8df36cf41e..383e1cbcf3 100644 --- a/packages/protocol/src/rest/file.ts +++ b/packages/protocol/src/rest/file.ts @@ -2,8 +2,7 @@ * POST /v1/files * Request: multipart/form-data with `file` (binary), `name` * (optional override), `expires_in_sec` (optional). - * Response data: `FileMeta` (full envelope). - * Errors: 41301 (>50MB). + * Response data: `FileMeta` (full envelope). No upload size cap. * * GET /v1/files/{file_id} * Response: binary stream or envelope (40407 / 41003).