From f6d584e8a50a5622916f60d94a91b0d0ae7c345b Mon Sep 17 00:00:00 2001 From: 7Sageer <7sageer@djwcb.cn> Date: Mon, 22 Jun 2026 14:48:53 +0800 Subject: [PATCH] perf(kaos): stream readLines to avoid loading whole file into memory Read files in 64KiB chunks and split on the LF byte so peak memory stays bounded by the longest line instead of the whole file. readTail/readForward keep the same behavior and Total lines reporting. --- .changeset/stream-read-large-files.md | 5 ++ packages/kaos/src/local.ts | 38 +++++++--- packages/kaos/test/local.test.ts | 100 ++++++++++++++++++++++++++ 3 files changed, 133 insertions(+), 10 deletions(-) create mode 100644 .changeset/stream-read-large-files.md diff --git a/.changeset/stream-read-large-files.md b/.changeset/stream-read-large-files.md new file mode 100644 index 0000000000..185bfcc605 --- /dev/null +++ b/.changeset/stream-read-large-files.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Avoid loading entire large text files into memory when reading them. diff --git a/packages/kaos/src/local.ts b/packages/kaos/src/local.ts index 71fa16b527..01937729c6 100644 --- a/packages/kaos/src/local.ts +++ b/packages/kaos/src/local.ts @@ -408,17 +408,35 @@ export class LocalKaos implements Kaos { const resolved = this._resolvePath(path); const encoding = options?.encoding ?? 'utf-8'; const errors = options?.errors ?? 'strict'; - const buf = await readFile(resolved); - const content = decodeTextWithErrors(buf, encoding, errors); - const lines = content.split('\n'); - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - if (line === undefined) continue; - if (i < lines.length - 1) { - yield line + '\n'; - } else if (line !== '') { - yield line; + const fh = await open(resolved, 'r'); + try { + const chunkSize = 64 * 1024; + const buf = Buffer.alloc(chunkSize); + let pending = Buffer.alloc(0); + while (true) { + const { bytesRead } = await fh.read(buf, 0, buf.length, null); + if (bytesRead === 0) break; + const data = + pending.length > 0 + ? Buffer.concat([pending, buf.subarray(0, bytesRead)]) + : buf.subarray(0, bytesRead); + let lineStart = 0; + for (let i = 0; i < data.length; i += 1) { + // Split on the LF byte (0x0a). LF/CR are ASCII and never appear + // inside a multibyte UTF-8/UTF-16 sequence, so cutting here is + // always on a character boundary and safe to decode per line. + if (data[i] === 0x0a) { + yield decodeTextWithErrors(data.subarray(lineStart, i + 1), encoding, errors); + lineStart = i + 1; + } + } + pending = lineStart < data.length ? Buffer.from(data.subarray(lineStart)) : Buffer.alloc(0); } + if (pending.length > 0) { + yield decodeTextWithErrors(pending, encoding, errors); + } + } finally { + await fh.close(); } } diff --git a/packages/kaos/test/local.test.ts b/packages/kaos/test/local.test.ts index dd8b88b6c6..b0a8dd9193 100644 --- a/packages/kaos/test/local.test.ts +++ b/packages/kaos/test/local.test.ts @@ -214,6 +214,106 @@ describe('LocalKaos', () => { }); }); + describe('readLines streaming', () => { + it('preserves content exactly across representative line endings', async () => { + const fixtures: Array<[string, string]> = [ + ['multiline', 'line1\nline2\nline3\n'], + ['no trailing newline', 'line1\nline2'], + ['single line', 'only'], + ['single newline', '\n'], + ['empty', ''], + ['crlf', 'a\r\nb\r\n'], + ['lone cr', 'a\rB\n'], + ]; + for (const [name, content] of fixtures) { + const filePath = join(tempDir, `${name}.txt`); + await kaos.writeText(filePath, content); + const lines: string[] = []; + for await (const line of kaos.readLines(filePath)) { + lines.push(line); + } + expect(lines.join('')).toBe(content); + } + }); + + it('reads a large multi-chunk file line by line', async () => { + const filePath = join(tempDir, 'large.txt'); + const lineCount = 5000; + const content = + Array.from({ length: lineCount }, (_, i) => `line ${String(i)} ${'x'.repeat(40)}`).join( + '\n', + ) + '\n'; + await kaos.writeText(filePath, content); + + const lines: string[] = []; + for await (const line of kaos.readLines(filePath)) { + lines.push(line); + } + expect(lines.length).toBe(lineCount); + expect(lines.join('')).toBe(content); + }); + + it('preserves a multibyte character straddling the 64KiB chunk boundary', async () => { + const filePath = join(tempDir, 'boundary.txt'); + const emoji = '😀'; + const content = `${'a'.repeat(65535)}${emoji}\n`; + await kaos.writeText(filePath, content); + + const lines: string[] = []; + for await (const line of kaos.readLines(filePath)) { + lines.push(line); + } + expect(lines).toEqual([content]); + }); + + describe('errors parameter', () => { + // "中" + invalid 0xff + "文" + "\n" + const invalidBytes = Buffer.concat([ + Buffer.from([0xe4, 0xb8, 0xad]), + Buffer.from([0xff]), + Buffer.from([0xe6, 0x96, 0x87]), + Buffer.from([0x0a]), + ]); + + it('throws on invalid utf-8 with errors="strict"', async () => { + const filePath = join(tempDir, 'invalid-strict.txt'); + await kaos.writeBytes(filePath, invalidBytes); + await expect( + (async () => { + const lines: string[] = []; + for await (const line of kaos.readLines(filePath)) { + lines.push(line); + } + return lines; + })(), + ).rejects.toThrow(); + }); + + it('replaces invalid bytes with errors="replace"', async () => { + const filePath = join(tempDir, 'invalid-replace.txt'); + await kaos.writeBytes(filePath, invalidBytes); + const lines: string[] = []; + for await (const line of kaos.readLines(filePath, { errors: 'replace' })) { + lines.push(line); + } + const output = lines.join(''); + expect(output).toContain('\uFFFD'); + expect(output).toContain('中'); + expect(output).toContain('文'); + }); + + it('drops invalid bytes with errors="ignore"', async () => { + const filePath = join(tempDir, 'invalid-ignore.txt'); + await kaos.writeBytes(filePath, invalidBytes); + const lines: string[] = []; + for await (const line of kaos.readLines(filePath, { errors: 'ignore' })) { + lines.push(line); + } + expect(lines.join('')).toBe('中文\n'); + }); + }); + }); + describe('LF preservation', () => { it('should not convert LF to CRLF', async () => { const filePath = join(tempDir, 'lf.txt');