diff --git a/.changeset/stream-read-large-files.md b/.changeset/stream-read-large-files.md new file mode 100644 index 0000000000..185bfcc605 --- /dev/null +++ b/.changeset/stream-read-large-files.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Avoid loading entire large text files into memory when reading them. diff --git a/packages/kaos/src/local.ts b/packages/kaos/src/local.ts index 71fa16b527..01937729c6 100644 --- a/packages/kaos/src/local.ts +++ b/packages/kaos/src/local.ts @@ -408,17 +408,35 @@ export class LocalKaos implements Kaos { const resolved = this._resolvePath(path); const encoding = options?.encoding ?? 'utf-8'; const errors = options?.errors ?? 'strict'; - const buf = await readFile(resolved); - const content = decodeTextWithErrors(buf, encoding, errors); - const lines = content.split('\n'); - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - if (line === undefined) continue; - if (i < lines.length - 1) { - yield line + '\n'; - } else if (line !== '') { - yield line; + const fh = await open(resolved, 'r'); + try { + const chunkSize = 64 * 1024; + const buf = Buffer.alloc(chunkSize); + let pending = Buffer.alloc(0); + while (true) { + const { bytesRead } = await fh.read(buf, 0, buf.length, null); + if (bytesRead === 0) break; + const data = + pending.length > 0 + ? Buffer.concat([pending, buf.subarray(0, bytesRead)]) + : buf.subarray(0, bytesRead); + let lineStart = 0; + for (let i = 0; i < data.length; i += 1) { + // Split on the LF byte (0x0a). LF/CR are ASCII and never appear + // inside a multibyte UTF-8/UTF-16 sequence, so cutting here is + // always on a character boundary and safe to decode per line. + if (data[i] === 0x0a) { + yield decodeTextWithErrors(data.subarray(lineStart, i + 1), encoding, errors); + lineStart = i + 1; + } + } + pending = lineStart < data.length ? Buffer.from(data.subarray(lineStart)) : Buffer.alloc(0); } + if (pending.length > 0) { + yield decodeTextWithErrors(pending, encoding, errors); + } + } finally { + await fh.close(); } } diff --git a/packages/kaos/test/local.test.ts b/packages/kaos/test/local.test.ts index dd8b88b6c6..b0a8dd9193 100644 --- a/packages/kaos/test/local.test.ts +++ b/packages/kaos/test/local.test.ts @@ -214,6 +214,106 @@ describe('LocalKaos', () => { }); }); + describe('readLines streaming', () => { + it('preserves content exactly across representative line endings', async () => { + const fixtures: Array<[string, string]> = [ + ['multiline', 'line1\nline2\nline3\n'], + ['no trailing newline', 'line1\nline2'], + ['single line', 'only'], + ['single newline', '\n'], + ['empty', ''], + ['crlf', 'a\r\nb\r\n'], + ['lone cr', 'a\rB\n'], + ]; + for (const [name, content] of fixtures) { + const filePath = join(tempDir, `${name}.txt`); + await kaos.writeText(filePath, content); + const lines: string[] = []; + for await (const line of kaos.readLines(filePath)) { + lines.push(line); + } + expect(lines.join('')).toBe(content); + } + }); + + it('reads a large multi-chunk file line by line', async () => { + const filePath = join(tempDir, 'large.txt'); + const lineCount = 5000; + const content = + Array.from({ length: lineCount }, (_, i) => `line ${String(i)} ${'x'.repeat(40)}`).join( + '\n', + ) + '\n'; + await kaos.writeText(filePath, content); + + const lines: string[] = []; + for await (const line of kaos.readLines(filePath)) { + lines.push(line); + } + expect(lines.length).toBe(lineCount); + expect(lines.join('')).toBe(content); + }); + + it('preserves a multibyte character straddling the 64KiB chunk boundary', async () => { + const filePath = join(tempDir, 'boundary.txt'); + const emoji = '😀'; + const content = `${'a'.repeat(65535)}${emoji}\n`; + await kaos.writeText(filePath, content); + + const lines: string[] = []; + for await (const line of kaos.readLines(filePath)) { + lines.push(line); + } + expect(lines).toEqual([content]); + }); + + describe('errors parameter', () => { + // "中" + invalid 0xff + "文" + "\n" + const invalidBytes = Buffer.concat([ + Buffer.from([0xe4, 0xb8, 0xad]), + Buffer.from([0xff]), + Buffer.from([0xe6, 0x96, 0x87]), + Buffer.from([0x0a]), + ]); + + it('throws on invalid utf-8 with errors="strict"', async () => { + const filePath = join(tempDir, 'invalid-strict.txt'); + await kaos.writeBytes(filePath, invalidBytes); + await expect( + (async () => { + const lines: string[] = []; + for await (const line of kaos.readLines(filePath)) { + lines.push(line); + } + return lines; + })(), + ).rejects.toThrow(); + }); + + it('replaces invalid bytes with errors="replace"', async () => { + const filePath = join(tempDir, 'invalid-replace.txt'); + await kaos.writeBytes(filePath, invalidBytes); + const lines: string[] = []; + for await (const line of kaos.readLines(filePath, { errors: 'replace' })) { + lines.push(line); + } + const output = lines.join(''); + expect(output).toContain('\uFFFD'); + expect(output).toContain('中'); + expect(output).toContain('文'); + }); + + it('drops invalid bytes with errors="ignore"', async () => { + const filePath = join(tempDir, 'invalid-ignore.txt'); + await kaos.writeBytes(filePath, invalidBytes); + const lines: string[] = []; + for await (const line of kaos.readLines(filePath, { errors: 'ignore' })) { + lines.push(line); + } + expect(lines.join('')).toBe('中文\n'); + }); + }); + }); + describe('LF preservation', () => { it('should not convert LF to CRLF', async () => { const filePath = join(tempDir, 'lf.txt');