feat: harden LLM API fault tolerance against 429 and overload

.changeset/retry-fault-tolerance.md

@@ -0,0 +1,5 @@
+---
+"@moonshot-ai/kimi-code": patch
+---
+
+Retry provider 429, overload, and other transient errors more reliably, honoring the server Retry-After delay, and surface retries in `-p --output-format stream-json`.

apps/kimi-code/src/cli/run-prompt.ts

@@ -501,6 +501,7 @@ function runPromptTurn(
           return;
         case 'turn.step.retrying':
           outputWriter.discardAssistant();
+          outputWriter.writeRetrying(event);
           return;
         case 'assistant.delta':
           outputWriter.writeAssistantDelta(event.delta);
@@ -612,6 +613,7 @@ interface PromptTurnWriter {
     argumentsPart: string | undefined,
   ): void;
   writeToolResult(toolCallId: string, output: unknown): void;
+  writeRetrying(event: Extract<Event, { type: 'turn.step.retrying' }>): void;
   flushAssistant(): void;
   discardAssistant(): void;
   finish(): void;
@@ -648,6 +650,11 @@ class PromptTranscriptWriter implements PromptTurnWriter {
 
   writeToolResult(): void {}
 
+  // Text `-p` keeps retries silent: only the failed attempt's partial assistant
+  // text is discarded (handled by the caller). No human-readable retry line is
+  // emitted, matching the prior behavior.
+  writeRetrying(): void {}
+
   flushAssistant(): void {
     this.assistantWriter.finish();
   }
@@ -689,6 +696,18 @@ interface PromptJsonResumeMetaMessage {
   content: string;
 }
 
+interface PromptJsonRetryMetaMessage {
+  role: 'meta';
+  type: 'turn.step.retrying';
+  failed_attempt: number;
+  next_attempt: number;
+  max_attempts: number;
+  delay_ms: number;
+  error_name: string;
+  error_message: string;
+  status_code?: number;
+}
+
 function writeResumeHint(
   sessionId: string,
   outputFormat: PromptOutputFormat,
@@ -787,6 +806,24 @@ class PromptJsonWriter implements PromptTurnWriter {
     this.toolCalls.length = 0;
   }
 
+  writeRetrying(event: Extract<Event, { type: 'turn.step.retrying' }>): void {
+    // Emit a machine-readable meta line so stream-json consumers can observe
+    // provider retries. The failed attempt's partial assistant text was already
+    // discarded by the caller, so no half-formed assistant message leaks.
+    const message: PromptJsonRetryMetaMessage = {
+      role: 'meta',
+      type: 'turn.step.retrying',
+      failed_attempt: event.failedAttempt,
+      next_attempt: event.nextAttempt,
+      max_attempts: event.maxAttempts,
+      delay_ms: event.delayMs,
+      error_name: event.errorName,
+      error_message: event.errorMessage,
+      status_code: event.statusCode,
+    };
+    this.writeJsonLine(message);
+  }
+
   finish(): void {
     this.flushAssistant();
   }
@@ -806,7 +843,9 @@ class PromptJsonWriter implements PromptTurnWriter {
     return toolCall;
   }
 
-  private writeJsonLine(message: PromptJsonAssistantMessage | PromptJsonToolMessage): void {
+  private writeJsonLine(
+    message: PromptJsonAssistantMessage | PromptJsonToolMessage | PromptJsonRetryMetaMessage,
+  ): void {
     this.stdout.write(`${JSON.stringify(message)}\n`);
   }
 }

apps/kimi-code/test/cli/run-prompt.test.ts

@@ -666,6 +666,59 @@ describe('runPrompt', () => {
     );
   });
 
+  it('emits a stream-json meta line on retry and discards the failed attempt output', async () => {
+    mocks.session.prompt.mockImplementationOnce(async () => {
+      for (const handler of mocks.eventHandlers) {
+        handler(mocks.mainEvent({ type: 'turn.started', turnId: 10, origin: { kind: 'user' } }));
+        handler(mocks.mainEvent({ type: 'assistant.delta', turnId: 10, delta: 'partial attempt' }));
+        handler(
+          mocks.mainEvent({
+            type: 'turn.step.retrying',
+            turnId: 10,
+            step: 1,
+            stepId: 'step-uuid',
+            failedAttempt: 1,
+            nextAttempt: 2,
+            maxAttempts: 3,
+            delayMs: 300,
+            errorName: 'APIProviderRateLimitError',
+            errorMessage: 'llmproxy/openai/responses/resp_abc.json status_code=429',
+            statusCode: 429,
+          }),
+        );
+        handler(mocks.mainEvent({ type: 'assistant.delta', turnId: 10, delta: 'final answer' }));
+        handler(mocks.mainEvent({ type: 'turn.ended', turnId: 10, reason: 'completed' }));
+      }
+    });
+    const stdout = writer();
+    const stderr = writer();
+
+    await runPrompt(opts({ outputFormat: 'stream-json' }), '1.2.3-test', { stdout, stderr });
+
+    const retryMeta = JSON.stringify({
+      role: 'meta',
+      type: 'turn.step.retrying',
+      failed_attempt: 1,
+      next_attempt: 2,
+      max_attempts: 3,
+      delay_ms: 300,
+      error_name: 'APIProviderRateLimitError',
+      error_message: 'llmproxy/openai/responses/resp_abc.json status_code=429',
+      status_code: 429,
+    });
+    expect(stdout.text()).toBe(
+      [
+        retryMeta,
+        '{"role":"assistant","content":"final answer"}',
+        '{"role":"meta","type":"session.resume_hint","session_id":"ses_prompt","command":"kimi -r ses_prompt","content":"To resume this session: kimi -r ses_prompt"}',
+        '',
+      ].join('\n'),
+    );
+    // The failed attempt's partial text must not leak as an assistant line.
+    expect(stdout.text()).not.toContain('partial attempt');
+    expect(stderr.text()).toBe('');
+  });
+
   it('flushes stream-json assistant output before waiting for background tasks', async () => {
     let releaseWait: () => void = () => {};
     const waitGate = new Promise<void>((resolve) => {

packages/agent-core/src/loop/retry.ts

@@ -1,5 +1,4 @@
 import { sleep } from '@antfu/utils';
-import * as retry from 'retry';
 
 import type { Logger } from '#/logging/types';
 
@@ -10,9 +9,16 @@ import type { LLM, LLMChatParams, LLMChatResponse } from './llm';
 
 export const DEFAULT_MAX_RETRY_ATTEMPTS = 3;
 
-const RETRY_MIN_TIMEOUT_MS = 300;
-const RETRY_MAX_TIMEOUT_MS = 5000;
+const BASE_DELAY_MS = 500;
+// Per-attempt backoff cap (32s). With the default 3 attempts the ramp
+// (0.5s, 1s) never reaches the cap, so interactive runs are unaffected; it
+// only matters for high-attempt configs (e.g. eval harnesses with
+// `max_retries_per_step = 10`), where it lets retries ride out multi-minute
+// provider overload instead of giving up after a few seconds of backoff.
+const MAX_DELAY_MS = 32_000;
 const RETRY_FACTOR = 2;
+// Up to 25% jitter on top of the exponential base to avoid herd retries.
+const JITTER_FACTOR = 0.25;
 
 export interface ChatWithRetryInput {
   readonly llm: LLM;
@@ -49,7 +55,10 @@ export async function chatWithRetry(input: ChatWithRetryInput): Promise<LLMChatR
         throw error;
       }
 
-      const delayMs = delays[attempt - 1] ?? 0;
+      // A server `Retry-After` (carried on the error) overrides the computed
+      // backoff. The chosen delay is what gets reported on the
+      // `step.retrying` event via `delayMs` either way.
+      const delayMs = readRetryAfterMs(error) ?? delays[attempt - 1] ?? 0;
       input.params.signal.throwIfAborted();
       input.dispatchEvent({
         type: 'step.retrying',
@@ -104,13 +113,27 @@ function paramsForAttempt(
 }
 
 export function retryBackoffDelays(maxAttempts: number): number[] {
-  return retry.timeouts({
-    retries: Math.max(maxAttempts - 1, 0),
-    minTimeout: RETRY_MIN_TIMEOUT_MS,
-    maxTimeout: RETRY_MAX_TIMEOUT_MS,
-    factor: RETRY_FACTOR,
-    randomize: true,
-  });
+  // For attempt (1-based) the base delay is min(500ms * 2^(attempt-1), 32s),
+  // plus up to 25% jitter. Index i here is 0-based, so attempt = i + 1.
+  const count = Math.max(maxAttempts - 1, 0);
+  const delays: number[] = [];
+  for (let i = 0; i < count; i += 1) {
+    const base = Math.min(BASE_DELAY_MS * Math.pow(RETRY_FACTOR, i), MAX_DELAY_MS);
+    delays.push(base + Math.random() * JITTER_FACTOR * base);
+  }
+  return delays;
+}
+
+/**
+ * Server-requested backoff carried on a kosong `APIStatusError` (parsed from
+ * the `retry-after` response header). When present and positive it overrides
+ * the computed backoff — a server `Retry-After` directive takes precedence
+ * over the local exponential delay.
+ */
+function readRetryAfterMs(error: unknown): number | null {
+  if (typeof error !== 'object' || error === null) return null;
+  const value = (error as { retryAfterMs?: unknown }).retryAfterMs;
+  return typeof value === 'number' && value > 0 ? value : null;
 }
 
 export async function sleepForRetry(delayMs: number, signal: AbortSignal): Promise<void> {

packages/agent-core/test/loop/retry.test.ts

@@ -1,10 +1,15 @@
-import { APIConnectionError, emptyUsage, isRetryableGenerateError } from '@moonshot-ai/kosong';
+import {
+  APIConnectionError,
+  APIProviderRateLimitError,
+  emptyUsage,
+  isRetryableGenerateError,
+} from '@moonshot-ai/kosong';
 import { describe, expect, it } from 'vitest';
 
 import type { KimiConfig } from '#/config';
 import { ErrorCodes, KimiError } from '#/errors';
 import type { LLM, LLMChatParams, LLMChatResponse } from '#/loop/llm';
-import { chatWithRetry } from '#/loop/retry';
+import { chatWithRetry, retryBackoffDelays } from '#/loop/retry';
 import { ProviderManager } from '#/session/provider-manager';
 
 function okResponse(): LLMChatResponse {
@@ -137,6 +142,74 @@ describe('chatWithRetry: terminated stream drops', () => {
   });
 });
 
+describe('retryBackoffDelays', () => {
+  it('uses a 500ms base, factor-2 ramp, 32s cap, and up to +25% jitter', () => {
+    const delays = retryBackoffDelays(10);
+    expect(delays).toHaveLength(9);
+    // Max possible delay is the capped base (32s) plus 25% jitter = 40s.
+    for (const d of delays) {
+      expect(d).toBeGreaterThan(0);
+      expect(d).toBeLessThanOrEqual(40_000);
+    }
+    // First attempt base is 500ms (plus up to 25% jitter) -> within [500, 625].
+    expect(delays[0]).toBeGreaterThanOrEqual(500);
+    expect(delays[0]).toBeLessThanOrEqual(625);
+  });
+
+  it('reaches the 32s cap for high-attempt configs (overload ride-out)', () => {
+    // The ramp hits 32s by attempt 7 (500 * 2^6); across many draws the peak
+    // approaches the cap (32s..40s with jitter), well above the old 5s cap.
+    let maxSeen = 0;
+    for (let i = 0; i < 50; i += 1) {
+      for (const d of retryBackoffDelays(12)) {
+        maxSeen = Math.max(maxSeen, d);
+      }
+    }
+    expect(maxSeen).toBeGreaterThan(30_000);
+  });
+
+  it('keeps default-attempt retries quick so interactive runs are not slowed', () => {
+    // 3 attempts -> 2 delays at the bottom of the ramp (~0.5s / ~1s before
+    // jitter); their sum stays small.
+    const delays = retryBackoffDelays(3);
+    expect(delays).toHaveLength(2);
+    expect(delays.reduce((a, b) => a + b, 0)).toBeLessThan(3_000);
+  });
+});
+
+describe('chatWithRetry: honors server retry-after', () => {
+  it('uses the error retryAfterMs as the retry delay instead of the backoff', async () => {
+    let calls = 0;
+    const captured: Array<{ type: string; delayMs?: number }> = [];
+    const llm: LLM = {
+      systemPrompt: '',
+      modelName: 'mock',
+      isRetryableError: (e) => isRetryableGenerateError(e),
+      async chat(): Promise<LLMChatResponse> {
+        calls += 1;
+        if (calls === 1) {
+          // 429 carrying a server `retry-after` of 42ms. Kept tiny so the test
+          // sleeps only briefly, while still being distinguishable from the
+          // attempt-1 backoff (500..625ms) it must override.
+          throw new APIProviderRateLimitError('rate limited', null, 42);
+        }
+        return okResponse();
+      },
+    };
+    const input = makeInput(llm, new AbortController().signal);
+    await chatWithRetry({
+      ...input,
+      dispatchEvent: async (event) => {
+        captured.push(event as { type: string; delayMs?: number });
+      },
+    });
+
+    expect(calls).toBe(2);
+    const retrying = captured.find((e) => e.type === 'step.retrying');
+    expect(retrying?.delayMs).toBe(42);
+  });
+});
+
 function oauthConfig(): KimiConfig {
   return {
     defaultModel: 'kimi-code/kimi-for-coding',