diff --git a/.changeset/quota-exhausted-fail-fast.md b/.changeset/quota-exhausted-fail-fast.md new file mode 100644 index 0000000000..b2405dc210 --- /dev/null +++ b/.changeset/quota-exhausted-fail-fast.md @@ -0,0 +1,6 @@ +--- +"@moonshot-ai/kimi-code": patch +"@moonshot-ai/kimi-code-sdk": patch +--- + +Fail fast on quota/balance-exhausted HTTP 429 errors (e.g. Moonshot `exceeded_current_quota_error`, OpenAI `insufficient_quota`) instead of silently retrying for ~3 minutes. Transient rate-limit 429s keep the existing retry, backoff, and Retry-After behavior. diff --git a/docs/en/configuration/config-files.md b/docs/en/configuration/config-files.md index e8df6e99cf..e0630a650d 100644 --- a/docs/en/configuration/config-files.md +++ b/docs/en/configuration/config-files.md @@ -242,6 +242,8 @@ When the experiment is enabled, the configuration is validated as the session st `max_steps_per_turn` can be overridden by the `KIMI_LOOP_MAX_STEPS_PER_TURN` environment variable, and `max_retries_per_step` by `KIMI_LOOP_MAX_RETRIES_PER_STEP`; both take higher priority than the config file. +Retries only apply to transient failures — connection errors, timeouts, HTTP 429 rate limits, and 5xx server errors. A 429 caused by an exhausted quota or insufficient account balance is not retried and fails immediately, since it cannot succeed until the account is recharged. + ## `background` `background` controls the concurrency behavior of background tasks (launched via the `Bash` tool or the `Agent` tool's `run_in_background=true` parameter). diff --git a/docs/zh/configuration/config-files.md b/docs/zh/configuration/config-files.md index be8757e99b..3c7318e364 100644 --- a/docs/zh/configuration/config-files.md +++ b/docs/zh/configuration/config-files.md @@ -242,6 +242,8 @@ max_output_size = 8192 `max_steps_per_turn` 可被环境变量 `KIMI_LOOP_MAX_STEPS_PER_TURN` 覆盖，`max_retries_per_step` 可被 `KIMI_LOOP_MAX_RETRIES_PER_STEP` 覆盖，优先级均高于配置文件。 +重试仅针对瞬时故障——连接错误、超时、HTTP 429 限流和 5xx 服务端错误。账户额度耗尽或余额不足导致的 429 不会重试，会立即失败：在充值之前重试不可能成功。 + ## `background` `background` 控制后台任务（通过 `Bash` 工具或 `Agent` 工具的 `run_in_background=true` 参数启动）的并发数。 diff --git a/packages/agent-core-v2/src/kosong/contract/errors.ts b/packages/agent-core-v2/src/kosong/contract/errors.ts index f7b571ca4a..7347afc47e 100644 --- a/packages/agent-core-v2/src/kosong/contract/errors.ts +++ b/packages/agent-core-v2/src/kosong/contract/errors.ts @@ -117,6 +117,18 @@ export class APIProviderRateLimitError extends APIStatusError { } } +export class APIProviderQuotaExhaustedError extends APIStatusError { + constructor( + message: string, + requestId?: string | null, + retryAfterMs?: number | null, + traceId?: string | null, + ) { + super(429, message, requestId, retryAfterMs, traceId); + this.name = 'APIProviderQuotaExhaustedError'; + } +} + export class APIProviderOverloadedError extends APIStatusError { constructor( statusCode: number, @@ -236,6 +248,9 @@ export function isRetryableGenerateError(error: unknown): boolean { return true; } if (error instanceof APIStatusError) { + if (error instanceof APIProviderQuotaExhaustedError) { + return false; + } return [408, 409, 429, 500, 502, 503, 504, 529].includes(error.statusCode); } return error instanceof ChatProviderError && !isImageFormatError(error); @@ -424,6 +439,7 @@ export function isRecoverableRequestStructureError(error: unknown): boolean { } export function isProviderRateLimitError(error: unknown): boolean { + if (error instanceof APIProviderQuotaExhaustedError) return false; if (error instanceof APIProviderRateLimitError) return true; const statusCode = getStatusCode(error); @@ -464,6 +480,7 @@ export type ApiErrorKind = | 'context_overflow' | 'overloaded' | 'rate_limit' + | 'quota_exhausted' | 'auth' | '5xx_server' | '4xx_client' @@ -481,6 +498,9 @@ export function classifyApiError(error: unknown): ApiErrorClassification { const statusCode = getStatusCode(error); if (error instanceof APIContextOverflowError) return { kind: 'context_overflow', statusCode }; if (error instanceof APIProviderOverloadedError) return { kind: 'overloaded', statusCode }; + if (error instanceof APIProviderQuotaExhaustedError) { + return { kind: 'quota_exhausted', statusCode }; + } if (error instanceof APIStatusError) { if (isContextOverflowStatusError(error.statusCode, error.message)) { return { kind: 'context_overflow', statusCode }; diff --git a/packages/agent-core-v2/src/kosong/protocol/errors.ts b/packages/agent-core-v2/src/kosong/protocol/errors.ts index b3b2593741..573b6d0994 100644 --- a/packages/agent-core-v2/src/kosong/protocol/errors.ts +++ b/packages/agent-core-v2/src/kosong/protocol/errors.ts @@ -18,6 +18,7 @@ import { APIContextOverflowError, APIEmptyResponseError, APIProviderOverloadedError, + APIProviderQuotaExhaustedError, APIStatusError, APITimeoutError, ChatProviderError, @@ -90,11 +91,13 @@ export function translateProviderError(error: unknown): Error2 { ? ProtocolErrors.codes.CONTEXT_OVERFLOW : error instanceof APIProviderOverloadedError || error.statusCode === 529 ? ProtocolErrors.codes.PROVIDER_OVERLOADED - : error.statusCode === 429 - ? ProtocolErrors.codes.PROVIDER_RATE_LIMIT - : error.statusCode === 401 || error.statusCode === 403 - ? ProtocolErrors.codes.PROVIDER_AUTH_ERROR - : ProtocolErrors.codes.PROVIDER_API_ERROR; + : error instanceof APIProviderQuotaExhaustedError + ? ProtocolErrors.codes.PROVIDER_API_ERROR + : error.statusCode === 429 + ? ProtocolErrors.codes.PROVIDER_RATE_LIMIT + : error.statusCode === 401 || error.statusCode === 403 + ? ProtocolErrors.codes.PROVIDER_AUTH_ERROR + : ProtocolErrors.codes.PROVIDER_API_ERROR; return new Error2(code, sanitizeStatusErrorMessage(error.message), { name: error.name, cause: error, diff --git a/packages/agent-core-v2/src/kosong/protocol/protocolTrait.ts b/packages/agent-core-v2/src/kosong/protocol/protocolTrait.ts index fc55c3096b..7b4d2500d6 100644 --- a/packages/agent-core-v2/src/kosong/protocol/protocolTrait.ts +++ b/packages/agent-core-v2/src/kosong/protocol/protocolTrait.ts @@ -2,7 +2,7 @@ * `kosong/protocol` domain (L1) — the declarative trait surface. * * A `ProtocolTrait` is a stateless declaration of how one vendor deviates - * from a wire base: sixteen fully optional hooks plus rare metadata markers + * from a wire base: seventeen fully optional hooks plus rare metadata markers * (non-function fields like `strictThinkingValidation` that qualify how a * hook's behavior is governed, without adding a code path). A trait declares * a deviation only where one exists; a hook returning `undefined` always @@ -15,6 +15,14 @@ * chain in trait order, each receiving the previous stage's output. * `convertMessage` may additionally return `null` to drop the message. * - Single-value hooks are overwritten in trait order: last declarer wins. + * - `convertError` is consulted by the bases with each RAW failure exactly + * once — the SDK error on HTTP paths, the raw event on in-stream paths — + * after the abort guard (a cancellation never reaches it) and after the + * already-converted `ChatProviderError` pass-through. The hook exists + * because base conversion drops vendor-parsed detail such as the body + * `error.type`/`error.code`; it is where a vendor declares what its own + * wire errors mean (e.g. which 429s are a non-retryable quota + * exhaustion rather than a transient rate limit). * - `endpoint` / `defaultHeaders` / `provides` are construction-time * declarations aggregated by the contrib factories, not per-request hooks. * @@ -24,6 +32,7 @@ */ import type { ModelCapability } from '#/kosong/contract/capability'; +import type { ChatProviderError } from '#/kosong/contract/errors'; import type { Message, VideoURLPart } from '#/kosong/contract/message'; import type { GenerateOptions, @@ -133,6 +142,8 @@ export interface ProtocolTrait { /** Single-value: tool-call id rewrite policy, replacing the base policy. */ toolCallIdPolicy?(ctx: TraitContext): ToolCallIdPolicy | undefined; + convertError?(error: unknown, ctx: TraitContext): ChatProviderError | undefined; + /** * Per-turn thinking intent → generation-kwargs patch. Receives the kwargs * already seeded by earlier intents (cacheKey, sampling) and returns the @@ -239,3 +250,15 @@ export function traitDefaultHeaders( } return headers; } + +export function traitConvertError( + traits: readonly ResolvedTrait[], +): ((error: unknown) => ChatProviderError | undefined) | undefined { + let bound: ((error: unknown) => ChatProviderError | undefined) | undefined; + for (const { trait, context } of traits) { + if (trait.convertError === undefined) continue; + const declared = trait.convertError.bind(trait); + bound = (error) => declared(error, context); + } + return bound; +} diff --git a/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropic.contrib.ts b/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropic.contrib.ts index 21e6b79fff..4ac022d874 100644 --- a/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropic.contrib.ts +++ b/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropic.contrib.ts @@ -3,9 +3,9 @@ * Messages base (`id: 'anthropic'`). * * The factory aggregates construction-time trait declarations and composes - * the (single-hook) Anthropic hook set. No apiKey suppression is needed here: - * the Anthropic base never reads shell API-key environment variables, so - * there is no base env fallback to suppress. + * the Anthropic hook set. No apiKey suppression is needed here: the + * Anthropic base never reads shell API-key environment variables, so there + * is no base env fallback to suppress. */ import { registerProtocolBase } from '#/kosong/protocol/protocolBase'; diff --git a/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropic.ts b/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropic.ts index fc276bb066..952e125cb2 100644 --- a/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropic.ts +++ b/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropic.ts @@ -6,15 +6,20 @@ * headers vs the beta endpoint, and the thinking profile matrix (budget vs * adaptive) from `anthropic-profile`. * - * The only hook surface is `withThinking` — a vendor dialect running over - * this transport re-encodes the thinking intent and nothing else. When the - * per-turn thinking intent carries `keep`, the BASE overlays the - * context-management edit uniformly on top of whatever thinking encoding - * happened (hook or base path), so a trait never handles `keep` itself. + * The hook surface is `withThinking` plus `convertError`. `withThinking` + * lets a vendor dialect running over this transport re-encode the thinking + * intent; when the per-turn thinking intent carries `keep`, the BASE + * overlays the context-management edit uniformly on top of whatever + * thinking encoding happened (hook or base path), so a trait never handles + * `keep` itself. * * `convertAnthropicError`'s FIRST line is the contract's `throwIfAbortError` * guard: a user cancellation is THROWN as the standard abort DOMException at - * the very front of the classification chain. + * the very front of the classification chain. After the guard, + * already-converted `ChatProviderError`s pass through untouched; only then is + * the trait-composed `convertError` hook consulted, so a vendor riding this + * transport classifies each RAW SDK failure exactly once before the base + * rules run. */ import Anthropic, { @@ -119,19 +124,13 @@ interface AnthropicContextManagement { edits: Array<{ type: string; keep?: unknown }>; } -/** - * The base-internal hook set: the L1 `withThinking` hook with the context - * already bound away. It receives a defensive COPY of the seeded kwargs, so a - * hook can never mutate base state — and a construction-headers synthetic - * trait can never shadow a real dialect hook (the compositor picks the last - * declarer). - */ export interface AnthropicHooks { withThinking?( effort: ThinkingEffort, options: { readonly keep?: string }, generationKwargs: AnthropicGenerationKwargs, ): AnthropicGenerationKwargs | undefined; + convertError?: (error: unknown) => ChatProviderError | undefined; } export interface AnthropicOptions { @@ -513,11 +512,21 @@ function shouldKeepConvertedMessage(message: MessageParam): boolean { return message.role !== 'assistant' || message.content.length > 0; } -export function convertAnthropicError(error: unknown): ChatProviderError { +export function convertAnthropicError( + error: unknown, + convertErrorHook?: (error: unknown) => ChatProviderError | undefined, +): ChatProviderError { // Abort guard FIRST: throws (never returns) the standard abort DOMException // for any abort shape, so a user cancellation is never misclassified as a // retryable provider failure. throwIfAbortError(error); + if (error instanceof ChatProviderError) { + return error; + } + const hooked = convertErrorHook?.(error); + if (hooked !== undefined) { + return hooked; + } if (error instanceof AnthropicTimeoutError) { return new APITimeoutError(error.message); } @@ -554,7 +563,13 @@ class AnthropicStreamedMessage implements StreamedMessage { private _rawFinishReason: string | null = null; private readonly _iter: AsyncGenerator; - constructor(response: unknown, isStream: boolean) { + constructor( + response: unknown, + isStream: boolean, + private readonly _convertErrorHook?: + | ((error: unknown) => ChatProviderError | undefined) + | undefined, + ) { if (isStream) { this._iter = this._convertStreamResponse(response as AsyncIterable); } else { @@ -780,7 +795,7 @@ class AnthropicStreamedMessage implements StreamedMessage { } } } catch (error: unknown) { - throw convertAnthropicError(error); + throw convertAnthropicError(error, this._convertErrorHook); } } } @@ -1021,9 +1036,9 @@ export class AnthropicChatProvider implements ChatProvider { { ...createParams, stream: true } as unknown as MessageCreateParamsStreaming, finalRequestOptions, ); - return new AnthropicStreamedMessage(stream, true); + return new AnthropicStreamedMessage(stream, true, this._hooks?.convertError); } catch (error: unknown) { - throw convertAnthropicError(error); + throw convertAnthropicError(error, this._hooks?.convertError); } } @@ -1037,9 +1052,9 @@ export class AnthropicChatProvider implements ChatProvider { { ...createParams, stream: false } as unknown as MessageCreateParams, finalRequestOptions, ); - return new AnthropicStreamedMessage(response, false); + return new AnthropicStreamedMessage(response, false, this._hooks?.convertError); } catch (error: unknown) { - throw convertAnthropicError(error); + throw convertAnthropicError(error, this._hooks?.convertError); } } diff --git a/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropicHooks.ts b/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropicHooks.ts index d908d70ea2..bf01ad54b4 100644 --- a/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropicHooks.ts +++ b/packages/agent-core-v2/src/kosong/provider/bases/anthropic/anthropicHooks.ts @@ -2,24 +2,34 @@ * `kosong/provider` domain (L2) — the ONLY composition point from resolved * traits to the Anthropic hook set. * - * The Anthropic base has a single hook, `withThinking`. The compositor takes - * the LAST declarer and wraps it with a defensive kwargs copy — so a hook can - * never mutate base state, and a synthetic construction-headers trait (which - * never declares `withThinking`) can never shadow a real dialect hook. + * The Anthropic base has two hooks. `withThinking` takes the LAST declarer + * and wraps it with a defensive kwargs copy — so a hook can never mutate base + * state, and a synthetic construction-headers trait (which never declares + * `withThinking`) can never shadow a real dialect hook. `convertError` is the + * shared single-value binding from `traitConvertError` (last declarer wins), + * consulted by the base's error converter after the abort guard. */ -import type { ResolvedTrait } from '#/kosong/protocol/protocolTrait'; +import { traitConvertError, type ResolvedTrait } from '#/kosong/protocol/protocolTrait'; import type { AnthropicHooks } from './anthropic'; export function composeAnthropicHooks( traits: readonly ResolvedTrait[], ): AnthropicHooks | undefined { + const hooks: AnthropicHooks = {}; + const thinkingTraits = traits.filter(({ trait }) => trait.withThinking !== undefined); - if (thinkingTraits.length === 0) return undefined; - const { trait, context } = thinkingTraits.at(-1)!; - return { - withThinking: (effort, options, kwargs) => - trait.withThinking!(effort, options, { ...kwargs }, context), - }; + if (thinkingTraits.length > 0) { + const { trait, context } = thinkingTraits.at(-1)!; + hooks.withThinking = (effort, options, kwargs) => + trait.withThinking!(effort, options, { ...kwargs }, context); + } + + const convertError = traitConvertError(traits); + if (convertError !== undefined) { + hooks.convertError = convertError; + } + + return Object.keys(hooks).length > 0 ? hooks : undefined; } diff --git a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-common.ts b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-common.ts index 68eed46d31..d636c1cb1a 100644 --- a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-common.ts +++ b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-common.ts @@ -9,7 +9,13 @@ * guard: a user cancellation (SDK `APIUserAbortError`, bare `AbortError`, the * standard abort DOMException) is THROWN as the standard abort shape at the * very front of the classification chain — it can never be converted into, - * nor returned as, a retryable provider error. + * nor returned as, a retryable provider error. After the guard, + * already-converted `ChatProviderError`s pass through untouched; only then is + * the optional trait-composed `convertError` hook consulted, so a vendor + * classifies each RAW wire failure (e.g. quota 429s) exactly once before the + * base rules run. The base itself classifies only OpenAI's own documented + * `insufficient_quota` code as a non-retryable quota exhaustion — + * vendor-specific quota signals belong on the vendor's trait. */ import { @@ -21,6 +27,7 @@ import { import { APIConnectionError, + APIProviderQuotaExhaustedError, APITimeoutError, ChatProviderError, classifyBaseApiError, @@ -98,7 +105,21 @@ export function toolToOpenAI(tool: Tool): OpenAIToolParam { }; } -export function convertOpenAIError(error: unknown): ChatProviderError { +export function isOpenAIInsufficientQuotaCode(code: string | null | undefined): boolean { + return code === 'insufficient_quota'; +} + +function isOpenAIInsufficientQuotaError(error: OpenAIAPIError): boolean { + if (error.status !== 429) return false; + if (typeof error.code === 'string' && isOpenAIInsufficientQuotaCode(error.code)) return true; + if (typeof error.type === 'string' && isOpenAIInsufficientQuotaCode(error.type)) return true; + return error.message.toLowerCase().includes('insufficient_quota'); +} + +export function convertOpenAIError( + error: unknown, + convertErrorHook?: (error: unknown) => ChatProviderError | undefined, +): ChatProviderError { // Abort guard FIRST: throws (never returns) the standard abort DOMException // for any abort shape, so a user cancellation is never misclassified as a // retryable provider failure. @@ -106,6 +127,10 @@ export function convertOpenAIError(error: unknown): ChatProviderError { if (error instanceof ChatProviderError) { return error; } + const hooked = convertErrorHook?.(error); + if (hooked !== undefined) { + return hooked; + } if (error instanceof OpenAITimeoutError) { return new APITimeoutError(error.message); } @@ -114,13 +139,12 @@ export function convertOpenAIError(error: unknown): ChatProviderError { } if (error instanceof OpenAIAPIError && typeof error.status === 'number') { const reqId = error.requestID ?? null; - return normalizeAPIStatusError( - error.status, - error.message, - reqId, - parseRetryAfterMs(error.headers), - parseTraceId(error.headers), - ); + const retryAfterMs = parseRetryAfterMs(error.headers); + const traceId = parseTraceId(error.headers); + if (isOpenAIInsufficientQuotaError(error)) { + return new APIProviderQuotaExhaustedError(error.message, reqId, retryAfterMs, traceId); + } + return normalizeAPIStatusError(error.status, error.message, reqId, retryAfterMs, traceId); } if ( error instanceof OpenAIAPIError && diff --git a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-legacy.ts b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-legacy.ts index 7c7215005a..78f94c7e58 100644 --- a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-legacy.ts +++ b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-legacy.ts @@ -26,7 +26,7 @@ import OpenAI from 'openai'; -import { parseTraceId } from '#/kosong/contract/errors'; +import { parseTraceId, type ChatProviderError } from '#/kosong/contract/errors'; import type { ContentPart, Message, @@ -102,6 +102,7 @@ export const OPENAI_CHAT_TOOL_CALL_ID_POLICY: ToolCallIdPolicy = { */ export interface OpenAIChatCompletionsHooks { convertTool?: (tool: Tool) => Record | undefined; + convertError?: (error: unknown) => ChatProviderError | undefined; convertMessage?: ( message: Message, converted: Record, @@ -377,6 +378,9 @@ export class OpenAILegacyStreamedMessage implements StreamedMessage { private readonly _extractUsageHook?: | ((chunk: Record) => Record | null | undefined) | undefined, + private readonly _convertErrorHook?: + | ((error: unknown) => ChatProviderError | undefined) + | undefined, ) { if (isStream) { this._iter = this._convertStreamResponse( @@ -511,7 +515,7 @@ export class OpenAILegacyStreamedMessage implements StreamedMessage { } } } catch (error: unknown) { - throw convertOpenAIError(error); + throw convertOpenAIError(error, this._convertErrorHook); } } } @@ -685,9 +689,10 @@ export class OpenAILegacyChatProvider implements ChatProvider { this._reasoningKeyDialect, parseTraceId(response.headers), this._hooks?.extractUsage, + this._hooks?.convertError, ); } catch (error: unknown) { - throw convertOpenAIError(error); + throw convertOpenAIError(error, this._hooks?.convertError); } } diff --git a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-responses.contrib.ts b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-responses.contrib.ts index 863cd9ccc8..635521a643 100644 --- a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-responses.contrib.ts +++ b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-responses.contrib.ts @@ -9,7 +9,7 @@ */ import { registerProtocolBase } from '#/kosong/protocol/protocolBase'; -import { traitDefaultHeaders } from '#/kosong/protocol/protocolTrait'; +import { traitConvertError, traitDefaultHeaders } from '#/kosong/protocol/protocolTrait'; import { getOpenAIResponsesModelCapability, OpenAIResponsesChatProvider } from './openai-responses'; import { compactObject, firstProcessEnv, traitEndpoint, traitProvides } from './openaiHooks'; @@ -34,6 +34,7 @@ registerProtocolBase({ defaultHeaders: traitDefaultHeaders(traits), maxOutputTokens: config.providerOptions?.defaultMaxTokens, offEffort: config.providerOptions?.offEffort, + convertError: traitConvertError(traits), }), }); }, diff --git a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-responses.ts b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-responses.ts index dfb482f2dd..20d24559cc 100644 --- a/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-responses.ts +++ b/packages/agent-core-v2/src/kosong/provider/bases/openai/openai-responses.ts @@ -4,15 +4,20 @@ * Speaks the Responses wire format: `input` items, `instructions`, * `reasoning` blocks with encrypted content, and the native * `prompt_cache_key` field (a cache key is encoded directly — no hook - * needed). This base carries no hook surface today; per-turn intents are - * encoded inline in the fixed contract order. The developer-role model - * detection lives here. + * needed). Per-turn intents are encoded inline in the fixed contract order; + * the base's only hook surface is the trait-composed `convertError` option, + * consulted with each raw failure exactly once — the SDK error on HTTP + * paths, the raw event on in-stream error paths — before the base's own + * classification (already-converted errors crossing an outer catch pass + * through without re-consulting). The developer-role model detection lives + * here. */ import OpenAI from 'openai'; import { APIContextOverflowError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, ChatProviderError, isContextOverflowErrorCode, @@ -41,6 +46,7 @@ import { convertOpenAIError, hasModelPrefix, isMediaPart, + isOpenAIInsufficientQuotaCode, isOpenAIReasoningModel, OPENAI_REASONING_CAPABILITY, OPENAI_VISION_TOOL_CAPABILITY, @@ -251,12 +257,23 @@ function errorFromOpenAIResponsesEvent( code: string | null, message: string, param: string | null, + options?: { + readonly rawEvent?: unknown; + readonly convertErrorHook?: (error: unknown) => ChatProviderError | undefined; + }, ): ChatProviderError { const formatted = formatResponsesErrorEvent(code, message, param); const fullMessage = `${prefix}: ${formatted}`; + const hooked = options?.convertErrorHook?.(options.rawEvent ?? { code, message, param }); + if (hooked !== undefined) { + return hooked; + } if (isContextOverflowErrorCode(code)) { return new APIContextOverflowError(400, fullMessage); } + if (isOpenAIInsufficientQuotaCode(code)) { + return new APIProviderQuotaExhaustedError(fullMessage); + } if (code === 'rate_limit_exceeded' || readEmbeddedStatusCode(message) === 429) { return new APIProviderRateLimitError(fullMessage); } @@ -297,7 +314,10 @@ function parseNestedGatewayStreamError(message: string): }; } -function malformedStreamErrorEvent(message: string): ChatProviderError { +function malformedStreamErrorEvent( + message: string, + convertErrorHook?: (error: unknown) => ChatProviderError | undefined, +): ChatProviderError { const nested = parseNestedGatewayStreamError(message); if (nested !== undefined) { return errorFromOpenAIResponsesEvent( @@ -305,6 +325,7 @@ function malformedStreamErrorEvent(message: string): ChatProviderError { nested.code, nested.message, nested.param, + { convertErrorHook }, ); } @@ -313,6 +334,7 @@ function malformedStreamErrorEvent(message: string): ChatProviderError { null, message, null, + { convertErrorHook }, ); } @@ -356,6 +378,7 @@ export interface OpenAIResponsesOptions { defaultHeaders?: Record; toolMessageConversion?: ToolMessageConversion | undefined; clientFactory?: (auth: ProviderRequestAuth) => OpenAI; + convertError?: (error: unknown) => ChatProviderError | undefined; } export interface OpenAIResponsesGenerationKwargs { @@ -662,7 +685,13 @@ export class OpenAIResponsesStreamedMessage implements StreamedMessage { private _rawFinishReason: string | null = null; private readonly _iter: AsyncGenerator; - constructor(response: unknown, isStream: boolean) { + constructor( + response: unknown, + isStream: boolean, + private readonly _convertErrorHook?: + | ((error: unknown) => ChatProviderError | undefined) + | undefined, + ) { if (isStream) { this._iter = this._convertStreamResponse(response as AsyncIterable); } else { @@ -859,7 +888,7 @@ export class OpenAIResponsesStreamedMessage implements StreamedMessage { if (!hasOwn(chunk, 'type')) { const message = readStringField(chunk, 'message'); if (message !== undefined) { - throw malformedStreamErrorEvent(message); + throw malformedStreamErrorEvent(message, this._convertErrorHook); } } failResponsesDecode('stream event.type', 'must be a string.'); @@ -965,6 +994,7 @@ export class OpenAIResponsesStreamedMessage implements StreamedMessage { readNullableStringField(chunk, 'code') ?? null, message, readNullableStringField(chunk, 'param') ?? null, + { rawEvent: chunk, convertErrorHook: this._convertErrorHook }, ); } case 'response.failed': { @@ -976,6 +1006,7 @@ export class OpenAIResponsesStreamedMessage implements StreamedMessage { error.code, error.message, null, + { rawEvent: chunk, convertErrorHook: this._convertErrorHook }, ); } throw new ChatProviderError( @@ -987,7 +1018,7 @@ export class OpenAIResponsesStreamedMessage implements StreamedMessage { } } } catch (error: unknown) { - throw convertOpenAIError(error); + throw convertOpenAIError(error, this._convertErrorHook); } } } @@ -1007,6 +1038,7 @@ export class OpenAIResponsesChatProvider implements ChatProvider { private readonly _client: OpenAI | undefined; private readonly _httpClient: unknown; private readonly _clientFactory: ((auth: ProviderRequestAuth) => OpenAI) | undefined; + private readonly _convertErrorHook: ((error: unknown) => ChatProviderError | undefined) | undefined; constructor(options: OpenAIResponsesOptions) { const apiKey = options.apiKey ?? process.env['OPENAI_API_KEY']; @@ -1021,6 +1053,7 @@ export class OpenAIResponsesChatProvider implements ChatProvider { this._toolMessageConversion = options.toolMessageConversion ?? null; this._httpClient = options.httpClient; this._clientFactory = options.clientFactory; + this._convertErrorHook = options.convertError; if (options.maxOutputTokens !== undefined) { this._generationKwargs.max_output_tokens = options.maxOutputTokens; @@ -1149,9 +1182,9 @@ export class OpenAIResponsesChatProvider implements ChatProvider { create(params: unknown, opts?: unknown): Promise; } ).create(createParams, options?.signal ? { signal: options.signal } : undefined); - return new OpenAIResponsesStreamedMessage(response, this._stream); + return new OpenAIResponsesStreamedMessage(response, this._stream, this._convertErrorHook); } catch (error: unknown) { - throw convertOpenAIError(error); + throw convertOpenAIError(error, this._convertErrorHook); } } diff --git a/packages/agent-core-v2/src/kosong/provider/bases/openai/openaiHooks.ts b/packages/agent-core-v2/src/kosong/provider/bases/openai/openaiHooks.ts index a46b2e1cb0..f7a32cfe2e 100644 --- a/packages/agent-core-v2/src/kosong/provider/bases/openai/openaiHooks.ts +++ b/packages/agent-core-v2/src/kosong/provider/bases/openai/openaiHooks.ts @@ -73,6 +73,9 @@ export function composeOpenAIChatHooks( if (trait.convertTool !== undefined) { hooks.convertTool = (tool: Tool) => trait.convertTool!(tool, context); } + if (trait.convertError !== undefined) { + hooks.convertError = (error: unknown) => trait.convertError!(error, context); + } if (trait.toolCallIdPolicy !== undefined) { hooks.toolCallIdPolicy = () => trait.toolCallIdPolicy!(context); } diff --git a/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi-errors.ts b/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi-errors.ts new file mode 100644 index 0000000000..c7a6db5701 --- /dev/null +++ b/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi-errors.ts @@ -0,0 +1,86 @@ +/** + * `kosong/provider` domain (L2) — Kimi vendor error classification, declared + * on the Kimi traits via the `convertError` hook. + * + * This module owns the vendor-specific knowledge of how the Moonshot backend + * signals quota/balance exhaustion on a 429: the structured body + * `error.type`/`error.code` value `exceeded_current_quota_error`, and the + * observed billing wordings for gateways that flatten the body to text + * ("You exceeded your current token quota: … please check your account + * balance", "Your account … is suspended due to insufficient balance, please + * recharge your account …", and arrears phrasing). Every pattern is anchored + * to billing wording — deliberately no bare /quota/ or /balance/, so + * transient throttle messages like "token quota per minute" keep classifying + * as retryable rate limits. The classifier reads the raw SDK error + * structurally (status / code / type / message), so it works over both the + * OpenAI and Anthropic transports Kimi registers on: the OpenAI SDK hoists + * the body's `error.code`/`error.type` to the top level, while the Anthropic + * SDK keeps the full body on `.error` (`{type: 'error', error: {type}}`), so + * candidate codes are collected from `error` → `.error` → `.error.error`. + * Anything not positively recognized answers `undefined`, keeping the base + * classification. + */ + +import { + APIProviderQuotaExhaustedError, + parseRetryAfterMs, + parseTraceId, +} from '#/kosong/contract/errors'; + +const KIMI_QUOTA_EXHAUSTED_ERROR_CODES = new Set(['exceeded_current_quota_error']); + +const KIMI_QUOTA_EXHAUSTED_MESSAGE_PATTERNS = [ + /exceeded your current (?:token )?quota/, + /check your account balance/, + /insufficient balance/, + /recharge your account|please recharge/, + /account (?:is )?in arrears/, +] as const; + +function readStringProp(value: object, key: string): string | undefined { + const raw = (value as Record)[key]; + return typeof raw === 'string' ? raw : undefined; +} + +function readErrorObjectProp(value: object): object | undefined { + const raw = (value as Record)['error']; + return typeof raw === 'object' && raw !== null ? raw : undefined; +} + +function collectErrorCodes(error: object): string[] { + const codes: string[] = []; + let current: object | undefined = error; + for (let depth = 0; current !== undefined && depth < 3; depth += 1) { + const code = readStringProp(current, 'code'); + if (code !== undefined) codes.push(code); + const type = readStringProp(current, 'type'); + if (type !== undefined) codes.push(type); + current = readErrorObjectProp(current); + } + return codes; +} + +export function classifyKimiQuotaError(error: unknown): APIProviderQuotaExhaustedError | undefined { + if (typeof error !== 'object' || error === null) return undefined; + const status = (error as Record)['status']; + if (status !== 429) return undefined; + + const message = readStringProp(error, 'message') ?? ''; + const structuredHit = collectErrorCodes(error).some((code) => + KIMI_QUOTA_EXHAUSTED_ERROR_CODES.has(code), + ); + const lowerMessage = message.toLowerCase(); + const wordingHit = KIMI_QUOTA_EXHAUSTED_MESSAGE_PATTERNS.some((pattern) => + pattern.test(lowerMessage), + ); + if (!structuredHit && !wordingHit) return undefined; + + const requestId = readStringProp(error, 'requestID') ?? null; + const headers = (error as Record)['headers']; + return new APIProviderQuotaExhaustedError( + message, + requestId, + parseRetryAfterMs(headers), + parseTraceId(headers), + ); +} diff --git a/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi-files.ts b/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi-files.ts index 12c6a7ef8a..e28a0e1c7a 100644 --- a/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi-files.ts +++ b/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi-files.ts @@ -4,6 +4,9 @@ * The file-upload companion of the video-upload trait: uploads a video (from * a filesystem path or in-memory bytes) to the Kimi files endpoint and * returns the `ms://` video URL part the wire messages reference. + * Upload failures classify through the same Kimi quota classifier the traits + * declare (this client runs outside any composed hook context), falling back + * to the base OpenAI conversion. */ import { Blob, File } from 'node:buffer'; @@ -23,6 +26,7 @@ import { requireProviderApiKey, resolveAuthBackedClient, } from '../../bases/request-auth'; +import { classifyKimiQuotaError } from './kimi-errors'; export interface KimiUploadOptions { auth?: ProviderRequestAuth; @@ -99,7 +103,7 @@ export class KimiFiles { options?.signal ? { signal: options.signal } : undefined, )) as unknown as { id: string }; } catch (error: unknown) { - throw convertOpenAIError(error); + throw convertOpenAIError(error, classifyKimiQuotaError); } return { diff --git a/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi.contrib.ts b/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi.contrib.ts index 967e9a8fa2..b95d79fcbd 100644 --- a/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi.contrib.ts +++ b/packages/agent-core-v2/src/kosong/provider/providers/kimi/kimi.contrib.ts @@ -44,6 +44,11 @@ * either at the top level (the base's default location) or inside * `choices[0].usage`; returning `undefined` defers to the base default * when neither position carries one; + * - errors: `convertError` classifies Moonshot's quota/balance-exhausted + * 429s (structured `exceeded_current_quota_error` type/code, billing + * wordings) as the non-retryable `APIProviderQuotaExhaustedError` via + * `classifyKimiQuotaError` from `./kimi-errors`, before the base's own + * classification would mint a retryable rate limit; * - video upload: `uploadVideo` uploads through the Kimi files API * (`KimiFiles` from `./kimi-files`), memoized per trait context with a * WeakMap — one composition (one resolved ctx) gets one files client, @@ -53,10 +58,12 @@ * `output_config.effort`, and the interleaved-thinking beta is stripped * from the seeded beta list. The `keep` dimension needs no trait handling * — the Anthropic base overlays the context-management edit itself. The - * trait deliberately does NOT declare `strictThinkingValidation`: over - * this foreign transport the backend may accept efforts the local catalog - * metadata does not list, so client-side validation stays lenient - * (warning + pass-through). + * trait declares the same `convertError` quota classification as the + * OpenAI registration (the classifier reads the SDK error structurally, + * so it is transport-agnostic). It deliberately does NOT declare + * `strictThinkingValidation`: over this foreign transport the backend may + * accept efforts the local catalog metadata does not list, so client-side + * validation stays lenient (warning + pass-through). * * Vendor-level facts — the endpoint fallback chain, full host-header * forwarding, and OAuth-catalog model discovery — are shared constants @@ -82,6 +89,7 @@ import type { import { type OpenAIToolParam, toolToOpenAI } from '../../bases/openai/openai-common'; import { registerProviderDefinition } from '../../providerDefinition'; +import { classifyKimiQuotaError } from './kimi-errors'; import { KimiFiles } from './kimi-files'; import { normalizeKimiToolSchema } from './kimi-schema'; @@ -176,6 +184,8 @@ export const kimiOpenAITrait: ProtocolTrait = { defaultBaseUrl: KIMI_DEFAULT_BASE_URL, }), + convertError: (error) => classifyKimiQuotaError(error), + cacheKey: (key) => ({ prompt_cache_key: key }), withThinking: (effort, options, generationKwargs) => { @@ -274,6 +284,8 @@ export const kimiOpenAITrait: ProtocolTrait = { }; export const kimiAnthropicTrait: ProtocolTrait = { + convertError: (error) => classifyKimiQuotaError(error), + withThinking: (effort, _options, generationKwargs) => { const seeded = generationKwargs['betaFeatures']; const betaFeatures = (Array.isArray(seeded) ? (seeded as string[]) : []).filter( diff --git a/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts b/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts index da498e9af5..75905c1892 100644 --- a/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts +++ b/packages/agent-core-v2/test/app/llmProtocol/errors.test.ts @@ -8,6 +8,7 @@ import { APIContextOverflowError, APIEmptyResponseError, APIProviderOverloadedError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIRequestTooLargeError, APIStatusError, @@ -707,3 +708,23 @@ describe('isProviderRateLimitError', () => { expect(isProviderRateLimitError(new Error('context length exceeded'))).toBe(false); }); }); + +describe('quota-exhausted error contract', () => { + it.each([ + 'Too many requests', + 'request reached user+model max RPM: 50', + 'Your account org-0123456789abcdef is suspended due to insufficient balance, please recharge your account or check your plan and billing details', + ])('keeps the vendor-neutral 429 normalization a rate limit for "%s"', (message) => { + const error = normalizeAPIStatusError(429, message); + expect(error).toBeInstanceOf(APIProviderRateLimitError); + expect(error).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + }); + + it('is neither retryable nor a provider rate limit', () => { + const quota = new APIProviderQuotaExhaustedError('quota exhausted', 'req-quota', 1); + expect(isRetryableGenerateError(quota)).toBe(false); + expect(isProviderRateLimitError(quota)).toBe(false); + expect(isRetryableGenerateError(new APIProviderRateLimitError('rate limited'))).toBe(true); + expect(isProviderRateLimitError(new APIProviderRateLimitError('rate limited'))).toBe(true); + }); +}); diff --git a/packages/agent-core-v2/test/app/protocol/errors.test.ts b/packages/agent-core-v2/test/app/protocol/errors.test.ts index ba326f603c..fb5cb26630 100644 --- a/packages/agent-core-v2/test/app/protocol/errors.test.ts +++ b/packages/agent-core-v2/test/app/protocol/errors.test.ts @@ -6,6 +6,7 @@ import { APIContextOverflowError, APIEmptyResponseError, APIProviderOverloadedError, + APIProviderQuotaExhaustedError, APIStatusError, APITimeoutError, ChatProviderError, @@ -152,4 +153,18 @@ describe('translateProviderError', () => { ); }); }); + + describe('quota-exhausted 429', () => { + it('maps to provider.api_error, not provider.rate_limit', () => { + const translated = translateProviderError( + new APIProviderQuotaExhaustedError( + 'Your account is suspended due to insufficient balance, please recharge your account', + 'req-quota', + ), + ); + expect(translated.code).toBe('provider.api_error'); + expect(translated.message).toContain('recharge'); + expect(translated.details).toMatchObject({ statusCode: 429, requestId: 'req-quota' }); + }); + }); }); diff --git a/packages/agent-core-v2/test/kosong/contract/errors.test.ts b/packages/agent-core-v2/test/kosong/contract/errors.test.ts index b4ba79cef8..2bf3efc758 100644 --- a/packages/agent-core-v2/test/kosong/contract/errors.test.ts +++ b/packages/agent-core-v2/test/kosong/contract/errors.test.ts @@ -14,6 +14,7 @@ import { APIContextOverflowError, APIEmptyResponseError, APIProviderOverloadedError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIStatusError, APITimeoutError, @@ -146,6 +147,10 @@ describe('classifyApiError', () => { kind: 'rate_limit', statusCode: 429, }); + expect(classifyApiError(new APIProviderQuotaExhaustedError('quota exhausted'))).toEqual({ + kind: 'quota_exhausted', + statusCode: 429, + }); expect(classifyApiError(new APIProviderOverloadedError(529, 'Overloaded'))).toEqual({ kind: 'overloaded', statusCode: 529, diff --git a/packages/agent-core-v2/test/kosong/protocol/protocolTrait.test.ts b/packages/agent-core-v2/test/kosong/protocol/protocolTrait.test.ts index f50fcf1d22..965444738a 100644 --- a/packages/agent-core-v2/test/kosong/protocol/protocolTrait.test.ts +++ b/packages/agent-core-v2/test/kosong/protocol/protocolTrait.test.ts @@ -1,6 +1,6 @@ /** - * `kosong/protocol` trait surface — the sixteen-hook declaration shape and - * the `traitDefaultHeaders` aggregation helper. + * `kosong/protocol` trait surface — the seventeen-hook declaration shape + * and the `traitDefaultHeaders` / `traitConvertError` aggregation helpers. * * Locks the trait contract: every hook is optional and takes `TraitContext` * as its last parameter, and header aggregation runs in trait order with @@ -11,11 +11,13 @@ import { describe, expect, it } from 'vitest'; import { + traitConvertError, traitDefaultHeaders, type ProtocolTrait, type ResolvedTrait, type TraitContext, } from '#/kosong/protocol/protocolTrait'; +import { ChatProviderError } from '#/kosong/contract/errors'; import type { ProtocolAdapterConfig } from '#/kosong/protocol/protocol'; const config: ProtocolAdapterConfig = { protocol: 'openai', modelName: 'test-model' }; @@ -26,12 +28,13 @@ function resolved(trait: ProtocolTrait): ResolvedTrait { } describe('ProtocolTrait', () => { - it('declares exactly the sixteen optional hooks', () => { + it('declares exactly the seventeen optional hooks', () => { const fullTrait: ProtocolTrait = { provides: () => undefined, endpoint: () => undefined, defaultHeaders: () => undefined, convertTool: () => undefined, + convertError: () => undefined, convertMessage: (_message, converted) => converted, mergeHistory: () => undefined, buildParams: () => undefined, @@ -49,6 +52,7 @@ describe('ProtocolTrait', () => { 'buildParams', 'cacheKey', 'capability', + 'convertError', 'convertMessage', 'convertTool', 'defaultHeaders', @@ -107,3 +111,31 @@ describe('traitDefaultHeaders', () => { expect(seen[0]?.providerId).toBe('vendor-x'); }); }); + +describe('traitConvertError', () => { + it('returns undefined when nothing declares the hook', () => { + expect(traitConvertError([])).toBeUndefined(); + expect(traitConvertError([resolved({})])).toBeUndefined(); + }); + + it('binds the last declarer with its context', () => { + const seen: TraitContext[] = []; + const first: ProtocolTrait = { convertError: () => new ChatProviderError('first') }; + const second: ProtocolTrait = { + convertError: (_error, ctx) => { + seen.push(ctx); + return new ChatProviderError('second'); + }, + }; + const entry = resolved(second); + const bound = traitConvertError([resolved(first), entry]); + expect(bound!(new Error('raw'))?.message).toBe('second'); + expect(seen).toHaveLength(1); + expect(seen[0]).toBe(entry.context); + }); + + it('propagates undefined so bases keep their classification', () => { + const bound = traitConvertError([resolved({ convertError: () => undefined })]); + expect(bound!(new Error('raw'))).toBeUndefined(); + }); +}); diff --git a/packages/agent-core-v2/test/kosong/provider/composition.test.ts b/packages/agent-core-v2/test/kosong/provider/composition.test.ts index 331281ef5a..c7ebdb9feb 100644 --- a/packages/agent-core-v2/test/kosong/provider/composition.test.ts +++ b/packages/agent-core-v2/test/kosong/provider/composition.test.ts @@ -40,8 +40,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { APIError as AnthropicAPIError } from '@anthropic-ai/sdk'; + import { isUnknownCapability } from '#/kosong/contract/capability'; -import { APIConnectionError } from '#/kosong/contract/errors'; +import { + APIConnectionError, + APIProviderQuotaExhaustedError, + APIProviderRateLimitError, + isRetryableGenerateError, +} from '#/kosong/contract/errors'; import type { Message } from '#/kosong/contract/message'; import type { ChatProvider, @@ -734,6 +741,68 @@ describe('per-turn intent wire encoding (behavior probes)', () => { }); }); +describe('quota-exhausted classification through the real composition (behavior probes)', () => { + const MOONSHOT_QUOTA_BODY = { + type: 'error', + error: { + type: 'exceeded_current_quota_error', + message: + 'Your account is suspended due to insufficient balance, please recharge your account', + }, + }; + + function mockQuota429Client(provider: ChatProvider): void { + const client = sdkClient(provider) as { + messages: { create: unknown }; + beta: { messages: { create: unknown } }; + }; + const reject = vi.fn().mockImplementation(() => { + throw AnthropicAPIError.generate( + 429, + MOONSHOT_QUOTA_BODY, + 'Too many requests', + new Headers(), + ); + }); + client.messages.create = reject; + client.beta.messages.create = reject; + } + + it('fails fast on a Moonshot quota 429 over the (kimi, anthropic) composition', async () => { + const provider = registry.createChatProvider({ + protocol: 'anthropic', + providerType: 'kimi', + modelName: 'kimi-for-coding', + apiKey: 'sk-probe', + }); + mockQuota429Client(provider); + + const caught = await provider.generate('', [], PROBE_HISTORY).then( + () => undefined, + (error: unknown) => error, + ); + expect(caught).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(caught)).toBe(false); + }); + + it('keeps the same 429 a retryable rate limit on a plain anthropic composition', async () => { + const provider = registry.createChatProvider({ + protocol: 'anthropic', + modelName: 'claude-opus-4-6', + apiKey: 'sk-probe', + }); + mockQuota429Client(provider); + + const caught = await provider.generate('', [], PROBE_HISTORY).then( + () => undefined, + (error: unknown) => error, + ); + expect(caught).toBeInstanceOf(APIProviderRateLimitError); + expect(caught).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(caught)).toBe(true); + }); +}); + describe('reasoning dialect (behavior probes)', () => { it('yields think parts from the `reasoning` wire field', async () => { const provider = registry.createChatProvider({ diff --git a/packages/agent-core-v2/test/kosong/provider/errors.test.ts b/packages/agent-core-v2/test/kosong/provider/errors.test.ts index 7bdcbdeb24..89749b9233 100644 --- a/packages/agent-core-v2/test/kosong/provider/errors.test.ts +++ b/packages/agent-core-v2/test/kosong/provider/errors.test.ts @@ -9,18 +9,36 @@ * throw at the front of the classification chain, not a return; * - non-abort errors still classify normally; * - `isRetryableGenerateError` is false for the abort shape. + * + * Also probes quota-exhausted classification at the provider boundary. The + * base converters stay vendor-neutral (only OpenAI's own documented + * `insufficient_quota` code fails fast there); Moonshot's quota signals are + * declared on the Kimi traits via the `convertError` hook, composed with + * last-declarer-wins semantics and consulted — after the abort guard — with + * the raw failure at every catch seam, including the Responses in-stream + * error-event path. */ +import { APIError as AnthropicAPIError } from '@anthropic-ai/sdk'; +import { APIError as OpenAIAPIError } from 'openai'; import { describe, expect, it } from 'vitest'; import { + APIProviderQuotaExhaustedError, + APIProviderRateLimitError, APIStatusError, ChatProviderError, createAbortError, isRetryableGenerateError, } from '#/kosong/contract/errors'; +import type { ProtocolAdapterConfig } from '#/kosong/protocol/protocol'; +import { traitConvertError, type TraitContext } from '#/kosong/protocol/protocolTrait'; import { convertAnthropicError } from '#/kosong/provider/bases/anthropic/anthropic'; import { convertOpenAIError } from '#/kosong/provider/bases/openai/openai-common'; +import { OpenAIResponsesStreamedMessage } from '#/kosong/provider/bases/openai/openai-responses'; +import { composeOpenAIChatHooks } from '#/kosong/provider/bases/openai/openaiHooks'; +import { kimiAnthropicTrait, kimiOpenAITrait } from '#/kosong/provider/providers/kimi/kimi.contrib'; +import { classifyKimiQuotaError } from '#/kosong/provider/providers/kimi/kimi-errors'; // Structurally an SDK user-abort: recognized by constructor name, the same // way the OpenAI and Anthropic SDKs name their abort error class. @@ -80,3 +98,208 @@ describe('non-abort classification still works', () => { expect(isRetryableGenerateError(status)).toBe(true); }); }); + +async function* streamEvents(events: readonly Record[]) { + yield* events; +} + +async function consume(stream: AsyncIterable): Promise { + for await (const _part of stream) { + void _part; + } +} + +const QUOTA_MESSAGE = + 'Your account org-0123456789abcdef is suspended due to insufficient balance, please recharge your account or check your plan and billing details'; + +const TOKEN_QUOTA_MESSAGE = + 'You exceeded your current token quota: 31275, please check your account balance'; + +function moonshotQuota429(message: string, type?: string): OpenAIAPIError { + return new OpenAIAPIError( + 429, + type === undefined ? undefined : { message, type }, + `429 ${message}`, + new Headers(), + ); +} + +describe('classifyKimiQuotaError (Kimi trait classifier)', () => { + it('classifies a structured exceeded_current_quota_error body as quota-exhausted', () => { + const error = classifyKimiQuotaError( + moonshotQuota429('Too many requests', 'exceeded_current_quota_error'), + ); + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(error)).toBe(false); + }); + + it.each([QUOTA_MESSAGE, TOKEN_QUOTA_MESSAGE])( + 'falls back to billing wording "%s" when no structured body is present', + (message) => { + const error = classifyKimiQuotaError(moonshotQuota429(message)); + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(error)).toBe(false); + }, + ); + + it.each(['Too many requests', 'your token quota per minute was exceeded'])( + 'answers undefined for transient 429 "%s"', + (message) => { + expect(classifyKimiQuotaError(moonshotQuota429(message))).toBeUndefined(); + }, + ); + + it('answers undefined for non-429 and non-SDK shapes', () => { + expect( + classifyKimiQuotaError(new OpenAIAPIError(403, undefined, QUOTA_MESSAGE, new Headers())), + ).toBeUndefined(); + expect(classifyKimiQuotaError(new Error(QUOTA_MESSAGE))).toBeUndefined(); + expect(classifyKimiQuotaError(undefined)).toBeUndefined(); + }); + + it('classifies the Anthropic SDK error shape (body nested under .error)', () => { + const source = AnthropicAPIError.generate( + 429, + { type: 'error', error: { type: 'exceeded_current_quota_error', message: 'quota gone' } }, + 'Too many requests', + new Headers(), + ); + const error = classifyKimiQuotaError(source); + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(error)).toBe(false); + }); + + it('is declared as the convertError hook on both Kimi traits', () => { + const context: TraitContext = { + config: { protocol: 'openai', providerType: 'kimi', modelName: '' } as ProtocolAdapterConfig, + providerId: 'kimi', + }; + const source = moonshotQuota429('Too many requests', 'exceeded_current_quota_error'); + expect(kimiOpenAITrait.convertError!(source, context)).toBeInstanceOf( + APIProviderQuotaExhaustedError, + ); + expect(kimiAnthropicTrait.convertError!(source, context)).toBeInstanceOf( + APIProviderQuotaExhaustedError, + ); + }); +}); + +describe('convertError hook consult at the OpenAI boundary', () => { + const kimiContext: TraitContext = { + config: { protocol: 'openai', providerType: 'kimi', modelName: '' } as ProtocolAdapterConfig, + providerId: 'kimi', + }; + + it('prefers the composed hook over the vendor-neutral base classification', () => { + const hooks = composeOpenAIChatHooks([{ trait: kimiOpenAITrait, context: kimiContext }]); + const source = moonshotQuota429(QUOTA_MESSAGE); + + expect(convertOpenAIError(source)).toBeInstanceOf(APIProviderRateLimitError); + expect(convertOpenAIError(source, hooks?.convertError)).toBeInstanceOf( + APIProviderQuotaExhaustedError, + ); + }); + + it('binds convertError with last-declarer-wins semantics', () => { + const bound = traitConvertError([ + { trait: { convertError: () => new ChatProviderError('first') }, context: kimiContext }, + { trait: { convertError: () => new ChatProviderError('second') }, context: kimiContext }, + ]); + expect(bound!(new Error('anything'))?.message).toBe('second'); + }); + + it('still throws the standard abort shape when a hook is present', () => { + expectStandardAbort(() => + convertOpenAIError(createAbortError(), () => new ChatProviderError('never')), + ); + }); + + it('passes already-converted errors through without re-consulting the hook', () => { + const calls: unknown[] = []; + const converted = new APIProviderQuotaExhaustedError('already classified'); + const result = convertOpenAIError(converted, (error) => { + calls.push(error); + return new ChatProviderError('re-classified'); + }); + expect(result).toBe(converted); + expect(calls).toHaveLength(0); + }); +}); + +describe('OpenAI base quota classification (vendor-neutral)', () => { + it("fails fast on OpenAI's own insufficient_quota code without any hook", () => { + const source = new OpenAIAPIError( + 429, + { + message: 'You exceeded your current quota, please check your plan and billing details.', + type: 'insufficient_quota', + }, + '429 You exceeded your current quota, please check your plan and billing details.', + new Headers(), + ); + + const error = convertOpenAIError(source); + + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(error)).toBe(false); + }); + + it('keeps vendor billing wordings a retryable rate limit without a hook', () => { + const error = convertOpenAIError(moonshotQuota429(QUOTA_MESSAGE)); + expect(error).toBeInstanceOf(APIProviderRateLimitError); + expect(isRetryableGenerateError(error)).toBe(true); + }); +}); + +describe('OpenAI Responses quota-exhausted conversion', () => { + it('fails fast on a streamed insufficient_quota error event', async () => { + const stream = new OpenAIResponsesStreamedMessage( + streamEvents([ + { + type: 'error', + code: 'insufficient_quota', + message: 'You exceeded your current quota, please check your plan and billing details.', + param: null, + }, + ]), + true, + ); + + const caught = await consume(stream).then( + () => undefined, + (error: unknown) => error, + ); + expect(caught).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(caught)).toBe(false); + }); + + it('consults a convertError hook with the raw in-stream error event', async () => { + const seen: unknown[] = []; + const stream = new OpenAIResponsesStreamedMessage( + streamEvents([ + { + type: 'error', + code: 'vendor_quota_gone', + message: 'vendor says the quota is gone', + param: null, + }, + ]), + true, + (error) => { + seen.push(error); + return (error as { code?: string }).code === 'vendor_quota_gone' + ? new APIProviderQuotaExhaustedError('vendor quota exhausted') + : undefined; + }, + ); + + const caught = await consume(stream).then( + () => undefined, + (error: unknown) => error, + ); + expect(caught).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect((caught as APIProviderQuotaExhaustedError).message).toBe('vendor quota exhausted'); + expect(seen).toHaveLength(1); + expect(seen[0]).toMatchObject({ type: 'error', code: 'vendor_quota_gone' }); + }); +}); diff --git a/packages/agent-core-v2/test/kosong/provider/kimi.test.ts b/packages/agent-core-v2/test/kosong/provider/kimi.test.ts index b6a53d941d..b1f7304227 100644 --- a/packages/agent-core-v2/test/kosong/provider/kimi.test.ts +++ b/packages/agent-core-v2/test/kosong/provider/kimi.test.ts @@ -20,14 +20,23 @@ * `extra_body.thinking` encoding, no 128k ceiling, `prompt_cache_key`, * and the `strictThinkingValidation` marker; * - `kimiAnthropicTrait` (the `(kimi, anthropic)` registration): thinking - * encoding and interleaved-thinking beta stripping. + * encoding and interleaved-thinking beta stripping; + * - `KimiFiles`: an upload failure classifies through + * `classifyKimiQuotaError`, so a Moonshot quota 429 from the files API + * fails fast instead of converting to a retryable rate limit. */ -import { describe, expect, it } from 'vitest'; +import { APIError as OpenAIAPIError } from 'openai'; +import { describe, expect, it, vi } from 'vitest'; +import { + APIProviderQuotaExhaustedError, + isRetryableGenerateError, +} from '#/kosong/contract/errors'; import type { Message } from '#/kosong/contract/message'; import type { Tool } from '#/kosong/contract/tool'; import type { ProtocolTrait, TraitContext } from '#/kosong/protocol/protocolTrait'; +import { KimiFiles } from '#/kosong/provider/providers/kimi/kimi-files'; import { convertKimiTool, kimiAnthropicTrait, @@ -288,6 +297,7 @@ describe('trait objects are plain declarations', () => { expect(hookNames(kimiOpenAITrait).toSorted()).toEqual([ 'buildParams', 'cacheKey', + 'convertError', 'convertMessage', 'convertTool', 'endpoint', @@ -298,7 +308,7 @@ describe('trait objects are plain declarations', () => { 'withMaxCompletionTokens', 'withThinking', ]); - expect(hookNames(kimiAnthropicTrait)).toEqual(['withThinking']); + expect(hookNames(kimiAnthropicTrait).toSorted()).toEqual(['convertError', 'withThinking']); }); it('marks only the native-transport thinking trait as strict-validation (v1 parity)', () => { @@ -308,3 +318,33 @@ describe('trait objects are plain declarations', () => { expect(kimiAnthropicTrait.strictThinkingValidation).toBeUndefined(); }); }); + +describe('KimiFiles upload error conversion', () => { + it('fails fast on a Moonshot quota-exhausted 429 from the files API', async () => { + const quotaError = new OpenAIAPIError( + 429, + { + message: 'Your account is suspended due to insufficient balance, please recharge', + type: 'exceeded_current_quota_error', + }, + '429 quota exhausted', + new Headers(), + ); + const files = new KimiFiles({ + baseUrl: 'https://api.example/v1', + clientFactory: () => ({ files: { create: vi.fn().mockRejectedValue(quotaError) } }) as never, + }); + + const caught = await files + .uploadVideo( + { data: Buffer.from([1, 2, 3]), mimeType: 'video/mp4' }, + { auth: { apiKey: 'request-token' } }, + ) + .then( + () => undefined, + (error: unknown) => error, + ); + expect(caught).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(caught)).toBe(false); + }); +}); diff --git a/packages/agent-core/src/agent/turn/index.ts b/packages/agent-core/src/agent/turn/index.ts index 11e20b054b..5b31756cbc 100644 --- a/packages/agent-core/src/agent/turn/index.ts +++ b/packages/agent-core/src/agent/turn/index.ts @@ -5,6 +5,7 @@ import { APIConnectionError, APIContextOverflowError, APIEmptyResponseError, + APIProviderQuotaExhaustedError, APIStatusError, APITimeoutError, inputTotal, @@ -1533,6 +1534,11 @@ interface ApiErrorClassification { } function classifyApiError(error: unknown, summary: KimiErrorPayload): ApiErrorClassification { + // Quota/balance exhaustion shares status 429 with rate limits but fails + // fast instead of retrying — keep the two apart in telemetry. + if (error instanceof APIProviderQuotaExhaustedError) { + return { errorType: 'quota_exhausted', statusCode: error.statusCode }; + } const statusCode = apiStatusCode(error) ?? summaryStatusCode(summary); if (statusCode !== undefined) { if (statusCode === 429) return { errorType: 'rate_limit', statusCode }; diff --git a/packages/agent-core/src/errors/serialize.ts b/packages/agent-core/src/errors/serialize.ts index 17ba995343..632334c8d7 100644 --- a/packages/agent-core/src/errors/serialize.ts +++ b/packages/agent-core/src/errors/serialize.ts @@ -1,6 +1,7 @@ import { APIConnectionError, APIEmptyResponseError, + APIProviderQuotaExhaustedError, APIStatusError, APITimeoutError, ChatProviderError, @@ -58,6 +59,10 @@ export function makeErrorPayload( * Recognized errors: * - `KimiError`: passthrough. * - `APIStatusError`: 429 -> rate_limit, 401 -> auth_error, otherwise -> api_error. + * Exception: a quota-exhausted 429 maps to api_error (retryable: false) — + * the rate_limit code would re-mint a rate-limit error across the wire + * boundary and drive the swarm requeue/suspend loop, which cannot help + * until the account is recharged. * - `APIConnectionError` / `APITimeoutError`: connection_error. * - `ChatProviderError`: api_error. * @@ -77,11 +82,13 @@ export function toKimiErrorPayload(error: unknown): KimiErrorPayload { if (error instanceof APIStatusError) { const code: KimiErrorCode = - error.statusCode === 429 - ? ErrorCodes.PROVIDER_RATE_LIMIT - : error.statusCode === 401 - ? ErrorCodes.PROVIDER_AUTH_ERROR - : ErrorCodes.PROVIDER_API_ERROR; + error instanceof APIProviderQuotaExhaustedError + ? ErrorCodes.PROVIDER_API_ERROR + : error.statusCode === 429 + ? ErrorCodes.PROVIDER_RATE_LIMIT + : error.statusCode === 401 + ? ErrorCodes.PROVIDER_AUTH_ERROR + : ErrorCodes.PROVIDER_API_ERROR; return { code, message: sanitizeStatusErrorMessage(error.message), diff --git a/packages/agent-core/src/session/provider-manager.ts b/packages/agent-core/src/session/provider-manager.ts index e8028c76c1..2bdb29a425 100644 --- a/packages/agent-core/src/session/provider-manager.ts +++ b/packages/agent-core/src/session/provider-manager.ts @@ -1,6 +1,11 @@ import type { Logger } from '#/logging/types'; import type { ProviderConfig as KosongProviderConfig, ModelCapability, ProviderRequestAuth } from '@moonshot-ai/kosong'; -import { APIStatusError, getModelCapability, UNKNOWN_CAPABILITY } from '@moonshot-ai/kosong'; +import { + APIStatusError, + classifyKimiQuotaError, + getModelCapability, + UNKNOWN_CAPABILITY, +} from '@moonshot-ai/kosong'; import { parseKimiCodeCustomHeaders } from '@moonshot-ai/kimi-code-oauth'; import { effectiveModelAlias, @@ -284,7 +289,12 @@ function toKosongProviderConfig( ...(maxOutputSize !== undefined ? { defaultMaxTokens: maxOutputSize } : {}), supportEfforts, ...(adaptiveThinking !== undefined ? { adaptiveThinking } : {}), - ...(provider.type === 'kimi' ? { kimiThinking: true } : {}), + // Kimi routed over the Anthropic transport keeps its vendor error + // classification: a Moonshot quota-exhausted 429 must fail fast here + // exactly as it does on the Kimi OpenAI transport. + ...(provider.type === 'kimi' + ? { kimiThinking: true, convertError: classifyKimiQuotaError } + : {}), ...(betaApi !== undefined ? { betaApi } : {}), // Session affinity: Anthropic's analog of OpenAI `prompt_cache_key` is // `metadata.user_id` on the Messages API (cache-affinity / end-user id). diff --git a/packages/agent-core/test/errors/serialize.test.ts b/packages/agent-core/test/errors/serialize.test.ts index 43e56ff717..db095b0613 100644 --- a/packages/agent-core/test/errors/serialize.test.ts +++ b/packages/agent-core/test/errors/serialize.test.ts @@ -1,4 +1,4 @@ -import { APIStatusError } from '@moonshot-ai/kosong'; +import { APIProviderQuotaExhaustedError, APIStatusError } from '@moonshot-ai/kosong'; import { describe, expect, it } from 'vitest'; import { toKimiErrorPayload } from '#/errors/serialize'; @@ -48,3 +48,21 @@ describe('toKimiErrorPayload — APIStatusError message sanitization', () => { ); }); }); + +describe('toKimiErrorPayload — quota-exhausted 429', () => { + it('maps a quota-exhausted 429 to provider.api_error, not provider.rate_limit', () => { + // provider.rate_limit is retryable and re-minted as a rate-limit error + // across the wire boundary, which drives the swarm requeue/suspend loop; + // quota exhaustion must carry the non-retryable generic code instead. + const payload = toKimiErrorPayload( + new APIProviderQuotaExhaustedError( + 'Your account is suspended due to insufficient balance, please recharge your account', + 'req-quota', + ), + ); + expect(payload.code).toBe('provider.api_error'); + expect(payload.retryable).toBe(false); + expect(payload.message).toContain('recharge'); + expect(payload.details).toMatchObject({ statusCode: 429, requestId: 'req-quota' }); + }); +}); diff --git a/packages/agent-core/test/harness/runtime-provider.test.ts b/packages/agent-core/test/harness/runtime-provider.test.ts index 1d20903234..96afbf4f15 100644 --- a/packages/agent-core/test/harness/runtime-provider.test.ts +++ b/packages/agent-core/test/harness/runtime-provider.test.ts @@ -1,3 +1,4 @@ +import { classifyKimiQuotaError } from '@moonshot-ai/kosong'; import { describe, expect, it } from 'vitest'; import type { KimiConfig, ModelAlias } from '../../src/config'; @@ -1257,6 +1258,11 @@ describe('per-model protocol routing', () => { model: 'kimi-for-coding', baseUrl: 'https://api.example', }); + // Kimi over the Anthropic transport keeps its vendor error classifier — + // a Moonshot quota 429 must fail fast on this route too. + expect( + (resolved.provider as { convertError?: (error: unknown) => unknown }).convertError, + ).toBe(classifyKimiQuotaError); }); it('keeps a model without protocol on the provider wire type and leaves the REST base intact', () => { @@ -1295,6 +1301,10 @@ describe('per-model protocol routing', () => { model: 'claude-sonnet-4-5', baseUrl: 'https://api.anthropic.example/v1', }); + // A plain anthropic provider carries no Kimi vendor classifier. + expect( + (resolved.provider as { convertError?: (error: unknown) => unknown }).convertError, + ).toBeUndefined(); }); }); diff --git a/packages/agent-core/test/loop/retry.test.ts b/packages/agent-core/test/loop/retry.test.ts index 9c38ed7811..8166cfc116 100644 --- a/packages/agent-core/test/loop/retry.test.ts +++ b/packages/agent-core/test/loop/retry.test.ts @@ -1,5 +1,6 @@ import { APIConnectionError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, emptyUsage, isRetryableGenerateError, @@ -210,6 +211,43 @@ describe('chatWithRetry: default retry budget', () => { }); }); +describe('chatWithRetry: quota-exhausted 429 fails fast', () => { + it('does not retry a quota-exhausted 429 even when it carries retry-after', async () => { + // Same status as a rate limit, but exhausted quota/balance never clears + // on its own — the error must surface after a single attempt instead of + // burning the whole default budget. The 1ms retry-after proves a server + // backoff hint does not re-enable retries either. + let calls = 0; + const captured: Array<{ type: string }> = []; + const llm: LLM = { + systemPrompt: '', + modelName: 'mock', + isRetryableError: (e) => isRetryableGenerateError(e), + async chat(): Promise { + calls += 1; + throw new APIProviderQuotaExhaustedError( + 'Your account is suspended due to insufficient balance, please recharge your account', + null, + 1, + ); + }, + }; + const input = makeInput(llm, new AbortController().signal); + + await expect( + chatWithRetry({ + ...input, + dispatchEvent: async (event) => { + captured.push(event as { type: string }); + }, + }), + ).rejects.toMatchObject({ name: 'APIProviderQuotaExhaustedError' }); + + expect(calls).toBe(1); + expect(captured.filter((e) => e.type === 'step.retrying')).toHaveLength(0); + }); +}); + describe('chatWithRetry: honors server retry-after', () => { it('uses the error retryAfterMs as the retry delay instead of the backoff', async () => { let calls = 0; diff --git a/packages/kosong/src/errors.ts b/packages/kosong/src/errors.ts index 0bdb54b96c..1ad2c00742 100644 --- a/packages/kosong/src/errors.ts +++ b/packages/kosong/src/errors.ts @@ -119,6 +119,33 @@ export class APIProviderRateLimitError extends APIStatusError { } } +/** + * HTTP 429 that specifically means the account's quota or balance is + * exhausted, as opposed to a transient rate limit. Deliberately NOT a + * subclass of `APIProviderRateLimitError`: a rate limit clears on its own + * (retry/requeue helps), while quota exhaustion is deterministic until the + * account is recharged — so this class is excluded from retry and from the + * rate-limit requeue/suspend paths. + * + * Observed shapes: Moonshot returns `error.type = + * "exceeded_current_quota_error"` with wording that varies by account state + * ("You exceeded your current token quota: ... please check your account + * balance" vs "Your account ... is suspended due to insufficient balance, + * please recharge your account ..."); OpenAI uses `insufficient_quota` as + * both `error.type` and `error.code`. + */ +export class APIProviderQuotaExhaustedError extends APIStatusError { + constructor( + message: string, + requestId?: string | null, + retryAfterMs?: number | null, + traceId?: string | null, + ) { + super(429, message, requestId, retryAfterMs, traceId); + this.name = 'APIProviderQuotaExhaustedError'; + } +} + /** * The API returned an empty response (no content, no tool calls). */ @@ -191,6 +218,12 @@ export function isRetryableGenerateError(error: unknown): boolean { return true; } if (error instanceof APIStatusError) { + // Quota/balance exhaustion is a 429 but deterministic until the account + // is recharged — retrying can never succeed, so it fails fast instead of + // burning the whole retry budget (~2-3 minutes of backoff). + if (error instanceof APIProviderQuotaExhaustedError) { + return false; + } // Transient statuses worth retrying: 408 (request timeout), 409 // (lock/conflict timeout), 429 (rate limit), 5xx (server errors) and 529 // (provider overloaded — the "engine is currently overloaded" case). @@ -551,6 +584,9 @@ export function isRecoverableRequestStructureError(error: unknown): boolean { } export function isProviderRateLimitError(error: unknown): boolean { + // Quota exhaustion is a 429 but not a rate limit: the rate-limit reactions + // (retry, requeue, suspend) cannot help until the account is recharged. + if (error instanceof APIProviderQuotaExhaustedError) return false; if (error instanceof APIProviderRateLimitError) return true; const statusCode = getStatusCode(error); diff --git a/packages/kosong/src/index.ts b/packages/kosong/src/index.ts index d999a02bc6..cd0440637b 100644 --- a/packages/kosong/src/index.ts +++ b/packages/kosong/src/index.ts @@ -32,6 +32,7 @@ export type { ProviderConfig, ProviderType } from './providers'; // kwargs, `thinking.keep` extra body). export { KimiChatProvider } from './providers/kimi'; export type { ExtraBody, GenerationKwargs, KimiOptions, ThinkingConfig } from './providers/kimi'; +export { classifyKimiQuotaError } from './providers/kimi-errors'; // Model capability matrix export { isUnknownCapability, UNKNOWN_CAPABILITY } from './capability'; @@ -70,6 +71,7 @@ export { APIConnectionError, APIContextOverflowError, APIEmptyResponseError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIRequestTooLargeError, APIStatusError, diff --git a/packages/kosong/src/providers/anthropic.ts b/packages/kosong/src/providers/anthropic.ts index 6b34aa5343..5af85106e2 100644 --- a/packages/kosong/src/providers/anthropic.ts +++ b/packages/kosong/src/providers/anthropic.ts @@ -122,6 +122,15 @@ export interface AnthropicOptions { */ betaApi?: boolean | undefined; clientFactory?: (auth: ProviderRequestAuth) => Anthropic; + /** + * Vendor error classification, consulted by `convertAnthropicError` with + * each raw SDK failure exactly once (after the abort guard and the + * already-converted pass-through) before the base rules run. `undefined` + * keeps the base classification. A Kimi provider routed over this + * transport passes `classifyKimiQuotaError` here so a quota-exhausted 429 + * fails fast instead of burning the retry budget. + */ + convertError?: (error: unknown) => ChatProviderError | undefined; } interface AnthropicGenerationKwargs { @@ -582,11 +591,26 @@ function shouldKeepConvertedMessage(message: MessageParam): boolean { return message.role !== 'assistant' || message.content.length > 0; } -export function convertAnthropicError(error: unknown): ChatProviderError { +export function convertAnthropicError( + error: unknown, + convertErrorHook?: (error: unknown) => ChatProviderError | undefined, +): ChatProviderError { // Abort guard FIRST: throws (never returns) the standard abort DOMException // for any abort shape, so a user cancellation is never misclassified as a // retryable provider failure. throwIfAbortError(error); + // Already-converted errors pass through untouched — the vendor hook below + // sees each raw failure exactly once. + if (error instanceof ChatProviderError) { + return error; + } + // Vendor classification next: the hook sees the RAW SDK error (the base + // conversion below drops the parsed body detail), and `undefined` keeps + // the base classification. + const hooked = convertErrorHook?.(error); + if (hooked !== undefined) { + return hooked; + } // Check timeout before connection (APIConnectionTimeoutError extends APIConnectionError) if (error instanceof AnthropicTimeoutError) { return new APITimeoutError(error.message); @@ -629,7 +653,13 @@ class AnthropicStreamedMessage implements StreamedMessage { private _rawFinishReason: string | null = null; private readonly _iter: AsyncGenerator; - constructor(response: unknown, isStream: boolean) { + constructor( + response: unknown, + isStream: boolean, + private readonly _convertErrorHook?: + | ((error: unknown) => ChatProviderError | undefined) + | undefined, + ) { if (isStream) { this._iter = this._convertStreamResponse(response as AsyncIterable); } else { @@ -873,7 +903,7 @@ class AnthropicStreamedMessage implements StreamedMessage { // message_stop: nothing to do } } catch (error: unknown) { - throw convertAnthropicError(error); + throw convertAnthropicError(error, this._convertErrorHook); } } } @@ -901,6 +931,7 @@ export class AnthropicChatProvider implements ChatProvider { private _adaptiveThinking: boolean | undefined; private readonly _supportEfforts: readonly string[] | undefined; private readonly _kimiThinking: boolean; + private readonly _convertErrorHook: ((error: unknown) => ChatProviderError | undefined) | undefined; private _betaApi: boolean; private _explicitMaxTokens: boolean; @@ -911,6 +942,7 @@ export class AnthropicChatProvider implements ChatProvider { this._adaptiveThinking = options.adaptiveThinking; this._supportEfforts = options.supportEfforts; this._kimiThinking = options.kimiThinking ?? false; + this._convertErrorHook = options.convertError; this._betaApi = options.betaApi ?? false; this._apiKey = options.apiKey === undefined || options.apiKey.length === 0 ? undefined : options.apiKey; @@ -1101,9 +1133,9 @@ export class AnthropicChatProvider implements ChatProvider { { ...createParams, stream: true } as unknown as MessageCreateParamsStreaming, finalRequestOptions, ); - return new AnthropicStreamedMessage(stream, true); + return new AnthropicStreamedMessage(stream, true, this._convertErrorHook); } catch (error: unknown) { - throw convertAnthropicError(error); + throw convertAnthropicError(error, this._convertErrorHook); } } @@ -1118,9 +1150,9 @@ export class AnthropicChatProvider implements ChatProvider { { ...createParams, stream: false } as unknown as MessageCreateParams, finalRequestOptions, ); - return new AnthropicStreamedMessage(response, false); + return new AnthropicStreamedMessage(response, false, this._convertErrorHook); } catch (error: unknown) { - throw convertAnthropicError(error); + throw convertAnthropicError(error, this._convertErrorHook); } } diff --git a/packages/kosong/src/providers/kimi-errors.ts b/packages/kosong/src/providers/kimi-errors.ts new file mode 100644 index 0000000000..88109a1f22 --- /dev/null +++ b/packages/kosong/src/providers/kimi-errors.ts @@ -0,0 +1,87 @@ +import { APIProviderQuotaExhaustedError, parseRetryAfterMs, parseTraceId } from '#/errors'; + +// Structured error `type`/`code` value that means the Moonshot account's +// quota or balance is exhausted (as opposed to a transient rate limit): the +// backend sets `exceeded_current_quota_error` as the body `error.type`. +const KIMI_QUOTA_EXHAUSTED_ERROR_CODES = new Set(['exceeded_current_quota_error']); + +// Message fallback for gateways that flatten the body to text, matched +// against the lowercased message of a 429. Every pattern is anchored to +// billing wording — deliberately no bare /quota/ or /balance/, which would +// also match transient throttle messages like "token quota per minute". +// Grounded in observed Moonshot bodies: "You exceeded your current token +// quota: ... please check your account balance" and "Your account ... is +// suspended due to insufficient balance, please recharge your account or +// check your plan and billing details". +const KIMI_QUOTA_EXHAUSTED_MESSAGE_PATTERNS = [ + /exceeded your current (?:token )?quota/, + /check your account balance/, + /insufficient balance/, + /recharge your account|please recharge/, + /account (?:is )?in arrears/, +] as const; + +function readStringProp(value: object, key: string): string | undefined { + const raw = (value as Record)[key]; + return typeof raw === 'string' ? raw : undefined; +} + +function readErrorObjectProp(value: object): object | undefined { + const raw = (value as Record)['error']; + return typeof raw === 'object' && raw !== null ? raw : undefined; +} + +// Collect every candidate `code`/`type` string the SDK error may carry. The +// OpenAI SDK hoists the body's `error.code`/`error.type` to the top level and +// keeps the inner error object on `.error`; the Anthropic SDK keeps the FULL +// body on `.error` (`{type: 'error', error: {type, message}}`), so the quota +// type sits two levels deep. Walking `error` → `.error` → `.error.error` +// covers both shapes without SDK imports. +function collectErrorCodes(error: object): string[] { + const codes: string[] = []; + let current: object | undefined = error; + for (let depth = 0; current !== undefined && depth < 3; depth += 1) { + const code = readStringProp(current, 'code'); + if (code !== undefined) codes.push(code); + const type = readStringProp(current, 'type'); + if (type !== undefined) codes.push(type); + current = readErrorObjectProp(current); + } + return codes; +} + +/** + * Classify a raw provider failure as Moonshot's quota/balance-exhausted 429, + * or answer `undefined` to keep the base classification. This is the Kimi + * vendor's error knowledge, kept out of the shared OpenAI conversion: the + * Kimi provider (and the Kimi files client) passes it to + * `convertOpenAIError` as the vendor hook, consulted after the abort guard + * with the raw SDK error — the base conversion would otherwise drop the + * SDK-parsed body `error.type`/`error.code` this reads. + */ +export function classifyKimiQuotaError( + error: unknown, +): APIProviderQuotaExhaustedError | undefined { + if (typeof error !== 'object' || error === null) return undefined; + const status = (error as Record)['status']; + if (status !== 429) return undefined; + + const message = readStringProp(error, 'message') ?? ''; + const structuredHit = collectErrorCodes(error).some((code) => + KIMI_QUOTA_EXHAUSTED_ERROR_CODES.has(code), + ); + const lowerMessage = message.toLowerCase(); + const wordingHit = KIMI_QUOTA_EXHAUSTED_MESSAGE_PATTERNS.some((pattern) => + pattern.test(lowerMessage), + ); + if (!structuredHit && !wordingHit) return undefined; + + const requestId = readStringProp(error, 'requestID') ?? null; + const headers = (error as Record)['headers']; + return new APIProviderQuotaExhaustedError( + message, + requestId, + parseRetryAfterMs(headers), + parseTraceId(headers), + ); +} diff --git a/packages/kosong/src/providers/kimi-files.ts b/packages/kosong/src/providers/kimi-files.ts index 75ca48e5eb..ef40938ee2 100644 --- a/packages/kosong/src/providers/kimi-files.ts +++ b/packages/kosong/src/providers/kimi-files.ts @@ -8,6 +8,7 @@ import type { ProviderRequestAuth, VideoUploadInput } from '#/provider'; import type OpenAI from 'openai'; import OpenAIClient from 'openai'; +import { classifyKimiQuotaError } from './kimi-errors'; import { convertOpenAIError } from './openai-common'; import { mergeRequestHeaders, @@ -128,7 +129,7 @@ export class KimiFiles { options?.signal ? { signal: options.signal } : undefined, )) as unknown as { id: string }; } catch (error: unknown) { - throw convertOpenAIError(error); + throw convertOpenAIError(error, classifyKimiQuotaError); } return { diff --git a/packages/kosong/src/providers/kimi.ts b/packages/kosong/src/providers/kimi.ts index c228eba91d..4496a432d9 100644 --- a/packages/kosong/src/providers/kimi.ts +++ b/packages/kosong/src/providers/kimi.ts @@ -16,6 +16,7 @@ import type { Tool } from '#/tool'; import type { TokenUsage } from '#/usage'; import OpenAI from 'openai'; +import { classifyKimiQuotaError } from './kimi-errors'; import { KimiFiles } from './kimi-files'; import { convertChatCompletionStreamToolCall, @@ -400,7 +401,7 @@ class KimiStreamedMessage implements StreamedMessage { } } } catch (error: unknown) { - throw convertOpenAIError(error); + throw convertOpenAIError(error, classifyKimiQuotaError); } } } @@ -579,7 +580,7 @@ export class KimiChatProvider implements ChatProvider { this._reasoningKeyDialect, ); } catch (error: unknown) { - throw convertOpenAIError(error); + throw convertOpenAIError(error, classifyKimiQuotaError); } } diff --git a/packages/kosong/src/providers/openai-common.ts b/packages/kosong/src/providers/openai-common.ts index 5f94a9fc66..91d2e80f4e 100644 --- a/packages/kosong/src/providers/openai-common.ts +++ b/packages/kosong/src/providers/openai-common.ts @@ -1,5 +1,6 @@ import { APIConnectionError, + APIProviderQuotaExhaustedError, APITimeoutError, ChatProviderError, classifyBaseApiError, @@ -97,14 +98,46 @@ export function toolToOpenAI(tool: Tool): OpenAIToolParam { * chain — it can never be converted into, nor returned as, a retryable * provider error. */ -export function convertOpenAIError(error: unknown): ChatProviderError { +// OpenAI's own documented signal that the account quota/balance is exhausted: +// the API sets `insufficient_quota` as both the body `error.type` and +// `error.code` on a 429. This is protocol knowledge of the OpenAI wire — the +// equivalent vendor-specific signals (e.g. Moonshot's +// `exceeded_current_quota_error`) live with their vendor and reach this +// converter through the optional `convertErrorHook` instead. +export function isOpenAIInsufficientQuotaCode(code: string | null | undefined): boolean { + return code === 'insufficient_quota'; +} + +function isOpenAIInsufficientQuotaError(error: OpenAIAPIError): boolean { + if (error.status !== 429) return false; + if (typeof error.code === 'string' && isOpenAIInsufficientQuotaCode(error.code)) return true; + if (typeof error.type === 'string' && isOpenAIInsufficientQuotaCode(error.type)) return true; + // Gateways sometimes flatten the JSON body into the message text; the + // literal code string is unambiguous there, unlike prose wordings. + return error.message.toLowerCase().includes('insufficient_quota'); +} + +export function convertOpenAIError( + error: unknown, + convertErrorHook?: (error: unknown) => ChatProviderError | undefined, +): ChatProviderError { // Abort guard FIRST: throws (never returns) the standard abort DOMException // for any abort shape, so a user cancellation is never misclassified as a // retryable provider failure. throwIfAbortError(error); + // Already-converted errors pass through untouched — they never re-enter + // vendor classification, so the hook below sees each raw failure exactly + // once even when a stream-minted error crosses an outer catch. if (error instanceof ChatProviderError) { return error; } + // Vendor classification next: the hook sees the RAW error (the base + // conversion below drops the SDK-parsed body `error.code`/`error.type`), + // and `undefined` keeps the base classification. + const hooked = convertErrorHook?.(error); + if (hooked !== undefined) { + return hooked; + } // v6: APIConnectionTimeoutError extends APIConnectionError, check timeout first if (error instanceof OpenAITimeoutError) { return new APITimeoutError(error.message); @@ -115,13 +148,14 @@ export function convertOpenAIError(error: unknown): ChatProviderError { // APIError with a status code => status error if (error instanceof OpenAIAPIError && typeof error.status === 'number') { const reqId = error.requestID ?? null; - return normalizeAPIStatusError( - error.status, - error.message, - reqId, - parseRetryAfterMs(error.headers), - parseTraceId(error.headers), - ); + const retryAfterMs = parseRetryAfterMs(error.headers); + const traceId = parseTraceId(error.headers); + // Quota/balance exhaustion is a 429 but deterministic until the account + // is recharged — it must not classify as a retryable rate limit. + if (isOpenAIInsufficientQuotaError(error)) { + return new APIProviderQuotaExhaustedError(error.message, reqId, retryAfterMs, traceId); + } + return normalizeAPIStatusError(error.status, error.message, reqId, retryAfterMs, traceId); } // Base APIError with no status and no body => transport-layer failure. // When the error has a body (e.g. SSE error events from the server), diff --git a/packages/kosong/src/providers/openai-responses.ts b/packages/kosong/src/providers/openai-responses.ts index 66770f1cdd..bd4e86e08a 100644 --- a/packages/kosong/src/providers/openai-responses.ts +++ b/packages/kosong/src/providers/openai-responses.ts @@ -1,5 +1,6 @@ import { APIContextOverflowError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, ChatProviderError, isContextOverflowErrorCode, @@ -23,6 +24,7 @@ import { usesOpenAIResponsesDeveloperRole } from './capability-registry'; import { convertOpenAIError, isMediaPart, + isOpenAIInsufficientQuotaCode, TOOL_RESULT_MEDIA_PLACEHOLDER, TOOL_RESULT_MEDIA_PROMPT, type ToolMessageConversion, @@ -251,6 +253,14 @@ function errorFromOpenAIResponsesEvent( if (isContextOverflowErrorCode(code)) { return new APIContextOverflowError(400, fullMessage); } + // Quota/balance exhaustion first — otherwise an `insufficient_quota` event + // falls through to the base ChatProviderError (whose unclassified fallback + // is retryable), and a quota message with an embedded status_code=429 would + // classify as a retryable rate limit. Only OpenAI's own documented code is + // recognized here; vendor-specific quota signals live with their vendor. + if (isOpenAIInsufficientQuotaCode(code)) { + return new APIProviderQuotaExhaustedError(fullMessage); + } if (code === 'rate_limit_exceeded' || readEmbeddedStatusCode(message) === 429) { return new APIProviderRateLimitError(fullMessage); } diff --git a/packages/kosong/test/anthropic-errors.test.ts b/packages/kosong/test/anthropic-errors.test.ts index 81fabdced1..6502c737e5 100644 --- a/packages/kosong/test/anthropic-errors.test.ts +++ b/packages/kosong/test/anthropic-errors.test.ts @@ -1,6 +1,7 @@ import { APIConnectionError, APIContextOverflowError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIStatusError, APITimeoutError, @@ -8,6 +9,7 @@ import { isRetryableGenerateError, } from '#/errors'; import { convertAnthropicError, AnthropicChatProvider } from '#/providers/anthropic'; +import { classifyKimiQuotaError } from '#/providers/kimi-errors'; import { APIConnectionError as AnthropicConnectionError, APIConnectionTimeoutError as AnthropicTimeoutError, @@ -465,3 +467,60 @@ describe('stream error propagation', () => { expect(isRetryableGenerateError(caught)).toBe(true); }); }); + +describe('convertAnthropicError: quota-exhausted 429 via the convertError hook', () => { + const QUOTA_BODY = { + type: 'error', + error: { + type: 'exceeded_current_quota_error', + message: + 'Your account org-0123456789abcdef is suspended due to insufficient balance, please recharge your account or check your plan and billing details', + }, + }; + + function quota429(): unknown { + return AnthropicAPIError.generate(429, QUOTA_BODY, 'Too many requests', new Headers()); + } + + it('keeps vendor quota signals a rate limit without the vendor hook', () => { + const result = convertAnthropicError(quota429()); + expect(result).toBeInstanceOf(APIProviderRateLimitError); + expect(isRetryableGenerateError(result)).toBe(true); + }); + + it('classifies the Kimi quota body as quota-exhausted through the hook', () => { + const result = convertAnthropicError(quota429(), classifyKimiQuotaError); + expect(result).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(result)).toBe(false); + }); + + it('passes already-converted errors through without re-consulting the hook', () => { + const calls: unknown[] = []; + const converted = new APIProviderQuotaExhaustedError('already classified'); + const result = convertAnthropicError(converted, (error) => { + calls.push(error); + return new ChatProviderError('re-classified'); + }); + expect(result).toBe(converted); + expect(calls).toHaveLength(0); + }); + + it('the provider threads options.convertError to its generate catch', async () => { + const provider = new AnthropicChatProvider({ + model: 'k25', + apiKey: 'test-key', + defaultMaxTokens: 1024, + stream: false, + convertError: classifyKimiQuotaError, + }); + (provider as any)._client.messages.create = vi.fn().mockRejectedValue(quota429()); + + await expect( + provider.generate( + '', + [], + [{ role: 'user', content: [{ type: 'text', text: 'Hi' }], toolCalls: [] }], + ), + ).rejects.toThrow(APIProviderQuotaExhaustedError); + }); +}); diff --git a/packages/kosong/test/errors.test.ts b/packages/kosong/test/errors.test.ts index 2e39e971ed..c2ad36f898 100644 --- a/packages/kosong/test/errors.test.ts +++ b/packages/kosong/test/errors.test.ts @@ -2,6 +2,7 @@ import { APIConnectionError, APIContextOverflowError, APIEmptyResponseError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIRequestTooLargeError, APIStatusError, @@ -678,3 +679,56 @@ describe('isImageFormatError', () => { ).toBe(false); }); }); + +describe('APIProviderQuotaExhaustedError', () => { + it('extends APIStatusError and preserves HTTP details', () => { + const err = new APIProviderQuotaExhaustedError('quota exhausted', 'req-quota', 12_500); + expect(err).toBeInstanceOf(APIStatusError); + expect(err).toBeInstanceOf(ChatProviderError); + expect(err).not.toBeInstanceOf(APIProviderRateLimitError); + expect(err.name).toBe('APIProviderQuotaExhaustedError'); + expect(err.statusCode).toBe(429); + expect(err.requestId).toBe('req-quota'); + expect(err.retryAfterMs).toBe(12_500); + }); +}); + +describe('normalizeAPIStatusError: 429 stays vendor-neutral', () => { + // The shared normalization never decides what a vendor's 429 means: quota + // classification lives with the vendor (`classifyKimiQuotaError`, the + // OpenAI base's own insufficient_quota check), so even billing wordings + // normalize to a retryable rate limit here. + it.each([ + 'Too many requests', + 'request reached user+model max RPM: 50', + 'your token quota per minute was exceeded', + 'Your account org-0123456789abcdef is suspended due to insufficient balance, please recharge your account or check your plan and billing details', + ])('normalizes 429 "%s" to APIProviderRateLimitError', (message) => { + const error = normalizeAPIStatusError(429, message); + expect(error).toBeInstanceOf(APIProviderRateLimitError); + expect(error).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + }); + + it('keeps billing wording on other statuses a generic status error', () => { + const error = normalizeAPIStatusError(403, 'insufficient balance'); + expect(error).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(error.constructor).toBe(APIStatusError); + }); +}); + +describe('quota-exhausted retry and rate-limit semantics', () => { + it('is not retryable while a plain rate limit stays retryable', () => { + expect(isRetryableGenerateError(new APIProviderQuotaExhaustedError('quota exhausted'))).toBe( + false, + ); + expect(isRetryableGenerateError(new APIProviderRateLimitError('rate limited'))).toBe(true); + expect(isRetryableGenerateError(new APIStatusError(429, 'rate limited'))).toBe(true); + }); + + it('is not a provider rate limit despite carrying status 429', () => { + expect(isProviderRateLimitError(new APIProviderQuotaExhaustedError('quota exhausted'))).toBe( + false, + ); + expect(isProviderRateLimitError(new APIProviderRateLimitError('rate limited'))).toBe(true); + }); +}); diff --git a/packages/kosong/test/kimi-files.test.ts b/packages/kosong/test/kimi-files.test.ts index ede39f74a1..27ebff9c10 100644 --- a/packages/kosong/test/kimi-files.test.ts +++ b/packages/kosong/test/kimi-files.test.ts @@ -2,8 +2,10 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; +import { APIProviderQuotaExhaustedError, isRetryableGenerateError } from '#/errors'; import { KimiChatProvider } from '#/providers/kimi'; import { KimiFiles } from '#/providers/kimi-files'; +import { APIError as OpenAIAPIError } from 'openai'; import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; function createProvider(): KimiChatProvider { @@ -197,4 +199,35 @@ describe('KimiFiles', () => { ).rejects.toThrow(/video/i); }); }); + + describe('upload error conversion', () => { + it('fails fast on a Moonshot quota-exhausted 429 from the files API', async () => { + const quotaError = new OpenAIAPIError( + 429, + { + message: 'Your account is suspended due to insufficient balance, please recharge', + type: 'exceeded_current_quota_error', + }, + '429 quota exhausted', + new Headers(), + ); + const files = new KimiFiles({ + baseUrl: 'https://api.example/v1', + clientFactory: () => + ({ files: { create: vi.fn().mockRejectedValue(quotaError) } }) as never, + }); + + const caught = await files + .uploadVideo( + { data: Buffer.from([1, 2, 3]), mimeType: 'video/mp4' }, + { auth: { apiKey: 'request-token' } }, + ) + .then( + () => undefined, + (error: unknown) => error, + ); + expect(caught).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(caught)).toBe(false); + }); + }); }); diff --git a/packages/kosong/test/kimi.test.ts b/packages/kosong/test/kimi.test.ts index 23f3fe2475..35601942b9 100644 --- a/packages/kosong/test/kimi.test.ts +++ b/packages/kosong/test/kimi.test.ts @@ -1,9 +1,13 @@ +import { APIError as AnthropicAPIError } from '@anthropic-ai/sdk'; +import { APIProviderQuotaExhaustedError, isRetryableGenerateError } from '#/errors'; import { generate } from '#/generate'; import type { ContentPart, Message, ToolCall } from '#/message'; import { extractUsageFromChunk, KimiChatProvider } from '#/providers/kimi'; +import { classifyKimiQuotaError } from '#/providers/kimi-errors'; import { extractUsage } from '#/providers/openai-common'; import type { GenerateOptions } from '#/provider'; import type { Tool } from '#/tool'; +import { APIError as OpenAIAPIError } from 'openai'; import { describe, it, expect, vi } from 'vitest'; function makeChatCompletionResponse(model: string = 'test-model') { @@ -2168,3 +2172,63 @@ describe('extractUsage', () => { expect(extractUsage(undef)).toBeNull(); }); }); + +describe('classifyKimiQuotaError', () => { + const QUOTA_MESSAGE = + 'Your account org-0123456789abcdef is suspended due to insufficient balance, please recharge your account or check your plan and billing details'; + const TOKEN_QUOTA_MESSAGE = + 'You exceeded your current token quota: 31275, please check your account balance'; + + function quota429(message: string, type?: string): OpenAIAPIError { + return new OpenAIAPIError( + 429, + type === undefined ? undefined : { message, type }, + `429 ${message}`, + new Headers(), + ); + } + + it('classifies a structured exceeded_current_quota_error body as quota-exhausted', () => { + const error = classifyKimiQuotaError( + quota429('Too many requests', 'exceeded_current_quota_error'), + ); + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(error)).toBe(false); + }); + + it.each([QUOTA_MESSAGE, TOKEN_QUOTA_MESSAGE])( + 'falls back to billing wording "%s" without a structured body', + (message) => { + expect(classifyKimiQuotaError(quota429(message))).toBeInstanceOf( + APIProviderQuotaExhaustedError, + ); + }, + ); + + it.each(['Too many requests', 'your token quota per minute was exceeded'])( + 'answers undefined for transient 429 "%s"', + (message) => { + expect(classifyKimiQuotaError(quota429(message))).toBeUndefined(); + }, + ); + + it('answers undefined for non-429 and non-SDK shapes', () => { + expect( + classifyKimiQuotaError(new OpenAIAPIError(403, undefined, QUOTA_MESSAGE, new Headers())), + ).toBeUndefined(); + expect(classifyKimiQuotaError(new Error(QUOTA_MESSAGE))).toBeUndefined(); + expect(classifyKimiQuotaError(undefined)).toBeUndefined(); + }); + + it('classifies the Anthropic SDK error shape (body nested under .error)', () => { + const source = AnthropicAPIError.generate( + 429, + { type: 'error', error: { type: 'exceeded_current_quota_error', message: 'quota gone' } }, + 'Too many requests', + new Headers(), + ); + const error = classifyKimiQuotaError(source); + expect(error).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(error)).toBe(false); + }); +}); diff --git a/packages/kosong/test/openai-common-errors.test.ts b/packages/kosong/test/openai-common-errors.test.ts index 30b082d332..0b7b7ca898 100644 --- a/packages/kosong/test/openai-common-errors.test.ts +++ b/packages/kosong/test/openai-common-errors.test.ts @@ -1,6 +1,7 @@ import { APIConnectionError, APIContextOverflowError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIStatusError, APITimeoutError, @@ -9,6 +10,7 @@ import { normalizeAPIStatusError, } from '#/errors'; import type { ContentPart } from '#/message'; +import { classifyKimiQuotaError } from '#/providers/kimi-errors'; import { convertContentPart, convertOpenAIError, @@ -422,3 +424,62 @@ describe('convertOpenAIError: non-Error values', () => { expect(result.message).toContain('plain error'); }); }); + +describe('convertOpenAIError: quota-exhausted 429', () => { + const QUOTA_MESSAGE = + 'Your account org-0123456789abcdef is suspended due to insufficient balance, please recharge your account or check your plan and billing details'; + + it("classifies OpenAI's own insufficient_quota code without any vendor hook", () => { + // insufficient_quota is OpenAI's documented signal on its own wire, so + // the base converter recognizes it directly. + const err = new OpenAIAPIError( + 429, + { message: 'You exceeded your current quota.', type: 'insufficient_quota' }, + '429 You exceeded your current quota.', + new Headers(), + ); + const result = convertOpenAIError(err); + expect(result).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect((result as APIProviderQuotaExhaustedError).statusCode).toBe(429); + expect(isRetryableGenerateError(result)).toBe(false); + }); + + it('keeps vendor quota signals a rate limit without the vendor hook', () => { + // Moonshot's structured type and billing wordings are vendor knowledge — + // the shared base must not decide what another vendor's 429 means. + const err = new OpenAIAPIError( + 429, + { message: QUOTA_MESSAGE, type: 'exceeded_current_quota_error' }, + `429 ${QUOTA_MESSAGE}`, + new Headers(), + ); + const result = convertOpenAIError(err); + expect(result).toBeInstanceOf(APIProviderRateLimitError); + expect(result).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + }); + + it('classifies vendor quota signals through the convertError hook', () => { + const err = new OpenAIAPIError( + 429, + { message: QUOTA_MESSAGE, type: 'exceeded_current_quota_error' }, + `429 ${QUOTA_MESSAGE}`, + new Headers(), + ); + const result = convertOpenAIError(err, classifyKimiQuotaError); + expect(result).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(result)).toBe(false); + }); + + it('keeps a transient structured 429 an APIProviderRateLimitError', () => { + const err = new OpenAIAPIError( + 429, + { message: 'Too many requests', type: 'rate_limit_reached_error' }, + 'Too many requests', + new Headers(), + ); + const result = convertOpenAIError(err, classifyKimiQuotaError); + expect(result).toBeInstanceOf(APIProviderRateLimitError); + expect(result).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(result)).toBe(true); + }); +}); diff --git a/packages/kosong/test/openai-responses.test.ts b/packages/kosong/test/openai-responses.test.ts index c7f9b3ee97..d8c7a6efcb 100644 --- a/packages/kosong/test/openai-responses.test.ts +++ b/packages/kosong/test/openai-responses.test.ts @@ -1,8 +1,10 @@ import { APIContextOverflowError, + APIProviderQuotaExhaustedError, APIProviderRateLimitError, APIStatusError, ChatProviderError, + isRetryableGenerateError, } from '#/errors'; import { generate } from '#/generate'; import type { ContentPart, Message, StreamedMessagePart, ToolCall } from '#/message'; @@ -2040,6 +2042,65 @@ describe('OpenAIResponsesChatProvider', () => { expect((caughtError as Error).message).toContain('status_code=429'); }); + it('fails fast on response.failed with an insufficient_quota code', async () => { + // Quota exhaustion arriving as a Responses stream event carries no HTTP + // status; without the structured-code check it would fall through to the + // base ChatProviderError, whose unclassified fallback is retryable — and + // burn the whole retry budget on an error that cannot succeed. + const events = [ + { + type: 'response.failed', + response: { + id: 'resp_quota', + status: 'failed', + error: { + code: 'insufficient_quota', + message: 'You exceeded your current quota, please check your plan and billing details.', + }, + }, + }, + ]; + const stream = new OpenAIResponsesStreamedMessage(makeAsyncIterable(events), true); + + let caughtError: unknown; + try { + await collectStreamParts(stream); + } catch (error) { + caughtError = error; + } + + expect(caughtError).toBeInstanceOf(APIProviderQuotaExhaustedError); + expect((caughtError as APIProviderQuotaExhaustedError).statusCode).toBe(429); + expect(isRetryableGenerateError(caughtError)).toBe(false); + }); + + it('keeps an embedded status_code=429 with vendor billing wording a rate limit', async () => { + // Vendor billing wordings are no longer recognized by the base — quota + // classification for a vendor's own errors lives on that vendor's + // convertError hook, and no vendor rides this transport here. + const events = [ + { + type: 'error', + code: 'upstream_error', + message: + 'llmproxy/openai/responses/resp_q.json status_code=429 Your account is suspended due to insufficient balance, please recharge your account', + param: null, + }, + ]; + const stream = new OpenAIResponsesStreamedMessage(makeAsyncIterable(events), true); + + let caughtError: unknown; + try { + await collectStreamParts(stream); + } catch (error) { + caughtError = error; + } + + expect(caughtError).toBeInstanceOf(APIProviderRateLimitError); + expect(caughtError).not.toBeInstanceOf(APIProviderQuotaExhaustedError); + expect(isRetryableGenerateError(caughtError)).toBe(true); + }); + it('rejects malformed stream events with a non-string type even when message is present', async () => { const events = [ {