feat(providers): rotate requests across multiple API keys per provider

.env.template

@@ -358,9 +358,16 @@
 # Add more instances with <PROVIDER>_<SUFFIX>_*.
 # Example: OPENAI_EAST_API_KEY and OPENAI_EAST_BASE_URL register provider openai-east.
 # Underscores in the suffix become hyphens in the provider name.
+#
+# Give ONE provider several API keys with <PROVIDER>_API_KEY_<n>, numbered from 2.
+# Requests then rotate across the keys round robin, which lifts the per-key rate
+# limit. Works for any provider that authenticates with an API key.
+# Caveat: rotation defeats provider prompt caching, because providers scope the
+# cache to the key that filled it. Rotate to raise rate limits, not to cut cost.
 
 # OpenAI
 # OPENAI_API_KEY=sk-...
+# OPENAI_API_KEY_2=sk-...
 # OPENAI_BASE_URL=https://api.openai.com/v1
 
 # Anthropic

CLAUDE.md

@@ -132,5 +132,6 @@ Full reference: `.env.template` and `config/config.yaml`
 - **Resilience:** Configured via `config/config.yaml` - global `resilience.retry.*` and `resilience.circuit_breaker.*` defaults with optional per-provider overrides under `providers.<name>.resilience.retry.*` and `providers.<name>.resilience.circuit_breaker.*`. Retry defaults: `max_retries` (3), `initial_backoff` (1s), `max_backoff` (30s), `backoff_factor` (2.0), `jitter_factor` (0.1). Circuit breaker defaults: `failure_threshold` (5), `success_threshold` (2), `timeout` (30s). Breaker state is per-process, not shown in the dashboard, and exported as the `gomodel_circuit_breaker_state` gauge when metrics are enabled.
 - **Metrics:** `METRICS_ENABLED` (false), `METRICS_ENDPOINT` (/metrics)
 - **Guardrails:** Definitions are persisted in the `guardrail_definitions` store and managed via the admin API/dashboard; `config/config.yaml` entries are validated and upserted into that store at startup (a seed, not the source of truth). `GUARDRAILS_ENABLED` env var gates the feature.
+- **Provider API key rotation:** Any API-key provider accepts several keys: `<PROVIDER>[_SUFFIX]_API_KEY_<n>` env vars (numbered from 2; `_1` is accepted as a synonym for the unsuffixed key) or `providers.<name>.api_keys` in `config.yaml` (merged after `api_key`, de-duplicated, unresolved `${...}` entries dropped; env replaces the whole YAML list). Two or more keys turn on round-robin rotation, drawn per outbound HTTP request — including retries, so a 429'd request retries under the next key. Realtime websocket sessions pick a key per session. Counters are in-memory per instance. The trailing number names a key, not a provider: `OPENAI_API_KEY_2` is key 2 of `openai`, while `OPENAI_REGION_2_API_KEY` is the sole key of provider `openai-region-2`. **Rotation defeats provider prompt caching** (providers scope the cache to the key that filled it); use it to lift per-key rate limits, not to save cost. Keyless (Ollama, vLLM) and non-API-key providers (Vertex, Bedrock) are unaffected.
 - **Providers:** `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `ANTHROPIC_DEFAULT_MAX_TOKENS` (optional default `max_tokens` for Anthropic-translated requests that omit it; default 4096), `GEMINI_API_KEY`, `USE_GOOGLE_GEMINI_NATIVE_API` (true by default; false uses Gemini's OpenAI-compatible chat API), `XAI_API_KEY`, `GROQ_API_KEY`, `FIREWORKS_API_KEY`, `FIREWORKS_BASE_URL` (optional Fireworks AI endpoint override; default `https://api.fireworks.ai/inference/v1`), `OPENROUTER_API_KEY`, `OPENROUTER_SITE_URL`/`OPENROUTER_APP_NAME` (optional OpenRouter attribution headers), `ZAI_API_KEY`, `ZAI_BASE_URL` (optional Z.ai endpoint override), `MINIMAX_API_KEY`, `MINIMAX_BASE_URL` (optional MiniMax endpoint override), `XIAOMI_API_KEY`, `XIAOMI_BASE_URL` (optional Xiaomi MiMo endpoint override), `OPENCODE_GO_API_KEY`, `OPENCODE_GO_BASE_URL` (optional OpenCode Go/Zen endpoint override; default `https://opencode.ai/zen/go/v1`), `OPENCODE_GO_MESSAGES_MODELS` (optional comma-separated model IDs routed to the Anthropic-native `/messages` endpoint instead of `/chat/completions`; default `qwen3.7-max`), `BAILIAN_API_KEY`, `BAILIAN_BASE_URL` (optional Bailian base URL for region switching; default `https://dashscope.aliyuncs.com/compatible-mode/v1`), `AZURE_API_KEY`, `AZURE_BASE_URL` (Azure OpenAI deployment base URL), `AZURE_API_VERSION` (optional Azure API version), `ORACLE_API_KEY` (Oracle API key), `ORACLE_BASE_URL` (Oracle OpenAI-compatible base URL), `<PROVIDER>[_SUFFIX]_MODELS` (comma-separated configured model list for any provider type), `OLLAMA_BASE_URL`, `VLLM_BASE_URL`, `VLLM_API_KEY` (optional upstream vLLM bearer token)
 - **Provider model metadata:** `providers.<name>.models` accepts either model IDs (strings) or `{id, metadata}` objects. When `metadata` is supplied (`display_name`, `context_window`, `max_output_tokens`, `modes`, `capabilities`, `pricing`, …) it is merged onto the remote ai-model-list entry during enrichment, with operator values winning per-field. Primary use case: advertising context windows, capabilities, and pricing for local models (Ollama) and other custom endpoints whose IDs are not in the upstream registry.

config/config.example.yaml

@@ -266,6 +266,13 @@ providers:
   openai:
     type: openai
     api_key: "sk-..."
+    # Several keys for one provider rotate round robin, raising the per-key rate
+    # limit. Equivalent env vars: OPENAI_API_KEY_2, OPENAI_API_KEY_3, ...
+    # Caveat: rotation defeats provider prompt caching, since providers scope the
+    # cache to the key that filled it.
+    # api_keys:
+    #   - "${OPENAI_API_KEY_2}"
+    #   - "${OPENAI_API_KEY_3}"
     # Per-provider resilience overrides (optional).
     # Only specified fields override the global defaults above.
     # resilience:

config/providers.go

@@ -4,8 +4,12 @@ package config
 // overrides, credential filtering, or resilience merging. Exported so the
 // providers package can resolve it into a fully-configured ProviderConfig.
 type RawProviderConfig struct {
-	Type                     string               `yaml:"type"`
-	APIKey                   string               `yaml:"api_key"`
+	Type   string `yaml:"type"`
+	APIKey string `yaml:"api_key"`
+	// APIKeys lists additional API keys for this provider. When more than one
+	// key is resolved (counting APIKey), requests rotate across them round
+	// robin. Set it via `api_keys:` or the `<PROVIDER>_API_KEY_<n>` env vars.
+	APIKeys                  []string             `yaml:"api_keys"`
 	BaseURL                  string               `yaml:"base_url"`
 	APIVersion               string               `yaml:"api_version"`
 	Backend                  string               `yaml:"backend"`

docs/docs.json

@@ -123,6 +123,7 @@
                 "icon": "plug",
                 "pages": [
                     "providers/overview",
+                    "providers/key-rotation",
                     "providers/anthropic",
                     "providers/gemini",
                     "providers/deepseek",

docs/providers/key-rotation.mdx

@@ -0,0 +1,130 @@
+---
+title: "API Key Rotation"
+description: "Give one provider several API keys and spread requests across them round robin to lift per-key rate limits."
+icon: "key-round"
+---
+
+A provider normally holds one API key, and every request to it uses that key.
+Configure more than one and GoModel spreads requests across them round robin, so
+each key carries a fraction of the traffic. This raises the per-key rate limits
+you hit before the provider starts returning `429`.
+
+Rotation is off until a second key is configured. One key behaves exactly as it
+always has.
+
+<Warning>
+  **Rotation defeats provider prompt caching.** Providers scope their prompt
+  cache to the API key that created it, so consecutive requests sent under
+  different keys never hit a warm cache. Expect higher cost and slower
+  time-to-first-token on workloads with long, repeated prefixes — system
+  prompts, few-shot examples, long documents, or coding agents replaying a
+  conversation.
+
+  Rotate when you are rate limited. Do not rotate to save money on cached
+  prompts; it does the opposite.
+</Warning>
+
+## Configure
+
+Add keys with numbered environment variables. `<PROVIDER>_API_KEY` is the first
+key; `<PROVIDER>_API_KEY_2`, `_3`, and so on add the rest.
+
+```bash
+OPENAI_API_KEY=sk-first
+OPENAI_API_KEY_2=sk-second
+OPENAI_API_KEY_3=sk-third
+```
+
+Three keys, so requests to `openai` cycle `sk-first` → `sk-second` →
+`sk-third` → `sk-first` → …
+
+Or in `config.yaml`:
+
+```yaml
+providers:
+  openai:
+    type: openai
+    api_keys:
+      - "${OPENAI_API_KEY}"
+      - "${OPENAI_API_KEY_2}"
+      - "${OPENAI_API_KEY_3}"
+```
+
+`api_key` and `api_keys` may be combined; `api_key` is used first.
+
+Any provider that authenticates with an API key can rotate — `ANTHROPIC_API_KEY_2`,
+`GEMINI_API_KEY_2`, `GROQ_API_KEY_2`, and so on. Providers that authenticate
+another way (Vertex AI and Bedrock, which use Google and AWS credentials) ignore
+these variables.
+
+## Rules
+
+- Numbering starts at `2`, since `<PROVIDER>_API_KEY` is key 1. Spelling out
+  `<PROVIDER>_API_KEY_1` instead of the unsuffixed form also works.
+- Gaps are fine. Setting only `_API_KEY` and `_API_KEY_3` gives two keys.
+- Duplicate keys are collapsed, so a key repeated across two variables does not
+  take a double share of the traffic.
+- Keys are used in the order configured: the unsuffixed key first, then
+  ascending by number.
+- Environment variables replace the provider's whole `api_keys` list from
+  `config.yaml` rather than merging into it.
+- A key that does not resolve — an `${UNSET_VAR}` placeholder — is dropped. A
+  provider left with no keys at all is not registered.
+
+<Note>
+  Keys rotate per outbound HTTP request, including retries. A request that a
+  provider rejects with `429` is retried under the next key rather than the one
+  that was just throttled.
+</Note>
+
+## Rotation on suffixed providers
+
+Suffixed environment variables register a *separate provider* of the same type.
+The two mechanisms compose, and the trailing number is what tells them apart:
+
+```bash
+OPENAI_EU_API_KEY=sk-eu-first
+OPENAI_EU_API_KEY_2=sk-eu-second
+```
+
+This is provider `openai-eu` with two keys. The trailing `_2` names a key.
+
+```bash
+OPENAI_REGION_2_API_KEY=sk-only
+```
+
+This is provider `openai-region-2` with one key. Here the `2` is part of the
+provider suffix, not a key number.
+
+## What rotation covers
+
+Every call GoModel makes to the provider draws the next key: chat completions,
+responses, embeddings, audio, images, batches, and provider-native
+[passthrough routes](/features/passthrough-api). Realtime websocket sessions pick a
+key when the session opens and keep it for the life of that session.
+
+Rotation is per gateway process. Counters live in memory and reset on restart,
+so `N` replicas each rotate independently — which is fine, since the goal is to
+spread load rather than to sequence it exactly.
+
+## Choosing between rotation and separate providers
+
+Both let you use several keys of one provider type. They solve different problems.
+
+| | Key rotation | Suffixed providers |
+| --- | --- | --- |
+| Config | `OPENAI_API_KEY_2` | `OPENAI_EAST_API_KEY` |
+| Model IDs | unchanged (`gpt-4o`) | provider-qualified (`openai-east/gpt-4o`) |
+| Routing | automatic, round robin | caller picks, or a [virtual model](/features/virtual-models) balances |
+| Prompt caching | lost | preserved per provider |
+| Use it for | lifting rate limits on one account | separate accounts, regions, or base URLs |
+
+If you want several keys *and* prompt caching, register them as suffixed
+providers and put a `round_robin` [virtual model](/features/virtual-models) in
+front. Each provider keeps its own key, so each keeps its own warm cache, at the
+cost of qualified model IDs and more configuration.
+
+## Verify
+
+Rotation is not reported by the API. To confirm it, watch the provider's own
+usage dashboard: traffic should divide roughly evenly across the keys.

docs/providers/overview.mdx

@@ -75,6 +75,10 @@ support, not every individual model capability exposed by an upstream provider.
   for providers that define a list, skipping their upstream `/models` calls.
 - **vLLM** — set `VLLM_API_KEY` only if the upstream server was started with
   `--api-key`.
+- **Multiple API keys for one provider** — set `OPENAI_API_KEY_2`,
+  `OPENAI_API_KEY_3`, and so on to spread requests across keys round robin and
+  lift per-key rate limits. This costs you provider prompt caching; see
+  [API key rotation](/providers/key-rotation).
 - **Multiple instances of one provider type** — without `config.yaml`, use
   suffixed env vars such as `OPENAI_EAST_API_KEY` and `OPENAI_EAST_BASE_URL`;
   add `OPENAI_EAST_MODELS` to configure that instance's model list. This

internal/embedding/embedding.go

@@ -13,6 +13,7 @@ import (
 	"github.com/goccy/go-json"
 
 	"gomodel/config"
+	"gomodel/internal/providers"
 )
 
 const defaultTimeout = 120 * time.Second
@@ -55,9 +56,12 @@ func NewEmbedder(cfg config.EmbedderConfig, resolvedProviders map[string]config.
 	} else if typ == "gemini" {
 		model = normalizeGeminiEmbeddingModel(model)
 	}
+	// APIKey leads APIKeys and is de-duplicated away when it repeats there, so
+	// this works whether raw came from provider resolution (which normalizes
+	// both fields) or was built by hand with only APIKey set.
 	return &apiEmbedder{
 		endpointURL: endpointURL,
-		apiKey:      raw.APIKey,
+		keys:        providers.NewKeyring(append([]string{raw.APIKey}, raw.APIKeys...)...),
 		model:       model,
 		httpClient:  &http.Client{Timeout: defaultTimeout},
 	}, nil
@@ -93,9 +97,12 @@ func openAIEmbeddingsEndpointURL(base string) (string, error) {
 }
 
 // apiEmbedder calls POST …/v1/embeddings on any OpenAI-compatible endpoint.
+// It shares the provider's configured key set but keeps its own rotation
+// counter, since it does not route through the provider's HTTP client.
+// Embeddings have no prompt cache to lose, so rotating here is free.
 type apiEmbedder struct {
 	endpointURL string
-	apiKey      string
+	keys        *providers.Keyring
 	model       string
 	httpClient  *http.Client
 }
@@ -124,8 +131,8 @@ func (e *apiEmbedder) Embed(ctx context.Context, text string) ([]float32, error)
 		return nil, fmt.Errorf("embedding: build request: %w", err)
 	}
 	req.Header.Set("Content-Type", "application/json")
-	if e.apiKey != "" {
-		req.Header.Set("Authorization", "Bearer "+e.apiKey)
+	if apiKey := e.keys.Next(); apiKey != "" {
+		req.Header.Set("Authorization", "Bearer "+apiKey)
 	}
 	resp, err := e.httpClient.Do(req)
 	if err != nil {

internal/embedding/embedding_test.go

@@ -137,8 +137,8 @@ func TestAPIEmbedder_UsesProviderCredentials(t *testing.T) {
 	if !ok {
 		t.Fatalf("expected *apiEmbedder, got %T", emb)
 	}
-	if a.apiKey != "gsk-abc" {
-		t.Errorf("expected apiKey gsk-abc, got %q", a.apiKey)
+	if got := a.keys.Primary(); got != "gsk-abc" {
+		t.Errorf("expected primary key gsk-abc, got %q", got)
 	}
 	if want := "https://api.groq.com/openai/v1/embeddings"; a.endpointURL != want {
 		t.Errorf("endpointURL = %q, want %q", a.endpointURL, want)

internal/providers/anthropic/anthropic.go

@@ -45,7 +45,7 @@ var allowedAnthropicImageMediaTypes = map[string]struct{}{
 // Provider implements the core.Provider interface for Anthropic
 type Provider struct {
 	client *llmclient.Client
-	apiKey string
+	keys   *providers.Keyring
 
 	batchEndpointsMu sync.RWMutex
 	// batchResultEndpoints keeps endpoint hints by provider batch id and custom_id.
@@ -56,7 +56,7 @@ type Provider struct {
 // New creates a new Anthropic provider.
 func New(providerCfg providers.ProviderConfig, opts providers.ProviderOptions) core.Provider {
 	p := &Provider{
-		apiKey:               providerCfg.APIKey,
+		keys:                 opts.Keyring(providerCfg.APIKey),
 		batchResultEndpoints: make(map[string]map[string]string),
 	}
 	clientCfg := llmclient.Config{
@@ -77,7 +77,7 @@ func NewWithHTTPClient(apiKey string, httpClient *http.Client, hooks llmclient.H
 		httpClient = http.DefaultClient
 	}
 	p := &Provider{
-		apiKey:               apiKey,
+		keys:                 providers.NewKeyring(apiKey),
 		batchResultEndpoints: make(map[string]map[string]string),
 	}
 	cfg := llmclient.DefaultConfig("anthropic", defaultBaseURL)
@@ -155,9 +155,10 @@ func (p *Provider) getBatchResultEndpoints(batchID string) map[string]string {
 	return cloned
 }
 
-// setHeaders sets the required headers for Anthropic API requests
+// setHeaders sets the required headers for Anthropic API requests. It runs once
+// per outbound request, so p.keys.Next advances the rotation per call.
 func (p *Provider) setHeaders(req *http.Request) {
-	req.Header.Set("x-api-key", p.apiKey)
+	req.Header.Set("x-api-key", p.keys.Next())
 	req.Header.Set("anthropic-version", anthropicAPIVersion)
 
 	// Forward request ID if present in context

internal/providers/anthropic/anthropic_test.go

@@ -22,8 +22,8 @@ func TestNew(t *testing.T) {
 	// Use NewWithHTTPClient to get concrete type for internal testing
 	provider := NewWithHTTPClient(apiKey, nil, llmclient.Hooks{})
 
-	if provider.apiKey != apiKey {
-		t.Errorf("apiKey = %q, want %q", provider.apiKey, apiKey)
+	if got := provider.keys.Primary(); got != apiKey {
+		t.Errorf("primary key = %q, want %q", got, apiKey)
 	}
 	if provider.client == nil {
 		t.Error("client should not be nil")

internal/providers/azure/azure.go

@@ -29,13 +29,14 @@ type Provider struct {
 	resourceProvider       *openai.CompatibleProvider
 	openAIResourceProvider *openai.CompatibleProvider
 	apiVersion             string
-	apiKey                 string // retained to inject the api-key header on the realtime target
+	keys                   *providers.Keyring // retained to inject the api-key header on the realtime target
 }
 
 func New(providerCfg providers.ProviderConfig, opts providers.ProviderOptions) core.Provider {
 	baseURL := providers.ResolveBaseURL(providerCfg.BaseURL, "https://example.invalid")
 	apiVersion := providers.ResolveAPIVersion(providerCfg.APIVersion, defaultAPIVersion)
-	p := &Provider{apiVersion: apiVersion, apiKey: providerCfg.APIKey}
+	// All three clients share opts.Keys, so the rotation is even across them.
+	p := &Provider{apiVersion: apiVersion, keys: opts.Keyring(providerCfg.APIKey)}
 	clientCfg := openai.CompatibleProviderConfig{
 		ProviderName: "azure",
 		BaseURL:      baseURL,
@@ -52,7 +53,7 @@ func New(providerCfg providers.ProviderConfig, opts providers.ProviderOptions) c
 }
 
 func NewWithHTTPClient(apiKey string, httpClient *http.Client, hooks llmclient.Hooks) *Provider {
-	p := &Provider{apiVersion: defaultAPIVersion, apiKey: apiKey}
+	p := &Provider{apiVersion: defaultAPIVersion, keys: providers.NewKeyring(apiKey)}
 	cfg := openai.CompatibleProviderConfig{
 		ProviderName: "azure",
 		BaseURL:      "https://example.invalid",

internal/providers/azure/realtime.go

@@ -122,8 +122,8 @@ func (p *Provider) realtimeRoot(insecureScheme, secureScheme string) (*url.URL,
 
 func (p *Provider) realtimeAuthHeaders() http.Header {
 	headers := http.Header{}
-	if p.apiKey != "" {
-		headers.Set("api-key", p.apiKey)
+	if apiKey := p.keys.Next(); apiKey != "" {
+		headers.Set("api-key", apiKey)
 	}
 	return headers
 }

internal/providers/bailian/bailian.go

@@ -43,28 +43,28 @@ type Provider struct {
 	*openai.BatchSurface
 	*openai.FileSurface
 	compatible *openai.CompatibleProvider
-	apiKey     string // retained to inject auth on the realtime websocket target
+	keys       *providers.Keyring // retained to inject auth on the realtime websocket target
 }
 
 // New creates a new Bailian provider from a resolved ProviderConfig.
 func New(cfg providers.ProviderConfig, opts providers.ProviderOptions) core.Provider {
 	compat := openai.NewCompatibleProvider(cfg.APIKey, opts, compatibleConfig(providers.ResolveBaseURL(cfg.BaseURL, defaultBaseURL)))
-	return newProvider(compat, cfg.APIKey)
+	return newProvider(compat, opts.Keyring(cfg.APIKey))
 }
 
 // NewWithHTTPClient creates a new Bailian provider with a custom HTTP client.
 // If httpClient is nil, http.DefaultClient is used.
 func NewWithHTTPClient(apiKey string, httpClient *http.Client, hooks llmclient.Hooks) *Provider {
 	compat := openai.NewCompatibleProviderWithHTTPClient(apiKey, httpClient, hooks, compatibleConfig(defaultBaseURL))
-	return newProvider(compat, apiKey)
+	return newProvider(compat, providers.NewKeyring(apiKey))
 }
 
-func newProvider(compat *openai.CompatibleProvider, apiKey string) *Provider {
+func newProvider(compat *openai.CompatibleProvider, keys *providers.Keyring) *Provider {
 	return &Provider{
 		BatchSurface: openai.NewBatchSurface(compat),
 		FileSurface:  openai.NewFileSurface(compat),
 		compatible:   compat,
-		apiKey:       apiKey,
+		keys:         keys,
 	}
 }