diff --git a/README.md b/README.md index e9c3a67f..82f47d13 100644 --- a/README.md +++ b/README.md @@ -272,7 +272,8 @@ docker run --rm -p 8080:8080 --env-file .env gomodel | Endpoint | Method | Description | | --------------------- | ------ | ----------------------------------------------- | -| `/health` | GET | Health check | +| `/health` | GET | Liveness check (always 200 while the process serves) | +| `/health/ready` | GET | Readiness check: pings storage (503 if down) and Redis cache (degraded, still 200) | | `/metrics` | GET | Prometheus metrics (experimental, when enabled) | | `/swagger/index.html` | GET | Swagger UI (when enabled) | diff --git a/cmd/gomodel/docs/docs.go b/cmd/gomodel/docs/docs.go index d4de51c9..e3a345cf 100644 --- a/cmd/gomodel/docs/docs.go +++ b/cmd/gomodel/docs/docs.go @@ -1591,6 +1591,33 @@ const docTemplate = `{ } } }, + "/health/ready": { + "get": { + "produces": [ + "application/json" + ], + "tags": [ + "system" + ], + "summary": "Readiness check", + "responses": { + "200": { + "description": "ready or degraded", + "schema": { + "type": "object", + "additionalProperties": true + } + }, + "503": { + "description": "not ready", + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, "/p/{provider}/{endpoint}": { "get": { "description": "Runtime-configurable passthrough endpoint under /p/{provider}/{endpoint}; enabled by default via server.enable_passthrough_routes. The endpoint path is opaque and may proxy JSON, binary, or SSE responses with upstream status codes preserved. For multi-segment provider endpoints, clients that rely on OpenAPI-generated path handling should URL-encode embedded slashes in the endpoint parameter. A leading v1/ segment is normalized away by default so /p/{provider}/v1/... and /p/{provider}/... map to the same upstream path relative to the provider base URL.", @@ -3457,6 +3484,67 @@ const docTemplate = `{ ] } }, + "/v1/realtime": { + "get": { + "description": "Upgrades to a websocket and relays an OpenAI-compatible realtime (speech-to-speech) session to the provider that owns the model named in the ?model= query parameter. Provider credentials are injected by the gateway.", + "tags": [ + "realtime" + ], + "summary": "Open a realtime session", + "parameters": [ + { + "type": "string", + "description": "Model that owns the realtime session", + "name": "model", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "Optional provider hint", + "name": "provider", + "in": "query" + } + ], + "responses": { + "101": { + "description": "Switching Protocols", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/core.OpenAIErrorEnvelope" + } + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/core.OpenAIErrorEnvelope" + } + }, + "501": { + "description": "Not Implemented", + "schema": { + "$ref": "#/definitions/core.OpenAIErrorEnvelope" + } + }, + "502": { + "description": "Bad Gateway", + "schema": { + "$ref": "#/definitions/core.OpenAIErrorEnvelope" + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ] + } + }, "/v1/responses": { "post": { "consumes": [ @@ -7009,7 +7097,7 @@ var SwaggerInfo = &swag.Spec{ BasePath: "/", Schemes: []string{"http"}, Title: "GoModel API", - Description: "AI gateway routing requests to multiple LLM providers (OpenAI, Anthropic, Gemini, Groq, OpenRouter, DeepSeek, Z.ai, xAI, MiniMax, Xiaomi MiMo, OpenCode Go, Bailian, Oracle, Ollama). Drop-in OpenAI-compatible API.", + Description: "AI gateway routing requests to multiple LLM providers (OpenAI, Anthropic, Gemini, Groq, OpenRouter, DeepSeek, Z.ai, xAI, MiniMax, Xiaomi MiMo, OpenCode Go, Oracle, Ollama, Bailian). Drop-in OpenAI-compatible API.", InfoInstanceName: "swagger", SwaggerTemplate: docTemplate, LeftDelim: "{{", diff --git a/cmd/gomodel/flags.go b/cmd/gomodel/flags.go index f24738e8..50c089a2 100644 --- a/cmd/gomodel/flags.go +++ b/cmd/gomodel/flags.go @@ -8,12 +8,20 @@ import ( "time" ) -const defaultHealthTimeout = 2 * time.Second +const ( + defaultHealthTimeout = 2 * time.Second + // defaultReadyTimeout is larger than the server's per-probe readinessProbeTimeout + // so a slow dependency yields a clean not_ready/degraded response instead of + // the client cutting the connection first. + defaultReadyTimeout = 4 * time.Second +) type cliOptions struct { Version bool Health bool HealthTimeout time.Duration + Ready bool + ReadyTimeout time.Duration } func parseCLI(args []string, output io.Writer) (cliOptions, error) { @@ -21,8 +29,10 @@ func parseCLI(args []string, output io.Writer) (cliOptions, error) { flags := flag.NewFlagSet("gomodel", flag.ContinueOnError) flags.SetOutput(output) flags.BoolVar(&opts.Version, "version", false, "Print version information") - flags.BoolVar(&opts.Health, "health", false, "Check the local GoModel health endpoint and exit") + flags.BoolVar(&opts.Health, "health", false, "Check the local GoModel health (liveness) endpoint and exit") flags.DurationVar(&opts.HealthTimeout, "health-timeout", defaultHealthTimeout, "Timeout for --health") + flags.BoolVar(&opts.Ready, "ready", false, "Check the local GoModel readiness endpoint and exit") + flags.DurationVar(&opts.ReadyTimeout, "ready-timeout", defaultReadyTimeout, "Timeout for --ready") if err := flags.Parse(args); err != nil { return opts, err } diff --git a/cmd/gomodel/health.go b/cmd/gomodel/health.go index 989bef9c..d719e460 100644 --- a/cmd/gomodel/health.go +++ b/cmd/gomodel/health.go @@ -29,7 +29,26 @@ func runHealthProbe(timeout time.Duration) error { return checkHealthEndpoint(context.Background(), client, endpoint) } +func runReadyProbe(timeout time.Duration) error { + result, err := config.Load() + if err != nil { + return fmt.Errorf("load config: %w", err) + } + + endpoint := probeURL(result.Config.Server, "/health/ready") + if timeout <= 0 { + timeout = defaultReadyTimeout + } + + client := &http.Client{Timeout: timeout} + return checkReadyEndpoint(context.Background(), client, endpoint) +} + func healthProbeURL(server config.ServerConfig) string { + return probeURL(server, "/health") +} + +func probeURL(server config.ServerConfig, path string) string { port := strings.TrimSpace(server.Port) if port == "" { port = "8080" @@ -38,7 +57,7 @@ func healthProbeURL(server config.ServerConfig) string { u := url.URL{ Scheme: "http", Host: net.JoinHostPort("127.0.0.1", port), - Path: config.JoinBasePath(server.BasePath, "/health"), + Path: config.JoinBasePath(server.BasePath, path), } return u.String() } @@ -70,3 +89,34 @@ func checkHealthEndpoint(ctx context.Context, client *http.Client, endpoint stri } return nil } + +// checkReadyEndpoint treats both "ready" and "degraded" as serviceable (exit 0) +// and only fails on "not_ready" (HTTP 503) — a degraded gateway still handles +// traffic, so it should remain in rotation. +func checkReadyEndpoint(ctx context.Context, client *http.Client, endpoint string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return fmt.Errorf("build readiness request: %w", err) + } + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("request readiness endpoint: %w", err) + } + defer resp.Body.Close() + + var body struct { + Status string `json:"status"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 1024)).Decode(&body); err != nil { + return fmt.Errorf("decode readiness response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("readiness endpoint returned HTTP %d (status %q)", resp.StatusCode, body.Status) + } + if body.Status != "ready" && body.Status != "degraded" { + return fmt.Errorf("readiness status is %q", body.Status) + } + return nil +} diff --git a/cmd/gomodel/main.go b/cmd/gomodel/main.go index 74b9c486..216710e3 100644 --- a/cmd/gomodel/main.go +++ b/cmd/gomodel/main.go @@ -77,7 +77,7 @@ func startApplication(application lifecycleApp, addr string) error { // @title GoModel API // @version 1.0 -// @description AI gateway routing requests to multiple LLM providers (OpenAI, Anthropic, Gemini, Groq, OpenRouter, DeepSeek, Z.ai, xAI, MiniMax, Xiaomi MiMo, Oracle, Ollama, Bailian). Drop-in OpenAI-compatible API. +// @description AI gateway routing requests to multiple LLM providers (OpenAI, Anthropic, Gemini, Groq, OpenRouter, DeepSeek, Z.ai, xAI, MiniMax, Xiaomi MiMo, OpenCode Go, Oracle, Ollama, Bailian). Drop-in OpenAI-compatible API. // @BasePath / // @schemes http // @securityDefinitions.apikey BearerAuth @@ -104,6 +104,14 @@ func main() { os.Exit(0) } + if opts.Ready { + if err := runReadyProbe(opts.ReadyTimeout); err != nil { + fmt.Fprintf(os.Stderr, "readiness check failed: %v\n", err) + os.Exit(1) + } + os.Exit(0) + } + if err := configureLogging(os.Stderr); err != nil { fmt.Fprintf(os.Stderr, "failed to configure logging: %v\n", err) os.Exit(1) diff --git a/cmd/gomodel/ready_test.go b/cmd/gomodel/ready_test.go new file mode 100644 index 00000000..02fc4fc0 --- /dev/null +++ b/cmd/gomodel/ready_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "gomodel/config" +) + +func TestReadyProbeURL(t *testing.T) { + tests := []struct { + name string + server config.ServerConfig + expected string + }{ + { + name: "default base path", + server: config.ServerConfig{Port: "8080", BasePath: "/"}, + expected: "http://127.0.0.1:8080/health/ready", + }, + { + name: "prefixed base path", + server: config.ServerConfig{Port: "9090", BasePath: "/g"}, + expected: "http://127.0.0.1:9090/g/health/ready", + }, + { + name: "empty port falls back to default", + server: config.ServerConfig{BasePath: "/"}, + expected: "http://127.0.0.1:8080/health/ready", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := probeURL(tt.server, "/health/ready") + if got != tt.expected { + t.Fatalf("probeURL() = %q, want %q", got, tt.expected) + } + }) + } +} + +func TestCheckReadyEndpoint(t *testing.T) { + tests := []struct { + name string + statusCode int + body string + wantErr string + }{ + { + name: "ready", + statusCode: http.StatusOK, + body: `{"status":"ready"}`, + }, + { + name: "degraded still serviceable", + statusCode: http.StatusOK, + body: `{"status":"degraded"}`, + }, + { + name: "not ready", + statusCode: http.StatusServiceUnavailable, + body: `{"status":"not_ready"}`, + wantErr: "HTTP 503", + }, + { + name: "bad response body", + statusCode: http.StatusOK, + body: `not json`, + wantErr: "decode readiness response", + }, + { + name: "unexpected status with 200", + statusCode: http.StatusOK, + body: `{"status":"starting"}`, + wantErr: `readiness status is "starting"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tt.statusCode) + _, _ = fmt.Fprint(w, tt.body) + })) + defer server.Close() + + err := checkReadyEndpoint(context.Background(), server.Client(), server.URL) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("checkReadyEndpoint() error = %v, want nil", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("checkReadyEndpoint() error = %v, want substring %q", err, tt.wantErr) + } + }) + } +} diff --git a/docs/advanced/cli.mdx b/docs/advanced/cli.mdx index 74d4186d..6742c196 100644 --- a/docs/advanced/cli.mdx +++ b/docs/advanced/cli.mdx @@ -13,8 +13,10 @@ the examples below use the long form. | Flag | Description | Default | | ------------------ | ------------------------------------------------------- | ------- | | `--version` | Print version information and exit | — | -| `--health` | Probe the local `/health` endpoint and exit | — | +| `--health` | Probe the local `/health` (liveness) endpoint and exit | — | | `--health-timeout` | Maximum time to wait for the `--health` probe | `2s` | +| `--ready` | Probe the local `/health/ready` (readiness) endpoint and exit | — | +| `--ready-timeout` | Maximum time to wait for the `--ready` probe | `4s` | ## Version @@ -59,3 +61,37 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ Orchestrators (Docker, Compose, Kubernetes) can then detect and restart an unhealthy gateway. + +## Readiness probe + +`--ready` probes `/health/ready`, which reports whether the instance should +receive traffic. Unlike liveness, readiness checks the dependencies the gateway +owns: + +- **Storage** is required. If the backend (SQLite/PostgreSQL/MongoDB) is + unreachable, readiness reports `not_ready` (HTTP `503`). +- **Redis exact cache** (when configured) is a performance optimization. If it + is unreachable, readiness reports `degraded` but stays HTTP `200` — the + gateway still serves requests. + +Upstream **provider** reachability is deliberately excluded: a provider outage +must not pull a healthy gateway out of rotation. + +```bash +gomodel --ready +``` + +- Exits `0` when the endpoint returns HTTP `200` (`ready` or `degraded`). +- Exits non-zero on `not_ready` (HTTP `503`), connection refused, or an + unexpected status value. + +```json +{ "status": "ready", "components": { "storage": "ok", "cache": "ok" } } +``` + +Liveness (`--health`) is the right signal for a Docker `HEALTHCHECK` (restart on +crash). Readiness (`/health/ready`) is the right signal for a Kubernetes +`readinessProbe` (gate traffic) — point it at the HTTP endpoint directly or run +`gomodel --ready` as an exec probe. Keep `--ready-timeout` larger than the +server's internal per-dependency probe timeout (`2s`) so a slow backend returns +a clean `not_ready` instead of the client timing out first. diff --git a/docs/openapi.json b/docs/openapi.json index 28928493..3f09b592 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -1,7 +1,7 @@ { "openapi": "3.0.0", "info": { - "description": "AI gateway routing requests to multiple LLM providers (OpenAI, Anthropic, Gemini, Groq, OpenRouter, DeepSeek, Z.ai, xAI, MiniMax, Xiaomi MiMo, OpenCode Go, Bailian, Oracle, Ollama). Drop-in OpenAI-compatible API.", + "description": "AI gateway routing requests to multiple LLM providers (OpenAI, Anthropic, Gemini, Groq, OpenRouter, DeepSeek, Z.ai, xAI, MiniMax, Xiaomi MiMo, OpenCode Go, Oracle, Ollama, Bailian). Drop-in OpenAI-compatible API.", "title": "GoModel API", "contact": {}, "version": "1.0" @@ -2110,6 +2110,43 @@ } } }, + "/health/ready": { + "get": { + "tags": [ + "system" + ], + "summary": "Readiness check", + "responses": { + "200": { + "description": "ready or degraded", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + }, + "503": { + "description": "not ready", + "content": { + "application/json": { + "schema": { + "type": "object", + "additionalProperties": true + } + } + } + } + }, + "x-mint": { + "metadata": { + "sidebarTitle": "/health/ready" + } + } + } + }, "/p/{provider}/{endpoint}": { "get": { "description": "Runtime-configurable passthrough endpoint under /p/{provider}/{endpoint}; enabled by default via server.enable_passthrough_routes. The endpoint path is opaque and may proxy JSON, binary, or SSE responses with upstream status codes preserved. For multi-segment provider endpoints, clients that rely on OpenAPI-generated path handling should URL-encode embedded slashes in the endpoint parameter. A leading v1/ segment is normalized away by default so /p/{provider}/v1/... and /p/{provider}/... map to the same upstream path relative to the provider base URL.", @@ -5237,6 +5274,96 @@ } } }, + "/v1/realtime": { + "get": { + "description": "Upgrades to a websocket and relays an OpenAI-compatible realtime (speech-to-speech) session to the provider that owns the model named in the ?model= query parameter. Provider credentials are injected by the gateway.", + "tags": [ + "realtime" + ], + "summary": "Open a realtime session", + "parameters": [ + { + "description": "Model that owns the realtime session", + "name": "model", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Optional provider hint", + "name": "provider", + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "101": { + "description": "Switching Protocols", + "content": { + "*/*": { + "schema": { + "type": "string" + } + } + } + }, + "400": { + "description": "Bad Request", + "content": { + "*/*": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "*/*": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + } + } + }, + "501": { + "description": "Not Implemented", + "content": { + "*/*": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + } + } + }, + "502": { + "description": "Bad Gateway", + "content": { + "*/*": { + "schema": { + "$ref": "#/components/schemas/core.OpenAIErrorEnvelope" + } + } + } + } + }, + "security": [ + { + "BearerAuth": [] + } + ], + "x-mint": { + "metadata": { + "sidebarTitle": "/v1/realtime" + } + } + } + }, "/v1/responses": { "post": { "tags": [ diff --git a/internal/app/app.go b/internal/app/app.go index 090eab64..1fc60831 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -414,6 +414,13 @@ func New(ctx context.Context, cfg Config) (*App, error) { SwaggerEnabled: swaggerEnabled, } + // Wire the readiness storage probe. Storage is a required dependency, so a + // failed ping makes /health/ready report not_ready (503). When no storage + // backend is active, readiness simply collapses to liveness. + if hc, ok := sharedStorage.(storage.HealthChecker); ok { + serverCfg.StorageProbe = hc + } + // Initialize admin API and dashboard (behind separate feature flags) adminCfg := appCfg.Admin if !adminCfg.EndpointsEnabled && adminCfg.UIEnabled { @@ -481,6 +488,13 @@ func New(ctx context.Context, cfg Config) (*App, error) { closers = append(closers, rcm.Close) serverCfg.ResponseCacheMiddleware = rcm + // Wire the readiness cache probe only when a Redis-backed exact cache is + // configured. The cache is a performance optimization, so a failed ping + // reports degraded (200) rather than blocking traffic. + if rcm.UsesRedis() { + serverCfg.CacheProbe = rcm + } + internalGuardrailExecutor := server.NewInternalChatCompletionExecutor(provider, server.InternalChatCompletionExecutorConfig{ ModelResolver: app.aliases.Service, ModelAuthorizer: app.modelOverrides.Service, diff --git a/internal/cache/redis.go b/internal/cache/redis.go index 4b8a3f32..4a9b8270 100644 --- a/internal/cache/redis.go +++ b/internal/cache/redis.go @@ -74,6 +74,14 @@ func (s *RedisStore) Set(ctx context.Context, key string, value []byte, ttl time return nil } +// Ping verifies connectivity to Redis. +func (s *RedisStore) Ping(ctx context.Context) error { + if s.client == nil { + return fmt.Errorf("redis client is not initialized") + } + return s.client.Ping(ctx).Err() +} + // Close closes the Redis connection. func (s *RedisStore) Close() error { if s.client != nil { diff --git a/internal/cache/store.go b/internal/cache/store.go index a76894bc..ee35d131 100644 --- a/internal/cache/store.go +++ b/internal/cache/store.go @@ -16,6 +16,13 @@ type Store interface { Close() error } +// Pinger is an optional capability for stores backed by a network service that +// can verify connectivity. Network-backed stores (RedisStore) implement it; +// in-memory stores (MapStore) do not, since they are always reachable. +type Pinger interface { + Ping(ctx context.Context) error +} + // MapStore is an in-memory Store for testing. type MapStore struct { mu sync.RWMutex diff --git a/internal/responsecache/readiness_test.go b/internal/responsecache/readiness_test.go new file mode 100644 index 00000000..1ddf0202 --- /dev/null +++ b/internal/responsecache/readiness_test.go @@ -0,0 +1,61 @@ +package responsecache + +import ( + "context" + "errors" + "testing" + "time" + + "gomodel/internal/cache" +) + +// pingableStore is a cache.Store that also implements cache.Pinger. +type pingableStore struct { + *cache.MapStore + err error +} + +func (p pingableStore) Ping(context.Context) error { return p.err } + +func TestUsesRedisAndPing(t *testing.T) { + t.Run("nil middleware", func(t *testing.T) { + var m *ResponseCacheMiddleware + if m.UsesRedis() { + t.Error("UsesRedis() = true, want false") + } + if err := m.Ping(context.Background()); err != nil { + t.Errorf("Ping() = %v, want nil", err) + } + }) + + t.Run("non-pinger store is not a readiness component", func(t *testing.T) { + m := NewResponseCacheMiddlewareWithStore(cache.NewMapStore(), time.Minute) + if m.UsesRedis() { + t.Error("UsesRedis() = true for in-memory store, want false") + } + if err := m.Ping(context.Background()); err != nil { + t.Errorf("Ping() = %v, want nil", err) + } + }) + + t.Run("pinger store reachable", func(t *testing.T) { + m := NewResponseCacheMiddlewareWithStore(pingableStore{MapStore: cache.NewMapStore()}, time.Minute) + if !m.UsesRedis() { + t.Error("UsesRedis() = false for pinger store, want true") + } + if err := m.Ping(context.Background()); err != nil { + t.Errorf("Ping() = %v, want nil", err) + } + }) + + t.Run("pinger store down", func(t *testing.T) { + want := errors.New("redis down") + m := NewResponseCacheMiddlewareWithStore(pingableStore{MapStore: cache.NewMapStore(), err: want}, time.Minute) + if !m.UsesRedis() { + t.Error("UsesRedis() = false for pinger store, want true") + } + if err := m.Ping(context.Background()); !errors.Is(err, want) { + t.Errorf("Ping() = %v, want %v", err, want) + } + }) +} diff --git a/internal/responsecache/responsecache.go b/internal/responsecache/responsecache.go index 94d758df..a199ee84 100644 --- a/internal/responsecache/responsecache.go +++ b/internal/responsecache/responsecache.go @@ -221,6 +221,30 @@ func (m *ResponseCacheMiddleware) HandleInternalRequest( }, nil } +// UsesRedis reports whether a Redis-backed exact (simple) cache is configured. +// Only then is the cache a meaningful readiness component worth probing. +func (m *ResponseCacheMiddleware) UsesRedis() bool { + if m == nil || m.simple == nil { + return false + } + _, ok := m.simple.store.(cache.Pinger) + return ok +} + +// Ping verifies connectivity to the Redis-backed exact cache. It returns nil +// when no Redis cache is configured, so callers should gate on UsesRedis first +// if they need to distinguish "not configured" from "reachable". +func (m *ResponseCacheMiddleware) Ping(ctx context.Context) error { + if m == nil || m.simple == nil { + return nil + } + pinger, ok := m.simple.store.(cache.Pinger) + if !ok { + return nil + } + return pinger.Ping(ctx) +} + // Close waits for any in-flight cache writes to complete, then releases cache resources. func (m *ResponseCacheMiddleware) Close() error { if m == nil { diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 474bf510..cab71786 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -44,6 +44,8 @@ type Handler struct { realtimeEnabled bool responseCache *responsecache.ResponseCacheMiddleware guardrailsHash string + storageProbe ReadinessProbe + cacheProbe ReadinessProbe translatedSvc *translatedInferenceService // snapshot of handler fields at first use; server.New sets cache/hash before traffic translatedSvcOnce sync.Once diff --git a/internal/server/http.go b/internal/server/http.go index 2243411b..2603c1cc 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -84,6 +84,16 @@ type Config struct { ResponseCacheMiddleware *responsecache.ResponseCacheMiddleware // Optional: response cache middleware for cacheable endpoints GuardrailsHash string // Optional: SHA-256 hash of active guardrail rules; stored in context post-patch for semantic cache IPExtractor echo.IPExtractor // Optional: trusted client IP extraction strategy for proxied deployments + StorageProbe ReadinessProbe // Optional: primary storage connectivity check; failure makes /health/ready report not_ready (503) + CacheProbe ReadinessProbe // Optional: Redis cache connectivity check; failure makes /health/ready report degraded (200, non-blocking) +} + +// ReadinessProbe verifies that a dependency the gateway owns is reachable. +// It is deliberately narrow so the server stays decoupled from concrete storage +// and cache types. Upstream provider reachability is intentionally NOT a probe: +// an external provider outage must not pull a healthy gateway out of rotation. +type ReadinessProbe interface { + Ping(ctx context.Context) error } // New creates a new HTTP server @@ -135,6 +145,8 @@ func New(provider core.RoutableProvider, cfg *Config) *Server { handler.keepOnlyAliasesAtModelsEndpoint = cfg.KeepOnlyAliasesAtModelsEndpoint handler.responseCache = cfg.ResponseCacheMiddleware handler.guardrailsHash = cfg.GuardrailsHash + handler.storageProbe = cfg.StorageProbe + handler.cacheProbe = cfg.CacheProbe } if cfg != nil && cfg.EnabledPassthroughProviders != nil { handler.setEnabledPassthroughProviders(cfg.EnabledPassthroughProviders) @@ -159,7 +171,7 @@ func New(provider core.RoutableProvider, cfg *Config) *Server { } // Build list of paths that skip authentication - authSkipPaths := []string{"/health"} + authSkipPaths := []string{"/health", "/health/ready"} // Determine metrics path metricsPath := "/metrics" @@ -282,6 +294,7 @@ func New(provider core.RoutableProvider, cfg *Config) *Server { // Public routes e.GET("/health", handler.Health) + e.GET("/health/ready", handler.Ready) registerSwagger(e, cfg) if cfg != nil && cfg.MetricsEnabled { e.GET(metricsPath, echo.WrapHandler(promhttp.Handler())) diff --git a/internal/server/readiness.go b/internal/server/readiness.go new file mode 100644 index 00000000..c3eec04c --- /dev/null +++ b/internal/server/readiness.go @@ -0,0 +1,91 @@ +package server + +import ( + "context" + "log/slog" + "net/http" + "time" + + "github.com/labstack/echo/v5" +) + +// readinessProbeTimeout caps each dependency check. It is intentionally shorter +// than the CLI --ready timeout (and the Docker/orchestrator probe timeout) so a +// slow dependency yields a clean not_ready/degraded response instead of the +// client cutting the connection on its own timeout. +const readinessProbeTimeout = 2 * time.Second + +// Readiness component and status values. +const ( + readyStatusReady = "ready" + readyStatusDegraded = "degraded" + readyStatusNotReady = "not_ready" + + readyComponentOK = "ok" + readyComponentDown = "down" +) + +// readinessResponse is the JSON body returned by GET /health/ready. +type readinessResponse struct { + Status string `json:"status"` + Components map[string]string `json:"components,omitempty"` +} + +// Ready handles GET /health/ready +// +// Readiness reports whether this instance should receive traffic. It probes +// dependencies the gateway owns: +// - Storage is required: if it is unreachable the gateway cannot serve +// requests, so the response is not_ready (HTTP 503). +// - The Redis exact cache is a performance optimization: if it is unreachable +// the gateway still serves requests, so the response is degraded (HTTP 200). +// +// Upstream provider reachability is deliberately excluded — a provider outage +// must not pull a healthy gateway out of rotation. Use GET /health for liveness. +// +// @Summary Readiness check +// @Tags system +// @Produce json +// @Success 200 {object} map[string]interface{} "ready or degraded" +// @Failure 503 {object} map[string]interface{} "not ready" +// @Router /health/ready [get] +func (h *Handler) Ready(c *echo.Context) error { + components := map[string]string{} + status := readyStatusReady + + if h.storageProbe != nil { + if err := pingWithTimeout(c.Request().Context(), h.storageProbe); err != nil { + components["storage"] = readyComponentDown + status = readyStatusNotReady + slog.Warn("readiness: storage probe failed", "error", err) + } else { + components["storage"] = readyComponentOK + } + } + + if h.cacheProbe != nil { + if err := pingWithTimeout(c.Request().Context(), h.cacheProbe); err != nil { + components["cache"] = readyComponentDown + if status == readyStatusReady { + status = readyStatusDegraded + } + slog.Warn("readiness: cache probe failed", "error", err) + } else { + components["cache"] = readyComponentOK + } + } + + code := http.StatusOK + if status == readyStatusNotReady { + code = http.StatusServiceUnavailable + } + return c.JSON(code, readinessResponse{Status: status, Components: components}) +} + +// pingWithTimeout runs a readiness probe with a bounded timeout, also honoring +// cancellation of the request context (whichever fires first). +func pingWithTimeout(ctx context.Context, probe ReadinessProbe) error { + ctx, cancel := context.WithTimeout(ctx, readinessProbeTimeout) + defer cancel() + return probe.Ping(ctx) +} diff --git a/internal/server/readiness_test.go b/internal/server/readiness_test.go new file mode 100644 index 00000000..3c968467 --- /dev/null +++ b/internal/server/readiness_test.go @@ -0,0 +1,104 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" +) + +type fakeProbe struct { + err error +} + +func (f fakeProbe) Ping(context.Context) error { return f.err } + +func TestReadyEndpoint(t *testing.T) { + tests := []struct { + name string + config *Config + wantStatusCode int + wantStatus string + wantComponents map[string]string + }{ + { + name: "no probes collapses to ready", + config: &Config{}, + wantStatusCode: http.StatusOK, + wantStatus: "ready", + wantComponents: map[string]string{}, + }, + { + name: "storage ok", + config: &Config{StorageProbe: fakeProbe{}}, + wantStatusCode: http.StatusOK, + wantStatus: "ready", + wantComponents: map[string]string{"storage": "ok"}, + }, + { + name: "storage down is not ready", + config: &Config{StorageProbe: fakeProbe{err: errors.New("boom")}}, + wantStatusCode: http.StatusServiceUnavailable, + wantStatus: "not_ready", + wantComponents: map[string]string{"storage": "down"}, + }, + { + name: "cache down is degraded but still serving", + config: &Config{StorageProbe: fakeProbe{}, CacheProbe: fakeProbe{err: errors.New("boom")}}, + wantStatusCode: http.StatusOK, + wantStatus: "degraded", + wantComponents: map[string]string{"storage": "ok", "cache": "down"}, + }, + { + name: "storage down dominates cache ok", + config: &Config{StorageProbe: fakeProbe{err: errors.New("boom")}, CacheProbe: fakeProbe{}}, + wantStatusCode: http.StatusServiceUnavailable, + wantStatus: "not_ready", + wantComponents: map[string]string{"storage": "down", "cache": "ok"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := New(&mockProvider{}, tt.config) + + req := httptest.NewRequest(http.MethodGet, "/health/ready", nil) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if rec.Code != tt.wantStatusCode { + t.Fatalf("status code = %d, want %d (body: %s)", rec.Code, tt.wantStatusCode, rec.Body.String()) + } + + var body readinessResponse + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode body: %v", err) + } + if body.Status != tt.wantStatus { + t.Errorf("status = %q, want %q", body.Status, tt.wantStatus) + } + for comp, want := range tt.wantComponents { + if got := body.Components[comp]; got != want { + t.Errorf("component %q = %q, want %q", comp, got, want) + } + } + if len(body.Components) != len(tt.wantComponents) { + t.Errorf("components = %v, want %v", body.Components, tt.wantComponents) + } + }) + } +} + +func TestReadyEndpointSkipsAuth(t *testing.T) { + srv := New(&mockProvider{}, &Config{MasterKey: "secret", StorageProbe: fakeProbe{}}) + + req := httptest.NewRequest(http.MethodGet, "/health/ready", nil) + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("readiness without auth = %d, want 200 (body: %s)", rec.Code, rec.Body.String()) + } +} diff --git a/internal/storage/mongodb.go b/internal/storage/mongodb.go index ee783292..8963161e 100644 --- a/internal/storage/mongodb.go +++ b/internal/storage/mongodb.go @@ -97,3 +97,11 @@ func (s *mongoStorage) Database() *mongo.Database { func (s *mongoStorage) Client() *mongo.Client { return s.client } + +// Ping verifies connectivity to MongoDB. +func (s *mongoStorage) Ping(ctx context.Context) error { + if s.client == nil { + return fmt.Errorf("mongodb client is not initialized") + } + return s.client.Ping(ctx, nil) +} diff --git a/internal/storage/postgresql.go b/internal/storage/postgresql.go index 3002b642..3dfd6228 100644 --- a/internal/storage/postgresql.go +++ b/internal/storage/postgresql.go @@ -60,3 +60,11 @@ func (s *postgresStorage) Close() error { func (s *postgresStorage) Pool() *pgxpool.Pool { return s.pool } + +// Ping verifies connectivity to PostgreSQL. +func (s *postgresStorage) Ping(ctx context.Context) error { + if s.pool == nil { + return fmt.Errorf("postgresql pool is not initialized") + } + return s.pool.Ping(ctx) +} diff --git a/internal/storage/sqlite.go b/internal/storage/sqlite.go index b33d4d44..ac44c587 100644 --- a/internal/storage/sqlite.go +++ b/internal/storage/sqlite.go @@ -1,6 +1,7 @@ package storage import ( + "context" "database/sql" "fmt" "os" @@ -61,6 +62,14 @@ func (s *sqliteStorage) DB() *sql.DB { return s.db } +// Ping verifies connectivity to the SQLite database. +func (s *sqliteStorage) Ping(ctx context.Context) error { + if s.db == nil { + return fmt.Errorf("sqlite database is not initialized") + } + return s.db.PingContext(ctx) +} + func (s *sqliteStorage) Close() error { if s.db != nil { return s.db.Close() diff --git a/internal/storage/sqlite_test.go b/internal/storage/sqlite_test.go index e9363148..79d2f115 100644 --- a/internal/storage/sqlite_test.go +++ b/internal/storage/sqlite_test.go @@ -9,6 +9,29 @@ import ( "time" ) +func TestSQLitePing(t *testing.T) { + store, err := NewSQLite(SQLiteConfig{Path: filepath.Join(t.TempDir(), "ping.db")}) + if err != nil { + t.Fatalf("failed to create SQLite storage: %v", err) + } + + hc, ok := store.(HealthChecker) + if !ok { + t.Fatalf("SQLite storage does not implement HealthChecker") + } + + if err := hc.Ping(context.Background()); err != nil { + t.Fatalf("Ping() error = %v, want nil", err) + } + + if err := store.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + if err := hc.Ping(context.Background()); err == nil { + t.Fatal("Ping() after Close() = nil, want error") + } +} + func TestSQLiteConcurrentWriteSafety(t *testing.T) { store, err := NewSQLite(SQLiteConfig{Path: filepath.Join(t.TempDir(), "test.db")}) if err != nil { diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 62447cf3..43ab1931 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -64,6 +64,13 @@ type Storage interface { Close() error } +// HealthChecker is implemented by storage backends that can verify +// connectivity to the underlying database. All concrete backends satisfy it; +// readiness checks type-assert against this interface. +type HealthChecker interface { + Ping(ctx context.Context) error +} + // SQLiteStorage exposes a SQLite database handle. type SQLiteStorage interface { Storage