diff --git a/apps/cli/src/next/commands/start/ui/StartDashboardView.unit.test.ts b/apps/cli/src/next/commands/start/ui/StartDashboardView.unit.test.ts index 57b1ad2fa7..331009b352 100644 --- a/apps/cli/src/next/commands/start/ui/StartDashboardView.unit.test.ts +++ b/apps/cli/src/next/commands/start/ui/StartDashboardView.unit.test.ts @@ -1,16 +1,17 @@ import { describe, expect, test } from "vitest"; +import { StackServiceState, type StackServiceStatus } from "@supabase/stack/effect"; import { ConnectionInfo } from "./ConnectionInfo.tsx"; -function state(name: string, status: string) { - return { +function state(name: string, status: StackServiceStatus, error: string | null = null) { + return new StackServiceState({ name, status, pid: null, exitCode: null, restartCount: 0, startedAt: null, - error: null, - } as any; + error, + }); } function collectNodes(node: unknown): Array { @@ -60,7 +61,7 @@ describe("StartDashboardView", () => { if (!("StartDashboardView" in dashboardModule)) return; const element = dashboardModule.StartDashboardView({ - states: [state("postgres", "Failed")], + states: [state("postgres", "Failed", "Health check failed and restart budget was exhausted")], info: { url: "http://127.0.0.1:54321", dbUrl: "postgresql://postgres:postgres@127.0.0.1:54322/postgres", diff --git a/packages/process-compose/docs/architecture.md b/packages/process-compose/docs/architecture.md index a7b97e0f5a..4e652d1237 100644 --- a/packages/process-compose/docs/architecture.md +++ b/packages/process-compose/docs/architecture.md @@ -40,7 +40,7 @@ construct its layer from a validated `ResolvedGraph`, a `ChildProcessSpawner` Ad - executable, arguments, environment, and working directory; - dependencies and a dependency wait timeout; -- optional HTTP, TCP, or exec health check; +- optional HTTP, TCP, or exec health check with separate startup and liveness failure thresholds; - shutdown signal and grace period; - restart policy and restart budget; - `started` and `healthy` lifecycle hooks; @@ -98,6 +98,10 @@ service maintained, and `stopped` records an explicit stop. Desired state is ind intermediate observed states and is what prevents an explicit stop from being undone by restart policy. +`pid` identifies only a currently live process. Process exit, supervisory termination, hook +failure, restart, and forced shutdown clear it. A health-triggered termination does not fabricate +an `exitCode`; the code remains `null` unless the process itself reports an exit. + `ServiceState` extends `Data.Class`, so Effect's `Equal.equals` can compare values structurally. It does not make two separately allocated objects equal under JavaScript `===`, and `SubscriptionRef.set` is not itself a distinct-update filter. Callers that want to suppress @@ -127,7 +131,8 @@ For each requested definition, `Orchestrator` runs this sequence: 9. Stream stdout and stderr into `LogBuffer`. 10. Run the health loop, or run `healthy` hooks immediately when no health check exists, then publish `Healthy`. -11. Race process exit against an unhealthy-restart request and apply restart policy. +11. Race process exit against unhealthy or hook failure, finalize the child, and apply restart + policy to process-exit and unhealthy causes. The service keeps the same state stream across restart generations. Restart backoff is `min(30 seconds, 2^(restartCount - 1))`. @@ -140,7 +145,8 @@ declaration order. Each hook has a timeout (default: 30 seconds) and either: - `fail`: publish `HookFailed` and stop running later hooks for that trigger; - `ignore`: append a diagnostic log and continue. -The supplied logger writes tagged stdout/stderr lines into the service's normal log stream. +The supplied logger writes tagged stdout/stderr lines into the service's normal log stream. A +failing hook finalizes its process generation before publishing terminal `Failed` state. ## Health and readiness @@ -151,17 +157,25 @@ The supplied logger writes tagged stdout/stderr lines into the service's normal - exec: successful for exit code `0`. After `initialDelaySeconds`, probes repeat every `periodSeconds`. Each attempt is bounded by -`timeoutSeconds`; consecutive success and failure counters reset each other. The current -implementation calls `onHealthy` after `successThreshold`, and calls `onUnhealthy` after -`failureThreshold` only after that same process generation has first become healthy. Consequently, -initial probe failures leave a service in `Running` rather than publishing `Unhealthy`; callers -must not treat the current generic `waitReady` Interface as a startup deadline. +`timeoutSeconds`; consecutive success and failure counters reset each other. Before a process +generation has ever become healthy, `startupFailureThreshold` controls when it becomes +`Unhealthy`. It defaults to `failureThreshold` for compatibility. After the first healthy result, +all later failures use `failureThreshold`, including after an unhealthy-to-healthy recovery. +Initial probe failures are therefore observable rather than leaving the service indefinitely in +`Running`. + +An unhealthy process uses the same pure restart-budget decision as a process exit. When restart is +enabled and the budget is exhausted, the supervisor terminates the child and publishes `Failed` +with `pid: null`, `exitCode: null`, and a stable health-exhaustion error. With restart policy `no`, +the live process remains observable as `Unhealthy`. `waitReady(name)` is intentionally unbounded. For long-running definitions it succeeds at -`Healthy` and fails at a non-restarting terminal state. For `restart: "no"` one-shot definitions, -successful completion is readiness. `waitAllReady()` considers only definitions whose desired -state is `running`, so intentionally inactive definitions do not block lazy callers. Higher-level -Modules own any finite user-facing deadline. +`Healthy` and fails at a non-restarting terminal state. A `restart: "no"` definition without a +health check is treated as one-shot work, where successful completion is readiness; a +health-checked definition remains subject to health readiness even when restart is disabled. +`waitAllReady()` considers only definitions whose desired state is `running`, so intentionally +inactive definitions do not block lazy callers. Higher-level Modules own any finite user-facing +deadline. ## Restart policies @@ -187,7 +201,9 @@ interrupts its lifecycle fiber. The generation finalizer: Whole-graph stop sets desired state first and then stops dependents before dependencies. The global shutdown budget defaults to 60 seconds. If that budget expires, the orchestrator logs the timeout -and clears all fibers; it does not fail with `ShutdownTimeoutError`. +and force-terminates active children before waiting for teardown to finish. Services that were +running reach terminal `Stopped` state with `pid: null` and exit code `143`; stop does not fail with +`ShutdownTimeoutError`. In-process cleanup and orphan supervision solve different failure modes: diff --git a/packages/process-compose/src/HealthProbe.ts b/packages/process-compose/src/HealthProbe.ts index 708a6b5b63..8664cc6100 100644 --- a/packages/process-compose/src/HealthProbe.ts +++ b/packages/process-compose/src/HealthProbe.ts @@ -69,13 +69,15 @@ export const runHealthProbe = (config: { const timeout = hc.timeoutSeconds ?? defaults.healthCheck.timeoutSeconds; const successThreshold = hc.successThreshold ?? defaults.healthCheck.successThreshold; const failureThreshold = hc.failureThreshold ?? defaults.healthCheck.failureThreshold; + const startupFailureThreshold = hc.startupFailureThreshold ?? failureThreshold; if (initialDelay > 0) { yield* Effect.sleep(Duration.seconds(initialDelay)); } const counters = yield* Ref.make({ successes: 0, failures: 0 }); - let isHealthy = false; + let phase: "Starting" | "Healthy" | "Unhealthy" = "Starting"; + let hasEverBeenHealthy = false; yield* Effect.repeat( Effect.gen(function* () { @@ -86,8 +88,9 @@ export const runHealthProbe = (config: { successes: c.successes + 1, failures: 0, })); - if (!isHealthy && successes + 1 >= successThreshold) { - isHealthy = true; + if (phase !== "Healthy" && successes + 1 >= successThreshold) { + phase = "Healthy"; + hasEverBeenHealthy = true; yield* config.callbacks.onHealthy(); } } else { @@ -95,8 +98,11 @@ export const runHealthProbe = (config: { successes: 0, failures: c.failures + 1, })); - if (isHealthy && failures + 1 >= failureThreshold) { - isHealthy = false; + const activeFailureThreshold = hasEverBeenHealthy + ? failureThreshold + : startupFailureThreshold; + if (phase !== "Unhealthy" && failures + 1 >= activeFailureThreshold) { + phase = "Unhealthy"; yield* config.callbacks.onUnhealthy(); } } diff --git a/packages/process-compose/src/HealthProbe.unit.test.ts b/packages/process-compose/src/HealthProbe.unit.test.ts index c4bcf4f6cb..a2ac95c4e7 100644 --- a/packages/process-compose/src/HealthProbe.unit.test.ts +++ b/packages/process-compose/src/HealthProbe.unit.test.ts @@ -15,6 +15,37 @@ const platformLayer = BunChildProcessSpawnerLayer.pipe( Layer.provide(Layer.mergeAll(BunFileSystemLayer, BunPathLayer)), ); +const sequenceProbeLayer = (results: ReadonlyArray) => { + let calls = 0; + return { + layer: Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make(() => + Effect.sync(() => { + const result = results[calls] ?? results.at(-1) ?? false; + calls++; + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(2000 + calls), + stdout: Stream.empty, + stderr: Stream.empty, + all: Stream.empty, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result ? 0 : 1)), + isRunning: Effect.succeed(false), + stdin: Sink.drain, + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ), + ), + get calls() { + return calls; + }, + }; +}; + const setupProbe = (probe: ProbeConfig, overrides?: Partial) => Effect.gen(function* () { let healthy = false; @@ -190,6 +221,103 @@ describe("HealthProbe", () => { }).pipe(Effect.provide(platformLayer)), ); + it.live("uses failureThreshold for startup when no startup threshold is configured", () => { + const probe = sequenceProbeLayer([false]); + return Effect.gen(function* () { + const { unhealthySignal, config } = yield* setupProbe( + { _tag: "Exec", command: "check", args: [] }, + { failureThreshold: 2 }, + ); + const fiber = yield* Effect.forkChild(runHealthProbe(config)); + yield* Deferred.await(unhealthySignal).pipe(Effect.timeout(Duration.seconds(1))); + expect(probe.calls).toBe(2); + yield* Fiber.interrupt(fiber); + }).pipe(Effect.provide(probe.layer)); + }); + + it.live("allows a larger startup threshold than the liveness threshold", () => { + const probe = sequenceProbeLayer([false]); + return Effect.gen(function* () { + const { unhealthySignal, config } = yield* setupProbe( + { _tag: "Exec", command: "check", args: [] }, + { startupFailureThreshold: 4, failureThreshold: 2 }, + ); + const fiber = yield* Effect.forkChild(runHealthProbe(config)); + yield* Deferred.await(unhealthySignal).pipe(Effect.timeout(Duration.seconds(1))); + expect(probe.calls).toBe(4); + yield* Fiber.interrupt(fiber); + }).pipe(Effect.provide(probe.layer)); + }); + + it.live("recovers on the final startup probe without becoming unhealthy", () => { + const probe = sequenceProbeLayer([false, false, true]); + return Effect.gen(function* () { + const { healthySignal, unhealthySignal, config } = yield* setupProbe( + { _tag: "Exec", command: "check", args: [] }, + { startupFailureThreshold: 3, failureThreshold: 1 }, + ); + const fiber = yield* Effect.forkChild(runHealthProbe(config)); + yield* Deferred.await(healthySignal).pipe(Effect.timeout(Duration.seconds(1))); + expect(probe.calls).toBe(3); + expect(yield* Deferred.isDone(unhealthySignal)).toBe(false); + yield* Fiber.interrupt(fiber); + }).pipe(Effect.provide(probe.layer)); + }); + + it.live("uses the liveness threshold after the first healthy transition", () => { + const probe = sequenceProbeLayer([true, false, false]); + return Effect.gen(function* () { + const { healthySignal, unhealthySignal, config } = yield* setupProbe( + { _tag: "Exec", command: "check", args: [] }, + { startupFailureThreshold: 5, failureThreshold: 2 }, + ); + const fiber = yield* Effect.forkChild(runHealthProbe(config)); + yield* Deferred.await(healthySignal).pipe(Effect.timeout(Duration.seconds(1))); + yield* Deferred.await(unhealthySignal).pipe(Effect.timeout(Duration.seconds(1))); + expect(probe.calls).toBe(3); + yield* Fiber.interrupt(fiber); + }).pipe(Effect.provide(probe.layer)); + }); + + it.live("does not re-enable startup tolerance after an unhealthy recovery", () => { + const probe = sequenceProbeLayer([true, false, false, true, false, false]); + return Effect.gen(function* () { + let healthyTransitions = 0; + let unhealthyTransitions = 0; + const secondUnhealthy = yield* Deferred.make(); + const fiber = yield* Effect.forkChild( + runHealthProbe({ + name: "test", + healthCheck: { + probe: { _tag: "Exec", command: "check", args: [] }, + periodSeconds: 0.01, + startupFailureThreshold: 5, + failureThreshold: 2, + }, + callbacks: { + onHealthy: () => + Effect.sync(() => { + healthyTransitions++; + }), + onUnhealthy: () => + Effect.gen(function* () { + unhealthyTransitions++; + if (unhealthyTransitions === 2) { + yield* Deferred.succeed(secondUnhealthy, void 0); + } + }), + }, + }), + ); + + yield* Deferred.await(secondUnhealthy).pipe(Effect.timeout(Duration.seconds(1))); + expect(probe.calls).toBe(6); + expect(healthyTransitions).toBe(2); + expect(unhealthyTransitions).toBe(2); + yield* Fiber.interrupt(fiber); + }).pipe(Effect.provide(probe.layer)); + }); + it.live("respects initialDelaySeconds before first probe", () => Effect.gen(function* () { const { healthySignal, config } = yield* setupProbe( diff --git a/packages/process-compose/src/Orchestrator.ts b/packages/process-compose/src/Orchestrator.ts index ed232dfcaa..dd1014cbd7 100644 --- a/packages/process-compose/src/Orchestrator.ts +++ b/packages/process-compose/src/Orchestrator.ts @@ -17,10 +17,14 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { buildGraph, type ResolvedGraph } from "./DependencyGraph.ts"; import { type HealthProbeCallbacks, runHealthProbe } from "./HealthProbe.ts"; import { LogBuffer } from "./LogBuffer.ts"; +import { + decideRestart, + type LifecycleCause, + UNHEALTHY_RESTART_EXHAUSTED_ERROR, +} from "./RestartDecision.ts"; import type { HookTrigger, OrchestratorConfig, - RestartPolicy, ServiceDef, ServiceStartOptions, } from "./ServiceDef.ts"; @@ -38,6 +42,19 @@ import { type ServiceEvent, transition } from "./ServiceTransition.ts"; const DIAGNOSTIC_LOG_LINES = 20; +const willRestartAfterExit = (def: ServiceDef, state: ServiceState): boolean => { + if (state.exitCode === null) return false; + return ( + decideRestart({ + cause: { _tag: "ProcessExit", exitCode: state.exitCode }, + policy: def.restart ?? defaults.restart, + restartCount: state.restartCount, + maxRestarts: def.maxRestarts ?? defaults.maxRestarts, + desired: state.desired, + })._tag === "Restart" + ); +}; + const waitForProcessToStop = (handle: { readonly isRunning: Effect.Effect; }): Effect.Effect => @@ -110,6 +127,7 @@ export class Orchestrator extends Context.Service< interface ServiceRuntime { readonly state: SubscriptionRef.SubscriptionRef; + effectiveDef: ServiceDef; } const services = new Map(); @@ -118,6 +136,7 @@ export class Orchestrator extends Context.Service< const stateRef = yield* SubscriptionRef.make(initial(def.name)); services.set(def.name, { state: stateRef, + effectiveDef: def, }); } @@ -156,7 +175,7 @@ export class Orchestrator extends Context.Service< ); // Helper: run all hooks for a given trigger in sequence - const runHooks = (def: ServiceDef, trigger: HookTrigger): Effect.Effect => + const runHooks = (def: ServiceDef, trigger: HookTrigger): Effect.Effect => Effect.gen(function* () { const hooks = (def.hooks ?? []).filter((h) => h.on === trigger); for (const hook of hooks) { @@ -167,11 +186,7 @@ export class Orchestrator extends Context.Service< .run(log) .pipe(Effect.timeout(Duration.seconds(timeout)), Effect.exit); if (Exit.isFailure(result) && (hook.failurePolicy ?? "fail") === "fail") { - yield* sendEvent(def.name, { - _tag: "HookFailed", - error: `Hook (on:${trigger}) failed: ${Cause.pretty(result.cause)}`, - }); - return; + return `Hook (on:${trigger}) failed: ${Cause.pretty(result.cause)}`; } if (Exit.isFailure(result)) { yield* logBuffer.append( @@ -181,13 +196,13 @@ export class Orchestrator extends Context.Service< ); } } + return null; }); type SpawnResult = - | { readonly _tag: "Exited"; readonly exitCode: number } - | { readonly _tag: "UnhealthyRestart" }; - - const shouldRestartOnUnhealthy = (policy: RestartPolicy): boolean => policy !== "no"; + | { readonly _tag: "ProcessExit"; readonly exitCode: number } + | { readonly _tag: "Unhealthy" } + | { readonly _tag: "HookFailed"; readonly error: string }; // The full lifecycle loop for a single service const runService = ( @@ -226,17 +241,41 @@ export class Orchestrator extends Context.Service< (state.status === "Stopped" && state.exitCode === 0)), ); } else if (condition === "healthy") { - yield* waitForState( + const ready = yield* waitForState( dependency, - (state) => state.desired === "running" && state.status === "Healthy", + (state) => + state.desired === "running" && + (state.status === "Healthy" || + ((state.status === "Failed" || state.status === "Stopped") && + !willRestartAfterExit(dependency.effectiveDef, state))), ); + if (ready.status !== "Healthy") { + yield* sendEvent(def.name, { + _tag: "DependencyFailed", + error: `Dependency ${depDef.name} failed: ${ + ready.error ?? + (ready.exitCode === null + ? "stopped before becoming healthy" + : `exited with code ${ready.exitCode} before becoming healthy`) + }`, + }); + return; + } } else if (condition === "completed") { const completed = yield* waitForState( dependency, (state) => - state.exitCode !== null && - (state.status === "Stopped" || state.status === "Failed"), + (state.status === "Failed" || + (state.status === "Stopped" && state.exitCode !== null)) && + !willRestartAfterExit(dependency.effectiveDef, state), ); + if (completed.status === "Failed") { + yield* sendEvent(def.name, { + _tag: "DependencyFailed", + error: `Dependency ${depDef.name} failed: ${completed.error ?? "unknown failure"}`, + }); + return; + } if (completed.exitCode !== 0) { yield* sendEvent(def.name, { _tag: "DependencyFailed", @@ -262,7 +301,7 @@ export class Orchestrator extends Context.Service< const spawnOnce = (): Effect.Effect => Effect.scoped( Effect.gen(function* () { - const unhealthyRestart = Deferred.makeUnsafe(); + const generationResult = Deferred.makeUnsafe(); const supervised = usesSupervisor(def); // Build command @@ -343,12 +382,9 @@ export class Orchestrator extends Context.Service< // Keep the service in Starting until its started hooks pass, // so Running is the stable dependency signal. - yield* runHooks(def, "started"); - const stateAfterStartedHooks = SubscriptionRef.getUnsafe( - services.get(def.name)!.state, - ); - if (stateAfterStartedHooks.status === "Failed") { - return { _tag: "Exited", exitCode: 1 } as SpawnResult; + const startedHookError = yield* runHooks(def, "started"); + if (startedHookError !== null) { + return { _tag: "HookFailed", error: startedHookError }; } yield* sendEvent(def.name, { _tag: "ProcessSpawned", @@ -387,9 +423,13 @@ export class Orchestrator extends Context.Service< const service = services.get(def.name); if (service === undefined) return; const current = SubscriptionRef.getUnsafe(service.state); - if (current.status === "Running") { - yield* runHooks(def, "healthy"); - if (SubscriptionRef.getUnsafe(service.state).status === "Failed") { + if (current.status === "Running" || current.status === "Unhealthy") { + const healthyHookError = yield* runHooks(def, "healthy"); + if (healthyHookError !== null) { + yield* Deferred.succeed(generationResult, { + _tag: "HookFailed", + error: healthyHookError, + }); return; } } @@ -403,8 +443,8 @@ export class Orchestrator extends Context.Service< `[health-check-failed] Service "${def.name}" became unhealthy. Recent output:`, `[health-check-failed] Service "${def.name}" became unhealthy (no recent log output).`, ); - if (shouldRestartOnUnhealthy(restartPolicy)) { - yield* Deferred.succeed(unhealthyRestart, void 0); + if (restartPolicy !== "no") { + yield* Deferred.succeed(generationResult, { _tag: "Unhealthy" }); } }), }; @@ -417,14 +457,11 @@ export class Orchestrator extends Context.Service< Effect.forkChild, ); } else { - yield* runHooks(def, "healthy"); - const service = services.get(def.name); - if ( - service !== undefined && - SubscriptionRef.getUnsafe(service.state).status !== "Failed" - ) { - yield* sendEvent(def.name, { _tag: "HealthCheckPassed" }); + const healthyHookError = yield* runHooks(def, "healthy"); + if (healthyHookError !== null) { + return { _tag: "HookFailed", error: healthyHookError }; } + yield* sendEvent(def.name, { _tag: "HealthCheckPassed" }); } // Race process exit against unhealthy restart signal. @@ -433,11 +470,11 @@ export class Orchestrator extends Context.Service< // as exit code 143 (128 + SIGTERM). const waitForExit = handle.exitCode.pipe( Effect.map( - (code): SpawnResult => ({ _tag: "Exited", exitCode: code as number }), + (code): SpawnResult => ({ _tag: "ProcessExit", exitCode: Number(code) }), ), Effect.catch( (): Effect.Effect => - Effect.succeed({ _tag: "Exited", exitCode: 143 }), + Effect.succeed({ _tag: "ProcessExit", exitCode: 143 }), ), ); const waitForObservedOneShotExit = @@ -448,7 +485,7 @@ export class Orchestrator extends Context.Service< Effect.timeout(Duration.millis(100)), Effect.catch( (): Effect.Effect => - Effect.succeed({ _tag: "Exited", exitCode: 0 }), + Effect.succeed({ _tag: "ProcessExit", exitCode: 0 }), ), ), ), @@ -458,9 +495,7 @@ export class Orchestrator extends Context.Service< return yield* Effect.raceAll([ waitForExit, waitForObservedOneShotExit, - Deferred.await(unhealthyRestart).pipe( - Effect.map((): SpawnResult => ({ _tag: "UnhealthyRestart" })), - ), + Deferred.await(generationResult), ]); }), ); @@ -480,7 +515,7 @@ export class Orchestrator extends Context.Service< // Handle spawn result const handleResult = (r: SpawnResult) => Effect.gen(function* () { - if (r._tag === "Exited") { + if (r._tag === "ProcessExit") { if (r.exitCode !== 0 && r.exitCode !== 143) { yield* appendRecentServiceLogs( def.name, @@ -489,27 +524,39 @@ export class Orchestrator extends Context.Service< ); } yield* sendEvent(def.name, { _tag: "ProcessExited", exitCode: r.exitCode }); + } else if (r._tag === "HookFailed") { + yield* sendEvent(def.name, { _tag: "HookFailed", error: r.error }); + } else { + yield* sendEvent(def.name, { _tag: "ProcessTerminated" }); } - // UnhealthyRestart: process killed by scope closure, skip ProcessExited + // Unhealthy is already recorded by the probe. Scope finalization + // terminates its process without inventing a process exit code. }); yield* handleResult(result); - // Restart loop - const shouldRestart = (r: SpawnResult): boolean => { + const restartDecision = (r: SpawnResult) => { const svc = services.get(def.name); - if (svc === undefined || SubscriptionRef.getUnsafe(svc.state).desired !== "running") { - return false; + const desired = + svc === undefined ? "inactive" : SubscriptionRef.getUnsafe(svc.state).desired; + if (r._tag === "HookFailed") { + return { _tag: "Terminate", reason: "PolicyDisabled" } as const; } - if (r._tag === "UnhealthyRestart") return true; - if (restartPolicy === "no") return false; - if (restartPolicy === "always") return true; - if (restartPolicy === "unless-stopped") return true; - if (restartPolicy === "on-failure") return r.exitCode !== 0; - return false; + const cause: LifecycleCause = + r._tag === "ProcessExit" + ? { _tag: "ProcessExit", exitCode: r.exitCode } + : { _tag: "Unhealthy" }; + return decideRestart({ + cause, + policy: restartPolicy, + restartCount, + maxRestarts, + desired, + }); }; - while (shouldRestart(result) && (maxRestarts === 0 || restartCount < maxRestarts)) { - restartCount++; + let decision = restartDecision(result); + while (decision._tag === "Restart") { + restartCount = decision.restartCount; yield* sendEvent(def.name, { _tag: "RestartTriggered", restartCount }); @@ -517,7 +564,7 @@ export class Orchestrator extends Context.Service< // be reserved safely for the duration of this restart's backoff. yield* prepareStart(); - if (result._tag === "UnhealthyRestart") { + if (result._tag === "Unhealthy") { yield* appendRecentServiceLogs( def.name, `[restart] Service "${def.name}" is restarting after an unhealthy health check. Recent output:`, @@ -535,11 +582,29 @@ export class Orchestrator extends Context.Service< result = yield* spawnOnce(); yield* handleResult(result); + decision = restartDecision(result); + } + + if ( + result._tag === "Unhealthy" && + decision._tag === "Terminate" && + decision.reason === "BudgetExhausted" + ) { + yield* sendEvent(def.name, { + _tag: "UnhealthyRestartExhausted", + error: UNHEALTHY_RESTART_EXHAUSTED_ERROR, + }); } }); const runServiceSafe = (def: ServiceDef, options?: ServiceStartOptions) => - runService(def, options).pipe( + Effect.sync(() => { + const service = services.get(def.name); + if (service !== undefined) { + service.effectiveDef = def; + } + }).pipe( + Effect.andThen(runService(def, options)), Effect.catch((error) => sendEvent(def.name, { _tag: "SpawnFailed", @@ -596,16 +661,8 @@ export class Orchestrator extends Context.Service< Effect.suspend(() => { const svc = services.get(def.name); if (!svc) return Effect.void; - const restartPolicy = def.restart ?? defaults.restart; - const maxRestarts = def.maxRestarts ?? defaults.maxRestarts; - const willRestartAfterExit = (state: ServiceState): boolean => { - if (state.desired !== "running" || state.exitCode === null) return false; - if (maxRestarts !== 0 && state.restartCount >= maxRestarts) return false; - if (restartPolicy === "always" || restartPolicy === "unless-stopped") return true; - return restartPolicy === "on-failure" && state.exitCode !== 0; - }; - const current = SubscriptionRef.getUnsafe(svc.state); + const effectiveDef = svc.effectiveDef; if (current.desired !== "running") { return Effect.fail( new ServiceReadyError({ @@ -617,7 +674,7 @@ export class Orchestrator extends Context.Service< }), ); } - if (current.status === "Failed" && !willRestartAfterExit(current)) { + if (current.status === "Failed" && !willRestartAfterExit(effectiveDef, current)) { return Effect.fail( new ServiceReadyError({ name: def.name, @@ -626,7 +683,10 @@ export class Orchestrator extends Context.Service< ); } - if (restartPolicy === "no") { + if ( + (effectiveDef.restart ?? defaults.restart) === "no" && + effectiveDef.healthCheck == null + ) { return waitForState( svc, (state) => state.status === "Failed" || state.status === "Stopped", @@ -653,7 +713,7 @@ export class Orchestrator extends Context.Service< (state) => state.status === "Healthy" || ((state.status === "Failed" || state.status === "Stopped") && - !willRestartAfterExit(state)), + !willRestartAfterExit(effectiveDef, state)), ).pipe( Effect.flatMap((ready) => { if (ready.status === "Healthy") return Effect.void; @@ -701,7 +761,8 @@ export class Orchestrator extends Context.Service< d.name !== name && service !== undefined && SubscriptionRef.getUnsafe(service.state).desired !== "stopped" && - restartPolicy === "no" + restartPolicy === "no" && + d.healthCheck == null ) { continue; } diff --git a/packages/process-compose/src/Orchestrator.unit.test.ts b/packages/process-compose/src/Orchestrator.unit.test.ts index 41b75d47b5..954ce355b6 100644 --- a/packages/process-compose/src/Orchestrator.unit.test.ts +++ b/packages/process-compose/src/Orchestrator.unit.test.ts @@ -636,6 +636,48 @@ describe("Orchestrator", () => { }).pipe(Effect.provide(layer), Effect.scoped); }); + it.live("startService restarts a stopped health-checked dependency", () => { + const { layer, proc } = setupOrchestrator( + [ + svc("db", { + restart: "no", + healthCheck: { + probe: { _tag: "Exec", command: "check", args: [] }, + periodSeconds: 0.01, + }, + }), + svc("api", { + dependencies: [{ service: "db", condition: "healthy" }], + }), + svc("worker", { + dependencies: [{ service: "db", condition: "healthy" }], + dependencyTimeoutSeconds: 1, + }), + ], + { + exitDelay: "5 seconds", + perService: { + db: { exitCode: 0, exitDelay: "50 millis" }, + check: { exitCode: 0, exitDelay: "1 millis" }, + }, + }, + ); + + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.startService("api"); + yield* proc.waitForSpawn("api"); + yield* waitForStopped(orc, "db"); + + yield* orc.startService("worker"); + yield* proc.waitForSpawn("db", 2); + yield* proc.waitForSpawn("worker"); + + expect(proc.spawned.filter((spawn) => spawn.command === "db")).toHaveLength(2); + expect((yield* orc.getState("worker")).status).not.toBe("Failed"); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + it.live("startService reruns a directly requested successful one-shot", () => { const { layer, proc } = setupOrchestrator([svc("setup", { restart: "no" })]); return Effect.gen(function* () { @@ -1072,6 +1114,7 @@ describe("Orchestrator", () => { const state = yield* orc.getState("a"); expect(state.status).toBe("Stopped"); expect(state.exitCode).toBe(0); + expect(state.pid).toBeNull(); }).pipe(Effect.provide(layer), Effect.scoped); }); @@ -1088,6 +1131,7 @@ describe("Orchestrator", () => { const state = yield* orc.getState("a"); expect(state.status).toBe("Failed"); expect(state.exitCode).toBe(1); + expect(state.pid).toBeNull(); expect(log.entries.some((e) => e.line.includes("[process-exited]"))).toBe(true); expect(log.entries.some((e) => e.line.includes("about to fail"))).toBe(true); }).pipe(Effect.provide(layer), Effect.scoped); @@ -1122,6 +1166,183 @@ describe("Orchestrator", () => { }).pipe(Effect.provide(layer), Effect.scoped); }); + it.live("healthy dependency fails immediately when health restarts are exhausted", () => { + const { layer } = setupOrchestrator( + [ + svc("db", { + restart: "always", + maxRestarts: 1, + healthCheck: { + probe: { _tag: "Exec", command: "check", args: [] }, + periodSeconds: 0.01, + startupFailureThreshold: 1, + failureThreshold: 1, + }, + }), + svc("api", { + restart: "no", + dependencies: [{ service: "db", condition: "healthy" }], + dependencyTimeoutSeconds: 5, + }), + ], + { + exitDelay: "5 seconds", + perService: { check: { exitCode: 1, exitDelay: "1 millis" } }, + }, + ); + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + const state = yield* waitForFailed(orc, "api"); + expect(state.error).toContain("Dependency db failed"); + expect(state.error).not.toContain("Timed out"); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + + it.live("healthy dependency waits through a transient failed restart", () => { + let databaseSpawns = 0; + const { layer, proc } = setupOrchestrator( + [ + svc("db", { + restart: "always", + maxRestarts: 1, + healthCheck: { + probe: { _tag: "Exec", command: "check", args: [] }, + initialDelaySeconds: 0.05, + periodSeconds: 0.05, + }, + }), + svc("api", { + restart: "no", + dependencies: [{ service: "db", condition: "healthy" }], + dependencyTimeoutSeconds: 5, + }), + ], + { + perService: { + db: { + exitCode: 1, + getExitDelay: () => (++databaseSpawns === 1 ? "10 millis" : "5 seconds"), + }, + check: { exitCode: 0, exitDelay: "1 millis" }, + }, + }, + ); + + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + yield* proc.waitForSpawn("api"); + + expect(proc.spawned.filter((spawn) => spawn.command === "db")).toHaveLength(2); + expect((yield* orc.getState("api")).status).not.toBe("Failed"); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + + it.live("healthy dependency fails after clean-exit restarts are exhausted", () => { + const { layer, proc } = setupOrchestrator( + [ + svc("db", { + restart: "always", + maxRestarts: 1, + healthCheck: { + probe: { _tag: "Exec", command: "check", args: [] }, + initialDelaySeconds: 5, + }, + }), + svc("api", { + restart: "no", + dependencies: [{ service: "db", condition: "healthy" }], + dependencyTimeoutSeconds: 3, + }), + ], + { + perService: { + db: { exitCode: 0, exitDelay: "10 millis" }, + check: { exitCode: 0, exitDelay: "1 millis" }, + }, + }, + ); + + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + + const state = yield* waitForFailed(orc, "api"); + expect(state.error).toContain("Dependency db failed"); + expect(state.error).not.toContain("Timed out"); + expect(proc.spawned.filter((spawn) => spawn.command === "db")).toHaveLength(2); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + + it.live("completed dependency waits through a transient failed restart", () => { + let setupSpawns = 0; + const { layer, proc } = setupOrchestrator( + [ + svc("setup", { + restart: "on-failure", + maxRestarts: 1, + }), + svc("app", { + restart: "no", + dependencies: [{ service: "setup", condition: "completed" }], + dependencyTimeoutSeconds: 5, + }), + ], + { + perService: { + setup: { + getExitCode: () => (++setupSpawns === 1 ? 1 : 0), + exitDelay: "10 millis", + }, + app: { exitDelay: "5 seconds" }, + }, + }, + ); + + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + yield* proc.waitForSpawn("app"); + + expect(proc.spawned.filter((spawn) => spawn.command === "setup")).toHaveLength(2); + expect((yield* orc.getState("app")).status).not.toBe("Failed"); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + + it.live("dependency waits use the running generation restart policy", () => { + const { layer, proc } = setupOrchestrator( + [ + svc("setup", { restart: "no" }), + svc("app", { + restart: "no", + dependencies: [{ service: "setup", condition: "completed" }], + dependencyTimeoutSeconds: 1, + }), + ], + { + perService: { + setup: { exitCode: 1, exitDelay: "200 millis" }, + app: { exitDelay: "5 seconds" }, + }, + }, + ); + + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + yield* proc.waitForSpawn("setup"); + + // The replacement applies on the next explicit restart. The running + // generation still uses restart:no and must remain authoritative. + yield* orc.updateServiceDefinition("setup", svc("setup", { restart: "always" })); + + const state = yield* waitForFailed(orc, "app"); + expect(state.error).toContain("Dependency setup failed"); + expect(state.error).not.toContain("Timed out"); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + it.live("no timeout when dependency resolves before deadline", () => { const { layer } = setupOrchestrator( [ @@ -1170,6 +1391,39 @@ describe("Orchestrator", () => { expect(state.error).toContain("Timed out"); }).pipe(Effect.provide(layer), Effect.scoped); }); + + it.live("completed dependency fails immediately when health restarts are exhausted", () => { + const { layer } = setupOrchestrator( + [ + svc("setup", { + restart: "always", + maxRestarts: 1, + healthCheck: { + probe: { _tag: "Exec", command: "check", args: [] }, + periodSeconds: 0.01, + startupFailureThreshold: 1, + failureThreshold: 1, + }, + }), + svc("app", { + restart: "no", + dependencies: [{ service: "setup", condition: "completed" }], + dependencyTimeoutSeconds: 5, + }), + ], + { + exitDelay: "5 seconds", + perService: { check: { exitCode: 1, exitDelay: "1 millis" } }, + }, + ); + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + const state = yield* waitForFailed(orc, "app"); + expect(state.error).toContain("Dependency setup failed"); + expect(state.error).not.toContain("Timed out"); + }).pipe(Effect.provide(layer), Effect.scoped); + }); }); describe("failure diagnostics", () => { @@ -1269,6 +1523,88 @@ describe("Orchestrator", () => { }).pipe(Effect.provide(layer), Effect.scoped); }); + it.live("runs on:healthy hook when an unhealthy service recovers", () => { + let checkCalls = 0; + let healthyHookRuns = 0; + const { layer } = setupOrchestrator( + [ + svc("a", { + restart: "no", + healthCheck: { + probe: { _tag: "Exec", command: "check", args: [] }, + periodSeconds: 0.01, + startupFailureThreshold: 1, + successThreshold: 1, + failureThreshold: 1, + }, + hooks: [ + { + on: "healthy", + run: () => + Effect.sync(() => { + healthyHookRuns++; + }), + }, + ], + }), + ], + { + exitDelay: "5 seconds", + perService: { + check: { + exitDelay: "1 millis", + getExitCode: () => (++checkCalls === 1 ? 1 : 0), + }, + }, + }, + ); + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + yield* waitForState(orc, "a", (state) => state.status === "Unhealthy", "Unhealthy"); + yield* waitForHealthy(orc, "a"); + expect(healthyHookRuns).toBe(1); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + + it.live("publishes a failed recovery hook as a terminal failure", () => { + let checkCalls = 0; + const { layer, proc } = setupOrchestrator( + [ + svc("a", { + restart: "no", + healthCheck: { + probe: { _tag: "Exec", command: "check", args: [] }, + periodSeconds: 0.01, + startupFailureThreshold: 1, + successThreshold: 1, + failureThreshold: 1, + }, + hooks: [{ on: "healthy", run: () => Effect.fail(new Error("recovery failed")) }], + }), + ], + { + exitDelay: "5 seconds", + perService: { + check: { + exitDelay: "1 millis", + getExitCode: () => (++checkCalls === 1 ? 1 : 0), + }, + }, + }, + ); + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + yield* waitForState(orc, "a", (state) => state.status === "Unhealthy", "Unhealthy"); + const state = yield* waitForFailed(orc, "a"); + expect(proc.killed.some((record) => record.command === "a")).toBe(true); + expect(state.pid).toBeNull(); + expect(state.error).toContain("on:healthy"); + expect(state.error).toContain("recovery failed"); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + it.live("dependent waits for on:started hook to complete before starting", () => { const order: string[] = []; const { layer } = setupOrchestrator( @@ -1656,6 +1992,10 @@ describe("Orchestrator", () => { yield* orc.stop(); const elapsed = Date.now() - before; expect(elapsed).toBeLessThan(3000); + const state = yield* orc.getState("stuck"); + expect(state.status).toBe("Stopped"); + expect(state.pid).toBeNull(); + expect(state.exitCode).toBe(143); expect(proc.killed).toEqual(["SIGTERM", "SIGKILL"]); }).pipe(Effect.provide(layer), Effect.scoped); }); @@ -1857,14 +2197,112 @@ describe("Orchestrator", () => { const orc = yield* Orchestrator; yield* orc.start(); yield* proc.waitForSpawn("a", 2); - yield* waitForState(orc, "a", (state) => state.status === "Unhealthy", "Unhealthy"); + const state = yield* waitForState( + orc, + "a", + (candidate) => candidate.status === "Failed", + "Failed", + ); // maxRestarts=1 means original spawn + 1 restart = 2 total const mainSpawns = proc.spawned.filter((s) => s.command === "a"); expect(mainSpawns.length).toBe(2); + expect(state.pid).toBeNull(); + expect(state.exitCode).toBeNull(); + expect(state.error).toBe("Health check failed and restart budget was exhausted"); + + const readyError = yield* orc.waitReady("a").pipe(Effect.flip); + expect(readyError._tag).toBe("ServiceReadyError"); + if (readyError._tag === "ServiceReadyError") { + expect(readyError.reason).toBe("Health check failed and restart budget was exhausted"); + } + }).pipe(Effect.provide(layer), Effect.scoped); + }); + + it.live("applies startup failures before a service has ever been healthy", () => { + const { layer, proc } = setupOrchestrator( + [ + svc("a", { + restart: "always", + maxRestarts: 1, + healthCheck: { + probe: { _tag: "Exec", command: "check", args: [] }, + periodSeconds: 0.01, + startupFailureThreshold: 2, + failureThreshold: 1, + }, + }), + ], + { + exitDelay: "5 seconds", + perService: { check: { exitCode: 1, exitDelay: "1 millis" } }, + }, + ); + + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + const state = yield* waitForFailed(orc, "a"); + expect(proc.spawned.filter((spawn) => spawn.command === "a")).toHaveLength(2); + expect(state.pid).toBeNull(); + expect(state.exitCode).toBeNull(); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + + it.live("keeps an initially unhealthy no-restart service alive", () => { + const { layer, proc } = setupOrchestrator( + [ + svc("a", { + restart: "no", + healthCheck: { + probe: { _tag: "Exec", command: "check", args: [] }, + periodSeconds: 0.01, + failureThreshold: 1, + }, + }), + ], + { + exitDelay: "5 seconds", + perService: { check: { exitCode: 1, exitDelay: "1 millis" } }, + }, + ); + + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + const state = yield* waitForState( + orc, + "a", + (candidate) => candidate.status === "Unhealthy", + "Unhealthy", + ); + expect(state.pid).not.toBeNull(); + expect(state.exitCode).toBeNull(); + expect(proc.spawned.filter((spawn) => spawn.command === "a")).toHaveLength(1); }).pipe(Effect.provide(layer), Effect.scoped); }); }); + it.live("finalizes the child before publishing a healthy-hook failure", () => { + const { layer, proc } = setupOrchestrator( + [ + svc("a", { + hooks: [{ on: "healthy", run: () => Effect.fail(new Error("warmup failed")) }], + }), + ], + { exitDelay: "5 seconds" }, + ); + + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + const state = yield* waitForFailed(orc, "a"); + expect(proc.killed.some((record) => record.command === "a")).toBe(true); + expect(state.pid).toBeNull(); + expect(state.exitCode).toBeNull(); + expect(state.error).toContain("on:healthy"); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + describe("readiness", () => { it.live("waitReady resolves when long-running service becomes healthy", () => { const { layer } = setupOrchestrator([svc("a")], { @@ -1914,6 +2352,26 @@ describe("Orchestrator", () => { }).pipe(Effect.provide(layer), Effect.scoped); }); + it.live("waitReady uses the running generation restart policy", () => { + const { layer, proc } = setupOrchestrator([svc("a", { restart: "no" })], { + exitCode: 1, + exitDelay: "200 millis", + }); + + return Effect.gen(function* () { + const orc = yield* Orchestrator; + yield* orc.start(); + yield* proc.waitForSpawn("a"); + + // The replacement applies to the next generation; this one still + // terminates under restart:no and readiness must fail with it. + yield* orc.updateServiceDefinition("a", svc("a", { restart: "always" })); + + const exit = yield* orc.waitReady("a").pipe(Effect.exit, Effect.timeout("1 second")); + expect(Exit.isFailure(exit)).toBe(true); + }).pipe(Effect.provide(layer), Effect.scoped); + }); + it.live("waitReady resolves when one-shot service completes successfully", () => { const { layer } = setupOrchestrator([svc("a", { restart: "no" })], { exitCode: 0, diff --git a/packages/process-compose/src/RestartDecision.ts b/packages/process-compose/src/RestartDecision.ts new file mode 100644 index 0000000000..dc1d2fc5ae --- /dev/null +++ b/packages/process-compose/src/RestartDecision.ts @@ -0,0 +1,49 @@ +import type { RestartPolicy } from "./ServiceDef.ts"; +import type { ServiceDesiredState } from "./ServiceState.ts"; + +export type LifecycleCause = + | { readonly _tag: "ProcessExit"; readonly exitCode: number } + | { readonly _tag: "Unhealthy" }; + +export type RestartDecision = + | { readonly _tag: "Restart"; readonly restartCount: number } + | { + readonly _tag: "Terminate"; + readonly reason: "NotDesired" | "PolicyDisabled" | "BudgetExhausted"; + } + | { readonly _tag: "KeepRunningUnhealthy" }; + +export const UNHEALTHY_RESTART_EXHAUSTED_ERROR = + "Health check failed and restart budget was exhausted"; + +export function decideRestart(options: { + readonly cause: LifecycleCause; + readonly policy: RestartPolicy; + readonly restartCount: number; + readonly maxRestarts: number; + readonly desired: ServiceDesiredState; +}): RestartDecision { + if (options.desired !== "running") { + return { _tag: "Terminate", reason: "NotDesired" }; + } + + if (options.cause._tag === "Unhealthy" && options.policy === "no") { + return { _tag: "KeepRunningUnhealthy" }; + } + + const policyAllowsRestart = + options.policy === "always" || + options.policy === "unless-stopped" || + (options.policy === "on-failure" && + (options.cause._tag === "Unhealthy" || options.cause.exitCode !== 0)); + + if (!policyAllowsRestart) { + return { _tag: "Terminate", reason: "PolicyDisabled" }; + } + + if (options.maxRestarts !== 0 && options.restartCount >= options.maxRestarts) { + return { _tag: "Terminate", reason: "BudgetExhausted" }; + } + + return { _tag: "Restart", restartCount: options.restartCount + 1 }; +} diff --git a/packages/process-compose/src/RestartDecision.unit.test.ts b/packages/process-compose/src/RestartDecision.unit.test.ts new file mode 100644 index 0000000000..f8d751daa5 --- /dev/null +++ b/packages/process-compose/src/RestartDecision.unit.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { decideRestart, type LifecycleCause } from "./RestartDecision.ts"; +import type { RestartPolicy } from "./ServiceDef.ts"; + +const exit = (exitCode: number): LifecycleCause => ({ _tag: "ProcessExit", exitCode }); +const unhealthy: LifecycleCause = { _tag: "Unhealthy" }; + +describe("decideRestart", () => { + it.each([ + ["no", exit(0), "Terminate"], + ["no", exit(1), "Terminate"], + ["no", unhealthy, "KeepRunningUnhealthy"], + ["on-failure", exit(0), "Terminate"], + ["on-failure", exit(1), "Restart"], + ["on-failure", unhealthy, "Restart"], + ["always", exit(0), "Restart"], + ["always", exit(1), "Restart"], + ["always", unhealthy, "Restart"], + ["unless-stopped", exit(0), "Restart"], + ["unless-stopped", exit(1), "Restart"], + ["unless-stopped", unhealthy, "Restart"], + ] as const)("applies %s to %o", (policy, cause, expected) => { + expect( + decideRestart({ + policy, + cause, + restartCount: 0, + maxRestarts: 1, + desired: "running", + })._tag, + ).toBe(expected); + }); + + it.each(["no", "on-failure", "always", "unless-stopped"] as const)( + "never restarts %s after stop was requested", + (policy: RestartPolicy) => { + expect( + decideRestart({ + policy, + cause: exit(1), + restartCount: 0, + maxRestarts: 0, + desired: "stopped", + }), + ).toEqual({ _tag: "Terminate", reason: "NotDesired" }); + }, + ); + + it("treats zero maxRestarts as unlimited", () => { + expect( + decideRestart({ + policy: "always", + cause: unhealthy, + restartCount: 1_000, + maxRestarts: 0, + desired: "running", + }), + ).toEqual({ _tag: "Restart", restartCount: 1_001 }); + }); + + it("terminates when the restart budget is exhausted", () => { + expect( + decideRestart({ + policy: "always", + cause: unhealthy, + restartCount: 2, + maxRestarts: 2, + desired: "running", + }), + ).toEqual({ _tag: "Terminate", reason: "BudgetExhausted" }); + }); +}); diff --git a/packages/process-compose/src/ServiceDef.ts b/packages/process-compose/src/ServiceDef.ts index 3f9f857139..d015fcdae1 100644 --- a/packages/process-compose/src/ServiceDef.ts +++ b/packages/process-compose/src/ServiceDef.ts @@ -30,6 +30,8 @@ export interface HealthCheckConfig { readonly periodSeconds?: number; readonly timeoutSeconds?: number; readonly successThreshold?: number; + /** Consecutive failures allowed before the process has ever become healthy. */ + readonly startupFailureThreshold?: number; readonly failureThreshold?: number; } diff --git a/packages/process-compose/src/ServiceTransition.ts b/packages/process-compose/src/ServiceTransition.ts index 662022ba57..94c28528ed 100644 --- a/packages/process-compose/src/ServiceTransition.ts +++ b/packages/process-compose/src/ServiceTransition.ts @@ -16,6 +16,8 @@ export type ServiceEvent = } | { readonly _tag: "HealthCheckPassed" } | { readonly _tag: "HealthCheckFailed" } + | { readonly _tag: "ProcessTerminated" } + | { readonly _tag: "UnhealthyRestartExhausted"; readonly error: string } | { readonly _tag: "ProcessExited"; readonly exitCode: number } | { readonly _tag: "StopRequested" } | { @@ -39,6 +41,7 @@ const allowed = new Set<`${ServiceStatus}:${ServiceEvent["_tag"]}`>([ "Starting:StopRequested", "Starting:HookFailed", "Running:HealthCheckPassed", + "Running:HealthCheckFailed", "Running:ProcessExited", "Running:StopRequested", "Healthy:HealthCheckPassed", @@ -47,6 +50,7 @@ const allowed = new Set<`${ServiceStatus}:${ServiceEvent["_tag"]}`>([ "Healthy:StopRequested", "Unhealthy:HealthCheckPassed", "Unhealthy:ProcessExited", + "Unhealthy:ProcessTerminated", "Unhealthy:StopRequested", "Stopping:ProcessExited", "Stopped:RestartTriggered", @@ -54,11 +58,13 @@ const allowed = new Set<`${ServiceStatus}:${ServiceEvent["_tag"]}`>([ "Failed:ProcessExited", "Failed:StopRequested", "Unhealthy:RestartTriggered", + "Unhealthy:UnhealthyRestartExhausted", "Restarting:StopRequested", "Restarting:SpawnFailed", "Restarting:BackoffElapsed", "Running:HookFailed", "Healthy:HookFailed", + "Unhealthy:HookFailed", ]); // --------------------------------------------------------------------------- @@ -78,6 +84,8 @@ export const applyEvent = (state: ServiceState, event: ServiceEvent): ServiceSta return new ServiceState({ ...state, status: "Failed", + pid: null, + exitCode: null, error: event.error, }); @@ -95,12 +103,25 @@ export const applyEvent = (state: ServiceState, event: ServiceEvent): ServiceSta case "HealthCheckFailed": return new ServiceState({ ...state, status: "Unhealthy" }); + case "UnhealthyRestartExhausted": + return new ServiceState({ + ...state, + status: "Failed", + pid: null, + exitCode: null, + error: event.error, + }); + + case "ProcessTerminated": + return new ServiceState({ ...state, pid: null }); + case "ProcessExited": { const status: ServiceStatus = state.status === "Stopping" ? "Stopped" : event.exitCode === 0 ? "Stopped" : "Failed"; return new ServiceState({ ...state, status, + pid: null, exitCode: event.exitCode, }); } @@ -116,6 +137,7 @@ export const applyEvent = (state: ServiceState, event: ServiceEvent): ServiceSta return new ServiceState({ ...state, status: "Restarting", + pid: null, restartCount: event.restartCount, }); @@ -133,6 +155,7 @@ export const applyEvent = (state: ServiceState, event: ServiceEvent): ServiceSta return new ServiceState({ ...state, status: "Failed", + pid: null, error: event.error, }); } diff --git a/packages/process-compose/src/ServiceTransition.unit.test.ts b/packages/process-compose/src/ServiceTransition.unit.test.ts index 08a295e6f7..58b41524f6 100644 --- a/packages/process-compose/src/ServiceTransition.unit.test.ts +++ b/packages/process-compose/src/ServiceTransition.unit.test.ts @@ -28,13 +28,15 @@ describe("ServiceTransition", () => { }); it("Pending + DependencyFailed → Failed with error", () => { - const state = make("api"); + const state = make("api", { pid: 1234, exitCode: 1 }); const next = applyEvent(state, { _tag: "DependencyFailed", error: "db exited with code 1", }); expect(next).not.toBeNull(); expect(next!.status).toBe("Failed"); + expect(next!.pid).toBeNull(); + expect(next!.exitCode).toBeNull(); expect(next!.error).toBe("db exited with code 1"); }); @@ -52,11 +54,13 @@ describe("ServiceTransition", () => { }); it("Starting + SpawnFailed → Failed with error", () => { - const result = applyEvent(make("db", { status: "Starting" }), { + const result = applyEvent(make("db", { status: "Starting", pid: 1234, exitCode: 1 }), { _tag: "SpawnFailed", error: "spawn gate failed", }); expect(result?.status).toBe("Failed"); + expect(result?.pid).toBeNull(); + expect(result?.exitCode).toBeNull(); expect(result?.error).toBe("spawn gate failed"); }); @@ -91,6 +95,7 @@ describe("ServiceTransition", () => { expect(next).not.toBeNull(); expect(next!.status).toBe("Stopped"); expect(next!.exitCode).toBe(0); + expect(next!.pid).toBeNull(); }); it("Running + ProcessExited(1) → Failed", () => { @@ -99,6 +104,7 @@ describe("ServiceTransition", () => { expect(next).not.toBeNull(); expect(next!.status).toBe("Failed"); expect(next!.exitCode).toBe(1); + expect(next!.pid).toBeNull(); }); it("Running + StopRequested → Stopping", () => { @@ -108,6 +114,13 @@ describe("ServiceTransition", () => { expect(next!.status).toBe("Stopping"); }); + it("Running + HealthCheckFailed → Unhealthy", () => { + const state = make("db", { status: "Running", pid: 1234 }); + const next = applyEvent(state, { _tag: "HealthCheckFailed" }); + expect(next?.status).toBe("Unhealthy"); + expect(next?.pid).toBe(1234); + }); + it("Healthy + HealthCheckFailed → Unhealthy", () => { const state = make("db", { status: "Healthy", pid: 1234 }); const next = applyEvent(state, { _tag: "HealthCheckFailed" }); @@ -219,6 +232,27 @@ describe("ServiceTransition", () => { expect(next).not.toBeNull(); expect(next!.status).toBe("Restarting"); expect(next!.restartCount).toBe(1); + expect(next!.pid).toBeNull(); + }); + + it("Unhealthy + UnhealthyRestartExhausted → terminal Failed", () => { + const state = make("db", { status: "Unhealthy", pid: 1234, exitCode: null }); + const next = applyEvent(state, { + _tag: "UnhealthyRestartExhausted", + error: "Health check failed and restart budget was exhausted", + }); + expect(next?.status).toBe("Failed"); + expect(next?.pid).toBeNull(); + expect(next?.exitCode).toBeNull(); + expect(next?.error).toBe("Health check failed and restart budget was exhausted"); + }); + + it("Unhealthy + ProcessTerminated clears its no-longer-live pid", () => { + const state = make("db", { status: "Unhealthy", pid: 1234 }); + const next = applyEvent(state, { _tag: "ProcessTerminated" }); + expect(next?.status).toBe("Unhealthy"); + expect(next?.pid).toBeNull(); + expect(next?.exitCode).toBeNull(); }); it("Pending + StopRequested → Stopped (no process to kill)", () => { @@ -324,6 +358,7 @@ describe("ServiceTransition", () => { expect(next).not.toBeNull(); expect(next!.status).toBe("Failed"); expect(next!.error).toBe("migration failed"); + expect(next!.pid).toBeNull(); }); it("Healthy + HookFailed → Failed with error", () => { @@ -334,6 +369,15 @@ describe("ServiceTransition", () => { expect(next!.error).toBe("seed failed"); }); + it("Unhealthy + HookFailed → Failed with error", () => { + const state = make("db", { status: "Unhealthy", pid: 1234, startedAt: 1000 }); + const next = applyEvent(state, { _tag: "HookFailed", error: "recovery failed" }); + expect(next).not.toBeNull(); + expect(next!.status).toBe("Failed"); + expect(next!.error).toBe("recovery failed"); + expect(next!.pid).toBeNull(); + }); + it("Starting + HookFailed → Failed with error", () => { const state = make("db", { status: "Starting" }); const next = applyEvent(state, { _tag: "HookFailed", error: "startup failed" }); diff --git a/packages/stack/src/ApiProxy.ts b/packages/stack/src/ApiProxy.ts index c3ecf5e01a..632e467b09 100644 --- a/packages/stack/src/ApiProxy.ts +++ b/packages/stack/src/ApiProxy.ts @@ -9,7 +9,7 @@ import { HttpServerRequest, HttpServerResponse, } from "effect/unstable/http"; -import { StackServiceActivator } from "./ServiceActivation.ts"; +import { activationTimeoutSecondsForService, StackServiceActivator } from "./ServiceActivation.ts"; import type { ServiceName } from "./versions.ts"; export interface ProxyConfig { @@ -114,8 +114,6 @@ function addCorsHeaders( // status does not mean a function is servable yet. Briefly retry transport // failures on that route so a user's first call doesn't surface as a 502. const COLD_START_RETRY_SCHEDULE = Schedule.spaced("250 millis").pipe(Schedule.upTo({ times: 8 })); -const DEFAULT_SERVICE_ACTIVATION_TIMEOUT = Duration.seconds(30); - interface ProxyHandlerOptions { readonly service: ServiceName; readonly backendPort: number; @@ -141,7 +139,10 @@ function makeProxyHandler( const activation = yield* activator .activate(opts.service) .pipe( - Effect.timeout(config.activationTimeout ?? DEFAULT_SERVICE_ACTIVATION_TIMEOUT), + Effect.timeout( + config.activationTimeout ?? + Duration.seconds(activationTimeoutSecondsForService(opts.service)), + ), Effect.result, ); if (Result.isFailure(activation)) { diff --git a/packages/stack/src/DaemonServer.integration.test.ts b/packages/stack/src/DaemonServer.integration.test.ts index 3b5c25cd15..e16ec4ac7f 100644 --- a/packages/stack/src/DaemonServer.integration.test.ts +++ b/packages/stack/src/DaemonServer.integration.test.ts @@ -31,7 +31,17 @@ const POSTGRES_STATE = new StackServiceState({ error: null, }); -const MOCK_STATES: ReadonlyArray = [POSTGRES_STATE]; +const HEALTH_FAILED_STATE = new StackServiceState({ + name: "edge-runtime", + status: "Failed", + pid: null, + exitCode: null, + restartCount: 2, + startedAt: Date.now(), + error: "Health check failed and restart budget was exhausted", +}); + +const MOCK_STATES: ReadonlyArray = [POSTGRES_STATE, HEALTH_FAILED_STATE]; const MOCK_LOGS: ReadonlyArray = [ { timestamp: 1000, service: "postgres", stream: "stdout", line: "starting" }, @@ -190,9 +200,16 @@ describe("DaemonServer", () => { expect(res.status).toBe(200); const body = (await res.json()) as { info: StackInfo; services: StackServiceState[] }; expect(body.info).toEqual(MOCK_INFO); - expect(body.services).toHaveLength(1); + expect(body.services).toHaveLength(2); expect(body.services.at(0)?.name).toBe("postgres"); expect(body.services.at(0)?.status).toBe("Running"); + expect(body.services.at(1)).toMatchObject({ + name: "edge-runtime", + status: "Failed", + pid: null, + exitCode: null, + error: "Health check failed and restart budget was exhausted", + }); }); // ------------------------------------------------------------------------- diff --git a/packages/stack/src/RemoteStack.integration.test.ts b/packages/stack/src/RemoteStack.integration.test.ts index 19e16a9c79..dc520bab1e 100644 --- a/packages/stack/src/RemoteStack.integration.test.ts +++ b/packages/stack/src/RemoteStack.integration.test.ts @@ -44,7 +44,21 @@ const AUTH_STATE = new StackServiceState({ error: null, }); -const MOCK_STATES: ReadonlyArray = [POSTGRES_STATE, AUTH_STATE]; +const HEALTH_FAILED_STATE = new StackServiceState({ + name: "edge-runtime", + status: "Failed", + pid: null, + exitCode: null, + restartCount: 2, + startedAt: Date.now(), + error: "Health check failed and restart budget was exhausted", +}); + +const MOCK_STATES: ReadonlyArray = [ + POSTGRES_STATE, + AUTH_STATE, + HEALTH_FAILED_STATE, +]; const MOCK_LOGS: ReadonlyArray = [ { timestamp: 1000, service: "postgres", stream: "stdout", line: "starting" }, @@ -233,9 +247,16 @@ describe("RemoteStack integration", () => { const states = await clientRuntime.runPromise( Effect.flatMap(Stack, (stack) => stack.getAllStates()), ); - expect(states).toHaveLength(2); + expect(states).toHaveLength(3); expect(states.at(0)?.name).toBe("postgres"); expect(states.at(1)?.name).toBe("auth"); + expect(states.at(2)).toMatchObject({ + name: "edge-runtime", + status: "Failed", + pid: null, + exitCode: null, + error: "Health check failed and restart budget was exhausted", + }); }); test("getState returns a single service state", async () => { diff --git a/packages/stack/src/ServiceActivation.ts b/packages/stack/src/ServiceActivation.ts index a97d147489..0edb4dafb9 100644 --- a/packages/stack/src/ServiceActivation.ts +++ b/packages/stack/src/ServiceActivation.ts @@ -2,7 +2,8 @@ import { ServiceNotFoundError } from "@supabase/process-compose"; import type { ServiceReadyError } from "@supabase/process-compose"; import { Context, Effect, Layer } from "effect"; import { StackBuildError, StackNotRunningError } from "./errors.ts"; -import type { ServiceName } from "./versions.ts"; +import { stackServiceStartupBudgetSeconds } from "./services/health-budgets.ts"; +import { SERVICE_NAMES, type ServiceName } from "./versions.ts"; export interface ServiceActivationPolicy { /** Whether the public service must already be running when lazy startup completes. */ @@ -55,6 +56,25 @@ export const activationTargetsForService = ( return [...targets]; }; +const DEFAULT_ACTIVATION_TIMEOUT_FLOOR_SECONDS = 180; +const ACTIVATION_COORDINATION_MARGIN_SECONDS = 5; + +/** + * Bounds request-triggered lazy activation by the complete companion closure. + * The floor preserves the existing tolerance for services with shorter probe + * budgets, while longer transitive closures expand the timeout automatically. + */ +export const activationTimeoutSecondsForService = (service: ServiceName): number => { + const startupBudget = activationTargetsForService(SERVICE_NAMES, service).reduce( + (total, target) => total + stackServiceStartupBudgetSeconds[target], + 0, + ); + return Math.max( + DEFAULT_ACTIVATION_TIMEOUT_FLOOR_SECONDS, + startupBudget + ACTIVATION_COORDINATION_MARGIN_SECONDS, + ); +}; + /** Services exclusively owned by a public service for stop/restart operations. */ export const lifecycleTargetsForService = ( enabledServices: ReadonlyArray, diff --git a/packages/stack/src/ServiceActivation.unit.test.ts b/packages/stack/src/ServiceActivation.unit.test.ts index cc41dd31b7..83336d573d 100644 --- a/packages/stack/src/ServiceActivation.unit.test.ts +++ b/packages/stack/src/ServiceActivation.unit.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + activationTimeoutSecondsForService, activationTargetsForService, eagerServices, lifecycleTargetsForService, @@ -44,6 +45,12 @@ describe("service activation", () => { expect(activationTargetsForService(enabled, "studio")).toEqual(["analytics", "studio"]); }); + it("derives request activation timeouts from the transitive companion closure", () => { + expect(activationTimeoutSecondsForService("auth")).toBe(180); + expect(activationTimeoutSecondsForService("analytics")).toBe(554); + expect(activationTimeoutSecondsForService("studio")).toBe(825); + }); + it("does not assign shared public dependencies to their consumers", () => { expect(lifecycleTargetsForService(SERVICE_NAMES, "storage")).toEqual(["storage", "imgproxy"]); expect(lifecycleTargetsForService(SERVICE_NAMES, "analytics")).toEqual(["analytics", "vector"]); diff --git a/packages/stack/src/StackBuilder.ts b/packages/stack/src/StackBuilder.ts index 31ebab0db5..dc95e58583 100644 --- a/packages/stack/src/StackBuilder.ts +++ b/packages/stack/src/StackBuilder.ts @@ -34,6 +34,10 @@ import { } from "./services/storage.ts"; import { makeStudioServiceDocker } from "./services/studio.ts"; import { makeVectorServiceDocker } from "./services/vector.ts"; +import { + dependencyTimeoutSecondsForServices, + POSTGRES_INIT_COMPLETION_BUDGET_SECONDS, +} from "./services/health-budgets.ts"; import type { PreparedStackArtifacts } from "./StackPreparation.ts"; import type { StackServiceProjectionCatalog } from "./StackStateProjection.ts"; import type { AllocatedPorts } from "./PortAllocator.ts"; @@ -324,6 +328,16 @@ const dependsOnPostgres = (hasPostgresInit: boolean): ReadonlyArray, hasPostgresInit: boolean, @@ -554,6 +568,15 @@ export class StackBuilder extends Context.Service< ); const hasPostgresInit = postgresResolution.type === "binary"; const postgresDeps = dependsOnPostgres(hasPostgresInit); + const postgresConsumerDependencyTimeoutSeconds = + POSTGRES_DEPENDENCY_TIMEOUT_SECONDS + + (hasPostgresInit ? POSTGRES_INIT_COMPLETION_BUDGET_SECONDS : 0); + const storageDependencyTimeoutSeconds = + STORAGE_DEPENDENCY_TIMEOUT_SECONDS + + (hasPostgresInit ? POSTGRES_INIT_COMPLETION_BUDGET_SECONDS : 0); + const analyticsDependencyTimeoutSeconds = + ANALYTICS_DEPENDENCY_TIMEOUT_SECONDS + + (hasPostgresInit ? POSTGRES_INIT_COMPLETION_BUDGET_SECONDS : 0); const jwtJwks = generateJwks(config.jwtSecret); const defs: Array = [ @@ -587,6 +610,7 @@ export class StackBuilder extends Context.Service< dbPort: config.dbPort, autoExposeNewTables: config.postgres.autoExposeNewTables, }), + dependencyTimeoutSeconds: POSTGRES_DEPENDENCY_TIMEOUT_SECONDS, enabled: true, }); } @@ -624,6 +648,7 @@ export class StackBuilder extends Context.Service< : { dependencies: [{ service: "postgres", condition: "healthy" as const }], }), + dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, }); } @@ -662,6 +687,7 @@ export class StackBuilder extends Context.Service< apiPort: config.apiPort, dependencies: postgresDeps, })), + dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, }); } @@ -690,6 +716,7 @@ export class StackBuilder extends Context.Service< networkArgs: dockerNetworkArgs(platform.os, [config.edgeRuntime.port]), dependencies: postgresDeps, })), + dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, }); } @@ -731,6 +758,7 @@ export class StackBuilder extends Context.Service< networkArgs: dockerNetworkArgs(platform.os, [config.realtime.port]), dependencies: postgresDeps, }), + dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, }); } @@ -758,6 +786,7 @@ export class StackBuilder extends Context.Service< dependencies: postgresDeps, cleanupDataDirOnExit: hasAutoManagedPath(config, config.storage.dataDir), }), + dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, }); } @@ -774,6 +803,7 @@ export class StackBuilder extends Context.Service< networkArgs: dockerNetworkArgs(platform.os, [config.imgproxy.port]), dependencies: [{ service: "storage", condition: "healthy" }], }), + dependencyTimeoutSeconds: storageDependencyTimeoutSeconds, enabled: true, }); } @@ -790,6 +820,7 @@ export class StackBuilder extends Context.Service< networkArgs: dockerNetworkArgs(platform.os, [config.pgmeta.port]), dependencies: postgresDeps, }), + dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, }); } @@ -817,6 +848,7 @@ export class StackBuilder extends Context.Service< ]), dependencies: postgresDeps, }), + dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, }); } @@ -834,6 +866,7 @@ export class StackBuilder extends Context.Service< networkArgs: dockerNetworkArgs(platform.os, []), dependencies: [{ service: "analytics", condition: "healthy" }], }), + dependencyTimeoutSeconds: analyticsDependencyTimeoutSeconds, enabled: true, }); } @@ -869,6 +902,7 @@ export class StackBuilder extends Context.Service< ]), dependencies: postgresDeps, }), + dependencyTimeoutSeconds: postgresConsumerDependencyTimeoutSeconds, enabled: true, }); } @@ -903,6 +937,7 @@ export class StackBuilder extends Context.Service< { service: "analytics", condition: "healthy" }, ], }), + dependencyTimeoutSeconds: analyticsDependencyTimeoutSeconds, enabled: true, }); } diff --git a/packages/stack/src/StackBuilder.unit.test.ts b/packages/stack/src/StackBuilder.unit.test.ts index f8c7f40765..75935f65ac 100644 --- a/packages/stack/src/StackBuilder.unit.test.ts +++ b/packages/stack/src/StackBuilder.unit.test.ts @@ -11,6 +11,10 @@ import { nativePostgresNeedsDockerAccess } from "./StackBuilder.ts"; import type { AllocatedPorts } from "./PortAllocator.ts"; import { StackPreparation } from "./StackPreparation.ts"; import type { StackPreparationInput } from "./StackPreparation.ts"; +import { + dependencyTimeoutSecondsForServices, + POSTGRES_INIT_COMPLETION_BUDGET_SECONDS, +} from "./services/health-budgets.ts"; import { DEFAULT_VERSIONS } from "./versions.ts"; const testJwtSecret = "super-secret-jwt-token-with-at-least-32-characters"; @@ -204,6 +208,17 @@ describe("StackBuilder", () => { expect(names.indexOf("postgres-init")).toBeLessThan(names.indexOf("postgrest")); expect(names.indexOf("postgres-init")).toBeLessThan(names.indexOf("auth")); + const postgresDependencyTimeout = dependencyTimeoutSecondsForServices(["postgres"]); + expect( + graph.startOrder.find((service) => service.name === "postgres-init") + ?.dependencyTimeoutSeconds, + ).toBe(postgresDependencyTimeout); + for (const name of ["postgrest", "auth"]) { + expect( + graph.startOrder.find((service) => service.name === name)?.dependencyTimeoutSeconds, + ).toBe(postgresDependencyTimeout + POSTGRES_INIT_COMPLETION_BUDGET_SECONDS); + } + expect(serviceProjection.get("postgres")).toEqual({ visibility: "public" }); expect(serviceProjection.get("postgres-init")).toEqual({ visibility: "internal", @@ -345,6 +360,9 @@ describe("StackBuilder", () => { const authDef = graph.startOrder.find((s) => s.name === "auth"); expect(authDef?.dependencies).toEqual([{ service: "postgres", condition: "healthy" }]); + expect(authDef?.dependencyTimeoutSeconds).toBe( + dependencyTimeoutSecondsForServices(["postgres"]), + ); }).pipe(Effect.provide(layer)); }); diff --git a/packages/stack/src/StackLifecycleCoordinator.ts b/packages/stack/src/StackLifecycleCoordinator.ts index a9eeb0196d..86b4097bae 100644 --- a/packages/stack/src/StackLifecycleCoordinator.ts +++ b/packages/stack/src/StackLifecycleCoordinator.ts @@ -36,7 +36,7 @@ import { versionsForConfig, type ResolvedStackConfig, } from "./StackBuilder.ts"; -import { changedProjectedStates, projectStackStates } from "./StackStateProjection.ts"; +import { projectStackStates, type StackServiceProjectionCatalog } from "./StackStateProjection.ts"; import { StackServiceState } from "./StackServiceState.ts"; import type { EdgeRuntimeReloadConfig, StackInfo } from "./Stack.ts"; import { SERVICE_NAMES, type ServiceName } from "./versions.ts"; @@ -53,6 +53,7 @@ type LifecyclePhase = interface RuntimeState { readonly orchestrator: Orchestrator["Service"]; readonly graph: ResolvedGraph; + readonly serviceProjection: StackServiceProjectionCatalog; readonly cleanupTargets: CleanupTargets; } @@ -216,6 +217,7 @@ export class StackLifecycleCoordinator extends Context.Service< const stateRef = yield* SubscriptionRef.make(initialPublicStates(config)); const phaseRef = yield* Ref.make("idle"); const lifecycleLock = Semaphore.makeUnsafe(1); + const projectionLock = Semaphore.makeUnsafe(1); const logBufferServices = yield* Layer.buildWithScope(LogBuffer.layer, scope); const logBuffer = Context.get(logBufferServices, LogBuffer); @@ -231,6 +233,17 @@ export class StackLifecycleCoordinator extends Context.Service< : [...current, nextState]; }); + const syncProjectedStates = ( + orchestrator: Orchestrator["Service"], + serviceProjection: StackServiceProjectionCatalog, + ) => + Effect.gen(function* () { + const rawStates = yield* orchestrator.getAllStates(); + yield* Effect.forEach(projectStackStates(rawStates, serviceProjection), updateState, { + discard: true, + }); + }).pipe(projectionLock.withPermit); + const requireKnownService = (name: string) => Effect.gen(function* () { const currentStates = SubscriptionRef.getUnsafe(stateRef); @@ -388,40 +401,9 @@ export class StackLifecycleCoordinator extends Context.Service< const orchServices = yield* Layer.buildWithScope(orchLayer, scope); const orchestrator = Context.get(orchServices, Orchestrator); - const projectedStates = Stream.unwrap( - Effect.gen(function* () { - const rawInitialStates = yield* orchestrator.getAllStates(); - const initialProjected = projectStackStates(rawInitialStates, serviceProjection); - let rawStates = new Map( - rawInitialStates.map((state) => [state.name, state] as const), - ); - let projectedByName = new Map( - initialProjected.map((state) => [state.name, state] as const), - ); - - return Stream.concat( - Stream.fromIterable(initialProjected), - orchestrator.allStateChanges().pipe( - Stream.map((rawState) => { - rawStates.set(rawState.name, rawState); - const nextProjected = projectStackStates( - [...rawStates.values()], - serviceProjection, - ); - const changed = changedProjectedStates(projectedByName, nextProjected); - projectedByName = new Map( - nextProjected.map((state) => [state.name, state] as const), - ); - return changed; - }), - Stream.flatMap((states) => Stream.fromIterable(states)), - ), - ); - }), - ); - - yield* projectedStates.pipe( - Stream.runForEach((state) => updateState(state)), + yield* syncProjectedStates(orchestrator, serviceProjection); + yield* orchestrator.allStateChanges().pipe( + Stream.runForEach(() => syncProjectedStates(orchestrator, serviceProjection)), Effect.ignore, Effect.forkIn(scope), ); @@ -429,6 +411,7 @@ export class StackLifecycleCoordinator extends Context.Service< return { orchestrator, graph, + serviceProjection, cleanupTargets, } satisfies RuntimeState; }).pipe( @@ -528,6 +511,8 @@ export class StackLifecycleCoordinator extends Context.Service< ), ); const withLifecycleLock = lifecycleLock.withPermit; + const syncRuntimeProjectedStates = (runtime: RuntimeState) => + syncProjectedStates(runtime.orchestrator, runtime.serviceProjection); const serviceStartOptions = { beforeStart: (name: string) => portLease.reserve(portFieldsForService(name)), beforeSpawn: (name: string) => portLease.release(portFieldsForService(name)), @@ -590,18 +575,21 @@ export class StackLifecycleCoordinator extends Context.Service< readonly runtime: RuntimeState; readonly targets: ReadonlyArray; }) => - Effect.forEach( - targets, - (target) => - runtime.orchestrator - .waitReady(target) - .pipe( - Effect.catchTag("ServiceNotFoundError", (cause) => - Effect.fail(knownServiceError(target, cause)), + Effect.gen(function* () { + yield* Effect.forEach( + targets, + (target) => + runtime.orchestrator + .waitReady(target) + .pipe( + Effect.catchTag("ServiceNotFoundError", (cause) => + Effect.fail(knownServiceError(target, cause)), + ), ), - ), - { concurrency: "unbounded", discard: true }, - ); + { concurrency: "unbounded", discard: true }, + ); + yield* syncRuntimeProjectedStates(runtime); + }); const inspectStartedTargets = (root: ServiceName) => Effect.gen(function* () { const runtime = yield* ensureRuntime; @@ -698,6 +686,7 @@ export class StackLifecycleCoordinator extends Context.Service< } else { yield* runtime.orchestrator.start(serviceStartOptions); yield* runtime.orchestrator.waitAllReady(); + yield* syncRuntimeProjectedStates(runtime); } yield* Ref.set(phaseRef, "running"); }).pipe( @@ -851,6 +840,7 @@ export class StackLifecycleCoordinator extends Context.Service< yield* requireKnownServiceName(name); const runtime = yield* ensureRuntime; yield* runtime.orchestrator.waitReady(name); + yield* syncRuntimeProjectedStates(runtime); }), waitAllReady: () => Effect.gen(function* () { @@ -864,6 +854,7 @@ export class StackLifecycleCoordinator extends Context.Service< } const runtime = yield* ensureRuntime; yield* runtime.orchestrator.waitAllReady(); + yield* syncRuntimeProjectedStates(runtime); }), subscribeLogs: (name) => logBuffer.subscribe(name), subscribeAllLogs: (services) => diff --git a/packages/stack/src/StackServiceState.unit.test.ts b/packages/stack/src/StackServiceState.unit.test.ts new file mode 100644 index 0000000000..20f3b99b81 --- /dev/null +++ b/packages/stack/src/StackServiceState.unit.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { ServiceState } from "@supabase/process-compose"; +import { fromRawServiceState } from "./StackServiceState.ts"; + +describe("fromRawServiceState", () => { + it("preserves terminal health-failure state semantics", () => { + const projected = fromRawServiceState( + new ServiceState({ + name: "edge-runtime", + status: "Failed", + pid: null, + exitCode: null, + restartCount: 2, + startedAt: 1_000, + error: "Health check failed and restart budget was exhausted", + desired: "running", + }), + ); + + expect(projected).toMatchObject({ + status: "Failed", + pid: null, + exitCode: null, + error: "Health check failed and restart budget was exhausted", + }); + }); +}); diff --git a/packages/stack/src/StackStateProjection.ts b/packages/stack/src/StackStateProjection.ts index 1d878a5a49..51b25512e8 100644 --- a/packages/stack/src/StackStateProjection.ts +++ b/packages/stack/src/StackStateProjection.ts @@ -17,18 +17,6 @@ function isHelperActive(state: RawServiceState): boolean { return state.status !== "Stopped" && state.status !== "Failed"; } -function sameState(a: StackServiceState | undefined, b: StackServiceState): boolean { - return ( - a?.name === b.name && - a.status === b.status && - a.pid === b.pid && - a.exitCode === b.exitCode && - a.restartCount === b.restartCount && - a.startedAt === b.startedAt && - a.error === b.error - ); -} - function projectPublicState( raw: RawServiceState, rawByName: ReadonlyMap, @@ -95,10 +83,3 @@ export function projectStackState( ): StackServiceState | undefined { return projectStackStates(rawStates, catalog).find((state) => state.name === name); } - -export function changedProjectedStates( - previous: ReadonlyMap, - next: ReadonlyArray, -): ReadonlyArray { - return next.filter((state) => !sameState(previous.get(state.name), state)); -} diff --git a/packages/stack/src/services/analytics.ts b/packages/stack/src/services/analytics.ts index 78c62f895e..9f2f742438 100644 --- a/packages/stack/src/services/analytics.ts +++ b/packages/stack/src/services/analytics.ts @@ -1,5 +1,6 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; interface DockerAnalyticsOptions { readonly image: string; @@ -34,9 +35,7 @@ const analyticsHealthCheck = (port: number): ServiceDef["healthCheck"] => ({ path: "/health", scheme: "http", }, - initialDelaySeconds: 10, - periodSeconds: 1, - failureThreshold: 60, + ...stackHealthBudgets.analytics, }); export const makeAnalyticsServiceDocker = (opts: DockerAnalyticsOptions): ServiceDef => { diff --git a/packages/stack/src/services/auth.ts b/packages/stack/src/services/auth.ts index b3e0360c83..355b8265c6 100644 --- a/packages/stack/src/services/auth.ts +++ b/packages/stack/src/services/auth.ts @@ -1,5 +1,6 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerServiceCleanup, dockerServiceOrphanCleanup } from "./docker-cleanup.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; interface AuthServiceOptions { readonly dbPort: number; @@ -66,8 +67,7 @@ const authHealthCheck = (port: number) => ({ path: "/health", scheme: "http" as const, }, - periodSeconds: 0.5, - failureThreshold: 20, + ...stackHealthBudgets.auth, }); export const makeAuthServiceNative = (opts: NativeAuthOptions): ServiceDef => ({ diff --git a/packages/stack/src/services/edge-runtime.ts b/packages/stack/src/services/edge-runtime.ts index 4132cf4787..66f6d52b9d 100644 --- a/packages/stack/src/services/edge-runtime.ts +++ b/packages/stack/src/services/edge-runtime.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import type { ServiceDef } from "@supabase/process-compose"; import { dockerRunService, hostHttpHealthCheck, type ServiceDependency } from "./service-utils.ts"; import bootstrapSource from "./edge-runtime-main.ts" with { type: "text" }; +import { stackHealthBudgets } from "./health-budgets.ts"; interface EdgeRuntimeOptions { readonly runtimeRoot: string; @@ -61,9 +62,7 @@ const edgeRuntimeArgs = ( const edgeRuntimeHealthCheck = (port: number): ServiceDef["healthCheck"] => hostHttpHealthCheck(port, "/_internal/health", { - initialDelaySeconds: 1, - periodSeconds: 0.5, - failureThreshold: 30, + ...stackHealthBudgets.edgeRuntime, }); export const makeEdgeRuntimeServiceNative = (opts: NativeEdgeRuntimeOptions): ServiceDef => { diff --git a/packages/stack/src/services/health-budgets.ts b/packages/stack/src/services/health-budgets.ts new file mode 100644 index 0000000000..7c623bfb6d --- /dev/null +++ b/packages/stack/src/services/health-budgets.ts @@ -0,0 +1,137 @@ +import { defaults, type HealthCheckConfig } from "@supabase/process-compose"; +import type { ServiceName } from "../versions.ts"; + +type HealthBudget = Required< + Pick< + HealthCheckConfig, + "initialDelaySeconds" | "periodSeconds" | "startupFailureThreshold" | "failureThreshold" + > +> & + Pick; + +/** Cold-start tolerance and tighter post-start liveness thresholds. */ +export const stackHealthBudgets = { + postgresNative: { + initialDelaySeconds: 0, + periodSeconds: 0.5, + startupFailureThreshold: 120, + failureThreshold: 30, + }, + postgresDocker: { + initialDelaySeconds: 1, + periodSeconds: 0.5, + startupFailureThreshold: 120, + failureThreshold: 30, + }, + auth: { + initialDelaySeconds: 0, + periodSeconds: 0.5, + startupFailureThreshold: 60, + failureThreshold: 20, + }, + postgrest: { + initialDelaySeconds: 0, + periodSeconds: 0.5, + startupFailureThreshold: 60, + failureThreshold: 20, + }, + edgeRuntime: { + initialDelaySeconds: 1, + periodSeconds: 0.5, + startupFailureThreshold: 60, + failureThreshold: 30, + }, + mailpit: { + initialDelaySeconds: 1, + periodSeconds: 0.5, + startupFailureThreshold: 60, + failureThreshold: 30, + }, + realtime: { + initialDelaySeconds: 1, + periodSeconds: 0.5, + startupFailureThreshold: 60, + failureThreshold: 30, + }, + storage: { + initialDelaySeconds: 1, + periodSeconds: 0.5, + startupFailureThreshold: 60, + failureThreshold: 30, + }, + imgproxy: { + initialDelaySeconds: 1, + periodSeconds: 0.5, + startupFailureThreshold: 60, + failureThreshold: 30, + }, + pgmeta: { + initialDelaySeconds: 1, + periodSeconds: 0.5, + startupFailureThreshold: 60, + failureThreshold: 30, + }, + analytics: { + initialDelaySeconds: 10, + periodSeconds: 1, + startupFailureThreshold: 120, + failureThreshold: 60, + }, + vector: { + initialDelaySeconds: 1, + periodSeconds: 1, + startupFailureThreshold: 60, + failureThreshold: 30, + }, + pooler: { + initialDelaySeconds: 2, + periodSeconds: 1, + startupFailureThreshold: 90, + failureThreshold: 60, + }, + studio: { + initialDelaySeconds: 2, + periodSeconds: 1, + startupFailureThreshold: 90, + failureThreshold: 60, + }, +} as const satisfies Record; + +export const healthStartupBudgetSeconds = (budget: HealthBudget): number => { + const attempts = budget.startupFailureThreshold; + const probeTimeoutSeconds = budget.timeoutSeconds ?? defaults.healthCheck.timeoutSeconds; + return ( + budget.initialDelaySeconds + + attempts * probeTimeoutSeconds + + (attempts - 1) * budget.periodSeconds + ); +}; + +/** Worst-case initial health budget for each public service. */ +export const stackServiceStartupBudgetSeconds = { + postgres: Math.max( + healthStartupBudgetSeconds(stackHealthBudgets.postgresNative), + healthStartupBudgetSeconds(stackHealthBudgets.postgresDocker), + ), + postgrest: healthStartupBudgetSeconds(stackHealthBudgets.postgrest), + auth: healthStartupBudgetSeconds(stackHealthBudgets.auth), + "edge-runtime": healthStartupBudgetSeconds(stackHealthBudgets.edgeRuntime), + realtime: healthStartupBudgetSeconds(stackHealthBudgets.realtime), + storage: healthStartupBudgetSeconds(stackHealthBudgets.storage), + imgproxy: healthStartupBudgetSeconds(stackHealthBudgets.imgproxy), + mailpit: healthStartupBudgetSeconds(stackHealthBudgets.mailpit), + pgmeta: healthStartupBudgetSeconds(stackHealthBudgets.pgmeta), + studio: healthStartupBudgetSeconds(stackHealthBudgets.studio), + analytics: healthStartupBudgetSeconds(stackHealthBudgets.analytics), + vector: healthStartupBudgetSeconds(stackHealthBudgets.vector), + pooler: healthStartupBudgetSeconds(stackHealthBudgets.pooler), +} as const satisfies Readonly>; + +const STARTUP_COORDINATION_MARGIN_SECONDS = 5; + +/** Maximum coordination allowance for the native one-shot database initialization step. */ +export const POSTGRES_INIT_COMPLETION_BUDGET_SECONDS = 60; + +export const dependencyTimeoutSecondsForServices = (services: ReadonlyArray): number => + services.reduce((total, service) => total + stackServiceStartupBudgetSeconds[service], 0) + + STARTUP_COORDINATION_MARGIN_SECONDS; diff --git a/packages/stack/src/services/health-budgets.unit.test.ts b/packages/stack/src/services/health-budgets.unit.test.ts new file mode 100644 index 0000000000..e9840ea45e --- /dev/null +++ b/packages/stack/src/services/health-budgets.unit.test.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from "vitest"; +import { + dependencyTimeoutSecondsForServices, + healthStartupBudgetSeconds, + stackHealthBudgets, + stackServiceStartupBudgetSeconds, +} from "./health-budgets.ts"; + +describe("stack health budgets", () => { + it("records startup and liveness policy for every health-checked service", () => { + const summarized = Object.fromEntries( + Object.entries(stackHealthBudgets).map(([name, budget]) => [ + name, + { + initialDelay: budget.initialDelaySeconds, + period: budget.periodSeconds, + startupThreshold: budget.startupFailureThreshold, + startupBudget: healthStartupBudgetSeconds(budget), + livenessThreshold: budget.failureThreshold, + }, + ]), + ); + + expect(summarized).toMatchInlineSnapshot(` + { + "analytics": { + "initialDelay": 10, + "livenessThreshold": 60, + "period": 1, + "startupBudget": 369, + "startupThreshold": 120, + }, + "auth": { + "initialDelay": 0, + "livenessThreshold": 20, + "period": 0.5, + "startupBudget": 149.5, + "startupThreshold": 60, + }, + "edgeRuntime": { + "initialDelay": 1, + "livenessThreshold": 30, + "period": 0.5, + "startupBudget": 150.5, + "startupThreshold": 60, + }, + "imgproxy": { + "initialDelay": 1, + "livenessThreshold": 30, + "period": 0.5, + "startupBudget": 150.5, + "startupThreshold": 60, + }, + "mailpit": { + "initialDelay": 1, + "livenessThreshold": 30, + "period": 0.5, + "startupBudget": 150.5, + "startupThreshold": 60, + }, + "pgmeta": { + "initialDelay": 1, + "livenessThreshold": 30, + "period": 0.5, + "startupBudget": 150.5, + "startupThreshold": 60, + }, + "pooler": { + "initialDelay": 2, + "livenessThreshold": 60, + "period": 1, + "startupBudget": 271, + "startupThreshold": 90, + }, + "postgresDocker": { + "initialDelay": 1, + "livenessThreshold": 30, + "period": 0.5, + "startupBudget": 300.5, + "startupThreshold": 120, + }, + "postgresNative": { + "initialDelay": 0, + "livenessThreshold": 30, + "period": 0.5, + "startupBudget": 299.5, + "startupThreshold": 120, + }, + "postgrest": { + "initialDelay": 0, + "livenessThreshold": 20, + "period": 0.5, + "startupBudget": 149.5, + "startupThreshold": 60, + }, + "realtime": { + "initialDelay": 1, + "livenessThreshold": 30, + "period": 0.5, + "startupBudget": 150.5, + "startupThreshold": 60, + }, + "storage": { + "initialDelay": 1, + "livenessThreshold": 30, + "period": 0.5, + "startupBudget": 150.5, + "startupThreshold": 60, + }, + "studio": { + "initialDelay": 2, + "livenessThreshold": 60, + "period": 1, + "startupBudget": 271, + "startupThreshold": 90, + }, + "vector": { + "initialDelay": 1, + "livenessThreshold": 30, + "period": 1, + "startupBudget": 180, + "startupThreshold": 60, + }, + } + `); + }); + + it("keeps dependency timeouts beyond each dependency startup path", () => { + expect(dependencyTimeoutSecondsForServices(["postgres"])).toBeGreaterThan( + stackServiceStartupBudgetSeconds.postgres, + ); + expect(dependencyTimeoutSecondsForServices(["postgres", "storage"])).toBeGreaterThan( + stackServiceStartupBudgetSeconds.postgres + stackServiceStartupBudgetSeconds.storage, + ); + expect(dependencyTimeoutSecondsForServices(["postgres", "analytics"])).toBeGreaterThan( + stackServiceStartupBudgetSeconds.postgres + stackServiceStartupBudgetSeconds.analytics, + ); + }); +}); diff --git a/packages/stack/src/services/imgproxy.ts b/packages/stack/src/services/imgproxy.ts index a943dad356..f38699f88e 100644 --- a/packages/stack/src/services/imgproxy.ts +++ b/packages/stack/src/services/imgproxy.ts @@ -1,5 +1,6 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerRunService, hostHttpHealthCheck, type ServiceDependency } from "./service-utils.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; interface DockerImgproxyOptions { readonly image: string; @@ -14,9 +15,7 @@ const IMGPROXY_STORAGE_DIR = "/var/lib/storage"; const imgproxyHealthCheck = (port: number): ServiceDef["healthCheck"] => hostHttpHealthCheck(port, "/health", { - initialDelaySeconds: 1, - periodSeconds: 0.5, - failureThreshold: 30, + ...stackHealthBudgets.imgproxy, }); export const makeImgproxyServiceDocker = (opts: DockerImgproxyOptions): ServiceDef => diff --git a/packages/stack/src/services/mailpit.ts b/packages/stack/src/services/mailpit.ts index 70e0176de6..e60f41a14e 100644 --- a/packages/stack/src/services/mailpit.ts +++ b/packages/stack/src/services/mailpit.ts @@ -1,5 +1,6 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerRunService, hostHttpHealthCheck } from "./service-utils.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; interface DockerMailpitOptions { readonly image: string; @@ -12,9 +13,7 @@ interface DockerMailpitOptions { const mailpitHealthCheck = (port: number): ServiceDef["healthCheck"] => hostHttpHealthCheck(port, "/readyz", { - initialDelaySeconds: 1, - periodSeconds: 0.5, - failureThreshold: 30, + ...stackHealthBudgets.mailpit, }); export const makeMailpitServiceDocker = (opts: DockerMailpitOptions): ServiceDef => diff --git a/packages/stack/src/services/pgmeta.ts b/packages/stack/src/services/pgmeta.ts index 38c4fd282e..67a258c6f2 100644 --- a/packages/stack/src/services/pgmeta.ts +++ b/packages/stack/src/services/pgmeta.ts @@ -1,5 +1,6 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; interface DockerPgmetaOptions { readonly image: string; @@ -19,9 +20,7 @@ const pgmetaHealthCheck = (port: number): ServiceDef["healthCheck"] => ({ path: "/health", scheme: "http", }, - initialDelaySeconds: 1, - periodSeconds: 0.5, - failureThreshold: 30, + ...stackHealthBudgets.pgmeta, }); export const makePgmetaServiceDocker = (opts: DockerPgmetaOptions): ServiceDef => diff --git a/packages/stack/src/services/pooler.ts b/packages/stack/src/services/pooler.ts index be75a7397b..7d81e3edda 100644 --- a/packages/stack/src/services/pooler.ts +++ b/packages/stack/src/services/pooler.ts @@ -1,5 +1,6 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; type PoolMode = "transaction" | "session"; @@ -28,9 +29,7 @@ const poolerHealthCheck = (port: number): ServiceDef["healthCheck"] => ({ path: "/api/health", scheme: "http", }, - initialDelaySeconds: 2, - periodSeconds: 1, - failureThreshold: 60, + ...stackHealthBudgets.pooler, }); const tenantScript = ( diff --git a/packages/stack/src/services/postgres.ts b/packages/stack/src/services/postgres.ts index bb3b5953f7..6968f989f8 100644 --- a/packages/stack/src/services/postgres.ts +++ b/packages/stack/src/services/postgres.ts @@ -5,6 +5,7 @@ import { dockerServiceOrphanCleanup, removePathOnOrphanCleanup, } from "./docker-cleanup.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; interface PostgresServiceOptions { readonly dataDir: string; @@ -89,8 +90,7 @@ const postgresHealthCheck = (binPath: string, port: number) => ({ LD_LIBRARY_PATH: `${binPath}/lib`, }, }, - periodSeconds: 0.5, - failureThreshold: 30, + ...stackHealthBudgets.postgresNative, }); /** @@ -107,9 +107,7 @@ const postgresDockerHealthCheck = (containerName: string, port: number) => ({ command: "docker", args: ["exec", containerName, "pg_isready", "-p", String(port), "-U", "postgres"], }, - initialDelaySeconds: 1, - periodSeconds: 0.5, - failureThreshold: 30, + ...stackHealthBudgets.postgresDocker, }); export const makePostgresService = (opts: NativePostgresOptions): ServiceDef => { diff --git a/packages/stack/src/services/postgrest.ts b/packages/stack/src/services/postgrest.ts index fffd349457..0562277927 100644 --- a/packages/stack/src/services/postgrest.ts +++ b/packages/stack/src/services/postgrest.ts @@ -1,5 +1,6 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerServiceCleanup, dockerServiceOrphanCleanup } from "./docker-cleanup.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; interface PostgrestServiceOptions { readonly dbPort: number; @@ -43,8 +44,7 @@ const postgrestHealthCheck = (port: number) => ({ path: "/", scheme: "http" as const, }, - periodSeconds: 0.5, - failureThreshold: 20, + ...stackHealthBudgets.postgrest, }); const postgrestDependencies = [{ service: "postgres-init", condition: "completed" as const }]; diff --git a/packages/stack/src/services/realtime.ts b/packages/stack/src/services/realtime.ts index 4f4c209777..a8bd42eacd 100644 --- a/packages/stack/src/services/realtime.ts +++ b/packages/stack/src/services/realtime.ts @@ -1,5 +1,6 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; interface DockerRealtimeOptions { readonly image: string; @@ -31,9 +32,7 @@ const realtimeHealthCheck = (port: number, tenantId: string): ServiceDef["health `http://127.0.0.1:${port}/api/ping`, ], }, - initialDelaySeconds: 1, - periodSeconds: 0.5, - failureThreshold: 30, + ...stackHealthBudgets.realtime, }); export const makeRealtimeServiceDocker = (opts: DockerRealtimeOptions): ServiceDef => diff --git a/packages/stack/src/services/storage.ts b/packages/stack/src/services/storage.ts index 056a27605e..87e21a6ab9 100644 --- a/packages/stack/src/services/storage.ts +++ b/packages/stack/src/services/storage.ts @@ -1,6 +1,7 @@ import type { ServiceDef } from "@supabase/process-compose"; import { removePathOnOrphanCleanup } from "./docker-cleanup.ts"; import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; interface DockerStorageOptions { readonly image: string; @@ -38,9 +39,7 @@ const storageHealthCheck = (port: number): ServiceDef["healthCheck"] => ({ path: "/status", scheme: "http", }, - initialDelaySeconds: 1, - periodSeconds: 0.5, - failureThreshold: 30, + ...stackHealthBudgets.storage, }); export const makeStorageServiceDocker = (opts: DockerStorageOptions): ServiceDef => diff --git a/packages/stack/src/services/studio.ts b/packages/stack/src/services/studio.ts index 7521fce3c7..4b139abc96 100644 --- a/packages/stack/src/services/studio.ts +++ b/packages/stack/src/services/studio.ts @@ -1,5 +1,6 @@ import type { ServiceDef } from "@supabase/process-compose"; import { dockerRunService, type ServiceDependency } from "./service-utils.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; interface DockerStudioOptions { readonly image: string; @@ -29,9 +30,7 @@ const studioHealthCheck = (port: number): ServiceDef["healthCheck"] => ({ path: "/api/platform/profile", scheme: "http", }, - initialDelaySeconds: 2, - periodSeconds: 1, - failureThreshold: 60, + ...stackHealthBudgets.studio, }); export const makeStudioServiceDocker = (opts: DockerStudioOptions): ServiceDef => diff --git a/packages/stack/src/services/vector.ts b/packages/stack/src/services/vector.ts index a66bf5f88a..90aa557f04 100644 --- a/packages/stack/src/services/vector.ts +++ b/packages/stack/src/services/vector.ts @@ -4,6 +4,7 @@ import { dockerRunService, type ServiceDependency, } from "./service-utils.ts"; +import { stackHealthBudgets } from "./health-budgets.ts"; interface DockerVectorOptions { readonly image: string; @@ -67,9 +68,7 @@ ${VECTOR_CONFIG(opts.serviceHost, opts.analyticsPort, opts.analyticsApiKey)}EOF "sh", ["-ec", "wget -q -O /dev/null http://127.0.0.1:9001/health"], { - initialDelaySeconds: 1, - periodSeconds: 1, - failureThreshold: 30, + ...stackHealthBudgets.vector, }, ), }); diff --git a/packages/stack/tests/createStack-docker.e2e.test.ts b/packages/stack/tests/createStack-docker.e2e.test.ts index 41eacc6e2b..f957492e72 100644 --- a/packages/stack/tests/createStack-docker.e2e.test.ts +++ b/packages/stack/tests/createStack-docker.e2e.test.ts @@ -4,11 +4,16 @@ import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { activationTimeoutSecondsForService } from "../src/ServiceActivation.ts"; import { createStack, type StackHandle } from "../src/node.ts"; +import { dependencyTimeoutSecondsForServices } from "../src/services/health-budgets.ts"; import { setupTestTable } from "./helpers/e2e.ts"; -const STACK_DOCKER_E2E_TEST_TIMEOUT_MS = 5_000; -const STACK_DOCKER_E2E_SETUP_TIMEOUT_MS = 90_000; +const STACK_DOCKER_E2E_TEST_TIMEOUT_MS = 180_000; +const STACK_DOCKER_E2E_SETUP_OVERHEAD_MS = 90_000; +const STACK_DOCKER_E2E_SETUP_TIMEOUT_MS = + dependencyTimeoutSecondsForServices(["postgres"]) * 1000 + STACK_DOCKER_E2E_SETUP_OVERHEAD_MS; +const ANALYTICS_COLD_START_TEST_TIMEOUT_MS = activationTimeoutSecondsForService("analytics") * 1000; function hasDockerDaemon(): boolean { try { @@ -32,8 +37,10 @@ dockerDescribe("createStack e2e (docker mode)", () => { stack = await createStack({ mode: "docker", + startupMode: "lazy", jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", postgres: { dataDir }, + analytics: {}, }); try { @@ -68,6 +75,8 @@ dockerDescribe("createStack e2e (docker mode)", () => { "runs the core services in Docker containers and serves health endpoints", { timeout: STACK_DOCKER_E2E_TEST_TIMEOUT_MS }, async () => { + await Promise.all([stack.startService("postgrest"), stack.startService("auth")]); + const runningImages = execSync("docker ps --format '{{.Image}}'").toString(); expect(runningImages).toContain("supabase/postgrest"); expect(runningImages).toContain("supabase/postgres"); @@ -90,10 +99,12 @@ dockerDescribe("createStack e2e (docker mode)", () => { "runs the edge runtime in Docker and serves the functions placeholder through the local gateway", { timeout: STACK_DOCKER_E2E_TEST_TIMEOUT_MS }, async () => { - const [runningImages, states, functionsRes] = await Promise.all([ + const functionsRes = await fetch(`${stack.url}/functions/v1/test`); + await stack.serviceReady("edge-runtime"); + + const [runningImages, states] = await Promise.all([ Promise.resolve(execSync("docker ps --format '{{.Image}}'").toString()), stack.getStatus(), - fetch(`${stack.url}/functions/v1/test`), ]); expect(runningImages).toContain("supabase/edge-runtime"); @@ -110,6 +121,28 @@ dockerDescribe("createStack e2e (docker mode)", () => { }, ); + test( + "cold-starts analytics through lazy service activation", + { timeout: ANALYTICS_COLD_START_TEST_TIMEOUT_MS }, + async () => { + expect(await stack.getServiceStatus("analytics")).toEqual( + expect.objectContaining({ status: "Dormant" }), + ); + + await stack.startService("analytics"); + + const [runningImages, states] = await Promise.all([ + Promise.resolve(execSync("docker ps --format '{{.Image}}'").toString()), + stack.getStatus(), + ]); + + expect(runningImages).toContain("supabase/logflare"); + expect(states).toEqual( + expect.arrayContaining([expect.objectContaining({ name: "analytics", status: "Healthy" })]), + ); + }, + ); + test( "supports the docker auth signup and session golden path", { timeout: STACK_DOCKER_E2E_TEST_TIMEOUT_MS },