fix(process-compose): make unhealthy lifecycle terminal

apps/cli/src/next/commands/start/ui/StartDashboardView.unit.test.ts

@@ -1,16 +1,17 @@
 import { describe, expect, test } from "vitest";
+import { StackServiceState, type StackServiceStatus } from "@supabase/stack/effect";
 import { ConnectionInfo } from "./ConnectionInfo.tsx";
 
-function state(name: string, status: string) {
-  return {
+function state(name: string, status: StackServiceStatus, error: string | null = null) {
+  return new StackServiceState({
     name,
     status,
     pid: null,
     exitCode: null,
     restartCount: 0,
     startedAt: null,
-    error: null,
-  } as any;
+    error,
+  });
 }
 
 function collectNodes(node: unknown): Array<unknown> {
@@ -60,7 +61,7 @@ describe("StartDashboardView", () => {
     if (!("StartDashboardView" in dashboardModule)) return;
 
     const element = dashboardModule.StartDashboardView({
-      states: [state("postgres", "Failed")],
+      states: [state("postgres", "Failed", "Health check failed and restart budget was exhausted")],
       info: {
         url: "http://127.0.0.1:54321",
         dbUrl: "postgresql://postgres:postgres@127.0.0.1:54322/postgres",

packages/process-compose/docs/architecture.md

@@ -40,7 +40,7 @@ construct its layer from a validated `ResolvedGraph`, a `ChildProcessSpawner` Ad
 
 - executable, arguments, environment, and working directory;
 - dependencies and a dependency wait timeout;
-- optional HTTP, TCP, or exec health check;
+- optional HTTP, TCP, or exec health check with separate startup and liveness failure thresholds;
 - shutdown signal and grace period;
 - restart policy and restart budget;
 - `started` and `healthy` lifecycle hooks;
@@ -98,6 +98,10 @@ service maintained, and `stopped` records an explicit stop. Desired state is ind
 intermediate observed states and is what prevents an explicit stop from being undone by restart
 policy.
 
+`pid` identifies only a currently live process. Process exit, supervisory termination, hook
+failure, restart, and forced shutdown clear it. A health-triggered termination does not fabricate
+an `exitCode`; the code remains `null` unless the process itself reports an exit.
+
 `ServiceState` extends `Data.Class`, so Effect's `Equal.equals` can compare values structurally. It
 does not make two separately allocated objects equal under JavaScript `===`, and
 `SubscriptionRef.set` is not itself a distinct-update filter. Callers that want to suppress
@@ -127,7 +131,8 @@ For each requested definition, `Orchestrator` runs this sequence:
 9. Stream stdout and stderr into `LogBuffer`.
 10. Run the health loop, or run `healthy` hooks immediately when no health check exists, then
     publish `Healthy`.
-11. Race process exit against an unhealthy-restart request and apply restart policy.
+11. Race process exit against unhealthy or hook failure, finalize the child, and apply restart
+    policy to process-exit and unhealthy causes.
 
 The service keeps the same state stream across restart generations. Restart backoff is
 `min(30 seconds, 2^(restartCount - 1))`.
@@ -140,7 +145,8 @@ declaration order. Each hook has a timeout (default: 30 seconds) and either:
 - `fail`: publish `HookFailed` and stop running later hooks for that trigger;
 - `ignore`: append a diagnostic log and continue.
 
-The supplied logger writes tagged stdout/stderr lines into the service's normal log stream.
+The supplied logger writes tagged stdout/stderr lines into the service's normal log stream. A
+failing hook finalizes its process generation before publishing terminal `Failed` state.
 
 ## Health and readiness
 
@@ -151,17 +157,25 @@ The supplied logger writes tagged stdout/stderr lines into the service's normal
 - exec: successful for exit code `0`.
 
 After `initialDelaySeconds`, probes repeat every `periodSeconds`. Each attempt is bounded by
-`timeoutSeconds`; consecutive success and failure counters reset each other. The current
-implementation calls `onHealthy` after `successThreshold`, and calls `onUnhealthy` after
-`failureThreshold` only after that same process generation has first become healthy. Consequently,
-initial probe failures leave a service in `Running` rather than publishing `Unhealthy`; callers
-must not treat the current generic `waitReady` Interface as a startup deadline.
+`timeoutSeconds`; consecutive success and failure counters reset each other. Before a process
+generation has ever become healthy, `startupFailureThreshold` controls when it becomes
+`Unhealthy`. It defaults to `failureThreshold` for compatibility. After the first healthy result,
+all later failures use `failureThreshold`, including after an unhealthy-to-healthy recovery.
+Initial probe failures are therefore observable rather than leaving the service indefinitely in
+`Running`.
+
+An unhealthy process uses the same pure restart-budget decision as a process exit. When restart is
+enabled and the budget is exhausted, the supervisor terminates the child and publishes `Failed`
+with `pid: null`, `exitCode: null`, and a stable health-exhaustion error. With restart policy `no`,
+the live process remains observable as `Unhealthy`.
 
 `waitReady(name)` is intentionally unbounded. For long-running definitions it succeeds at
-`Healthy` and fails at a non-restarting terminal state. For `restart: "no"` one-shot definitions,
-successful completion is readiness. `waitAllReady()` considers only definitions whose desired
-state is `running`, so intentionally inactive definitions do not block lazy callers. Higher-level
-Modules own any finite user-facing deadline.
+`Healthy` and fails at a non-restarting terminal state. A `restart: "no"` definition without a
+health check is treated as one-shot work, where successful completion is readiness; a
+health-checked definition remains subject to health readiness even when restart is disabled.
+`waitAllReady()` considers only definitions whose desired state is `running`, so intentionally
+inactive definitions do not block lazy callers. Higher-level Modules own any finite user-facing
+deadline.
 
 ## Restart policies
 
@@ -187,7 +201,9 @@ interrupts its lifecycle fiber. The generation finalizer:
 
 Whole-graph stop sets desired state first and then stops dependents before dependencies. The global
 shutdown budget defaults to 60 seconds. If that budget expires, the orchestrator logs the timeout
-and clears all fibers; it does not fail with `ShutdownTimeoutError`.
+and force-terminates active children before waiting for teardown to finish. Services that were
+running reach terminal `Stopped` state with `pid: null` and exit code `143`; stop does not fail with
+`ShutdownTimeoutError`.
 
 In-process cleanup and orphan supervision solve different failure modes:
 

packages/process-compose/src/HealthProbe.ts

@@ -69,13 +69,15 @@ export const runHealthProbe = (config: {
     const timeout = hc.timeoutSeconds ?? defaults.healthCheck.timeoutSeconds;
     const successThreshold = hc.successThreshold ?? defaults.healthCheck.successThreshold;
     const failureThreshold = hc.failureThreshold ?? defaults.healthCheck.failureThreshold;
+    const startupFailureThreshold = hc.startupFailureThreshold ?? failureThreshold;
 
     if (initialDelay > 0) {
       yield* Effect.sleep(Duration.seconds(initialDelay));
     }
 
     const counters = yield* Ref.make({ successes: 0, failures: 0 });
-    let isHealthy = false;
+    let phase: "Starting" | "Healthy" | "Unhealthy" = "Starting";
+    let hasEverBeenHealthy = false;
 
     yield* Effect.repeat(
       Effect.gen(function* () {
@@ -86,17 +88,21 @@ export const runHealthProbe = (config: {
             successes: c.successes + 1,
             failures: 0,
           }));
-          if (!isHealthy && successes + 1 >= successThreshold) {
-            isHealthy = true;
+          if (phase !== "Healthy" && successes + 1 >= successThreshold) {
+            phase = "Healthy";
+            hasEverBeenHealthy = true;
             yield* config.callbacks.onHealthy();
           }
         } else {
           const { failures } = yield* Ref.getAndUpdate(counters, (c) => ({
             successes: 0,
             failures: c.failures + 1,
           }));
-          if (isHealthy && failures + 1 >= failureThreshold) {
-            isHealthy = false;
+          const activeFailureThreshold = hasEverBeenHealthy
+            ? failureThreshold
+            : startupFailureThreshold;
+          if (phase !== "Unhealthy" && failures + 1 >= activeFailureThreshold) {
+            phase = "Unhealthy";
             yield* config.callbacks.onUnhealthy();
           }
         }

packages/process-compose/src/HealthProbe.unit.test.ts

@@ -15,6 +15,37 @@ const platformLayer = BunChildProcessSpawnerLayer.pipe(
   Layer.provide(Layer.mergeAll(BunFileSystemLayer, BunPathLayer)),
 );
 
+const sequenceProbeLayer = (results: ReadonlyArray<boolean>) => {
+  let calls = 0;
+  return {
+    layer: Layer.succeed(
+      ChildProcessSpawner.ChildProcessSpawner,
+      ChildProcessSpawner.make(() =>
+        Effect.sync(() => {
+          const result = results[calls] ?? results.at(-1) ?? false;
+          calls++;
+          return ChildProcessSpawner.makeHandle({
+            pid: ChildProcessSpawner.ProcessId(2000 + calls),
+            stdout: Stream.empty,
+            stderr: Stream.empty,
+            all: Stream.empty,
+            exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(result ? 0 : 1)),
+            isRunning: Effect.succeed(false),
+            stdin: Sink.drain,
+            kill: () => Effect.void,
+            unref: Effect.succeed(Effect.void),
+            getInputFd: () => Sink.drain,
+            getOutputFd: () => Stream.empty,
+          });
+        }),
+      ),
+    ),
+    get calls() {
+      return calls;
+    },
+  };
+};
+
 const setupProbe = (probe: ProbeConfig, overrides?: Partial<HealthCheckConfig>) =>
   Effect.gen(function* () {
     let healthy = false;
@@ -190,6 +221,103 @@ describe("HealthProbe", () => {
     }).pipe(Effect.provide(platformLayer)),
   );
 
+  it.live("uses failureThreshold for startup when no startup threshold is configured", () => {
+    const probe = sequenceProbeLayer([false]);
+    return Effect.gen(function* () {
+      const { unhealthySignal, config } = yield* setupProbe(
+        { _tag: "Exec", command: "check", args: [] },
+        { failureThreshold: 2 },
+      );
+      const fiber = yield* Effect.forkChild(runHealthProbe(config));
+      yield* Deferred.await(unhealthySignal).pipe(Effect.timeout(Duration.seconds(1)));
+      expect(probe.calls).toBe(2);
+      yield* Fiber.interrupt(fiber);
+    }).pipe(Effect.provide(probe.layer));
+  });
+
+  it.live("allows a larger startup threshold than the liveness threshold", () => {
+    const probe = sequenceProbeLayer([false]);
+    return Effect.gen(function* () {
+      const { unhealthySignal, config } = yield* setupProbe(
+        { _tag: "Exec", command: "check", args: [] },
+        { startupFailureThreshold: 4, failureThreshold: 2 },
+      );
+      const fiber = yield* Effect.forkChild(runHealthProbe(config));
+      yield* Deferred.await(unhealthySignal).pipe(Effect.timeout(Duration.seconds(1)));
+      expect(probe.calls).toBe(4);
+      yield* Fiber.interrupt(fiber);
+    }).pipe(Effect.provide(probe.layer));
+  });
+
+  it.live("recovers on the final startup probe without becoming unhealthy", () => {
+    const probe = sequenceProbeLayer([false, false, true]);
+    return Effect.gen(function* () {
+      const { healthySignal, unhealthySignal, config } = yield* setupProbe(
+        { _tag: "Exec", command: "check", args: [] },
+        { startupFailureThreshold: 3, failureThreshold: 1 },
+      );
+      const fiber = yield* Effect.forkChild(runHealthProbe(config));
+      yield* Deferred.await(healthySignal).pipe(Effect.timeout(Duration.seconds(1)));
+      expect(probe.calls).toBe(3);
+      expect(yield* Deferred.isDone(unhealthySignal)).toBe(false);
+      yield* Fiber.interrupt(fiber);
+    }).pipe(Effect.provide(probe.layer));
+  });
+
+  it.live("uses the liveness threshold after the first healthy transition", () => {
+    const probe = sequenceProbeLayer([true, false, false]);
+    return Effect.gen(function* () {
+      const { healthySignal, unhealthySignal, config } = yield* setupProbe(
+        { _tag: "Exec", command: "check", args: [] },
+        { startupFailureThreshold: 5, failureThreshold: 2 },
+      );
+      const fiber = yield* Effect.forkChild(runHealthProbe(config));
+      yield* Deferred.await(healthySignal).pipe(Effect.timeout(Duration.seconds(1)));
+      yield* Deferred.await(unhealthySignal).pipe(Effect.timeout(Duration.seconds(1)));
+      expect(probe.calls).toBe(3);
+      yield* Fiber.interrupt(fiber);
+    }).pipe(Effect.provide(probe.layer));
+  });
+
+  it.live("does not re-enable startup tolerance after an unhealthy recovery", () => {
+    const probe = sequenceProbeLayer([true, false, false, true, false, false]);
+    return Effect.gen(function* () {
+      let healthyTransitions = 0;
+      let unhealthyTransitions = 0;
+      const secondUnhealthy = yield* Deferred.make<void>();
+      const fiber = yield* Effect.forkChild(
+        runHealthProbe({
+          name: "test",
+          healthCheck: {
+            probe: { _tag: "Exec", command: "check", args: [] },
+            periodSeconds: 0.01,
+            startupFailureThreshold: 5,
+            failureThreshold: 2,
+          },
+          callbacks: {
+            onHealthy: () =>
+              Effect.sync(() => {
+                healthyTransitions++;
+              }),
+            onUnhealthy: () =>
+              Effect.gen(function* () {
+                unhealthyTransitions++;
+                if (unhealthyTransitions === 2) {
+                  yield* Deferred.succeed(secondUnhealthy, void 0);
+                }
+              }),
+          },
+        }),
+      );
+
+      yield* Deferred.await(secondUnhealthy).pipe(Effect.timeout(Duration.seconds(1)));
+      expect(probe.calls).toBe(6);
+      expect(healthyTransitions).toBe(2);
+      expect(unhealthyTransitions).toBe(2);
+      yield* Fiber.interrupt(fiber);
+    }).pipe(Effect.provide(probe.layer));
+  });
+
   it.live("respects initialDelaySeconds before first probe", () =>
     Effect.gen(function* () {
       const { healthySignal, config } = yield* setupProbe(