diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 534d216aade..49a3a31940f 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -583,8 +583,13 @@ export const makeServerLayer = Layer.unwrap( const server = yield* HttpServer.HttpServer; const address = server.address; if (typeof address === "string" || !("port" in address)) return; - yield* Effect.sleep("250 millis").pipe( - Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)), + // No settling delay before the first attempt: routes are already + // serving by the time activation opens this gate (the startup + // sequence awaits routesReady), and the retry schedule below + // covers anything this sleep used to hedge against. Every + // millisecond here is dead time on the path to remote + // reachability after a restart. + yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe( Effect.retry({ while: (error) => error._tag !== "EnvironmentHttpBadRequestError" && diff --git a/packages/client-runtime/src/state/server.test.ts b/packages/client-runtime/src/state/server.test.ts index d764b729fc3..8edecae5646 100644 --- a/packages/client-runtime/src/state/server.test.ts +++ b/packages/client-runtime/src/state/server.test.ts @@ -7,12 +7,16 @@ import { } from "@t3tools/contracts"; import { describe, expect, it } from "@effect/vitest"; import * as Cause from "effect/Cause"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; +import * as TestClock from "effect/testing/TestClock"; import { RpcClientError } from "effect/unstable/rpc"; import * as Socket from "effect/unstable/socket/Socket"; @@ -30,6 +34,7 @@ import { makeEnvironmentServerConfigState, isLegacyUpdateHandoffLoss, matchesServerUpdateReadyEvent, + nudgeReconnectDuringUpdateRestart, projectServerWelcome, resolveServerConfigValue, resolveServerUpdateProgressResult, @@ -71,6 +76,56 @@ function session(client: WsRpcProtocolClient): RpcSession { }; } +describe("update restart reconnect nudges", () => { + it.effect("retries once per backoff entry instead of only the first", () => + Effect.gen(function* () { + const retries = yield* Ref.make(0); + const states = [ + { phase: "backoff" }, + { phase: "connecting" }, + { phase: "backoff" }, + { phase: "backoff" }, + ]; + + yield* nudgeReconnectDuringUpdateRestart({ + stateChanges: Stream.fromIterable(states), + retryNow: Ref.update(retries, (count) => count + 1), + interval: Duration.zero, + }); + + // Three backoff entries, three nudges. The old one-shot behavior fired + // once and then let the supervisor's ladder stretch to 16-second gaps. + expect(yield* Ref.get(retries)).toBe(3); + }), + ); + + it.effect("paces nudges so a fast-failing connection cannot spin", () => + Effect.gen(function* () { + const retries = yield* Ref.make(0); + + const fiber = yield* Effect.forkChild( + nudgeReconnectDuringUpdateRestart({ + stateChanges: Stream.fromIterable([{ phase: "backoff" }, { phase: "backoff" }]), + retryNow: Ref.update(retries, (count) => count + 1), + }), + { startImmediately: true }, + ); + + // Each nudge waits out the interval first, so nothing fires immediately. + yield* TestClock.adjust(Duration.millis(999)); + expect(yield* Ref.get(retries)).toBe(0); + + yield* TestClock.adjust(Duration.millis(1)); + expect(yield* Ref.get(retries)).toBe(1); + + yield* TestClock.adjust(Duration.seconds(1)); + expect(yield* Ref.get(retries)).toBe(2); + + yield* Fiber.join(fiber); + }).pipe(Effect.provide(TestClock.layer())), + ); +}); + describe("server state projection", () => { it("only treats a legacy transport interruption as an unacknowledged handoff", () => { expect(isLegacyUpdateHandoffLoss(Cause.interrupt(1))).toBe(true); diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index 12639c4ed7b..8c61a939e9e 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -149,6 +149,35 @@ export function validateServerUpdateReadyEvent( ); } +/** + * Keeps reconnect attempts ~1s apart for the whole update restart. + * + * A restart takes the server down for ~15 seconds, but the supervisor's normal + * backoff ladder (1/2/4/8/16s) assumes an unexpected failure and lands attempts + * at ~3, 5, 9, 17 and 33 seconds — so a 15-second restart is observed as a + * 33-second "Resuming". Nudging on every backoff entry (not just the first) + * holds the retry cadence flat until the server answers again. The sleep before + * each nudge is the pacer: a connection that fails instantly re-enters backoff + * immediately and would otherwise spin a tight retry loop. + * + * Callers fork this as a child of the update command so it is interrupted as + * soon as the update settles, whether it succeeds, fails, or times out. + */ +export function nudgeReconnectDuringUpdateRestart(input: { + readonly stateChanges: Stream.Stream<{ readonly phase: string }, unknown>; + readonly retryNow: Effect.Effect; + readonly interval?: Duration.Duration; +}): Effect.Effect { + return input.stateChanges.pipe( + Stream.filter((state) => state.phase === "backoff"), + Stream.runForEach(() => + Effect.sleep(input.interval ?? Duration.seconds(1)).pipe(Effect.andThen(input.retryNow)), + ), + Effect.timeoutOption(SERVER_UPDATE_RESUME_TIMEOUT), + Effect.ignore, + ); +} + export function serverUpdateStateForProgressEvent( fromVersion: string, targetVersion: string, @@ -589,18 +618,13 @@ export function createServerEnvironmentAtoms( }), ); - // The update restart is intentional. As soon as the supervisor sees - // that first failed connection, discard any prior backoff debt and - // retry immediately instead of carrying an old 16-second delay. - yield* environmentRegistry.stateChanges(target.environmentId).pipe( - Stream.filter((state) => state.phase === "backoff"), - Stream.take(1), - Stream.runDrain, - Effect.andThen(environmentRegistry.retryNow(target.environmentId)), - Effect.timeoutOption(Duration.seconds(30)), - Effect.ignore, - Effect.forkChild, - ); + // The update restart is intentional and the server stays unreachable + // for the whole restart, so hold the retry cadence flat instead of + // letting the supervisor climb its backoff ladder. + yield* nudgeReconnectDuringUpdateRestart({ + stateChanges: environmentRegistry.stateChanges(target.environmentId), + retryNow: environmentRegistry.retryNow(target.environmentId), + }).pipe(Effect.forkChild); const resumed = yield* environmentRegistry .followStream(target.environmentId, subscribe(WS_METHODS.subscribeServerLifecycle, {}))