From 4dedb6a3e5e0f221b605c4d928f11ca3b0370e37 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Wed, 5 Aug 2026 05:09:41 -0700 Subject: [PATCH] fix: reconnect faster after remote server updates A remote update restarts the server for ~15 seconds, but the client shows "Resuming" for up to ~33s and the relay tunnel comes up ~5s behind the HTTP listener. The client forked a fiber that nudged the connection supervisor once, on the first backoff entry. That nudge fires at ~T+2s while the server is still down for another ~11s, so it is wasted, and the supervisor then climbs its 1/2/4/8/16s ladder with attempts landing at ~3, 5, 9, 17 and 33s. Nudge on every backoff entry instead, paced one second apart, so attempts stay ~1s apart for the whole restart window. The fiber stays a child of the update command, so it is interrupted as soon as the update settles. On the server, drop the 250ms sleep in front of the startup cloud link reconcile. Routes are already serving when activation opens that gate, and the retry schedule covers what the sleep hedged against. Co-Authored-By: Claude Fable 5 --- apps/server/src/server.ts | 9 ++- .../client-runtime/src/state/server.test.ts | 55 +++++++++++++++++++ packages/client-runtime/src/state/server.ts | 48 ++++++++++++---- 3 files changed, 98 insertions(+), 14 deletions(-) diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 534d216aade..49a3a31940f 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -583,8 +583,13 @@ export const makeServerLayer = Layer.unwrap( const server = yield* HttpServer.HttpServer; const address = server.address; if (typeof address === "string" || !("port" in address)) return; - yield* Effect.sleep("250 millis").pipe( - Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)), + // No settling delay before the first attempt: routes are already + // serving by the time activation opens this gate (the startup + // sequence awaits routesReady), and the retry schedule below + // covers anything this sleep used to hedge against. Every + // millisecond here is dead time on the path to remote + // reachability after a restart. + yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe( Effect.retry({ while: (error) => error._tag !== "EnvironmentHttpBadRequestError" && diff --git a/packages/client-runtime/src/state/server.test.ts b/packages/client-runtime/src/state/server.test.ts index d764b729fc3..8edecae5646 100644 --- a/packages/client-runtime/src/state/server.test.ts +++ b/packages/client-runtime/src/state/server.test.ts @@ -7,12 +7,16 @@ import { } from "@t3tools/contracts"; import { describe, expect, it } from "@effect/vitest"; import * as Cause from "effect/Cause"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; +import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; import * as SubscriptionRef from "effect/SubscriptionRef"; +import * as TestClock from "effect/testing/TestClock"; import { RpcClientError } from "effect/unstable/rpc"; import * as Socket from "effect/unstable/socket/Socket"; @@ -30,6 +34,7 @@ import { makeEnvironmentServerConfigState, isLegacyUpdateHandoffLoss, matchesServerUpdateReadyEvent, + nudgeReconnectDuringUpdateRestart, projectServerWelcome, resolveServerConfigValue, resolveServerUpdateProgressResult, @@ -71,6 +76,56 @@ function session(client: WsRpcProtocolClient): RpcSession { }; } +describe("update restart reconnect nudges", () => { + it.effect("retries once per backoff entry instead of only the first", () => + Effect.gen(function* () { + const retries = yield* Ref.make(0); + const states = [ + { phase: "backoff" }, + { phase: "connecting" }, + { phase: "backoff" }, + { phase: "backoff" }, + ]; + + yield* nudgeReconnectDuringUpdateRestart({ + stateChanges: Stream.fromIterable(states), + retryNow: Ref.update(retries, (count) => count + 1), + interval: Duration.zero, + }); + + // Three backoff entries, three nudges. The old one-shot behavior fired + // once and then let the supervisor's ladder stretch to 16-second gaps. + expect(yield* Ref.get(retries)).toBe(3); + }), + ); + + it.effect("paces nudges so a fast-failing connection cannot spin", () => + Effect.gen(function* () { + const retries = yield* Ref.make(0); + + const fiber = yield* Effect.forkChild( + nudgeReconnectDuringUpdateRestart({ + stateChanges: Stream.fromIterable([{ phase: "backoff" }, { phase: "backoff" }]), + retryNow: Ref.update(retries, (count) => count + 1), + }), + { startImmediately: true }, + ); + + // Each nudge waits out the interval first, so nothing fires immediately. + yield* TestClock.adjust(Duration.millis(999)); + expect(yield* Ref.get(retries)).toBe(0); + + yield* TestClock.adjust(Duration.millis(1)); + expect(yield* Ref.get(retries)).toBe(1); + + yield* TestClock.adjust(Duration.seconds(1)); + expect(yield* Ref.get(retries)).toBe(2); + + yield* Fiber.join(fiber); + }).pipe(Effect.provide(TestClock.layer())), + ); +}); + describe("server state projection", () => { it("only treats a legacy transport interruption as an unacknowledged handoff", () => { expect(isLegacyUpdateHandoffLoss(Cause.interrupt(1))).toBe(true); diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index 12639c4ed7b..8c61a939e9e 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -149,6 +149,35 @@ export function validateServerUpdateReadyEvent( ); } +/** + * Keeps reconnect attempts ~1s apart for the whole update restart. + * + * A restart takes the server down for ~15 seconds, but the supervisor's normal + * backoff ladder (1/2/4/8/16s) assumes an unexpected failure and lands attempts + * at ~3, 5, 9, 17 and 33 seconds — so a 15-second restart is observed as a + * 33-second "Resuming". Nudging on every backoff entry (not just the first) + * holds the retry cadence flat until the server answers again. The sleep before + * each nudge is the pacer: a connection that fails instantly re-enters backoff + * immediately and would otherwise spin a tight retry loop. + * + * Callers fork this as a child of the update command so it is interrupted as + * soon as the update settles, whether it succeeds, fails, or times out. + */ +export function nudgeReconnectDuringUpdateRestart(input: { + readonly stateChanges: Stream.Stream<{ readonly phase: string }, unknown>; + readonly retryNow: Effect.Effect; + readonly interval?: Duration.Duration; +}): Effect.Effect { + return input.stateChanges.pipe( + Stream.filter((state) => state.phase === "backoff"), + Stream.runForEach(() => + Effect.sleep(input.interval ?? Duration.seconds(1)).pipe(Effect.andThen(input.retryNow)), + ), + Effect.timeoutOption(SERVER_UPDATE_RESUME_TIMEOUT), + Effect.ignore, + ); +} + export function serverUpdateStateForProgressEvent( fromVersion: string, targetVersion: string, @@ -589,18 +618,13 @@ export function createServerEnvironmentAtoms( }), ); - // The update restart is intentional. As soon as the supervisor sees - // that first failed connection, discard any prior backoff debt and - // retry immediately instead of carrying an old 16-second delay. - yield* environmentRegistry.stateChanges(target.environmentId).pipe( - Stream.filter((state) => state.phase === "backoff"), - Stream.take(1), - Stream.runDrain, - Effect.andThen(environmentRegistry.retryNow(target.environmentId)), - Effect.timeoutOption(Duration.seconds(30)), - Effect.ignore, - Effect.forkChild, - ); + // The update restart is intentional and the server stays unreachable + // for the whole restart, so hold the retry cadence flat instead of + // letting the supervisor climb its backoff ladder. + yield* nudgeReconnectDuringUpdateRestart({ + stateChanges: environmentRegistry.stateChanges(target.environmentId), + retryNow: environmentRegistry.retryNow(target.environmentId), + }).pipe(Effect.forkChild); const resumed = yield* environmentRegistry .followStream(target.environmentId, subscribe(WS_METHODS.subscribeServerLifecycle, {}))