Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions apps/server/src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -583,8 +583,13 @@ export const makeServerLayer = Layer.unwrap(
const server = yield* HttpServer.HttpServer;
const address = server.address;
if (typeof address === "string" || !("port" in address)) return;
yield* Effect.sleep("250 millis").pipe(
Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)),
// No settling delay before the first attempt: routes are already
// serving by the time activation opens this gate (the startup
// sequence awaits routesReady), and the retry schedule below
// covers anything this sleep used to hedge against. Every
// millisecond here is dead time on the path to remote
// reachability after a restart.
yield* reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`).pipe(
Effect.retry({
while: (error) =>
error._tag !== "EnvironmentHttpBadRequestError" &&
Expand Down
55 changes: 55 additions & 0 deletions packages/client-runtime/src/state/server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,16 @@ import {
} from "@t3tools/contracts";
import { describe, expect, it } from "@effect/vitest";
import * as Cause from "effect/Cause";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as Fiber from "effect/Fiber";
import * as Option from "effect/Option";
import * as Queue from "effect/Queue";
import * as Ref from "effect/Ref";
import * as Stream from "effect/Stream";
import * as SubscriptionRef from "effect/SubscriptionRef";
import * as TestClock from "effect/testing/TestClock";
import { RpcClientError } from "effect/unstable/rpc";
import * as Socket from "effect/unstable/socket/Socket";

Expand All @@ -30,6 +34,7 @@ import {
makeEnvironmentServerConfigState,
isLegacyUpdateHandoffLoss,
matchesServerUpdateReadyEvent,
nudgeReconnectDuringUpdateRestart,
projectServerWelcome,
resolveServerConfigValue,
resolveServerUpdateProgressResult,
Expand Down Expand Up @@ -71,6 +76,56 @@ function session(client: WsRpcProtocolClient): RpcSession {
};
}

describe("update restart reconnect nudges", () => {
it.effect("retries once per backoff entry instead of only the first", () =>
Effect.gen(function* () {
const retries = yield* Ref.make(0);
const states = [
{ phase: "backoff" },
{ phase: "connecting" },
{ phase: "backoff" },
{ phase: "backoff" },
];

yield* nudgeReconnectDuringUpdateRestart({
stateChanges: Stream.fromIterable(states),
retryNow: Ref.update(retries, (count) => count + 1),
interval: Duration.zero,
});

// Three backoff entries, three nudges. The old one-shot behavior fired
// once and then let the supervisor's ladder stretch to 16-second gaps.
expect(yield* Ref.get(retries)).toBe(3);
}),
);

it.effect("paces nudges so a fast-failing connection cannot spin", () =>
Effect.gen(function* () {
const retries = yield* Ref.make(0);

const fiber = yield* Effect.forkChild(
nudgeReconnectDuringUpdateRestart({
stateChanges: Stream.fromIterable([{ phase: "backoff" }, { phase: "backoff" }]),
retryNow: Ref.update(retries, (count) => count + 1),
}),
{ startImmediately: true },
);

// Each nudge waits out the interval first, so nothing fires immediately.
yield* TestClock.adjust(Duration.millis(999));
expect(yield* Ref.get(retries)).toBe(0);

yield* TestClock.adjust(Duration.millis(1));
expect(yield* Ref.get(retries)).toBe(1);

yield* TestClock.adjust(Duration.seconds(1));
expect(yield* Ref.get(retries)).toBe(2);

yield* Fiber.join(fiber);
}).pipe(Effect.provide(TestClock.layer())),
);
});

describe("server state projection", () => {
it("only treats a legacy transport interruption as an unacknowledged handoff", () => {
expect(isLegacyUpdateHandoffLoss(Cause.interrupt(1))).toBe(true);
Expand Down
48 changes: 36 additions & 12 deletions packages/client-runtime/src/state/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,35 @@ export function validateServerUpdateReadyEvent(
);
}

/**
* Keeps reconnect attempts ~1s apart for the whole update restart.
*
* A restart takes the server down for ~15 seconds, but the supervisor's normal
* backoff ladder (1/2/4/8/16s) assumes an unexpected failure and lands attempts
* at ~3, 5, 9, 17 and 33 seconds — so a 15-second restart is observed as a
* 33-second "Resuming". Nudging on every backoff entry (not just the first)
* holds the retry cadence flat until the server answers again. The sleep before
* each nudge is the pacer: a connection that fails instantly re-enters backoff
* immediately and would otherwise spin a tight retry loop.
*
* Callers fork this as a child of the update command so it is interrupted as
* soon as the update settles, whether it succeeds, fails, or times out.
*/
export function nudgeReconnectDuringUpdateRestart(input: {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 High state/server.ts:166

The sleep-then-retry in nudgeReconnectDuringUpdateRestart can tear down a connection it just succeeded in establishing. After filtering a backoff event, the function sleeps for ~1s and then fires retryNow unconditionally. If the supervisor reconnects on its own during that sleep, the stale nudge still sends a retry signal, which causes monitorConnectedLease to return false and tear down the new lease — discarding the successful connection before the lifecycle ready event arrives. Consider re-checking the supervisor state (or making retryNow a no-op when not in backoff) before issuing the nudge.

🚀 Reply "fix it for me" or copy this AI Prompt for your agent:
In file @packages/client-runtime/src/state/server.ts around line 166:

The sleep-then-retry in `nudgeReconnectDuringUpdateRestart` can tear down a connection it just succeeded in establishing. After filtering a `backoff` event, the function sleeps for ~1s and then fires `retryNow` unconditionally. If the supervisor reconnects on its own during that sleep, the stale nudge still sends a retry signal, which causes `monitorConnectedLease` to return `false` and tear down the new lease — discarding the successful connection before the lifecycle `ready` event arrives. Consider re-checking the supervisor state (or making `retryNow` a no-op when not in `backoff`) before issuing the nudge.

readonly stateChanges: Stream.Stream<{ readonly phase: string }, unknown>;
readonly retryNow: Effect.Effect<void>;
readonly interval?: Duration.Duration;
}): Effect.Effect<void> {
return input.stateChanges.pipe(
Stream.filter((state) => state.phase === "backoff"),
Stream.runForEach(() =>
Effect.sleep(input.interval ?? Duration.seconds(1)).pipe(Effect.andThen(input.retryNow)),
),
Effect.timeoutOption(SERVER_UPDATE_RESUME_TIMEOUT),
Effect.ignore,
);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Stale reconnect nudge race

Medium Severity

nudgeReconnectDuringUpdateRestart sleeps on each backoff entry and then always calls retryNow, without checking whether the supervisor is still backing off. retryNow also tears down an active session, so a reconnect that succeeds during that sleep—common when SubscriptionRef.changes joins mid-backoff or the supervisor’s own 1s timer wins the race—gets interrupted and the resume wait stretches again.

Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 4dedb6a. Configure here.

}

export function serverUpdateStateForProgressEvent(
fromVersion: string,
targetVersion: string,
Expand Down Expand Up @@ -589,18 +618,13 @@ export function createServerEnvironmentAtoms<R, E>(
}),
);

// The update restart is intentional. As soon as the supervisor sees
// that first failed connection, discard any prior backoff debt and
// retry immediately instead of carrying an old 16-second delay.
yield* environmentRegistry.stateChanges(target.environmentId).pipe(
Stream.filter((state) => state.phase === "backoff"),
Stream.take(1),
Stream.runDrain,
Effect.andThen(environmentRegistry.retryNow(target.environmentId)),
Effect.timeoutOption(Duration.seconds(30)),
Effect.ignore,
Effect.forkChild,
);
// The update restart is intentional and the server stays unreachable
// for the whole restart, so hold the retry cadence flat instead of
// letting the supervisor climb its backoff ladder.
yield* nudgeReconnectDuringUpdateRestart({
stateChanges: environmentRegistry.stateChanges(target.environmentId),
retryNow: environmentRegistry.retryNow(target.environmentId),
}).pipe(Effect.forkChild);

const resumed = yield* environmentRegistry
.followStream(target.environmentId, subscribe(WS_METHODS.subscribeServerLifecycle, {}))
Expand Down
Loading