From 513de4c83b0e61606962ba869690e2c5942b31b5 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 31 Jul 2026 03:06:07 -0700 Subject: [PATCH 1/2] fix(server): self-update no longer rolls itself back on restart A boot-service self-update runs `systemctl --user restart` from inside the service it restarts. systemd's default KillMode=control-group sends SIGTERM to the whole cgroup, so the systemctl child died mid-restart, the updater read that as a restart failure, and its rollback restored the previous unit while the old server was still shutting down. Result: download and install succeed, then the server resumes on the old version ("The server did not resume on t3@"). Queue the restart with --no-block instead: systemctl exits before the stop signal lands, so its exit code only reflects whether systemd accepted the job. The rollback path now fires only on genuine rejections, which it was written for. Co-Authored-By: Claude Fable 5 --- apps/server/src/cloud/selfUpdate.test.ts | 4 ++-- apps/server/src/cloud/selfUpdate.ts | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 655228c047f..5a6f19941ac 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -545,7 +545,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { yield* TestClock.adjust(Duration.seconds(10)); assert.deepEqual(context.commands[3], { command: "systemctl", - args: ["--user", "restart", "t3code.service"], + args: ["--user", "restart", "--no-block", "t3code.service"], }); assert.lengthOf(context.spawns, 0); // systemd replaces the process; the server must not exit itself. @@ -583,7 +583,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { assert.deepEqual( context.commands.slice(-2).map((entry) => entry.args), [ - ["--user", "restart", BOOT_SERVICE_UNIT_FILE], + ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE], ["--user", "daemon-reload"], ], ); diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index f786cbea8cf..5c8560692ca 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -354,14 +354,19 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option targetVersion, }); // Restart after the acknowledgement has had time to cross any relay - // hop. If systemd rejects the handoff, restore the previous unit while - // this process is still alive and log the failure for diagnostics. + // hop. --no-block queues the restart job and exits before systemd + // stops this unit: a blocking restart's SIGTERM reaches the systemctl + // child (it shares this service's cgroup), which read as a restart + // failure and rolled the new unit back while the old server finished + // shutting down. With the handoff race gone, a non-zero exit or spawn + // error means systemd genuinely rejected the job while this process is + // still alive, so restoring the previous unit below stays correct. yield* scheduleRestart( Effect.gen(function* () { const restart = yield* runner .run({ command: "systemctl", - args: ["--user", "restart", BOOT_SERVICE_UNIT_FILE], + args: ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE], }) .pipe( Effect.mapError((cause) => From 001b2c4adde95d98b96b53d42f397290d4089332 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Fri, 31 Jul 2026 03:30:46 -0700 Subject: [PATCH 2/2] fix(server): hold the update lock through a queued restart handoff With --no-block the restart fiber completes while the old process is still shutting down; the unconditional ensuring released the in-flight lock at that point, letting a second update rewrite the unit mid-teardown. Release the lock only on the failure path, after the rollback has restored the previous unit. Co-Authored-By: Claude Fable 5 --- apps/server/src/cloud/selfUpdate.test.ts | 8 ++++++++ apps/server/src/cloud/selfUpdate.ts | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 5a6f19941ac..bfac916a59d 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -550,6 +550,14 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { assert.lengthOf(context.spawns, 0); // systemd replaces the process; the server must not exit itself. assert.equal(context.exitCount(), 0); + + // The queued restart returns while this process is still shutting + // down; the lock must stay held so a second update cannot rewrite the + // unit mid-teardown. + const concurrentError = yield* context.service + .update({ targetVersion: "0.0.30" }) + .pipe(Effect.flip); + assert.include(concurrentError.reason, "already in progress"); }).pipe(Effect.provide(TestClock.layer())), ); diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index 5c8560692ca..9dcb713e1a1 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -394,9 +394,13 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option Effect.catch((error) => Effect.logError("Server self-update could not restart the boot service.").pipe( Effect.annotateLogs({ targetVersion, error: error.reason }), + // Permit a retry only after the failed handoff was rolled + // back. A queued restart returns while this process is still + // shutting down; releasing the lock then would let a second + // update rewrite the unit mid-teardown. + Effect.andThen(Ref.set(inFlight, false)), ), ), - Effect.ensuring(Ref.set(inFlight, false)), ), ); } else {