fix(web): show server update progress through reconnect

apps/server/src/auth/RpcAuthorization.ts

@@ -32,6 +32,7 @@ export const RPC_REQUIRED_SCOPES = {
   [WS_METHODS.serverRefreshProviders]: AuthOrchestrationOperateScope,
   [WS_METHODS.serverUpdateProvider]: AuthOrchestrationOperateScope,
   [WS_METHODS.serverUpdateServer]: AuthOrchestrationOperateScope,
+  [WS_METHODS.serverUpdateServerWithProgress]: AuthOrchestrationOperateScope,
   [WS_METHODS.serverUpsertKeybinding]: AuthOrchestrationOperateScope,
   [WS_METHODS.serverRemoveKeybinding]: AuthOrchestrationOperateScope,
   [WS_METHODS.serverGetSettings]: AuthOrchestrationReadScope,

apps/server/src/cloud/selfUpdate.test.ts

@@ -26,6 +26,33 @@ import * as SelfUpdate from "./selfUpdate.ts";
 
 const NODE_PATH = "/usr/local/bin/node";
 
+const eventuallyFileString = Effect.fn("test.eventuallyFileString")(function* (
+  filePath: string,
+  expected: string,
+) {
+  const fs = yield* FileSystem.FileSystem;
+  for (let iteration = 0; iteration < 1_000; iteration += 1) {
+    const contents = yield* fs.readFileString(filePath);
+    if (contents === expected) {
+      return;
+    }
+    // The rollback performs real filesystem I/O on a detached fiber, which
+    // advancing TestClock does not await.
+    yield* Effect.yieldNow;
+  }
+  return yield* Effect.die(new Error(`Expected file contents were not observed at ${filePath}.`));
+});
+
+const eventuallyTrue = Effect.fn("test.eventuallyTrue")(function* (predicate: () => boolean) {
+  for (let iteration = 0; iteration < 1_000; iteration += 1) {
+    if (predicate()) {
+      return;
+    }
+    yield* Effect.yieldNow;
+  }
+  return yield* Effect.die(new Error("Expected condition was not observed."));
+});
+
 interface RecordedCommand {
   readonly command: string;
   readonly args: ReadonlyArray<string>;
@@ -457,8 +484,12 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => {
   it.effect("installs, preflights, and respawns a foreground server", () =>
     Effect.gen(function* () {
       const context = yield* makeContext();
-      const result = yield* context.service.update({ targetVersion: "0.0.29" });
+      const progress: Array<string> = [];
+      const result = yield* context.service.update({ targetVersion: "0.0.29" }, (stage) =>
+        Effect.sync(() => progress.push(stage)),
+      );
       assert.deepEqual(result, { targetVersion: "0.0.29", method: "respawn" });
+      assert.deepEqual(progress, ["downloading", "installing"]);
       assert.lengthOf(context.spawns, 1);
 
       const concurrentError = yield* context.service
@@ -506,10 +537,12 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => {
       assert.include(unit, `ExecStart=${NODE_PATH} ${pinnedEntry} serve`);
       assert.deepEqual(
         context.commands.map((entry) => entry.command),
-        ["npm", NODE_PATH, "systemctl", "systemctl"],
+        ["npm", NODE_PATH, "systemctl"],
       );
       assert.deepEqual(context.commands[2]?.args, ["--user", "daemon-reload"]);
 
+      // Restart waits until after the update acknowledgement can flush.
+      yield* TestClock.adjust(Duration.seconds(10));
       assert.deepEqual(context.commands[3], {
         command: "systemctl",
         args: ["--user", "restart", "t3code.service"],
@@ -542,9 +575,11 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => {
       );
       const previousUnit = yield* context.fs.readFileString(unitPath);
 
-      const first = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip);
-      assert.include(first.reason, "Restarting the systemd boot service failed");
-      assert.equal(yield* context.fs.readFileString(unitPath), previousUnit);
+      const first = yield* context.service.update({ targetVersion: "0.0.29" });
+      assert.deepEqual(first, { targetVersion: "0.0.29", method: "boot-service" });
+      yield* TestClock.adjust(Duration.seconds(10));
+      yield* eventuallyFileString(unitPath, previousUnit);
+      yield* eventuallyTrue(() => context.commands.at(-1)?.args[1] === "daemon-reload");
       assert.deepEqual(
         context.commands.slice(-2).map((entry) => entry.args),
         [

apps/server/src/cloud/selfUpdate.ts

@@ -6,6 +6,7 @@ import {
   ServerSelfUpdateError,
   type ServerSelfUpdateCapability,
   type ServerSelfUpdateInput,
+  type ServerSelfUpdateProgressStage,
   type ServerSelfUpdateResult,
 } from "@t3tools/contracts";
 import {
@@ -147,6 +148,7 @@ export class ServerSelfUpdate extends Context.Service<
   {
     readonly update: (
       input: ServerSelfUpdateInput,
+      reportProgress?: (stage: ServerSelfUpdateProgressStage) => Effect.Effect<void, never, never>,
     ) => Effect.Effect<ServerSelfUpdateResult, ServerSelfUpdateError>;
   }
 >()("t3/cloud/selfUpdate/ServerSelfUpdate") {}
@@ -227,7 +229,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option
 
   const update: ServerSelfUpdate["Service"]["update"] = Effect.fn(
     "cloud.server_self_update.update",
-  )(function* (input) {
+  )(function* (input, reportProgress = () => Effect.void) {
     if (capability === "desktop-managed") {
       return yield* failWith(
         "This server is managed by the T3 Code desktop app on its machine; update the desktop app to update it.",
@@ -250,6 +252,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option
     }
 
     return yield* Effect.gen(function* () {
+      yield* reportProgress("downloading");
       const runtimePaths = yield* ensurePinnedRuntimeInstalled({
         baseDir: serverConfig.baseDir,
         version: targetVersion,
@@ -260,6 +263,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option
         Effect.mapError((error) => failWith("Could not install the requested t3 version.", error)),
       );
 
+      yield* reportProgress("installing");
       // A broken artifact (failed native build, incompatible node) must be
       // caught while the current server is still alive to report it.
       const preflight = yield* runner
@@ -349,38 +353,45 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option
         yield* Effect.logInfo("Server self-update installed; restarting boot service.", {
           targetVersion,
         });
-        // A successful systemd restart stops this process, so the RPC is
-        // interrupted and the reconnecting client observes the new version.
-        // A rejected restart returns while the old process is still alive;
-        // restore the previous unit and report that failure through the RPC.
-        yield* Effect.gen(function* () {
-          const restart = yield* runner
-            .run({
-              command: "systemctl",
-              args: ["--user", "restart", BOOT_SERVICE_UNIT_FILE],
-            })
-            .pipe(
-              Effect.mapError((cause) =>
-                failWith("Could not restart the systemd boot service.", cause),
+        // Restart after the acknowledgement has had time to cross any relay
+        // hop. If systemd rejects the handoff, restore the previous unit while
+        // this process is still alive and log the failure for diagnostics.
+        yield* scheduleRestart(
+          Effect.gen(function* () {
+            const restart = yield* runner
+              .run({
+                command: "systemctl",
+                args: ["--user", "restart", BOOT_SERVICE_UNIT_FILE],
+              })
+              .pipe(
+                Effect.mapError((cause) =>
+                  failWith("Could not restart the systemd boot service.", cause),
+                ),
+              );
+            if (restart.code !== 0) {
+              return yield* failWith(
+                `Restarting the systemd boot service failed (exit code ${String(restart.code)}).`,
+              );
+            }
+          }).pipe(
+            Effect.catch((restartError) =>
+              writeUnitAtomically(unitPath, previousUnit).pipe(
+                Effect.andThen(reloadSystemd()),
+                Effect.mapError((rollbackError) =>
+                  failWith("Could not restore the previous systemd unit.", {
+                    restartError,
+                    rollbackError,
+                  }),
+                ),
+                Effect.andThen(Effect.fail(restartError)),
               ),
-            );
-          if (restart.code !== 0) {
-            return yield* failWith(
-              `Restarting the systemd boot service failed (exit code ${String(restart.code)}).`,
-            );
-          }
-        }).pipe(
-          Effect.catch((restartError) =>
-            writeUnitAtomically(unitPath, previousUnit).pipe(
-              Effect.andThen(reloadSystemd()),
-              Effect.mapError((rollbackError) =>
-                failWith("Could not restore the previous systemd unit.", {
-                  restartError,
-                  rollbackError,
-                }),
+            ),
+            Effect.catch((error) =>
+              Effect.logError("Server self-update could not restart the boot service.").pipe(
+                Effect.annotateLogs({ targetVersion, error: error.reason }),
               ),
-              Effect.andThen(Effect.fail(restartError)),
             ),
+            Effect.ensuring(Ref.set(inFlight, false)),
           ),
         );
       } else {

apps/server/src/environment/ServerEnvironment.ts

@@ -143,6 +143,9 @@ export const make = Effect.gen(function* () {
       threadSettlement: true,
       threadSnooze: true,
       ...(serverSelfUpdate === null ? {} : { serverSelfUpdate }),
+      ...(serverSelfUpdate === "boot-service" || serverSelfUpdate === "respawn"
+        ? { serverSelfUpdateProgress: true }
+        : {}),
     },
   };
 

apps/server/src/ws.ts

@@ -40,6 +40,8 @@ import {
   ProjectWriteFileError,
   RelayClientInstallFailedError,
   type RelayClientInstallProgressEvent,
+  type ServerSelfUpdateError,
+  type ServerSelfUpdateProgressEvent,
   type FilesystemBrowseFailure,
   FilesystemBrowseError,
   AssetWorkspaceContextNotFoundError,
@@ -1364,6 +1366,33 @@ const makeWsRpcLayer = (
           observeRpcEffect(WS_METHODS.serverUpdateServer, serverSelfUpdate.update(input), {
             "rpc.aggregate": "server",
           }),
+        [WS_METHODS.serverUpdateServerWithProgress]: (input) =>
+          observeRpcStream(
+            WS_METHODS.serverUpdateServerWithProgress,
+            Stream.callback<ServerSelfUpdateProgressEvent, ServerSelfUpdateError>((queue) =>
+              serverSelfUpdate
+                .update(input, (stage) =>
+                  Queue.offer(queue, {
+                    type: "progress",
+                    stage,
+                  }).pipe(Effect.asVoid),
+                )
+                .pipe(
+                  Effect.flatMap((result) =>
+                    Queue.offer(queue, {
+                      type: "complete",
+                      result,
+                    }),
+                  ),
+                  Effect.catchTags({
+                    ServerSelfUpdateError: (error) => Queue.fail(queue, error),
+                  }),
+                  Effect.andThen(Queue.end(queue)),
+                  Effect.forkScoped,
+                ),
+            ),
+            { "rpc.aggregate": "server" },
+          ),
         [WS_METHODS.serverUpsertKeybinding]: (rule) =>
           observeRpcEffect(
             WS_METHODS.serverUpsertKeybinding,

apps/web/src/components/ChatView.tsx

@@ -296,7 +296,7 @@ import {
   AlertDialogTitle,
 } from "./ui/alert-dialog";
 import { Tooltip, TooltipPopup, TooltipTrigger } from "./ui/tooltip";
-import { ServerUpdateAction } from "./ServerUpdateAction";
+import { ServerUpdateAction, ServerUpdateProgress } from "./ServerUpdateAction";
 import {
   buildVersionMismatchDismissalKey,
   dismissVersionMismatch,
@@ -1876,12 +1876,16 @@ function ChatViewContent(props: ChatViewProps) {
     hasMultipleRegisteredEnvironments && activeThread
       ? `${environmentById.get(activeThread.environmentId)?.label ?? serverConfig?.environment.label ?? activeThread.environmentId} server`
       : "server";
-  const versionMismatchEnvironmentId =
-    versionMismatch && activeThread ? activeThread.environmentId : null;
+  const serverUpdateEnvironmentId = activeThread?.environmentId ?? null;
   const versionMismatchSelfUpdate = resolveServerSelfUpdateCapability(serverConfig);
+  const serverUpdateState = useAtomValue(
+    serverEnvironment.updateStateAtom(serverUpdateEnvironmentId),
+  );
   const systemComposerBannerItems = useMemo<ComposerBannerStackItem[]>(() => {
     const items: ComposerBannerStackItem[] = [];
-    if (activeEnvironmentUnavailableState) {
+    const resumingServerUpdate =
+      serverUpdateState.status === "running" && serverUpdateState.stage === "resuming";
+    if (activeEnvironmentUnavailableState && !resumingServerUpdate) {
       const connection = activeEnvironmentUnavailableState.connection;
       const isReconnecting =
         connection.phase === "connecting" || connection.phase === "reconnecting";
@@ -1918,39 +1922,57 @@ function ChatViewContent(props: ChatViewProps) {
       });
     }
     if (
-      showVersionMismatchBanner &&
-      versionMismatch &&
-      versionMismatchDismissKey &&
-      versionMismatchEnvironmentId
+      serverUpdateEnvironmentId &&
+      (serverUpdateState.status !== "idle" ||
+        (showVersionMismatchBanner && versionMismatch && versionMismatchDismissKey))
     ) {
+      const updateInProgress = serverUpdateState.status === "running";
+      const updateFailed = serverUpdateState.status === "failed";
       items.push({
-        id: `version-mismatch:${versionMismatchDismissKey}`,
-        variant: "warning",
+        id: `server-version:${serverUpdateEnvironmentId}`,
+        variant: updateFailed ? "error" : "warning",
         icon: <TriangleAlertIcon />,
-        title: "Client and server versions differ",
-        description: (
-          <>
-            Client {versionMismatch.clientVersion} is connected to {versionMismatchServerLabel}{" "}
-            {versionMismatch.serverVersion}.{" "}
-            {serverUpdateGuidance(versionMismatchSelfUpdate, versionMismatchServerLabel)}
-          </>
-        ),
+        title:
+          updateInProgress || updateFailed
+            ? `${updateFailed ? "Could not update" : "Updating"} ${versionMismatchServerLabel}`
+            : "Client and server versions differ",
+        description:
+          updateInProgress || updateFailed ? (
+            <ServerUpdateProgress
+              fromVersion={serverUpdateState.fromVersion}
+              serverLabel={versionMismatchServerLabel}
+              state={serverUpdateState}
+            />
+          ) : versionMismatch ? (
+            <>
+              Client {versionMismatch.clientVersion} is connected to {versionMismatchServerLabel}{" "}
+              {versionMismatch.serverVersion}.{" "}
+              {serverUpdateGuidance(versionMismatchSelfUpdate, versionMismatchServerLabel)}
+            </>
+          ) : null,
         // The desktop-managed guidance is already the description; the action
         // slot would only repeat it.
         actions:
+          updateInProgress ||
+          !versionMismatch ||
           versionMismatchSelfUpdate === "desktop-managed" ? undefined : (
             <ServerUpdateAction
-              environmentId={versionMismatchEnvironmentId}
+              environmentId={serverUpdateEnvironmentId}
               serverLabel={versionMismatchServerLabel}
               selfUpdate={versionMismatchSelfUpdate}
               targetVersion={versionMismatch.clientVersion}
+              {...(updateFailed ? { label: "Retry update" } : {})}
             />
           ),
-        dismissLabel: "Dismiss version mismatch warning",
-        onDismiss: () => {
-          dismissVersionMismatch(versionMismatchDismissKey);
-          setDismissedVersionMismatchKey(versionMismatchDismissKey);
-        },
+        ...(updateInProgress || updateFailed || !versionMismatchDismissKey
+          ? {}
+          : {
+              dismissLabel: "Dismiss version mismatch warning",
+              onDismiss: () => {
+                dismissVersionMismatch(versionMismatchDismissKey);
+                setDismissedVersionMismatchKey(versionMismatchDismissKey);
+              },
+            }),
       });
     }
     return items;
@@ -1960,9 +1982,10 @@ function ChatViewContent(props: ChatViewProps) {
     navigate,
     setDismissedVersionMismatchKey,
     showVersionMismatchBanner,
+    serverUpdateState,
     versionMismatch,
     versionMismatchDismissKey,
-    versionMismatchEnvironmentId,
+    serverUpdateEnvironmentId,
     versionMismatchSelfUpdate,
     versionMismatchServerLabel,
   ]);