diff --git a/apps/server/package.json b/apps/server/package.json index 48ee9121b51..8e7b5b38591 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -16,7 +16,7 @@ "type": "module", "scripts": { "dev": "node --watch src/bin.ts", - "build:bundle": "vp pack", + "build:bundle": "vp pack && vp pack src/service-launcher.ts --out-dir dist --no-clean", "start": "node dist/bin.mjs", "typecheck": "tsgo --noEmit", "test": "vp test run" diff --git a/apps/server/scripts/cli.ts b/apps/server/scripts/cli.ts index 517f633577c..2de5b702a28 100644 --- a/apps/server/scripts/cli.ts +++ b/apps/server/scripts/cli.ts @@ -223,7 +223,11 @@ const publishCmd = Command.make( const packageJsonPath = path.join(serverDir, "package.json"); // Assert build assets exist - for (const relPath of ["dist/bin.mjs", "dist/client/index.html"]) { + for (const relPath of [ + "dist/bin.mjs", + "dist/service-launcher.mjs", + "dist/client/index.html", + ]) { const abs = path.join(serverDir, relPath); if (!(yield* fs.exists(abs))) { return yield* new ServerCliBuildAssetMissingError({ assetPath: abs }); diff --git a/apps/server/src/bin.ts b/apps/server/src/bin.ts index ab60749e389..d1bdcf90997 100644 --- a/apps/server/src/bin.ts +++ b/apps/server/src/bin.ts @@ -15,6 +15,7 @@ import { sharedServerCommandFlags } from "./cli/config.ts"; import { projectCommand } from "./cli/project.ts"; import { runServerCommand, serveCommand, startCommand } from "./cli/server.ts"; import { serviceCommand } from "./cli/service.ts"; +import { servicePreflightCommand } from "./cli/servicePreflight.ts"; const CliRuntimeLayer = Layer.mergeAll(NodeServices.layer, NetService.layer); @@ -53,6 +54,7 @@ export const makeCli = ({ cloudEnabled = hasCloudPublicConfig } = {}) => authCommand, projectCommand, serviceCommand, + servicePreflightCommand, cloudEnabled ? connectCommand : connectUnavailableCommand, ]), ); diff --git a/apps/server/src/cli/servicePreflight.ts b/apps/server/src/cli/servicePreflight.ts new file mode 100644 index 00000000000..60a457f388a --- /dev/null +++ b/apps/server/src/cli/servicePreflight.ts @@ -0,0 +1,17 @@ +import * as Console from "effect/Console"; +import * as Effect from "effect/Effect"; +import { Command, Flag } from "effect/unstable/cli"; + +import { runServicePreflight } from "../cloud/servicePreflight.ts"; + +export const servicePreflightCommand = Command.make("__service-preflight", { + databasePath: Flag.string("database-path"), + launcherProtocol: Flag.integer("launcher-protocol"), +}).pipe( + Command.withHidden, + Command.withHandler(({ databasePath, launcherProtocol }) => + Console.log(JSON.stringify(runServicePreflight({ databasePath, launcherProtocol }))).pipe( + Effect.asVoid, + ), + ), +); diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 46e9a1da987..b0ff61bba4b 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -1,558 +1,136 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; +import { expect, it } from "@effect/vitest"; +import { + HostProcessArguments, + HostProcessExecutablePath, + HostProcessPlatform, +} from "@t3tools/shared/hostProcess"; import * as ConfigProvider from "effect/ConfigProvider"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Path from "effect/Path"; -import * as Schema from "effect/Schema"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; -import { - HostProcessArguments, - HostProcessExecutablePath, - HostProcessPlatform, -} from "@t3tools/shared/hostProcess"; - -import { reconcileService } from "../cli/service.ts"; import * as ProcessRunner from "../processRunner.ts"; import * as BootService from "./bootService.ts"; +import { pinnedRuntimePaths } from "./pinnedRuntime.ts"; +import { parseServiceState } from "./serviceProtocol.ts"; -const isUnsupportedError = Schema.is(BootService.BootServiceUnsupportedError); -const isCommandError = Schema.is(BootService.BootServiceCommandError); - -interface RecordedCommand { - readonly command: string; - readonly args: ReadonlyArray; -} - -const makeRecordingRunnerLayer = ( - commands: Array, - options?: { - readonly failCommand?: string; - readonly failWhen?: (command: string, args: ReadonlyArray) => boolean; - }, -) => - Layer.succeed( - ProcessRunner.ProcessRunner, - ProcessRunner.ProcessRunner.of({ - run: (input) => - Effect.sync(() => { - assert.isUndefined(input.env); - commands.push({ command: input.command, args: input.args }); - const failed = - input.command === options?.failCommand || - options?.failWhen?.(input.command, input.args) === true; - return { - stdout: "", - stderr: failed ? `${input.command} exploded` : "", - code: ChildProcessSpawner.ExitCode(failed ? 1 : 0), - timedOut: false, - stdoutTruncated: false, - stderrTruncated: false, - }; - }), - }), - ); - -const makeHost = (entry: string): BootService.BootServiceHost => ({ - execPath: "/usr/local/bin/node", - cliEntryPath: entry, -}); - -const provideHostRefs = (home: string, platform: NodeJS.Platform = "linux") => - Effect.provide( - Layer.mergeAll( - Layer.succeed(HostProcessPlatform, platform), - ConfigProvider.layer(ConfigProvider.fromEnv({ env: { HOME: home } })), - ), - ); - -const makeTestContext = Effect.fn("test.makeTestContext")(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boot-service-test-" }); - // A real file for the stable-entry cases so status can confirm the entry - // point exists. - const stableEntry = path.join(root, "bin.mjs"); - yield* fs.writeFileString(stableEntry, "#!/usr/bin/env node\n"); - return { - fs, - path, - dirs: { - home: root, - baseDir: path.join(root, ".t3"), - logsDir: path.join(root, ".t3", "userdata", "logs"), - stableEntry, - }, - }; -}); - -it("renders a systemd unit with absolute paths and append-mode logging", () => { +it("keeps systemd pinned to the stable launcher rather than a versioned server", () => { const unit = BootService.renderBootServiceUnit({ - nodePath: "/usr/local/bin/node", - t3EntryPath: "/home/theo/.t3/runtime/versions/0.0.27/node_modules/t3/dist/bin.mjs", + nodePath: "/usr/bin/node", + launcherPath: "/home/theo/.t3/runtime/service-launcher.mjs", baseDir: "/home/theo/.t3", logPath: "/home/theo/.t3/userdata/logs/boot-service.log", unitPath: "/home/theo/.config/systemd/user/t3code.service", }); - assert.equal( - unit, - [ - "[Unit]", - "Description=T3 Code server", - "StartLimitIntervalSec=300", - "StartLimitBurst=5", - "", - "[Service]", - "Type=simple", - "WorkingDirectory=%h", - "Environment=T3CODE_HOME=/home/theo/.t3", - "Environment=T3_BOOT_SERVICE_UNIT=t3code.service", - "ExecStart=/usr/local/bin/node /home/theo/.t3/runtime/versions/0.0.27/node_modules/t3/dist/bin.mjs serve", - "Restart=always", - "RestartSec=5", - "StandardOutput=append:/home/theo/.t3/userdata/logs/boot-service.log", - "StandardError=append:/home/theo/.t3/userdata/logs/boot-service.log", - "", - "[Install]", - "WantedBy=default.target", - "", - ].join("\n"), - ); + expect(unit).toContain("ExecStart=/usr/bin/node /home/theo/.t3/runtime/service-launcher.mjs"); + expect(unit).toContain("KillMode=control-group"); + expect(unit).not.toContain("versions/1.2.3"); }); -it("quotes systemd values containing spaces and escapes percent specifiers", () => { - assert.equal(BootService.quoteSystemdValue("/plain/path"), "/plain/path"); - assert.equal(BootService.quoteSystemdValue("/home/me/T3 Data"), '"/home/me/T3 Data"'); - assert.equal(BootService.quoteSystemdValue("/opt/100%cpu"), "/opt/100%%cpu"); - - const unit = BootService.renderBootServiceUnit({ - nodePath: "/home/me/my tools/node", - t3EntryPath: "/home/me/T3 Data/bin.mjs", - baseDir: "/home/me/T3 Data", - logPath: "/home/me/100%logs/boot.log", - unitPath: "/home/me/.config/systemd/user/t3code.service", +const makeHarness = Effect.fn("test.make_boot_service_harness")(function* ( + platform: NodeJS.Platform = "linux", +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-boot-service-test-" }); + const baseDir = path.join(home, ".t3"); + const sourceLauncher = path.join(home, "service-launcher.mjs"); + const statePath = path.join(baseDir, "runtime", "service-state.json"); + yield* fs.writeFileString(sourceLauncher, "export {};\n"); + const runtime = pinnedRuntimePaths(path, baseDir, "1.2.3"); + yield* fs.makeDirectory(path.dirname(runtime.entryPath), { recursive: true }); + yield* fs.writeFileString(runtime.entryPath, "export {};\n"); + yield* fs.writeFileString(runtime.sentinelPath, "1.2.3\n"); + + const commands: string[] = []; + const control: { failCommand: string | undefined } = { failCommand: undefined }; + const runner = ProcessRunner.ProcessRunner.of({ + run: (input) => + Effect.sync(() => { + const command = `${input.command} ${input.args.join(" ")}`; + commands.push(command); + return { + stdout: input.args[1] === "--version" ? "t3 v1.2.3\n" : "", + stderr: "", + code: ChildProcessSpawner.ExitCode(command === control.failCommand ? 1 : 0), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }; + }), }); - assert.include(unit, 'ExecStart="/home/me/my tools/node" "/home/me/T3 Data/bin.mjs" serve'); - assert.include(unit, 'Environment=T3CODE_HOME="/home/me/T3 Data"'); - // append: paths take the rest of the line literally (spaces are fine, - // quoting is not), but % still goes through specifier expansion. - assert.include(unit, "StandardOutput=append:/home/me/100%%logs/boot.log"); - assert.include(unit, "StandardError=append:/home/me/100%%logs/boot.log"); -}); - -it("flags package-manager cache entry points as ephemeral", () => { - assert.isTrue( - BootService.isEphemeralCacheEntry("/home/theo/.npm/_npx/abc123/node_modules/t3/dist/bin.mjs"), - ); - assert.isTrue( - BootService.isEphemeralCacheEntry("C:\\Users\\theo\\AppData\\npm-cache\\_npx\\abc\\bin.mjs"), - ); - assert.isTrue( - BootService.isEphemeralCacheEntry( - "/home/theo/.cache/pnpm/dlx/abc/node_modules/t3/dist/bin.mjs", - ), - ); - assert.isTrue( - BootService.isEphemeralCacheEntry("/home/theo/.bun/install/cache/t3@0.0.27/dist/bin.mjs"), - ); - assert.isFalse(BootService.isEphemeralCacheEntry("/usr/local/lib/node_modules/t3/dist/bin.mjs")); - assert.isFalse( - BootService.isEphemeralCacheEntry( - "/home/theo/dev/pnpm/dlx-tools/t3/node_modules/t3/dist/bin.mjs", - ), - ); - assert.isFalse( - BootService.isEphemeralCacheEntry( - "/home/theo/.t3/runtime/versions/0.0.27/node_modules/t3/dist/bin.mjs", + const service = yield* BootService.make({ + baseDir, + logsDir: path.join(baseDir, "userdata", "logs"), + cliVersion: "1.2.3", + host: { + execPath: "/usr/bin/node", + cliEntryPath: path.join(home, "bin.mjs"), + launcherSourcePath: sourceLauncher, + }, + }).pipe( + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.provide( + Layer.mergeAll( + Layer.succeed(HostProcessPlatform, platform), + Layer.succeed(HostProcessExecutablePath, "/usr/bin/node"), + Layer.succeed(HostProcessArguments, ["/usr/bin/node", path.join(home, "bin.mjs")]), + ConfigProvider.layer(ConfigProvider.fromEnv({ env: { HOME: home } })), + ), ), ); + return { service, fs, statePath, commands, control }; }); -it.layer(NodeServices.layer)("BootService", (it) => { - it.effect("reconciles the standalone service once and is then idempotent", () => - Effect.gen(function* () { - const { dirs } = yield* makeTestContext(); - const commands: Array = []; - const service = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost(dirs.stableEntry), - }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); - - const first = yield* reconcileService().pipe( - Effect.provideService(BootService.BootService, service), - ); - assert.isTrue(first.changed); - if (!first.changed) return; - assert.isFalse(first.previouslyInstalled); - - const commandCount = commands.length; - const second = yield* reconcileService().pipe( - Effect.provideService(BootService.BootService, service), - ); - assert.isFalse(second.changed); - assert.lengthOf(commands, commandCount); - }), - ); - - it.effect("installs the unit, enables the service, and enables linger", () => - Effect.gen(function* () { - const { dirs, fs, path } = yield* makeTestContext(); - const commands: Array = []; - const service = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost(dirs.stableEntry), - }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); - - const plan = yield* service.install; - - // A stable entry point is reused directly — no npm install. - assert.equal(plan.t3EntryPath, dirs.stableEntry); - assert.deepEqual( - commands.map((entry) => [entry.command, ...entry.args].join(" ")), - [ - "systemctl --user daemon-reload", - "systemctl --user enable t3code.service", - // restart (not enable --now) so repairing a stale unit replaces a - // running process instead of leaving the old one until reboot. - "systemctl --user restart t3code.service", - "loginctl enable-linger", - ], - ); - - const unitPath = path.join(dirs.home, ".config", "systemd", "user", "t3code.service"); - const unit = yield* fs.readFileString(unitPath); - assert.include(unit, `ExecStart=/usr/local/bin/node ${dirs.stableEntry} serve`); - assert.include(unit, `Environment=T3CODE_HOME=${dirs.baseDir}`); - - const status = yield* service.status; - assert.isTrue(status.supported); - assert.isTrue(status.installed); - assert.isTrue(status.current); - - const removed = yield* service.uninstall; - assert.isTrue(removed); - assert.isFalse(yield* fs.exists(unitPath)); - const statusAfter = yield* service.status; - assert.isFalse(statusAfter.installed); - const removedAgain = yield* service.uninstall; - assert.isFalse(removedAgain); - }), - ); - - it.effect("pins a runtime via npm install when running from the npx cache", () => +it.layer(NodeServices.layer)("boot service install", (it) => { + it.effect("installs, reports current state, and uninstalls", () => Effect.gen(function* () { - const { dirs, fs, path } = yield* makeTestContext(); - const commands: Array = []; - const service = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost("/home/theo/.npm/_npx/abc/node_modules/t3/dist/bin.mjs"), - }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); - + const { service, fs, statePath, commands } = yield* makeHarness(); const plan = yield* service.install; - const runtimeDir = path.join(dirs.baseDir, "runtime", "versions", "0.0.27"); - assert.equal( - plan.t3EntryPath, - path.join(runtimeDir, "node_modules", "t3", "dist", "bin.mjs"), - ); - assert.deepEqual(commands[0], { - command: "npm", - args: ["install", "--prefix", runtimeDir, "--no-fund", "--no-audit", "t3@0.0.27"], + expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ + protocol: 1, + activeVersion: "1.2.3", }); - // Success is recorded via a sentinel so interrupted installs re-run. - assert.isTrue(yield* fs.exists(path.join(runtimeDir, ".install-complete"))); - }), - ); - - it.effect("reinstalls a pinned runtime when its entry point is missing", () => - Effect.gen(function* () { - const { dirs, fs, path } = yield* makeTestContext(); - const commands: Array = []; - const service = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost("/home/theo/.npm/_npx/abc/node_modules/t3/dist/bin.mjs"), - }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); - - const plan = yield* service.install; - yield* fs.makeDirectory(path.dirname(plan.t3EntryPath), { recursive: true }); - yield* fs.writeFileString(plan.t3EntryPath, "#!/usr/bin/env node\n"); - yield* fs.remove(plan.t3EntryPath); - commands.length = 0; - - yield* service.install; - - assert.isTrue(commands.some(({ command }) => command === "npm")); - }), - ); - - it.effect("reads executable metadata from host process references", () => - Effect.gen(function* () { - const { dirs } = yield* makeTestContext(); - const commands: Array = []; - const service = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - }).pipe( - Effect.provide(makeRecordingRunnerLayer(commands)), - provideHostRefs(dirs.home), - Effect.provideService(HostProcessExecutablePath, "/opt/node/bin/node"), - Effect.provideService(HostProcessArguments, ["/opt/node/bin/node", dirs.stableEntry]), - ); - - const plan = yield* service.install; - assert.equal(plan.nodePath, "/opt/node/bin/node"); - assert.equal(plan.t3EntryPath, dirs.stableEntry); - }), - ); - - it.effect("cleans up and fails when the pinned runtime install fails", () => - Effect.gen(function* () { - const { dirs, fs, path } = yield* makeTestContext(); - const commands: Array = []; - const service = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost("/home/theo/.npm/_npx/abc/node_modules/t3/dist/bin.mjs"), - }).pipe( - Effect.provide(makeRecordingRunnerLayer(commands, { failCommand: "npm" })), - provideHostRefs(dirs.home), - ); - - const error = yield* service.install.pipe(Effect.flip); - assert.isTrue(isCommandError(error)); - const runtimeDir = path.join(dirs.baseDir, "runtime", "versions", "0.0.27"); - // The half-installed tree must not be reused by the next attempt. - assert.isFalse(yield* fs.exists(runtimeDir)); - assert.isFalse(yield* fs.exists(path.join(runtimeDir, ".install-complete"))); - }), - ); - - it.effect("reports an installed-but-stale unit so the lifecycle can offer a repair", () => - Effect.gen(function* () { - const { dirs, fs, path } = yield* makeTestContext(); - const commands: Array = []; - const service = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost(dirs.stableEntry), - }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); - - const unitDir = path.join(dirs.home, ".config", "systemd", "user"); - yield* fs.makeDirectory(unitDir, { recursive: true }); + expect(yield* fs.readFileString(plan.launcherPath)).toBe("export {};\n"); + expect((yield* service.status).current).toBe(true); yield* fs.writeFileString( - path.join(unitDir, "t3code.service"), - "[Service]\nExecStart=/old/node /old/t3 serve\n", + statePath, + '{"protocol":1,"activeVersion":"1.2.3","update":{"id":"u","fromVersion":"1.2.3","targetVersion":"1.2.4","status":"pending"}}', ); - - const status = yield* service.status; - assert.isTrue(status.supported); - assert.isTrue(status.installed); - assert.isFalse(status.current); + expect((yield* service.status).current).toBe(false); + expect(yield* service.uninstall).toBe(true); + expect((yield* service.status).installed).toBe(false); + expect(commands.some((command) => command.startsWith("npm "))).toBe(false); }), ); - it.effect("reports a current unit as stale when its entry point is gone", () => + it.effect("restarts an installed service when repair fails", () => Effect.gen(function* () { - const { dirs, fs } = yield* makeTestContext(); - const commands: Array = []; - const service = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost(dirs.stableEntry), - }).pipe(Effect.provide(makeRecordingRunnerLayer(commands)), provideHostRefs(dirs.home)); - + const { service, commands, control } = yield* makeHarness(); yield* service.install; - assert.isTrue((yield* service.status).current); - - // The pinned runtime (or global bin) was deleted to reclaim space; the - // unit still matches byte-for-byte but would crashloop at boot. - yield* fs.remove(dirs.stableEntry); - const status = yield* service.status; - assert.isTrue(status.installed); - assert.isFalse(status.current); - }), - ); - - it.effect("fails on non-Linux platforms without touching the filesystem", () => - Effect.gen(function* () { - const { dirs, fs, path } = yield* makeTestContext(); - const commands: Array = []; - const service = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost("/usr/local/lib/node_modules/t3/dist/bin.mjs"), - }).pipe( - Effect.provide(makeRecordingRunnerLayer(commands)), - provideHostRefs(dirs.home, "darwin"), - ); - - const error = yield* service.install.pipe(Effect.flip); - assert.isTrue(isUnsupportedError(error)); - assert.lengthOf(commands, 0); - assert.isFalse( - yield* fs.exists(path.join(dirs.home, ".config", "systemd", "user", "t3code.service")), - ); - - const status = yield* service.status; - assert.isFalse(status.supported); - assert.isFalse(status.installed); - }), - ); - - it.effect("removes the unit file when an activation step fails", () => - Effect.gen(function* () { - const { dirs, fs, path } = yield* makeTestContext(); - const commands: Array = []; - const service = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost("/usr/local/lib/node_modules/t3/dist/bin.mjs"), - }).pipe( - Effect.provide(makeRecordingRunnerLayer(commands, { failCommand: "loginctl" })), - provideHostRefs(dirs.home), - ); + commands.length = 0; + control.failCommand = "systemctl --user daemon-reload"; const error = yield* service.install.pipe(Effect.flip); - assert.isTrue(isCommandError(error)); - // A leftover unit would make status report "installed" even though - // linger never happened. - assert.isFalse( - yield* fs.exists(path.join(dirs.home, ".config", "systemd", "user", "t3code.service")), - ); - const status = yield* service.status; - assert.isFalse(status.installed); - assert.isTrue( - commands.some( - ({ command, args }) => - command === "systemctl" && args.join(" ") === "--user disable --now t3code.service", - ), - ); + expect(error._tag).toBe("BootServiceCommandError"); + expect(commands.filter((command) => command.startsWith("systemctl "))).toEqual([ + "systemctl --user stop t3code.service", + "systemctl --user daemon-reload", + "systemctl --user restart t3code.service", + ]); }), ); - it.effect("restores the previous unit when a repair cannot activate", () => + it.effect("fails closed off Linux", () => Effect.gen(function* () { - const { dirs, fs, path } = yield* makeTestContext(); - const initialCommands: Array = []; - const initialService = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost(dirs.stableEntry), - }).pipe( - Effect.provide(makeRecordingRunnerLayer(initialCommands)), - provideHostRefs(dirs.home), - ); - yield* initialService.install; - - const unitPath = path.join(dirs.home, ".config", "systemd", "user", "t3code.service"); - const previousUnit = yield* fs.readFileString(unitPath); - const replacementEntry = path.join(dirs.home, "replacement-bin.mjs"); - yield* fs.writeFileString(replacementEntry, "#!/usr/bin/env node\n"); - const repairCommands: Array = []; - const repairService = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.28", - host: makeHost(replacementEntry), - }).pipe( - Effect.provide(makeRecordingRunnerLayer(repairCommands, { failCommand: "loginctl" })), - provideHostRefs(dirs.home), - ); - - const error = yield* repairService.install.pipe(Effect.flip); - - assert.isTrue(isCommandError(error)); - assert.equal(yield* fs.readFileString(unitPath), previousUnit); - assert.isTrue( - repairCommands.some( - ({ command, args }) => - command === "systemctl" && args.join(" ") === "--user restart t3code.service", - ), - ); - }), - ); - - it.effect("keeps the unit when stopping it during uninstall fails", () => - Effect.gen(function* () { - const { dirs, fs, path } = yield* makeTestContext(); - const installCommands: Array = []; - const installedService = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost(dirs.stableEntry), - }).pipe( - Effect.provide(makeRecordingRunnerLayer(installCommands)), - provideHostRefs(dirs.home), - ); - yield* installedService.install; - - const uninstallCommands: Array = []; - const failingService = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost(dirs.stableEntry), - }).pipe( - Effect.provide( - makeRecordingRunnerLayer(uninstallCommands, { - failWhen: (command, args) => - command === "systemctl" && args.includes("disable") && args.includes("--now"), - }), - ), - provideHostRefs(dirs.home), - ); - - const error = yield* failingService.uninstall.pipe(Effect.flip); - - assert.isTrue(isCommandError(error)); - assert.isTrue( - yield* fs.exists(path.join(dirs.home, ".config", "systemd", "user", "t3code.service")), - ); - }), - ); - - it.effect("appends failed steps to the boot-service log", () => - Effect.gen(function* () { - const { dirs, fs, path } = yield* makeTestContext(); - const commands: Array = []; - const service = yield* BootService.make({ - baseDir: dirs.baseDir, - logsDir: dirs.logsDir, - cliVersion: "0.0.27", - host: makeHost("/usr/local/lib/node_modules/t3/dist/bin.mjs"), - }).pipe( - Effect.provide(makeRecordingRunnerLayer(commands, { failCommand: "systemctl" })), - provideHostRefs(dirs.home), - ); - - const error = yield* service.install.pipe(Effect.flip); - assert.isTrue(isCommandError(error)); - if (!isCommandError(error)) return; - assert.equal(error.exitCode, 1); - assert.equal(error.stderrLength, "systemctl exploded".length); - - const logPath = path.join(dirs.logsDir, "boot-service.log"); - assert.isTrue(yield* fs.exists(logPath)); - assert.include(yield* fs.readFileString(logPath), "exit code 1"); + const { service } = yield* makeHarness("darwin"); + expect((yield* service.status).supported).toBe(false); + expect((yield* service.install.pipe(Effect.flip))._tag).toBe("BootServiceUnsupportedError"); }), ); }); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index d7e13e834f4..874b6b712a0 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -1,5 +1,10 @@ -import * as Context from "effect/Context"; +import { + HostProcessArguments, + HostProcessExecutablePath, + HostProcessPlatform, +} from "@t3tools/shared/hostProcess"; import * as Config from "effect/Config"; +import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -9,58 +14,29 @@ import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; -import { - HostProcessArguments, - HostProcessExecutablePath, - HostProcessPlatform, -} from "@t3tools/shared/hostProcess"; - import * as ProcessRunner from "../processRunner.ts"; -import { ensurePinnedRuntimeInstalled, pinnedRuntimePaths } from "./pinnedRuntime.ts"; - -/** - * Installs T3 Code as a per-user boot service. Linux-only for now: systemd - * user unit + loginctl enable-linger. The service runs a stable or pinned - * runtime — never an ephemeral `npx t3` cache whose eviction could break - * startup. - */ +import { + ensurePinnedRuntimeInstalled, + pinnedRuntimePaths, + PinnedRuntimeInstallError, +} from "./pinnedRuntime.ts"; +import { + SERVICE_LAUNCHER_FILE, + SERVICE_LAUNCHER_PROTOCOL, + SERVICE_STATE_FILE, + parseServiceState, + type ServiceState, +} from "./serviceProtocol.ts"; const BOOT_SERVICE_NAME = "t3code"; - export const BOOT_SERVICE_UNIT_FILE = `${BOOT_SERVICE_NAME}.service`; export const BOOT_SERVICE_UNIT_ENV = "T3_BOOT_SERVICE_UNIT"; -const EPHEMERAL_CACHE_SEGMENTS = [ - "/_npx/", // npx - "\\_npx\\", - "/pnpm/dlx/", // pnpm dlx (~/.cache/pnpm/dlx and $PNPM_HOME/.pnpm/dlx) - "/.pnpm/dlx/", - "/.bun/install/cache/", // bunx -]; - -/** - * `npx t3` (and pnpm dlx / bunx) run out of ephemeral package-manager - * caches that can be evicted at any time — a boot service must never point - * there. Global installs, repo checkouts, and the pinned runtime below are - * all stable. - */ -export function isEphemeralCacheEntry(entryPath: string): boolean { - return EPHEMERAL_CACHE_SEGMENTS.some((segment) => entryPath.includes(segment)); -} - -/** - * systemd expands `%` specifiers in most directive values, including the - * `append:` file paths, which take the rest of the line literally and must - * NOT be quoted. - */ +/** systemd expands `%` specifiers, including in unquoted append-log paths. */ export function escapeSystemdSpecifiers(value: string): string { return value.replaceAll("%", "%%"); } -/** - * systemd word-splits ExecStart and Environment values and expands `%` - * specifiers, so paths with spaces or percents must be quoted and escaped. - */ export function quoteSystemdValue(value: string): string { const escaped = escapeSystemdSpecifiers(value); return /[\s"'\\]/.test(escaped) @@ -69,31 +45,19 @@ export function quoteSystemdValue(value: string): string { } export interface BootServicePlan { - /** Absolute path of the node binary running this CLI. */ readonly nodePath: string; - /** Absolute path of the pinned t3 entry point the unit will run. */ - readonly t3EntryPath: string; + readonly launcherPath: string; readonly baseDir: string; readonly logPath: string; readonly unitPath: string; } -/** - * Pure so it is testable byte-for-byte. systemd user units run with a - * minimal environment: every path must be absolute, and the service must - * not rely on PATH, nvm shims, or shell profiles. Failures land in - * `logPath` because `systemctl --user` failures are otherwise invisible. - */ +/** Pure renderer: service units cannot rely on the user's shell or PATH. */ export function renderBootServiceUnit(plan: BootServicePlan): string { - // No After=network-online.target: it does not exist in the systemd *user* - // manager, so ordering on it is silently ignored. The server retries its - // relay connection, and Restart=always covers early-boot failures. + // The user manager has no reliable network-online target; server networking retries itself. return [ "[Unit]", "Description=T3 Code server", - // Give up after 5 crashes in 5 minutes so a persistently broken install - // (deleted runtime, broken workspace) stops instead of restarting every - // 5s forever and growing the unrotated append log without bound. "StartLimitIntervalSec=300", "StartLimitBurst=5", "", @@ -102,7 +66,8 @@ export function renderBootServiceUnit(plan: BootServicePlan): string { "WorkingDirectory=%h", `Environment=T3CODE_HOME=${quoteSystemdValue(plan.baseDir)}`, `Environment=${BOOT_SERVICE_UNIT_ENV}=${BOOT_SERVICE_UNIT_FILE}`, - `ExecStart=${quoteSystemdValue(plan.nodePath)} ${quoteSystemdValue(plan.t3EntryPath)} serve`, + `ExecStart=${quoteSystemdValue(plan.nodePath)} ${quoteSystemdValue(plan.launcherPath)}`, + "KillMode=control-group", "Restart=always", "RestartSec=5", `StandardOutput=append:${escapeSystemdSpecifiers(plan.logPath)}`, @@ -157,7 +122,6 @@ export type BootServiceError = export interface BootServiceStatus { readonly supported: boolean; readonly installed: boolean; - /** False when the installed unit no longer matches what install would write. */ readonly current: boolean; readonly unitPath: string; readonly logPath: string; @@ -166,12 +130,7 @@ export interface BootServiceStatus { export class BootService extends Context.Service< BootService, { - /** Installs the pinned runtime + unit, enables linger, starts the service. */ readonly install: Effect.Effect; - /** - * Stops and removes the unit; leaves the pinned runtime for reuse. - * Returns whether a unit was actually removed. - */ readonly uninstall: Effect.Effect; readonly status: Effect.Effect; } @@ -180,6 +139,7 @@ export class BootService extends Context.Service< export interface BootServiceHost { readonly execPath: string; readonly cliEntryPath: string; + readonly launcherSourcePath?: string; } export const make = Effect.fn("cloud.boot_service.make")(function* (input: { @@ -188,24 +148,45 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { readonly cliVersion: string; readonly host?: BootServiceHost; }) { - const hostExecPath = yield* HostProcessExecutablePath; const hostArguments = yield* HostProcessArguments; - const host = input.host ?? { - execPath: hostExecPath, - // When running the packed CLI this is dist/bin.mjs; when stable (global - // install, repo checkout) the boot service runs this same artifact. - cliEntryPath: hostArguments[1] ?? "", - }; + const hostExecPath = yield* HostProcessExecutablePath; const platform = yield* HostProcessPlatform; const homeDir = yield* Config.string("HOME").pipe(Config.withDefault("")); const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const runner = yield* ProcessRunner.ProcessRunner; + const host = input.host ?? { + execPath: hostExecPath, + cliEntryPath: hostArguments[1] ?? "", + }; const unitDir = path.join(homeDir, ".config", "systemd", "user"); const unitPath = path.join(unitDir, BOOT_SERVICE_UNIT_FILE); const logPath = path.join(input.logsDir, "boot-service.log"); + const launcherPath = path.join(input.baseDir, "runtime", SERVICE_LAUNCHER_FILE); + const statePath = path.join(input.baseDir, "runtime", SERVICE_STATE_FILE); + const launcherSourcePath = + host.launcherSourcePath ?? path.join(path.dirname(host.cliEntryPath), SERVICE_LAUNCHER_FILE); const runtimePaths = pinnedRuntimePaths(path, input.baseDir, input.cliVersion); + const writeDurably = (filePath: string, contents: string) => + Effect.scoped( + Effect.gen(function* () { + const directory = path.dirname(filePath); + yield* fs.makeDirectory(directory, { recursive: true }); + const tempPath = yield* fs.makeTempFileScoped({ directory, prefix: ".service-write-" }); + yield* fs.writeFileString(tempPath, contents, { mode: 0o600 }); + yield* (yield* fs.open(tempPath, { flag: "r" })).sync; + yield* fs.rename(tempPath, filePath); + yield* (yield* fs.open(directory, { flag: "r" })).sync; + }), + ).pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + const plan: BootServicePlan = { + nodePath: host.execPath, + launcherPath, + baseDir: input.baseDir, + logPath, + unitPath, + }; const requireSystemdLinux = Effect.gen(function* () { if (platform !== "linux" || homeDir === "") { @@ -244,150 +225,130 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { ); }); - /** - * Ensures plannedEntryPath exists before the unit points at it. A stable - * install (global bin, repo checkout) is used as-is; an ephemeral cache - * entry is replaced by `npm install --prefix`-ing the exact running - * version into /runtime/versions/. A real install (not a copy - * of bin.mjs) because t3 ships native deps like node-pty. - */ - const ensurePinnedRuntime = Effect.gen(function* () { - if (!isEphemeralCacheEntry(host.cliEntryPath)) { - return; - } + const install: BootService["Service"]["install"] = Effect.gen(function* () { + yield* requireSystemdLinux; + yield* fs + .makeDirectory(input.logsDir, { recursive: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + + // Prepare every immutable artifact before stopping the installed unit. yield* ensurePinnedRuntimeInstalled({ baseDir: input.baseDir, version: input.cliVersion, fs, path, runner, + validate: (runtime) => + runner + .run({ + command: host.execPath, + args: [runtime.entryPath, "--version"], + timeout: Duration.seconds(30), + }) + .pipe( + Effect.mapError( + (cause) => + new PinnedRuntimeInstallError({ + step: "verifying the pinned t3 runtime", + cause, + }), + ), + Effect.flatMap((result) => { + const reportedVersion = /\bv(\S+)\s*$/.exec(result.stdout)?.[1]; + return result.code === 0 && reportedVersion === input.cliVersion + ? Effect.void + : Effect.fail( + new PinnedRuntimeInstallError({ + step: "verifying the pinned t3 runtime", + exitCode: Number(result.code), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }), + ); + }), + ), }).pipe( Effect.mapError((error) => - error.step.startsWith("installing") + error._tag === "PinnedRuntimeInstallError" ? new BootServiceCommandError({ step: error.step, exitCode: error.exitCode, stdoutLength: error.stdoutLength, stderrLength: error.stderrLength, - cause: error.cause, + cause: error, }) : new BootServiceInstallError({ cause: error }), ), - Effect.tapError((error) => - DateTime.now.pipe( - Effect.flatMap((now) => - fs.writeFileString(logPath, `${DateTime.formatIso(now)} ${error.message}\n`, { - flag: "a", - }), - ), - Effect.ignore, - ), - ), ); - }); - - // Where the unit will point: derivable without touching the network, so - // status can compare units purely; install materializes it first. - const plannedEntryPath = isEphemeralCacheEntry(host.cliEntryPath) - ? runtimePaths.entryPath - : host.cliEntryPath; - const plan: BootServicePlan = { - nodePath: host.execPath, - t3EntryPath: plannedEntryPath, - baseDir: input.baseDir, - logPath, - unitPath, - }; - - const install: BootService["Service"]["install"] = Effect.gen(function* () { - yield* requireSystemdLinux; - yield* fs - .makeDirectory(input.logsDir, { recursive: true }) + const launcherSource = yield* fs + .readFileString(launcherSourcePath) .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); - yield* ensurePinnedRuntime; - - const previousUnit = yield* fs.exists(unitPath).pipe( - Effect.flatMap((exists) => - exists - ? fs.readFileString(unitPath).pipe(Effect.map(Option.some)) - : Effect.succeed(Option.none()), - ), - Effect.mapError((cause) => new BootServiceInstallError({ cause })), - ); - - yield* fs.makeDirectory(unitDir, { recursive: true }).pipe( - Effect.andThen(fs.writeFileString(unitPath, renderBootServiceUnit(plan))), - Effect.mapError((cause) => new BootServiceInstallError({ cause })), - ); + const installed = yield* fs + .exists(unitPath) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + if (installed) { + yield* runStep("stopping the installed service", "systemctl", [ + "--user", + "stop", + BOOT_SERVICE_UNIT_FILE, + ]); + } - // If any activation step fails, remove the unit again: a leftover file - // would make service status report it as installed even though it was - // never enabled or lingered. yield* Effect.gen(function* () { + yield* fs + .makeDirectory(unitDir, { recursive: true }) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); + yield* writeDurably(launcherPath, launcherSource); + yield* writeDurably( + statePath, + // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned document. + `${JSON.stringify( + { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: input.cliVersion, + } satisfies ServiceState, + null, + 2, + )}\n`, + ); + yield* writeDurably(unitPath, renderBootServiceUnit(plan)); + yield* runStep("reloading systemd user units", "systemctl", ["--user", "daemon-reload"]); yield* runStep("enabling the service", "systemctl", [ "--user", "enable", BOOT_SERVICE_UNIT_FILE, ]); - // restart rather than enable --now: --now does not replace an already - // running process, so repairing a stale unit would leave the old - // server running until reboot. restart also starts a stopped service. + yield* runStep("enabling lingering for this user", "loginctl", ["enable-linger"]); + // Start last. No administrative state write occurs after this succeeds. yield* runStep("starting the service", "systemctl", [ "--user", "restart", BOOT_SERVICE_UNIT_FILE, ]); - // Linger keeps the user manager (and this service) running without an - // open session — the whole point on a box reached over SSH. No - // username argument: loginctl defaults to the calling user, which is - // always right, while $USER can be stale (su without -l) or unset. - yield* runStep("enabling lingering for this user", "loginctl", ["enable-linger"]); - }).pipe(Effect.tapError(() => rollbackFailedInstall(previousUnit))); - + }).pipe( + Effect.tapError(() => + installed + ? runStep("restarting the service after a failed update", "systemctl", [ + "--user", + "restart", + BOOT_SERVICE_UNIT_FILE, + ]).pipe(Effect.ignore) + : Effect.void, + ), + ); return plan; }).pipe(Effect.withSpan("cloud.boot_service.install")); - // If activation fails partway (e.g. enable succeeds but restart/linger - // fails), leave nothing behind: disable removes the enable symlink, remove - // deletes the file, daemon-reload clears the stale definition — otherwise a - // dangling wants/ symlink logs "Failed to load unit" at every boot and the - // next lifecycle command misreports the state. - const rollbackFailedInstall = Effect.fn("cloud.boot_service.rollback_failed_install")(function* ( - previousUnit: Option.Option, - ) { - if (Option.isSome(previousUnit)) { - yield* fs.writeFileString(unitPath, previousUnit.value).pipe(Effect.ignore); - } else { - yield* runStep("cleaning up the service", "systemctl", [ - "--user", - "disable", - "--now", - BOOT_SERVICE_UNIT_FILE, - ]).pipe(Effect.ignore); - yield* fs.remove(unitPath).pipe(Effect.ignore); - } - yield* runStep("reloading systemd user units", "systemctl", ["--user", "daemon-reload"]).pipe( - Effect.ignore, - ); - if (Option.isSome(previousUnit)) { - yield* runStep("restoring the previous service", "systemctl", [ - "--user", - "restart", - BOOT_SERVICE_UNIT_FILE, - ]).pipe(Effect.ignore); - } - }); - const uninstall: BootService["Service"]["uninstall"] = Effect.gen(function* () { yield* requireSystemdLinux; - const exists = yield* fs - .exists(unitPath) - .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); - if (!exists) { + if ( + !(yield* fs + .exists(unitPath) + .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause })))) + ) return false; - } yield* runStep("stopping the service", "systemctl", [ "--user", "disable", @@ -405,18 +366,32 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { if (platform !== "linux" || homeDir === "") { return { supported: false, installed: false, current: false, unitPath, logPath }; } - const unitExists = yield* fs.exists(unitPath); - if (!unitExists) { + if (!(yield* fs.exists(unitPath))) { return { supported: true, installed: false, current: false, unitPath, logPath }; } - const unit = yield* fs.readFileString(unitPath); - // A unit is current only if it matches what install would write now (an - // older CLI wrote a different runtime/node path) AND the entry point it - // references still exists (a pinned runtime under ~/.t3 can be deleted to - // reclaim space). Either mismatch makes connect offer a repair. - const entryExists = yield* fs.exists(plannedEntryPath); - const current = unit === renderBootServiceUnit(plan) && entryExists; - return { supported: true, installed: true, current, unitPath, logPath }; + const [unit, launcherExists, runtimeEntryExists, runtimeSentinel, stateText] = + yield* Effect.all([ + fs.readFileString(unitPath), + fs.exists(launcherPath), + fs.exists(runtimePaths.entryPath), + fs.readFileString(runtimePaths.sentinelPath).pipe(Effect.option), + fs.readFileString(statePath).pipe(Effect.option), + ]); + const state = Option.isSome(stateText) ? parseServiceState(stateText.value) : undefined; + return { + supported: true, + installed: true, + current: + unit === renderBootServiceUnit(plan) && + launcherExists && + runtimeEntryExists && + Option.isSome(runtimeSentinel) && + runtimeSentinel.value.trim() === input.cliVersion && + state?.activeVersion === input.cliVersion && + state?.update?.status !== "pending", + unitPath, + logPath, + }; }).pipe( Effect.mapError((cause) => new BootServiceInstallError({ cause })), Effect.withSpan("cloud.boot_service.status"), diff --git a/apps/server/src/cloud/pinnedRuntime.test.ts b/apps/server/src/cloud/pinnedRuntime.test.ts index 9b46c0038ec..e6ec99e7e8c 100644 --- a/apps/server/src/cloud/pinnedRuntime.test.ts +++ b/apps/server/src/cloud/pinnedRuntime.test.ts @@ -2,8 +2,8 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; -import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; import * as Path from "effect/Path"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; @@ -11,119 +11,164 @@ import * as ProcessRunner from "../processRunner.ts"; import { ensurePinnedRuntimeInstalled, pinnedRuntimePaths, - removePinnedRuntimeInstallation, + PinnedRuntimeInstallError, } from "./pinnedRuntime.ts"; +const successfulRunner = (fs: FileSystem.FileSystem, path: Path.Path) => + ProcessRunner.ProcessRunner.of({ + run: (input) => + Effect.gen(function* () { + const prefixIndex = input.args.indexOf("--prefix"); + const stagingDir = input.args[prefixIndex + 1]; + if (stagingDir === undefined) return yield* Effect.die("missing npm --prefix"); + const entry = path.join(stagingDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entry), { recursive: true }).pipe(Effect.orDie); + yield* fs.writeFileString(entry, "export {};\n").pipe(Effect.orDie); + return { + stdout: "", + stderr: "", + code: ChildProcessSpawner.ExitCode(0), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }; + }), + }); + it.layer(NodeServices.layer)("ensurePinnedRuntimeInstalled", (it) => { - it.effect("serializes concurrent installs of the same runtime", () => + it.effect("validates a staging tree before atomically publishing it", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-test-" }); - const installStarted = yield* Deferred.make(); - const allowInstallToFinish = yield* Deferred.make(); - const paths = pinnedRuntimePaths(path, baseDir, "0.0.29"); - let npmRuns = 0; + const finalPaths = pinnedRuntimePaths(path, baseDir, "1.2.3"); + let validatedDirectory = ""; - const runner = ProcessRunner.ProcessRunner.of({ - run: (_input) => + const installed = yield* ensurePinnedRuntimeInstalled({ + baseDir, + version: "1.2.3", + fs, + path, + runner: successfulRunner(fs, path), + validate: (staging) => Effect.gen(function* () { - npmRuns += 1; - yield* Deferred.succeed(installStarted, undefined); - yield* Deferred.await(allowInstallToFinish); - yield* fs - .makeDirectory(path.dirname(paths.entryPath), { recursive: true }) - .pipe(Effect.orDie); - yield* fs.writeFileString(paths.entryPath, "export {};\n").pipe(Effect.orDie); - return { - stdout: "", - stderr: "", - code: ChildProcessSpawner.ExitCode(0), - timedOut: false, - stdoutTruncated: false, - stderrTruncated: false, - }; - }), + validatedDirectory = staging.versionDir; + assert.isFalse(yield* fs.exists(finalPaths.versionDir)); + assert.isTrue(yield* fs.exists(staging.entryPath)); + }).pipe(Effect.orDie), }); - const install = ensurePinnedRuntimeInstalled({ + + assert.notEqual(validatedDirectory, finalPaths.versionDir); + assert.deepEqual(installed, finalPaths); + assert.isTrue(yield* fs.exists(finalPaths.entryPath)); + assert.equal(yield* fs.readFileString(finalPaths.sentinelPath), "1.2.3\n"); + }), + ); + + it.effect("removes staging and leaves no final runtime when validation fails", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-test-" }); + const finalPaths = pinnedRuntimePaths(path, baseDir, "1.2.3"); + + yield* ensurePinnedRuntimeInstalled({ baseDir, - version: "0.0.29", + version: "1.2.3", fs, path, - runner, - }); + runner: successfulRunner(fs, path), + validate: () => + Effect.fail(new PinnedRuntimeInstallError({ step: "validating the staged runtime" })), + }).pipe(Effect.flip); - const first = yield* Effect.forkChild(install, { startImmediately: true }); - yield* Deferred.await(installStarted); - const second = yield* Effect.forkChild(install, { startImmediately: true }); - yield* Effect.yieldNow; - assert.equal(npmRuns, 1); + assert.isFalse(yield* fs.exists(finalPaths.versionDir)); + assert.deepEqual( + (yield* fs.readDirectory(path.dirname(finalPaths.versionDir))).filter((entry) => + entry.startsWith(".staging-"), + ), + [], + ); + }), + ); - yield* Deferred.succeed(allowInstallToFinish, undefined); - yield* Fiber.join(first); - yield* Fiber.join(second); + it.effect("replaces an incomplete pinned runtime", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-repair-" }); + const finalPaths = pinnedRuntimePaths(path, baseDir, "1.2.3"); + yield* fs.makeDirectory(finalPaths.versionDir, { recursive: true }); + yield* fs.writeFileString(path.join(finalPaths.versionDir, "partial"), "incomplete\n"); - assert.equal(npmRuns, 1); - assert.isTrue(yield* fs.exists(paths.sentinelPath)); - assert.isTrue(yield* fs.exists(paths.entryPath)); + yield* ensurePinnedRuntimeInstalled({ + baseDir, + version: "1.2.3", + fs, + path, + runner: successfulRunner(fs, path), + validate: () => Effect.void, + }); + + assert.isFalse(yield* fs.exists(path.join(finalPaths.versionDir, "partial"))); + assert.isTrue(yield* fs.exists(finalPaths.entryPath)); }), ); - it.effect("waits for an active install before removing its runtime", () => + it.effect("preserves a completed runtime when validation fails", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-test-" }); - const installStarted = yield* Deferred.make(); - const allowInstallToFinish = yield* Deferred.make(); - const paths = pinnedRuntimePaths(path, baseDir, "0.0.30"); - const runner = ProcessRunner.ProcessRunner.of({ - run: (_input) => + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-repair-" }); + const finalPaths = pinnedRuntimePaths(path, baseDir, "1.2.3"); + yield* fs.makeDirectory(path.dirname(finalPaths.entryPath), { recursive: true }); + yield* fs.writeFileString(finalPaths.entryPath, "broken\n"); + yield* fs.writeFileString(finalPaths.sentinelPath, "1.2.3\n"); + + let validations = 0; + yield* ensurePinnedRuntimeInstalled({ + baseDir, + version: "1.2.3", + fs, + path, + runner: successfulRunner(fs, path), + validate: (paths) => Effect.gen(function* () { - yield* Deferred.succeed(installStarted, undefined); - yield* Deferred.await(allowInstallToFinish); - yield* fs - .makeDirectory(path.dirname(paths.entryPath), { recursive: true }) - .pipe(Effect.orDie); - yield* fs.writeFileString(paths.entryPath, "export {};\n").pipe(Effect.orDie); - return { - stdout: "", - stderr: "", - code: ChildProcessSpawner.ExitCode(0), - timedOut: false, - stdoutTruncated: false, - stderrTruncated: false, - }; + validations += 1; + const source = yield* fs.readFileString(paths.entryPath).pipe(Effect.orDie); + if (source === "broken\n") { + return yield* new PinnedRuntimeInstallError({ step: "validating the runtime" }); + } }), - }); + }).pipe(Effect.flip); - const installFiber = yield* Effect.forkChild( - ensurePinnedRuntimeInstalled({ - baseDir, - version: "0.0.30", - fs, - path, - runner, - }), - { startImmediately: true }, - ); - yield* Deferred.await(installStarted); - const removeFiber = yield* Effect.forkChild( - removePinnedRuntimeInstallation({ - baseDir, - version: "0.0.30", - fs, - path, - }), - { startImmediately: true }, - ); - yield* Effect.yieldNow; - assert.isTrue(yield* fs.exists(paths.versionDir)); + assert.equal(validations, 1); + assert.equal(yield* fs.readFileString(finalPaths.entryPath), "broken\n"); + }), + ); + + it.effect("removes staging when installation is interrupted", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pinned-runtime-interrupt-" }); + const started = yield* Deferred.make(); + const runner = ProcessRunner.ProcessRunner.of({ + run: () => Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)), + }); + const install = yield* ensurePinnedRuntimeInstalled({ + baseDir, + version: "1.2.3", + fs, + path, + runner, + validate: () => Effect.void, + }).pipe(Effect.forkScoped); - yield* Deferred.succeed(allowInstallToFinish, undefined); - yield* Fiber.join(installFiber); - yield* Fiber.join(removeFiber); - assert.isFalse(yield* fs.exists(paths.versionDir)); + yield* Deferred.await(started); + yield* Fiber.interrupt(install); + const versionsDir = path.join(baseDir, "runtime", "versions"); + assert.deepEqual(yield* fs.readDirectory(versionsDir), []); }), ); }); diff --git a/apps/server/src/cloud/pinnedRuntime.ts b/apps/server/src/cloud/pinnedRuntime.ts index e3e095ce081..ba3b380b0ca 100644 --- a/apps/server/src/cloud/pinnedRuntime.ts +++ b/apps/server/src/cloud/pinnedRuntime.ts @@ -3,6 +3,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; +import * as Option from "effect/Option"; import * as Semaphore from "effect/Semaphore"; import * as ProcessRunner from "../processRunner.ts"; @@ -11,15 +12,14 @@ import * as ProcessRunner from "../processRunner.ts"; * A pinned runtime is an exact `t3@` npm-installed into * /runtime/versions/. The boot service points its systemd * unit here, and server self-update installs the target version here before - * switching over — never `npx t3`, whose cache is ephemeral and whose + * switching over, never `npx t3`, whose cache is ephemeral and whose * registry fetch at boot would make startup depend on the network. */ const PINNED_RUNTIME_DIR = "runtime"; const PINNED_RUNTIME_INSTALL_TIMEOUT = Duration.minutes(10); -// Boot-service setup and remote self-update share this module but can be -// constructed in separate layers. Serialize the complete check/install/ -// sentinel transaction across all callers in this process. +// Boot-service setup and remote update can construct separate layers. Serialize +// the complete install transaction across every caller in this process. const pinnedRuntimeInstallLock = Semaphore.makeUnsafe(1); export interface PinnedRuntimePaths { @@ -58,119 +58,165 @@ export class PinnedRuntimeInstallError extends Schema.TaggedErrorClass()( + "PinnedRuntimePreflightBlockedError", + { + version: Schema.String, + reason: Schema.String, + }, +) { + override get message(): string { + return this.reason; + } +} + /** * Installs `t3@` into the pinned runtime directory unless a complete * install is already there, and returns its paths. The sentinel is written - * only after npm exits 0; checking the entry file alone is not enough — npm + * only after npm exits 0; checking the entry file alone is not enough. npm * extracts files before running native builds (node-pty), so a killed * install leaves a plausible-looking but broken tree behind. */ -export const ensurePinnedRuntimeInstalled = Effect.fn("cloud.pinned_runtime.ensure_installed")( - function* (input: { - readonly baseDir: string; - readonly version: string; - readonly fs: FileSystem.FileSystem; - readonly path: Path.Path; - readonly runner: ProcessRunner.ProcessRunner["Service"]; - }) { - const { fs, runner } = input; - const paths = pinnedRuntimePaths(input.path, input.baseDir, input.version); +interface PinnedRuntimeInstallInput { + readonly baseDir: string; + readonly version: string; + readonly fs: FileSystem.FileSystem; + readonly path: Path.Path; + readonly runner: ProcessRunner.ProcessRunner["Service"]; + readonly validate: ( + paths: PinnedRuntimePaths, + ) => Effect.Effect; +} + +const installPinnedRuntime = Effect.fn("cloud.pinned_runtime.ensure_installed")(function* ( + input: PinnedRuntimeInstallInput, +) { + const { fs, runner } = input; + const paths = pinnedRuntimePaths(input.path, input.baseDir, input.version); + const [versionDirExists, entryExists, sentinel] = yield* Effect.all([ + fs.exists(paths.versionDir), + fs.exists(paths.entryPath), + fs.readFileString(paths.sentinelPath).pipe(Effect.option), + ]).pipe( + Effect.mapError( + (cause) => new PinnedRuntimeInstallError({ step: "checking the pinned runtime", cause }), + ), + ); + const alreadyPinned = + entryExists && Option.isSome(sentinel) && sentinel.value.trim() === input.version; + if (alreadyPinned) { + yield* input.validate(paths); + return paths; + } + if (versionDirExists) { + yield* fs.remove(paths.versionDir, { recursive: true, force: true }).pipe( + Effect.mapError( + (cause) => + new PinnedRuntimeInstallError({ + step: "removing an incomplete pinned runtime", + cause, + }), + ), + ); + } + + const versionsDir = input.path.dirname(paths.versionDir); + yield* fs.makeDirectory(versionsDir, { recursive: true }).pipe( + Effect.mapError( + (cause) => + new PinnedRuntimeInstallError({ + step: "preparing the pinned runtime directory", + cause, + }), + ), + ); + const stagingDir = yield* fs + .makeTempDirectory({ + directory: versionsDir, + prefix: ".staging-", + }) + .pipe( + Effect.mapError( + (cause) => + new PinnedRuntimeInstallError({ + step: "preparing the pinned runtime directory", + cause, + }), + ), + ); + const stagingPaths: PinnedRuntimePaths = { + versionDir: stagingDir, + entryPath: input.path.join(stagingDir, "node_modules", "t3", "dist", "bin.mjs"), + sentinelPath: input.path.join(stagingDir, ".install-complete"), + }; - return yield* pinnedRuntimeInstallLock.withPermit( - Effect.gen(function* () { - const alreadyPinned = yield* Effect.all([ - fs.exists(paths.sentinelPath), + return yield* Effect.gen(function* () { + const installStep = "installing the pinned t3 runtime (this can take a few minutes)"; + yield* runner + .run({ + command: "npm", + args: ["install", "--prefix", stagingDir, "--no-fund", "--no-audit", `t3@${input.version}`], + // Native dependencies may compile from source on slower machines. + timeout: PINNED_RUNTIME_INSTALL_TIMEOUT, + }) + .pipe( + Effect.mapError((cause) => new PinnedRuntimeInstallError({ step: installStep, cause })), + Effect.filterOrFail( + (result) => result.code === 0, + (result) => + new PinnedRuntimeInstallError({ + step: installStep, + exitCode: Number(result.code), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }), + ), + ); + + yield* input.validate(stagingPaths); + yield* fs + .writeFileString(stagingPaths.sentinelPath, `${input.version}\n`) + .pipe( + Effect.mapError( + (cause) => + new PinnedRuntimeInstallError({ step: "recording the completed install", cause }), + ), + ); + const published = yield* fs.rename(stagingDir, paths.versionDir).pipe( + Effect.as(true), + Effect.catch((cause) => + Effect.all([ fs.exists(paths.entryPath), + fs.readFileString(paths.sentinelPath).pipe(Effect.option), ]).pipe( - Effect.map(([sentinelExists, entryExists]) => sentinelExists && entryExists), - Effect.mapError( - (cause) => - new PinnedRuntimeInstallError({ step: "checking the pinned runtime", cause }), - ), - ); - if (alreadyPinned) { - return paths; - } - - yield* fs.remove(paths.versionDir, { recursive: true, force: true }).pipe( - Effect.andThen(fs.makeDirectory(paths.versionDir, { recursive: true })), Effect.mapError( - (cause) => + (checkCause) => new PinnedRuntimeInstallError({ - step: "preparing the pinned runtime directory", - cause, + step: "checking a concurrently published pinned runtime", + cause: checkCause, }), ), - ); - - const installStep = "installing the pinned t3 runtime (this can take a few minutes)"; - yield* runner - .run({ - command: "npm", - args: [ - "install", - "--prefix", - paths.versionDir, - "--no-fund", - "--no-audit", - `t3@${input.version}`, - ], - // Native deps (node-pty) can compile from source on slow boxes; the - // ProcessRunner default of 60s would kill a healthy install. - timeout: PINNED_RUNTIME_INSTALL_TIMEOUT, - }) - .pipe( - Effect.mapError((cause) => new PinnedRuntimeInstallError({ step: installStep, cause })), - Effect.filterOrFail( - (result) => result.code === 0, - (result) => - new PinnedRuntimeInstallError({ - step: installStep, - exitCode: Number(result.code), - stdoutLength: result.stdout.length, - stderrLength: result.stderr.length, - }), - ), - Effect.tapError(() => - fs.remove(paths.versionDir, { recursive: true, force: true }).pipe(Effect.ignore), - ), - ); - - yield* fs - .writeFileString(paths.sentinelPath, `${input.version}\n`) - .pipe( - Effect.mapError( - (cause) => - new PinnedRuntimeInstallError({ step: "recording the completed install", cause }), - ), - ); - - return paths; - }), - ); - }, -); - -/** Removes one pinned runtime while holding the same process-wide lock used - * by install/check/sentinel work, so cleanup cannot race another caller that - * is materializing or reusing the runtime tree. */ -export const removePinnedRuntimeInstallation = Effect.fn("cloud.pinned_runtime.remove")( - function* (input: { - readonly baseDir: string; - readonly version: string; - readonly fs: FileSystem.FileSystem; - readonly path: Path.Path; - }) { - const paths = pinnedRuntimePaths(input.path, input.baseDir, input.version); - yield* pinnedRuntimeInstallLock.withPermit( - input.fs - .remove(paths.versionDir, { recursive: true, force: true }) - .pipe( - Effect.mapError( - (cause) => - new PinnedRuntimeInstallError({ step: "removing the pinned runtime", cause }), + Effect.flatMap(([publishedEntryExists, publishedSentinel]) => + publishedEntryExists && + Option.isSome(publishedSentinel) && + publishedSentinel.value.trim() === input.version + ? Effect.succeed(false) + : Effect.fail( + new PinnedRuntimeInstallError({ + step: "publishing the pinned runtime", + cause, + }), + ), ), ), + ), ); - }, -); + if (!published) yield* input.validate(paths); + return paths; + }).pipe( + Effect.ensuring(fs.remove(stagingDir, { recursive: true, force: true }).pipe(Effect.ignore)), + ); +}); + +export const ensurePinnedRuntimeInstalled = (input: PinnedRuntimeInstallInput) => + pinnedRuntimeInstallLock.withPermit(installPinnedRuntime(input)); diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index bfac916a59d..6fe1d5a4a3c 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -1,632 +1,144 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; -import { assert, it } from "@effect/vitest"; -import * as Duration from "effect/Duration"; +import { expect, it } from "@effect/vitest"; +import { HostProcessExecutablePath } from "@t3tools/shared/hostProcess"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; -import * as Layer from "effect/Layer"; +import * as Fiber from "effect/Fiber"; import * as Path from "effect/Path"; -import * as TestClock from "effect/testing/TestClock"; import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner"; -import { - HostProcessArguments, - HostProcessEnvironment, - HostProcessExecutablePath, - HostProcessPlatform, -} from "@t3tools/shared/hostProcess"; - import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; -import { - BOOT_SERVICE_UNIT_ENV, - BOOT_SERVICE_UNIT_FILE, - renderBootServiceUnit, -} from "./bootService.ts"; -import * as SelfUpdate from "./selfUpdate.ts"; - -const NODE_PATH = "/usr/local/bin/node"; +import * as ServiceLauncherClient from "./serviceLauncherClient.ts"; +import * as ServerSelfUpdate from "./selfUpdate.ts"; + +interface HarnessOptions { + readonly mode?: "web" | "desktop"; + readonly managed?: boolean; + readonly preflight?: "ready" | "blocked"; + readonly requestUpdate?: ServiceLauncherClient.ServiceLauncherClient["Service"]["requestUpdate"]; +} -const eventuallyFileString = Effect.fn("test.eventuallyFileString")(function* ( - filePath: string, - expected: string, +const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( + options: HarnessOptions = {}, ) { const fs = yield* FileSystem.FileSystem; - for (let iteration = 0; iteration < 1_000; iteration += 1) { - const contents = yield* fs.readFileString(filePath); - if (contents === expected) { - return; - } - // The rollback performs real filesystem I/O on a detached fiber, which - // advancing TestClock does not await. - yield* Effect.yieldNow; - } - return yield* Effect.die(new Error(`Expected file contents were not observed at ${filePath}.`)); -}); - -const eventuallyTrue = Effect.fn("test.eventuallyTrue")(function* (predicate: () => boolean) { - for (let iteration = 0; iteration < 1_000; iteration += 1) { - if (predicate()) { - return; - } - yield* Effect.yieldNow; - } - return yield* Effect.die(new Error("Expected condition was not observed.")); -}); - -interface RecordedCommand { - readonly command: string; - readonly args: ReadonlyArray; -} - -const makeRecordingRunnerLayer = ( - commands: Array, - options?: { - readonly failWhen?: ((command: string, args: ReadonlyArray) => boolean) | undefined; - readonly stdoutFor?: - | ((command: string, args: ReadonlyArray) => string | undefined) - | undefined; - }, -) => - Layer.succeed( - ProcessRunner.ProcessRunner, - ProcessRunner.ProcessRunner.of({ - run: (input) => - Effect.sync(() => { - commands.push({ command: input.command, args: input.args }); - const failed = options?.failWhen?.(input.command, input.args) === true; - const versionFromPath = - input.command === NODE_PATH && input.args[1] === "--version" - ? /[/\\]runtime[/\\]versions[/\\]([^/\\]+)/.exec(input.args[0] ?? "")?.[1] - : undefined; + const path = yield* Path.Path; + const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-self-update-test-" }); + const order: string[] = []; + const runner = ProcessRunner.ProcessRunner.of({ + run: (input) => + Effect.gen(function* () { + if (input.command === "npm") { + order.push("install"); + const prefix = input.args[input.args.indexOf("--prefix") + 1]; + if (prefix === undefined) return yield* Effect.die("missing npm prefix"); + const entry = path.join(prefix, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entry), { recursive: true }).pipe(Effect.orDie); + yield* fs.writeFileString(entry, "export {};\n").pipe(Effect.orDie); return { - stdout: - options?.stdoutFor?.(input.command, input.args) ?? - (versionFromPath === undefined ? "" : `t3 v${versionFromPath}\n`), - stderr: failed ? `${input.command} exploded` : "", - code: ChildProcessSpawner.ExitCode(failed ? 1 : 0), + stdout: "", + stderr: "", + code: ChildProcessSpawner.ExitCode(0), timedOut: false, stdoutTruncated: false, stderrTruncated: false, }; - }), - }), - ); - -const provideHostRefs = (input: { - readonly platform: NodeJS.Platform; - readonly env: NodeJS.ProcessEnv; - readonly entryPath: string; -}) => - Effect.provide( - Layer.mergeAll( - Layer.succeed(HostProcessPlatform, input.platform), - Layer.succeed(HostProcessEnvironment, input.env), - Layer.succeed(HostProcessExecutablePath, NODE_PATH), - Layer.succeed(HostProcessArguments, [NODE_PATH, input.entryPath, "serve"]), - ), - ); - -it("recognizes published npm artifacts as swappable entry points", () => { - assert.isTrue(SelfUpdate.isPublishedCliEntry("/usr/local/lib/node_modules/t3/dist/bin.mjs")); - assert.isTrue( - SelfUpdate.isPublishedCliEntry("/home/theo/.npm/_npx/abc123/node_modules/t3/dist/bin.mjs"), - ); - assert.isTrue( - SelfUpdate.isPublishedCliEntry( - "C:\\Users\\theo\\AppData\\Roaming\\npm\\node_modules\\t3\\dist\\bin.mjs", - ), - ); - // Dev checkouts and the desktop bundle run apps/server/dist directly. - assert.isFalse(SelfUpdate.isPublishedCliEntry("/home/theo/dev/t3/apps/server/dist/bin.mjs")); - assert.isFalse(SelfUpdate.isPublishedCliEntry("")); -}); - -it.layer(NodeServices.layer)("resolveServerSelfUpdateCapability", (it) => { - const makeHome = Effect.fn("test.makeHome")(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-self-update-test-" }); - return { fs, path, home }; - }); - - const writeUnitReferencing = Effect.fn("test.writeUnitReferencing")(function* ( - home: string, - entryPath: string, - ) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const unitDir = path.join(home, ".config", "systemd", "user"); - yield* fs.makeDirectory(unitDir, { recursive: true }); - yield* fs.writeFileString( - path.join(unitDir, "t3code.service"), - renderBootServiceUnit({ - nodePath: NODE_PATH, - t3EntryPath: entryPath, - baseDir: path.join(home, ".t3"), - logPath: path.join(home, ".t3", "userdata", "logs", "boot-service.log"), - unitPath: path.join(unitDir, "t3code.service"), + } + order.push("preflight"); + const result = + options.preflight === "blocked" + ? { status: "blocked", version: "1.1.0", reason: "local update required" } + : { status: "ready", version: "1.1.0", launcherProtocol: 1 }; + return { + // @effect-diagnostics-next-line preferSchemaOverJson:off - fake child-process stdout. + stdout: JSON.stringify(result), + stderr: "", + code: ChildProcessSpawner.ExitCode(0), + timedOut: false, + stdoutTruncated: false, + stderrTruncated: false, + }; }), - ); }); - - it.effect("reports boot-service for the systemd-spawned unit process", () => - Effect.gen(function* () { - const { home, path } = yield* makeHome(); - const entryPath = path.join(home, ".t3/runtime/versions/0.0.28/node_modules/t3/dist/bin.mjs"); - yield* writeUnitReferencing(home, entryPath); - const method = yield* SelfUpdate.resolveServerSelfUpdateCapability({ - desktopManaged: false, - }).pipe( - provideHostRefs({ - platform: "linux", - env: { - HOME: home, - INVOCATION_ID: "abc123", - [BOOT_SERVICE_UNIT_ENV]: BOOT_SERVICE_UNIT_FILE, - }, - entryPath, - }), - ); - assert.equal(method, "boot-service"); - }), - ); - - it.effect("does not claim a systemd process owned by another unit", () => - Effect.gen(function* () { - const { home, path } = yield* makeHome(); - const entryPath = path.join(home, ".t3/runtime/versions/0.0.28/node_modules/t3/dist/bin.mjs"); - yield* writeUnitReferencing(home, entryPath); - const method = yield* SelfUpdate.resolveServerSelfUpdateCapability({ - desktopManaged: false, - }).pipe( - provideHostRefs({ - platform: "linux", - env: { HOME: home, INVOCATION_ID: "abc123" }, - entryPath, - }), - ); - assert.isNull(method); - }), - ); - - it.effect("reports respawn for a manual run of the pinned artifact", () => - Effect.gen(function* () { - const { home, path } = yield* makeHome(); - const entryPath = path.join(home, ".t3/runtime/versions/0.0.28/node_modules/t3/dist/bin.mjs"); - yield* writeUnitReferencing(home, entryPath); - // Same unit on disk, but no INVOCATION_ID: restarting the unit would - // not replace this process, so it must respawn itself instead. - const method = yield* SelfUpdate.resolveServerSelfUpdateCapability({ - desktopManaged: false, - }).pipe(provideHostRefs({ platform: "linux", env: { HOME: home }, entryPath })); - assert.equal(method, "respawn"); - }), - ); - - it.effect("reports respawn for a foreground npx artifact on darwin", () => - Effect.gen(function* () { - const { home } = yield* makeHome(); - const method = yield* SelfUpdate.resolveServerSelfUpdateCapability({ - desktopManaged: false, - }).pipe( - provideHostRefs({ - platform: "darwin", - env: { HOME: home }, - entryPath: `${home}/.npm/_npx/abc123/node_modules/t3/dist/bin.mjs`, - }), - ); - assert.equal(method, "respawn"); - }), - ); - - it.effect("reports desktop-managed for desktop-supervised backends", () => - Effect.gen(function* () { - const { home, path } = yield* makeHome(); - // Desktop ownership wins over every process-shape heuristic: even a - // systemd-looking pinned artifact belongs to the app that spawned it. - const entryPath = path.join(home, ".t3/runtime/versions/0.0.28/node_modules/t3/dist/bin.mjs"); - yield* writeUnitReferencing(home, entryPath); - const method = yield* SelfUpdate.resolveServerSelfUpdateCapability({ - desktopManaged: true, - }).pipe( - provideHostRefs({ - platform: "linux", - env: { - HOME: home, - INVOCATION_ID: "abc123", - [BOOT_SERVICE_UNIT_ENV]: BOOT_SERVICE_UNIT_FILE, - }, - entryPath, - }), - ); - assert.equal(method, "desktop-managed"); - }), - ); - - it.effect("reports no method for dev checkouts and Windows", () => - Effect.gen(function* () { - const { home } = yield* makeHome(); - const devMethod = yield* SelfUpdate.resolveServerSelfUpdateCapability({ - desktopManaged: false, - }).pipe( - provideHostRefs({ - platform: "darwin", - env: { HOME: home }, - entryPath: `${home}/dev/t3/apps/server/dist/bin.mjs`, - }), - ); - assert.isNull(devMethod); - const windowsMethod = yield* SelfUpdate.resolveServerSelfUpdateCapability({ - desktopManaged: false, - }).pipe( - provideHostRefs({ - platform: "win32", - env: { HOME: home }, - entryPath: "C:\\Users\\theo\\AppData\\Roaming\\npm\\node_modules\\t3\\dist\\bin.mjs", - }), - ); - assert.isNull(windowsMethod); - }), - ); -}); - -it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { - interface RecordedSpawn { - readonly command: string; - readonly args: ReadonlyArray; - } - - const makeContext = Effect.fn("test.makeContext")(function* (options?: { - readonly platform?: NodeJS.Platform; - readonly bootService?: boolean; - readonly desktopManaged?: boolean; - readonly entryPath?: string; - readonly failWhen?: (command: string, args: ReadonlyArray) => boolean; - readonly stdoutFor?: (command: string, args: ReadonlyArray) => string | undefined; - readonly failSpawn?: boolean; - }) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const home = yield* fs.makeTempDirectoryScoped({ prefix: "t3-self-update-test-" }); - const baseDir = path.join(home, ".t3"); - const entryPath = - options?.entryPath ?? - path.join(home, ".t3/runtime/versions/0.0.28/node_modules/t3/dist/bin.mjs"); - const env: NodeJS.ProcessEnv = - options?.bootService === true - ? { - HOME: home, - INVOCATION_ID: "abc123", - [BOOT_SERVICE_UNIT_ENV]: BOOT_SERVICE_UNIT_FILE, - } - : { HOME: home }; - if (options?.bootService === true) { - const unitDir = path.join(home, ".config", "systemd", "user"); - yield* fs.makeDirectory(unitDir, { recursive: true }); - yield* fs.writeFileString( - path.join(unitDir, "t3code.service"), - renderBootServiceUnit({ - nodePath: NODE_PATH, - t3EntryPath: entryPath, - baseDir, - logPath: path.join(baseDir, "userdata", "logs", "boot-service.log"), - unitPath: path.join(unitDir, "t3code.service"), - }), - ); - } - - const commands: Array = []; - const spawns: Array = []; - let exited = 0; - // layerTest always reports mode "web"; desktop-managed contexts overlay - // the mode the desktop app's bootstrap envelope would set. - const configLayer = - options?.desktopManaged === true - ? Layer.effect( - ServerConfig.ServerConfig, - Effect.gen(function* () { - const config = yield* ServerConfig.ServerConfig; - return { ...config, mode: "desktop" as const }; - }), - ).pipe(Layer.provide(ServerConfig.layerTest(home, baseDir))) - : ServerConfig.layerTest(home, baseDir); - const service = yield* SelfUpdate.make({ - host: { - spawnDetached: (command, args) => - Effect.sync(() => spawns.push({ command, args })).pipe( - Effect.andThen( - options?.failSpawn === true - ? Effect.fail( - new ProcessRunner.ProcessSpawnError({ - command, - argumentCount: args.length, - cause: new Error("detached spawn failed"), - }), - ) - : Effect.void, - ), - ), - exitProcess: () => { - exited += 1; - }, - }, - }).pipe( - Effect.provide( - Layer.mergeAll( - makeRecordingRunnerLayer(commands, { - failWhen: options?.failWhen, - stdoutFor: options?.stdoutFor, - }), - configLayer, - ), - ), - provideHostRefs({ platform: options?.platform ?? "linux", env, entryPath }), - ); - return { - fs, - path, - home, - baseDir, - entryPath, - commands, - spawns, - exitCount: () => exited, - service, - }; + const launcher = ServiceLauncherClient.ServiceLauncherClient.of({ + managed: options.managed ?? true, + trial: false, + requestUpdate: + options.requestUpdate ?? + (() => + Effect.sync(() => { + order.push("accept"); + return "launcher-id"; + })), + prepareTrial: Effect.sync((): undefined => undefined), }); - - it.effect("rejects dist-tags and other non-exact versions", () => - Effect.gen(function* () { - const context = yield* makeContext(); - const error = yield* context.service.update({ targetVersion: "latest" }).pipe(Effect.flip); - assert.include(error.reason, "not an exact t3 version"); - assert.lengthOf(context.commands, 0); - }), - ); - - it.effect("refuses to update a desktop-managed backend and points at the app", () => - Effect.gen(function* () { - const context = yield* makeContext({ desktopManaged: true, bootService: true }); - const error = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); - assert.include(error.reason, "desktop app"); - assert.lengthOf(context.commands, 0); - assert.lengthOf(context.spawns, 0); - }), - ); - - it.effect("fails without touching anything when no update method applies", () => - Effect.gen(function* () { - const context = yield* makeContext({ - entryPath: "/home/theo/dev/t3/apps/server/dist/bin.mjs", - }); - const error = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); - assert.include(error.reason, "cannot update itself"); - assert.lengthOf(context.commands, 0); - }), - ); - - it.effect("surfaces a failed npm install and never schedules a restart", () => - Effect.gen(function* () { - const context = yield* makeContext({ failWhen: (command) => command === "npm" }); - const error = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); - assert.equal(error.reason, "Could not install the requested t3 version."); - yield* TestClock.adjust(Duration.seconds(10)); - assert.lengthOf(context.spawns, 0); - assert.equal(context.exitCount(), 0); - }).pipe(Effect.provide(TestClock.layer())), + const config = yield* ServerConfig.ServerConfig.pipe( + Effect.provide(ServerConfig.layerTest(process.cwd(), baseDir)), ); - - it.effect("reinstalls the same version after a failed preflight", () => - Effect.gen(function* () { - let preflightAttempts = 0; - const context = yield* makeContext({ - failWhen: (command) => { - if (command !== NODE_PATH) return false; - preflightAttempts += 1; - return preflightAttempts === 1; - }, - }); - const versionDir = context.path.join(context.baseDir, "runtime", "versions", "0.0.29"); - const entryPath = context.path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); - yield* context.fs.makeDirectory(context.path.dirname(entryPath), { recursive: true }); - yield* context.fs.writeFileString(entryPath, "export {};\n"); - yield* context.fs.writeFileString( - context.path.join(versionDir, ".install-complete"), - "0.0.29\n", - ); - - const firstError = yield* context.service - .update({ targetVersion: "0.0.29" }) - .pipe(Effect.flip); - assert.include(firstError.reason, "failed its version check"); - assert.isFalse(yield* context.fs.exists(versionDir)); - - const result = yield* context.service.update({ targetVersion: "0.0.29" }); - assert.deepEqual(result, { targetVersion: "0.0.29", method: "respawn" }); - assert.deepEqual( - context.commands.map((entry) => entry.command), - [NODE_PATH, "npm", NODE_PATH], - ); - }).pipe(Effect.provide(TestClock.layer())), + const selfUpdate = yield* ServerSelfUpdate.make().pipe( + Effect.provideService(ProcessRunner.ProcessRunner, runner), + Effect.provideService(ServiceLauncherClient.ServiceLauncherClient, launcher), + Effect.provideService(HostProcessExecutablePath, "/usr/bin/node"), + Effect.provide(ServerConfig.layer({ ...config, mode: options.mode ?? "web" })), ); + return { selfUpdate, order }; +}); - it.effect("rejects and removes an installed runtime that reports the wrong version", () => +it.layer(NodeServices.layer)("server self update", (it) => { + it.effect("stages and preflights before asking the launcher for an update ID", () => Effect.gen(function* () { - const context = yield* makeContext({ - stdoutFor: (command, args) => - command === NODE_PATH && args[1] === "--version" ? "t3 v0.0.28\n" : undefined, + const { selfUpdate, order } = yield* makeHarness(); + expect(yield* selfUpdate.update({ targetVersion: "1.1.0" })).toEqual({ + targetVersion: "1.1.0", + method: "boot-service", + updateId: "launcher-id", }); - const versionDir = context.path.join(context.baseDir, "runtime", "versions", "0.0.29"); - - const error = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); - - assert.include(error.reason, "did not report the requested"); - assert.isFalse(yield* context.fs.exists(versionDir)); - assert.lengthOf(context.spawns, 0); + expect(order).toEqual(["install", "preflight", "accept"]); }), ); - it.effect("reports a detached replacement spawn failure and leaves updates retryable", () => + it.effect("rejects invalid versions and desktop-managed servers before staging", () => Effect.gen(function* () { - const context = yield* makeContext({ failSpawn: true }); - - const first = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); - assert.include(first.reason, "Could not start the replacement"); - - const second = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); - assert.include(second.reason, "Could not start the replacement"); - assert.notInclude(second.reason, "already in progress"); - assert.lengthOf(context.spawns, 2); - assert.equal(context.exitCount(), 0); + const web = yield* makeHarness(); + expect( + (yield* web.selfUpdate.update({ targetVersion: "latest" }).pipe(Effect.flip)).reason, + ).toBe("'latest' is not an exact t3 version."); + const desktop = yield* makeHarness({ mode: "desktop" }); + expect( + (yield* desktop.selfUpdate.update({ targetVersion: "1.1.0" }).pipe(Effect.flip)).reason, + ).toContain("desktop app"); + expect([...web.order, ...desktop.order]).toEqual([]); }), ); - it.effect("installs, preflights, and respawns a foreground server", () => + it.effect("preserves the preflight refusal reason", () => Effect.gen(function* () { - const context = yield* makeContext(); - const progress: Array = []; - const result = yield* context.service.update({ targetVersion: "0.0.29" }, (stage) => - Effect.sync(() => progress.push(stage)), + const { selfUpdate } = yield* makeHarness({ preflight: "blocked" }); + expect((yield* selfUpdate.update({ targetVersion: "1.1.0" }).pipe(Effect.flip)).reason).toBe( + "local update required", ); - assert.deepEqual(result, { targetVersion: "0.0.29", method: "respawn" }); - assert.deepEqual(progress, ["downloading", "installing"]); - assert.lengthOf(context.spawns, 1); - - const concurrentError = yield* context.service - .update({ targetVersion: "0.0.30" }) - .pipe(Effect.flip); - assert.include(concurrentError.reason, "already in progress"); - - const pinnedEntry = context.path.join( - context.baseDir, - "runtime/versions/0.0.29/node_modules/t3/dist/bin.mjs", - ); - assert.deepEqual( - context.commands.map((entry) => [entry.command, ...entry.args].join(" ")), - [ - `npm install --prefix ${context.path.join(context.baseDir, "runtime/versions/0.0.29")} --no-fund --no-audit t3@0.0.29`, - `${NODE_PATH} ${pinnedEntry} --version`, - ], - ); - - // The restart is deferred so the RPC acknowledgement flushes first. - yield* TestClock.adjust(Duration.seconds(10)); - assert.lengthOf(context.spawns, 1); - const spawn = context.spawns[0]; - assert.equal(spawn?.command, "/bin/sh"); - assert.include(spawn?.args ?? [], pinnedEntry); - // The replacement replays the original CLI arguments. - assert.include(spawn?.args ?? [], "serve"); - assert.equal(context.exitCount(), 1); - }).pipe(Effect.provide(TestClock.layer())), - ); - - it.effect("rewrites the systemd unit and restarts the boot service", () => - Effect.gen(function* () { - const context = yield* makeContext({ bootService: true }); - const result = yield* context.service.update({ targetVersion: "0.0.29" }); - assert.deepEqual(result, { targetVersion: "0.0.29", method: "boot-service" }); - - const pinnedEntry = context.path.join( - context.baseDir, - "runtime/versions/0.0.29/node_modules/t3/dist/bin.mjs", - ); - const unit = yield* context.fs.readFileString( - context.path.join(context.home, ".config", "systemd", "user", "t3code.service"), - ); - assert.include(unit, `ExecStart=${NODE_PATH} ${pinnedEntry} serve`); - assert.deepEqual( - context.commands.map((entry) => entry.command), - ["npm", NODE_PATH, "systemctl"], - ); - assert.deepEqual(context.commands[2]?.args, ["--user", "daemon-reload"]); - - // Restart waits until after the update acknowledgement can flush. - yield* TestClock.adjust(Duration.seconds(10)); - assert.deepEqual(context.commands[3], { - command: "systemctl", - args: ["--user", "restart", "--no-block", "t3code.service"], - }); - assert.lengthOf(context.spawns, 0); - // systemd replaces the process; the server must not exit itself. - assert.equal(context.exitCount(), 0); - - // The queued restart returns while this process is still shutting - // down; the lock must stay held so a second update cannot rewrite the - // unit mid-teardown. - const concurrentError = yield* context.service - .update({ targetVersion: "0.0.30" }) - .pipe(Effect.flip); - assert.include(concurrentError.reason, "already in progress"); - }).pipe(Effect.provide(TestClock.layer())), + }), ); - it.effect("restores the previous unit and permits a retry when systemd restart fails", () => + it.effect("allows only one update at a time", () => Effect.gen(function* () { - let failRestart = true; - const context = yield* makeContext({ - bootService: true, - failWhen: (command, args) => { - if (command !== "systemctl" || args[1] !== "restart" || !failRestart) { - return false; - } - failRestart = false; - return true; - }, + const requested = yield* Deferred.make(); + const accepted = yield* Deferred.make(); + const { selfUpdate } = yield* makeHarness({ + requestUpdate: () => + Deferred.succeed(requested, undefined).pipe(Effect.andThen(Deferred.await(accepted))), }); - const unitPath = context.path.join( - context.home, - ".config", - "systemd", - "user", - BOOT_SERVICE_UNIT_FILE, - ); - const previousUnit = yield* context.fs.readFileString(unitPath); - - const first = yield* context.service.update({ targetVersion: "0.0.29" }); - assert.deepEqual(first, { targetVersion: "0.0.29", method: "boot-service" }); - yield* TestClock.adjust(Duration.seconds(10)); - yield* eventuallyFileString(unitPath, previousUnit); - yield* eventuallyTrue(() => context.commands.at(-1)?.args[1] === "daemon-reload"); - assert.deepEqual( - context.commands.slice(-2).map((entry) => entry.args), - [ - ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE], - ["--user", "daemon-reload"], - ], - ); - - const retry = yield* context.service.update({ targetVersion: "0.0.30" }); - assert.deepEqual(retry, { targetVersion: "0.0.30", method: "boot-service" }); - }).pipe(Effect.provide(TestClock.layer())), - ); - - it.effect("restores the previous systemd unit when daemon-reload fails", () => - Effect.gen(function* () { - const context = yield* makeContext({ - bootService: true, - failWhen: (command) => command === "systemctl", + const first = yield* Effect.forkChild(selfUpdate.update({ targetVersion: "1.1.0" }), { + startImmediately: true, }); - const unitPath = context.path.join( - context.home, - ".config", - "systemd", - "user", - BOOT_SERVICE_UNIT_FILE, - ); - const previousUnit = yield* context.fs.readFileString(unitPath); - - const error = yield* context.service.update({ targetVersion: "0.0.29" }).pipe(Effect.flip); - assert.include(error.reason, "Reloading systemd units failed"); - assert.equal(yield* context.fs.readFileString(unitPath), previousUnit); - assert.deepEqual( - context.commands.map((entry) => entry.command), - ["npm", NODE_PATH, "systemctl", "systemctl"], + yield* Deferred.await(requested); + expect((yield* selfUpdate.update({ targetVersion: "1.1.1" }).pipe(Effect.flip)).reason).toBe( + "A server update is already in progress.", ); - - yield* TestClock.adjust(Duration.seconds(10)); - assert.lengthOf(context.spawns, 0); - assert.equal(context.exitCount(), 0); - }).pipe(Effect.provide(TestClock.layer())), + yield* Deferred.succeed(accepted, "launcher-id"); + expect((yield* Fiber.join(first)).updateId).toBe("launcher-id"); + }), ); }); diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index 9dcb713e1a1..58bb8411730 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -1,7 +1,3 @@ -// @effect-diagnostics nodeBuiltinImport:off -// node:child_process directly: the foreground-server replacement must be a -// detached fire-and-forget child that outlives this process, while Effect's -// ChildProcessSpawner ties every child to a scope that kills it. import { ServerSelfUpdateError, type ServerSelfUpdateCapability, @@ -9,13 +5,7 @@ import { type ServerSelfUpdateProgressStage, type ServerSelfUpdateResult, } from "@t3tools/contracts"; -import { - HostProcessArguments, - HostProcessEnvironment, - HostProcessExecutablePath, - HostProcessPlatform, -} from "@t3tools/shared/hostProcess"; -import * as NodeChildProcess from "node:child_process"; +import { HostProcessExecutablePath } from "@t3tools/shared/hostProcess"; import * as Context from "effect/Context"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; @@ -25,208 +15,52 @@ import * as Path from "effect/Path"; import * as Ref from "effect/Ref"; import * as ServerConfig from "../config.ts"; -import { writeFileStringAtomically } from "../atomicWrite.ts"; import * as ProcessRunner from "../processRunner.ts"; import { - BOOT_SERVICE_UNIT_ENV, - BOOT_SERVICE_UNIT_FILE, - quoteSystemdValue, - renderBootServiceUnit, -} from "./bootService.ts"; -import { ensurePinnedRuntimeInstalled, removePinnedRuntimeInstallation } from "./pinnedRuntime.ts"; - -/** - * Lets a connected client replace this server with another published `t3` - * version over RPC — the only update path that works when the user is not at - * the machine (phone against a home server, relay-managed box). The target - * version is npm-installed into the pinned runtime and verified before - * anything restarts, so a failed install leaves the running server untouched. - */ + ensurePinnedRuntimeInstalled, + PinnedRuntimeInstallError, + PinnedRuntimePreflightBlockedError, +} from "./pinnedRuntime.ts"; +import { decodeServicePreflightResult } from "./servicePreflight.ts"; +import * as ServiceLauncherClient from "./serviceLauncherClient.ts"; +import { isExactServiceVersion, SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; const PREFLIGHT_TIMEOUT = Duration.seconds(30); -/** Grace between acknowledging the RPC and killing the process, so the - response (and its relay hop) flushes before the socket drops. */ -const RESTART_DELAY = Duration.seconds(2); - -/** Exact npm versions only — never dist-tags — so the acknowledgement names - the version that was actually installed. Also keeps the value safe to - pass to npm and embed in filesystem paths. */ -const EXACT_VERSION_PATTERN = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/; - -export interface ServerSelfUpdateHost { - readonly execPath: string; - readonly cliEntryPath: string; - /** Original CLI arguments after the entry path, replayed on respawn. */ - readonly cliArgs: ReadonlyArray; - /** Resolves once the foreground replacement process has actually spawned. */ - readonly spawnDetached: ( - command: string, - args: ReadonlyArray, - ) => Effect.Effect; - readonly exitProcess: () => void; -} - -function normalizeEntryPath(entryPath: string): string { - return entryPath.replaceAll("\\", "/"); -} -/** - * Only a published npm artifact can be swapped for another version: dev - * checkouts (apps/server/dist) and the desktop app's bundled backend have no - * npm identity, and the desktop manages its own updates. - */ -export function isPublishedCliEntry(entryPath: string): boolean { - return normalizeEntryPath(entryPath).includes("/node_modules/t3/dist/"); -} - -/** - * The update path this process can offer, or null when only a manual - * relaunch works. "desktop-managed" — the T3 Code desktop app spawned this - * backend and owns its version; only updating the app updates it. - * "boot-service" — this is the systemd-supervised process from - * bootService.ts: rewrite the unit and let systemd swap it. "respawn" — a - * foreground POSIX process running a published artifact: replace it with a - * detached child. Windows foreground runs are unsupported for now (no - * equivalent of the detach-and-exec handoff below). - */ -export const resolveServerSelfUpdateCapability = Effect.fn( - "cloud.server_self_update.resolve_capability", -)(function* (input: { - /** True when the desktop app supervises this backend (mode "desktop"). */ +export function resolveServerSelfUpdateCapability(input: { readonly desktopManaged: boolean; -}) { - if (input.desktopManaged) { - return "desktop-managed" as const; - } - - const platform = yield* HostProcessPlatform; - const env = yield* HostProcessEnvironment; - const hostArguments = yield* HostProcessArguments; - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - - const entryPath = hostArguments[1] ?? ""; - if (entryPath === "") { - return null; - } - - const homeDir = env.HOME ?? ""; - if (platform === "linux" && homeDir !== "") { - const unitPath = path.join(homeDir, ".config", "systemd", "user", BOOT_SERVICE_UNIT_FILE); - const unitReferencesEntry = yield* fs.readFileString(unitPath).pipe( - Effect.map((unit) => unit.includes(quoteSystemdValue(entryPath))), - Effect.orElseSucceed(() => false), - ); - // INVOCATION_ID only proves that some systemd unit launched us. The - // explicit marker written into t3code.service identifies this unit as the - // supervisor that will replace the current process when restarted. - if ( - unitReferencesEntry && - (env.INVOCATION_ID ?? "") !== "" && - env[BOOT_SERVICE_UNIT_ENV] === BOOT_SERVICE_UNIT_FILE - ) { - return "boot-service" as const; - } - - // A process owned by another (or a legacy unmarked) systemd unit must not - // use the foreground respawn path: Restart=always could otherwise launch - // the old unit beside the detached replacement. - if ((env.INVOCATION_ID ?? "") !== "") { - return null; - } - } - - if ((platform === "linux" || platform === "darwin") && isPublishedCliEntry(entryPath)) { - return "respawn" as const; - } - - return null; -}); + readonly launcherManaged: boolean; +}): ServerSelfUpdateCapability | null { + if (input.desktopManaged) return "desktop-managed" as const; + return input.launcherManaged ? ("boot-service" as const) : null; +} export class ServerSelfUpdate extends Context.Service< ServerSelfUpdate, { readonly update: ( input: ServerSelfUpdateInput, - reportProgress?: (stage: ServerSelfUpdateProgressStage) => Effect.Effect, + reportProgress?: (stage: ServerSelfUpdateProgressStage) => Effect.Effect, ) => Effect.Effect; } >()("t3/cloud/selfUpdate/ServerSelfUpdate") {} -export const make = Effect.fn("cloud.server_self_update.make")(function* (options?: { - readonly host?: Partial; -}) { +export const make = Effect.fn("cloud.server_self_update.make")(function* () { const serverConfig = yield* ServerConfig.ServerConfig; + const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; + const runner = yield* ProcessRunner.ProcessRunner; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const runner = yield* ProcessRunner.ProcessRunner; - const env = yield* HostProcessEnvironment; - const hostExecPath = yield* HostProcessExecutablePath; - const hostArguments = yield* HostProcessArguments; - const capability: ServerSelfUpdateCapability | null = yield* resolveServerSelfUpdateCapability({ - desktopManaged: serverConfig.mode === "desktop", - }); - - const host: ServerSelfUpdateHost = { - execPath: options?.host?.execPath ?? hostExecPath, - cliEntryPath: options?.host?.cliEntryPath ?? hostArguments[1] ?? "", - cliArgs: options?.host?.cliArgs ?? hostArguments.slice(2), - spawnDetached: - options?.host?.spawnDetached ?? - ((command, args) => - Effect.callback((resume) => { - const spawnError = (cause: unknown) => - new ProcessRunner.ProcessSpawnError({ - command, - argumentCount: args.length, - cause, - }); - let child: NodeChildProcess.ChildProcess; - try { - child = NodeChildProcess.spawn(command, [...args], { - detached: true, - stdio: "ignore", - }); - } catch (cause) { - resume(Effect.fail(spawnError(cause))); - return; - } - - const onSpawnError = (cause: Error) => resume(Effect.fail(spawnError(cause))); - child.once("error", onSpawnError); - child.once("spawn", () => { - child.removeListener("error", onSpawnError); - // Keep asynchronous child errors from becoming uncaught after the - // successful spawn handoff has already been acknowledged. - child.on("error", () => undefined); - child.unref(); - resume(Effect.void); - }); - })), - exitProcess: options?.host?.exitProcess ?? (() => process.exit(0)), - }; - + const execPath = yield* HostProcessExecutablePath; const inFlight = yield* Ref.make(false); + const capability: ServerSelfUpdateCapability | null = + serverConfig.mode === "desktop" ? "desktop-managed" : launcher.managed ? "boot-service" : null; const failWith = (reason: string, cause?: unknown) => cause === undefined ? new ServerSelfUpdateError({ reason }) : new ServerSelfUpdateError({ reason, cause }); - /** Deferred so the RPC acknowledgement flushes before the process dies. - Detached from the request scope: the triggering connection is exactly - what the restart tears down. */ - const scheduleRestart = (restart: Effect.Effect) => - Effect.sleep(RESTART_DELAY).pipe( - Effect.andThen(restart), - Effect.forkDetach({ startImmediately: true }), - ); - const writeUnitAtomically = (filePath: string, contents: string) => - writeFileStringAtomically({ filePath, contents }).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Path.Path, path), - ); - const update: ServerSelfUpdate["Service"]["update"] = Effect.fn( "cloud.server_self_update.update", )(function* (input, reportProgress = () => Effect.void) { @@ -237,208 +71,123 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option } if (capability === null) { return yield* failWith( - "This server cannot update itself; relaunch it manually with the new version.", + "Remote updates require the T3 Code background service. Run `t3 service install` on the server machine.", ); } - const activeMethod = capability; + const targetVersion = input.targetVersion.trim(); - if (!EXACT_VERSION_PATTERN.test(targetVersion)) { + if (!isExactServiceVersion(targetVersion)) { return yield* failWith(`'${targetVersion}' is not an exact t3 version.`); } - - const alreadyRunning = yield* Ref.getAndSet(inFlight, true); - if (alreadyRunning) { + if (yield* Ref.getAndSet(inFlight, true)) { return yield* failWith("A server update is already in progress."); } return yield* Effect.gen(function* () { yield* reportProgress("downloading"); - const runtimePaths = yield* ensurePinnedRuntimeInstalled({ + const paths = yield* ensurePinnedRuntimeInstalled({ baseDir: serverConfig.baseDir, version: targetVersion, fs, path, runner, + validate: (runtime) => + runner + .run({ + command: execPath, + args: [ + runtime.entryPath, + "__service-preflight", + "--database-path", + serverConfig.dbPath, + "--launcher-protocol", + String(SERVICE_LAUNCHER_PROTOCOL), + ], + timeout: PREFLIGHT_TIMEOUT, + }) + .pipe( + Effect.mapError( + (cause) => + new PinnedRuntimeInstallError({ + step: "running the staged service preflight", + cause, + }), + ), + Effect.flatMap( + ( + result, + ): Effect.Effect< + void, + PinnedRuntimeInstallError | PinnedRuntimePreflightBlockedError + > => { + if (result.code !== 0) { + return Effect.fail( + new PinnedRuntimeInstallError({ + step: "running the staged service preflight", + exitCode: Number(result.code), + stdoutLength: result.stdout.length, + stderrLength: result.stderr.length, + }), + ); + } + let parsed: unknown; + try { + parsed = JSON.parse(result.stdout.trim()); + } catch (cause) { + return Effect.fail( + new PinnedRuntimeInstallError({ + step: "decoding the staged service preflight", + cause, + }), + ); + } + const preflight = decodeServicePreflightResult(parsed); + if (preflight === undefined || preflight.version !== targetVersion) { + return Effect.fail( + new PinnedRuntimeInstallError({ + step: "verifying the staged service preflight", + }), + ); + } + return preflight.status === "ready" + ? Effect.void + : Effect.fail( + new PinnedRuntimePreflightBlockedError({ + version: targetVersion, + reason: preflight.reason, + }), + ); + }, + ), + ), }).pipe( - Effect.mapError((error) => failWith("Could not install the requested t3 version.", error)), + Effect.mapError((error) => + error._tag === "PinnedRuntimePreflightBlockedError" + ? failWith(error.reason, error) + : failWith(`Could not prepare t3@${targetVersion}.`, error), + ), ); yield* reportProgress("installing"); - // A broken artifact (failed native build, incompatible node) must be - // caught while the current server is still alive to report it. - const preflight = yield* runner - .run({ - command: host.execPath, - args: [runtimePaths.entryPath, "--version"], - timeout: PREFLIGHT_TIMEOUT, - }) + const updateId = yield* launcher + .requestUpdate({ targetVersion }) .pipe( - Effect.mapError((cause) => - failWith(`Could not verify the installed t3@${targetVersion}.`, cause), - ), - ); - // Effect CLI's unstable formatVersion currently emits `${name} v${version}`. - // Extract the version token so surrounding presentation changes do not break updates. - const reportedVersion = /\bv(\S+)\s*$/.exec(preflight.stdout)?.[1]; - if (preflight.code !== 0 || reportedVersion !== targetVersion) { - // A completed npm install can still be unusable under this Node or on - // this machine. Remove its sentinel and tree so a retry of the same - // version performs a clean install instead of reusing a known-bad one. - yield* removePinnedRuntimeInstallation({ - baseDir: serverConfig.baseDir, - version: targetVersion, - fs, - path, - }).pipe( Effect.mapError((error) => - failWith(`Could not remove the failed t3@${targetVersion} installation.`, error), - ), - ); - return yield* failWith( - preflight.code !== 0 - ? `The installed t3@${targetVersion} failed its version check (exit code ${String(preflight.code)}).` - : `The installed runtime did not report the requested t3@${targetVersion} version.`, - ); - } - - if (activeMethod === "boot-service") { - const homeDir = env.HOME ?? ""; - const unitPath = path.join(homeDir, ".config", "systemd", "user", BOOT_SERVICE_UNIT_FILE); - const previousUnit = yield* fs - .readFileString(unitPath) - .pipe( - Effect.mapError((cause) => failWith("Could not read the current systemd unit.", cause)), - ); - // Same shape bootService.install writes, so host lifecycle commands - // still recognize the unit as current. - const unit = renderBootServiceUnit({ - nodePath: host.execPath, - t3EntryPath: runtimePaths.entryPath, - baseDir: serverConfig.baseDir, - logPath: path.join(serverConfig.logsDir, "boot-service.log"), - unitPath, - }); - yield* writeUnitAtomically(unitPath, unit).pipe( - Effect.mapError((cause) => failWith("Could not update the systemd unit.", cause)), - ); - - const reloadSystemd = Effect.fn("cloud.server_self_update.reload_systemd")(function* () { - const reload = yield* runner - .run({ command: "systemctl", args: ["--user", "daemon-reload"] }) - .pipe(Effect.mapError((cause) => failWith("Could not reload systemd units.", cause))); - if (reload.code !== 0) { - return yield* failWith( - `Reloading systemd units failed (exit code ${String(reload.code)}).`, - ); - } - }); - - yield* reloadSystemd().pipe( - Effect.catch((reloadError) => - writeUnitAtomically(unitPath, previousUnit).pipe( - Effect.mapError((rollbackCause) => - failWith("Could not restore the previous systemd unit.", { - reloadError, - rollbackCause, - }), - ), - // Systemd should still have the old unit in memory after the - // failed reload, but retry after restoring in case it applied a - // partial update before returning an error. - Effect.andThen(reloadSystemd().pipe(Effect.ignore)), - Effect.andThen(Effect.fail(reloadError)), - ), - ), - ); - yield* Effect.logInfo("Server self-update installed; restarting boot service.", { - targetVersion, - }); - // Restart after the acknowledgement has had time to cross any relay - // hop. --no-block queues the restart job and exits before systemd - // stops this unit: a blocking restart's SIGTERM reaches the systemctl - // child (it shares this service's cgroup), which read as a restart - // failure and rolled the new unit back while the old server finished - // shutting down. With the handoff race gone, a non-zero exit or spawn - // error means systemd genuinely rejected the job while this process is - // still alive, so restoring the previous unit below stays correct. - yield* scheduleRestart( - Effect.gen(function* () { - const restart = yield* runner - .run({ - command: "systemctl", - args: ["--user", "restart", "--no-block", BOOT_SERVICE_UNIT_FILE], - }) - .pipe( - Effect.mapError((cause) => - failWith("Could not restart the systemd boot service.", cause), - ), - ); - if (restart.code !== 0) { - return yield* failWith( - `Restarting the systemd boot service failed (exit code ${String(restart.code)}).`, - ); - } - }).pipe( - Effect.catch((restartError) => - writeUnitAtomically(unitPath, previousUnit).pipe( - Effect.andThen(reloadSystemd()), - Effect.mapError((rollbackError) => - failWith("Could not restore the previous systemd unit.", { - restartError, - rollbackError, - }), - ), - Effect.andThen(Effect.fail(restartError)), - ), - ), - Effect.catch((error) => - Effect.logError("Server self-update could not restart the boot service.").pipe( - Effect.annotateLogs({ targetVersion, error: error.reason }), - // Permit a retry only after the failed handoff was rolled - // back. A queued restart returns while this process is still - // shutting down; releasing the lock then would let a second - // update rewrite the unit mid-teardown. - Effect.andThen(Ref.set(inFlight, false)), - ), - ), - ), - ); - } else { - // Spawn the shim before acknowledging the RPC so ENOENT/EACCES and - // other launch failures leave this server alive and return a useful - // error. The shim itself waits until after the acknowledgement and - // deferred exit before binding the replacement server. - yield* host - .spawnDetached("/bin/sh", [ - "-c", - 'sleep 3; exec "$@"', - "t3-self-update", - host.execPath, - runtimePaths.entryPath, - ...host.cliArgs, - ]) - .pipe( - Effect.mapError((cause) => - failWith("Could not start the replacement t3 process.", cause), - ), - ); - yield* Effect.logInfo("Server self-update installed; respawning.", { targetVersion }); - yield* scheduleRestart( - Effect.try({ - try: () => host.exitProcess(), - catch: (cause) => failWith("Could not exit the replaced t3 process.", cause), - }).pipe( - Effect.catch((error) => - Effect.logError("Server self-update could not exit the replaced process.").pipe( - Effect.annotateLogs({ targetVersion, error: error.reason }), - Effect.ensuring(Ref.set(inFlight, false)), - ), + failWith( + error._tag === "ServiceLauncherRejectedError" + ? error.reason + : "Could not ask the service launcher to activate the prepared update.", + error, ), ), ); - } - return { targetVersion, method: activeMethod }; + yield* Effect.logInfo("Server update prepared; handing off to the service launcher.", { + updateId, + targetVersion, + runtimePath: paths.entryPath, + }); + return { targetVersion, method: "boot-service" as const, updateId }; }).pipe(Effect.onError(() => Ref.set(inFlight, false))); }); diff --git a/apps/server/src/cloud/serviceLauncherClient.test.ts b/apps/server/src/cloud/serviceLauncherClient.test.ts new file mode 100644 index 00000000000..6b9e926ff8d --- /dev/null +++ b/apps/server/src/cloud/serviceLauncherClient.test.ts @@ -0,0 +1,130 @@ +import { expect, it } from "@effect/vitest"; +import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; +import * as Effect from "effect/Effect"; +import * as Fiber from "effect/Fiber"; + +import { + SERVICE_LAUNCHER_CONTEXT_ENV, + type ServiceLauncherChildMessage, + type ServiceLauncherParentMessage, +} from "./serviceProtocol.ts"; +import * as ServiceLauncherClient from "./serviceLauncherClient.ts"; + +class FakeLauncherProcess { + readonly connected = true; + readonly env: Record; + readonly sent: ServiceLauncherChildMessage[] = []; + readonly #listeners = new Map) => void>>(); + + constructor(context: unknown) { + this.env = { [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) }; + } + + send = (message: ServiceLauncherChildMessage, callback?: (error: Error | null) => void) => { + this.sent.push(message); + callback?.(null); + return true; + }; + + on = (event: "message" | "disconnect", listener: (...args: ReadonlyArray) => void) => { + const listeners = this.#listeners.get(event) ?? new Set(); + listeners.add(listener); + this.#listeners.set(event, listeners); + }; + + off = (event: "message" | "disconnect", listener: (...args: ReadonlyArray) => void) => { + this.#listeners.get(event)?.delete(listener); + }; + + emit(message: ServiceLauncherParentMessage) { + for (const listener of this.#listeners.get("message") ?? []) listener(message); + } +} + +const makeClient = (host: FakeLauncherProcess, currentVersion: string) => + ServiceLauncherClient.make({ currentVersion }).pipe( + Effect.provideService(ServiceLauncherClient.ServiceLauncherHostProcess, host), + Effect.provideService(HostProcessEnvironment, host.env), + ); + +it.effect("waits for the launcher to durably commit the trial update ID", () => + Effect.gen(function* () { + const pending = { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status: "pending" as const, + }; + const host = new FakeLauncherProcess({ + protocol: 1, + childVersion: "1.1.0", + update: pending, + }); + const client = yield* makeClient(host, "1.1.0"); + const prepared = yield* Effect.forkChild(client.prepareTrial, { startImmediately: true }); + yield* Effect.yieldNow; + expect(host.sent).toEqual([{ type: "prepared", updateId: "update-1" }]); + + const committed = { + id: pending.id, + fromVersion: pending.fromVersion, + targetVersion: pending.targetVersion, + status: "committed" as const, + }; + host.emit({ type: "committed", updateId: committed.id }); + expect(yield* Fiber.join(prepared)).toEqual(committed); + }), +); + +it.effect("returns the launcher-generated ID only after update acceptance", () => + Effect.gen(function* () { + const host = new FakeLauncherProcess({ + protocol: 1, + childVersion: "1.0.0", + }); + const client = yield* makeClient(host, "1.0.0"); + const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), { + startImmediately: true, + }); + yield* Effect.yieldNow; + host.emit({ + type: "update-accepted", + updateId: "launcher-id", + }); + expect(yield* Fiber.join(requested)).toBe("launcher-id"); + }), +); + +it.effect("preserves a launcher rejection as a distinct error", () => + Effect.gen(function* () { + const host = new FakeLauncherProcess({ protocol: 1, childVersion: "1.0.0" }); + const client = yield* makeClient(host, "1.0.0"); + const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), { + startImmediately: true, + }); + yield* Effect.yieldNow; + host.emit({ type: "update-rejected", reason: "requires local update" }); + expect(yield* Fiber.join(requested).pipe(Effect.flip)).toMatchObject({ + _tag: "ServiceLauncherRejectedError", + targetVersion: "1.1.0", + reason: "requires local update", + }); + }), +); + +it.effect("rejects contradictory trial context instead of leaving activation closed", () => + Effect.gen(function* () { + const host = new FakeLauncherProcess({ + protocol: 1, + childVersion: "1.1.0", + update: { + id: "update-1", + fromVersion: "1.0.0", + targetVersion: "1.2.0", + status: "pending", + }, + }); + const error = yield* makeClient(host, "1.1.0").pipe(Effect.flip); + expect(error.message).toBe("The service launcher supplied invalid startup context."); + }), +); diff --git a/apps/server/src/cloud/serviceLauncherClient.ts b/apps/server/src/cloud/serviceLauncherClient.ts new file mode 100644 index 00000000000..760642c29f5 --- /dev/null +++ b/apps/server/src/cloud/serviceLauncherClient.ts @@ -0,0 +1,249 @@ +import type { ServerSelfUpdateOutcome } from "@t3tools/contracts"; +import { HostProcessEnvironment } from "@t3tools/shared/hostProcess"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +import packageJson from "../../package.json" with { type: "json" }; +import { + decodeServiceLauncherContext, + decodeServiceLauncherParentMessage, + SERVICE_LAUNCHER_CONTEXT_ENV, + type ServiceLauncherChildMessage, + type ServiceLauncherParentMessage, +} from "./serviceProtocol.ts"; + +export class ServiceLauncherClientError extends Schema.TaggedErrorClass()( + "ServiceLauncherClientError", + { + operation: Schema.Literals([ + "decode-context", + "version-mismatch", + "ipc-unavailable", + "unmanaged", + "send", + "disconnect", + "timeout", + ]), + cause: Schema.optional(Schema.Defect()), + }, +) { + override get message(): string { + switch (this.operation) { + case "decode-context": + return "The service launcher supplied invalid startup context."; + case "version-mismatch": + return "The service launcher started a different t3 version."; + case "ipc-unavailable": + return "The service launcher IPC channel is unavailable."; + case "unmanaged": + return "This server is not managed by the launcher."; + case "send": + return "Could not send a request to the service launcher."; + case "disconnect": + return "The service launcher disconnected before acknowledging the request."; + case "timeout": + return "The service launcher did not respond within 30 seconds."; + } + } +} + +export class ServiceLauncherRejectedError extends Schema.TaggedErrorClass()( + "ServiceLauncherRejectedError", + { + targetVersion: Schema.String, + reason: Schema.String, + }, +) { + override get message(): string { + return this.reason; + } +} + +interface ServiceLauncherProcess { + readonly connected: boolean; + readonly send: ( + message: ServiceLauncherChildMessage, + callback?: (error: Error | null) => void, + ) => boolean; + readonly on: ( + event: "message" | "disconnect", + listener: (...args: ReadonlyArray) => void, + ) => void; + readonly off: ( + event: "message" | "disconnect", + listener: (...args: ReadonlyArray) => void, + ) => void; +} + +export const ServiceLauncherHostProcess = Context.Reference( + "t3/cloud/serviceLauncherHostProcess", + { + defaultValue: () => ({ + connected: process.connected && process.send !== undefined, + send: (message, callback) => { + if (process.send === undefined) return false; + return callback === undefined ? process.send(message) : process.send(message, callback); + }, + on: (event, listener) => { + process.on(event, listener); + }, + off: (event, listener) => { + process.off(event, listener); + }, + }), + }, +); + +export class ServiceLauncherClient extends Context.Service< + ServiceLauncherClient, + { + readonly managed: boolean; + readonly trial: boolean; + readonly requestUpdate: (input: { + readonly targetVersion: string; + }) => Effect.Effect; + readonly prepareTrial: Effect.Effect< + ServerSelfUpdateOutcome | undefined, + ServiceLauncherClientError + >; + } +>()("t3/cloud/serviceLauncherClient") {} + +const resolveStartup = Effect.fn("cloud.service_launcher_client.resolve_startup")( + function* (options?: { readonly currentVersion?: string }) { + const host = yield* ServiceLauncherHostProcess; + const environment = yield* HostProcessEnvironment; + const currentVersion = options?.currentVersion ?? packageJson.version; + const rawContext = environment[SERVICE_LAUNCHER_CONTEXT_ENV]; + const context = rawContext === undefined ? undefined : decodeServiceLauncherContext(rawContext); + + if (rawContext !== undefined && context === undefined) { + return yield* new ServiceLauncherClientError({ operation: "decode-context" }); + } + if (context !== undefined && context.childVersion !== currentVersion) { + return yield* new ServiceLauncherClientError({ operation: "version-mismatch" }); + } + + const managed = context !== undefined && host.connected; + if (context !== undefined && !managed) { + return yield* new ServiceLauncherClientError({ operation: "ipc-unavailable" }); + } + + return { host, context, managed }; + }, +); + +export const resolveServiceLauncherMode = Effect.fn("cloud.service_launcher_client.resolve_mode")( + function* () { + const { context, managed } = yield* resolveStartup(); + return { managed, trial: context?.update?.status === "pending" }; + }, +); + +export const make = Effect.fn("cloud.service_launcher_client.make")(function* (options?: { + readonly currentVersion?: string; +}) { + const { host, context, managed } = yield* resolveStartup(options); + + const exchange = ( + message: ServiceLauncherChildMessage, + accept: (reply: ServiceLauncherParentMessage) => boolean, + ) => + Effect.callback((resume) => { + if (!managed) { + resume(Effect.fail(new ServiceLauncherClientError({ operation: "unmanaged" }))); + return; + } + + let settled = false; + const cleanup = () => { + host.off("message", onMessage); + host.off("disconnect", onDisconnect); + }; + const settle = ( + effect: Effect.Effect, + ) => { + if (settled) return; + settled = true; + cleanup(); + resume(effect); + }; + const onMessage = (...args: ReadonlyArray) => { + const reply = decodeServiceLauncherParentMessage(args[0]); + if (reply !== undefined && accept(reply)) settle(Effect.succeed(reply)); + }; + const onDisconnect = () => + settle(Effect.fail(new ServiceLauncherClientError({ operation: "disconnect" }))); + + host.on("message", onMessage); + host.on("disconnect", onDisconnect); + try { + host.send(message, (error) => { + if (error !== null) { + settle( + Effect.fail(new ServiceLauncherClientError({ operation: "send", cause: error })), + ); + } + }); + } catch (cause) { + settle(Effect.fail(new ServiceLauncherClientError({ operation: "send", cause }))); + } + + return Effect.sync(cleanup); + }).pipe( + Effect.timeoutOrElse({ + duration: "30 seconds", + orElse: () => Effect.fail(new ServiceLauncherClientError({ operation: "timeout" })), + }), + ); + + const requestUpdate = (input: { readonly targetVersion: string }) => + exchange( + { type: "request-update", ...input }, + (reply) => reply.type === "update-accepted" || reply.type === "update-rejected", + ).pipe( + Effect.flatMap((reply) => + reply.type === "update-accepted" + ? Effect.succeed(reply.updateId) + : reply.type === "update-rejected" + ? Effect.fail( + new ServiceLauncherRejectedError({ + targetVersion: input.targetVersion, + reason: reply.reason, + }), + ) + : Effect.die("service launcher returned an impossible update response"), + ), + ); + + const pending = context?.update?.status === "pending" ? context.update : undefined; + const outcome = + context?.update === undefined || context.update.status === "pending" + ? undefined + : context.update; + const prepareTrial = + pending !== undefined + ? exchange( + { type: "prepared", updateId: pending.id }, + (reply) => reply.type === "committed" && reply.updateId === pending.id, + ).pipe( + Effect.flatMap((reply) => { + if (reply.type !== "committed") { + return Effect.die("service launcher returned an impossible prepared response"); + } + return Effect.succeed({ ...pending, status: "committed" as const }); + }), + ) + : Effect.succeed(outcome); + + return ServiceLauncherClient.of({ + managed, + trial: pending !== undefined, + requestUpdate, + prepareTrial, + }); +}); + +export const layer = Layer.effect(ServiceLauncherClient, make()); diff --git a/apps/server/src/cloud/servicePreflight.test.ts b/apps/server/src/cloud/servicePreflight.test.ts new file mode 100644 index 00000000000..d2ce6db8de8 --- /dev/null +++ b/apps/server/src/cloud/servicePreflight.test.ts @@ -0,0 +1,47 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; +import * as NodeSqlite from "node:sqlite"; + +import { migrationManifest } from "../persistence/Migrations.ts"; +import { runServicePreflight } from "./servicePreflight.ts"; + +it.layer(NodeServices.layer)("service update preflight", (it) => { + it.effect("requires exact migration-manifest equality without mutating the database", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-preflight-test-" }); + const databasePath = path.join(root, "state.sqlite"); + const database = new NodeSqlite.DatabaseSync(databasePath); + database.exec("CREATE TABLE effect_sql_migrations (migration_id INTEGER, name TEXT)"); + const insert = database.prepare( + "INSERT INTO effect_sql_migrations (migration_id, name) VALUES (?, ?)", + ); + for (const [id, name] of migrationManifest) insert.run(id, name); + database.close(); + + expect(runServicePreflight({ databasePath, launcherProtocol: 1, version: "1.2.3" })).toEqual({ + status: "ready", + version: "1.2.3", + launcherProtocol: 1, + }); + + const changed = new NodeSqlite.DatabaseSync(databasePath); + changed.exec("DELETE FROM effect_sql_migrations WHERE migration_id = 35"); + changed.close(); + const blocked = runServicePreflight({ + databasePath, + launcherProtocol: 1, + version: "1.2.3", + }); + expect(blocked.status).toBe("blocked"); + if (blocked.status === "blocked") { + expect(blocked.reason).toContain("npx t3@1.2.3 service update"); + } + }), + ); +}); diff --git a/apps/server/src/cloud/servicePreflight.ts b/apps/server/src/cloud/servicePreflight.ts new file mode 100644 index 00000000000..1843e163881 --- /dev/null +++ b/apps/server/src/cloud/servicePreflight.ts @@ -0,0 +1,96 @@ +import * as NodeSqlite from "node:sqlite"; + +import packageJson from "../../package.json" with { type: "json" }; +import { migrationManifest } from "../persistence/Migrations.ts"; +import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; + +export type ServicePreflightResult = + | { + readonly status: "ready"; + readonly version: string; + readonly launcherProtocol: typeof SERVICE_LAUNCHER_PROTOCOL; + } + | { + readonly status: "blocked"; + readonly version: string; + readonly reason: string; + }; + +const localUpdateReason = (version: string) => + `This version includes a database update and cannot be installed remotely. Run \`npx t3@${version} service update\` on the server machine.`; + +const isMigrationRow = ( + value: unknown, +): value is { readonly migration_id: number; readonly name: string } => + typeof value === "object" && + value !== null && + "migration_id" in value && + typeof value.migration_id === "number" && + "name" in value && + typeof value.name === "string"; + +export function runServicePreflight(input: { + readonly databasePath: string; + readonly launcherProtocol: number; + readonly version?: string; +}): ServicePreflightResult { + const version = input.version ?? packageJson.version; + if (input.launcherProtocol !== SERVICE_LAUNCHER_PROTOCOL) { + return { + status: "blocked", + version, + reason: + "This release requires a newer T3 Code service launcher. Update it on the server machine.", + }; + } + + try { + const database = new NodeSqlite.DatabaseSync(input.databasePath, { readOnly: true }); + try { + const rows: ReadonlyArray = database + .prepare("SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id") + .all(); + const exact = + rows.length === migrationManifest.length && + rows.every((row, index) => { + const expected = migrationManifest[index]; + return ( + isMigrationRow(row) && row.migration_id === expected?.[0] && row.name === expected?.[1] + ); + }); + if (!exact) return { status: "blocked", version, reason: localUpdateReason(version) }; + } finally { + database.close(); + } + } catch { + return { status: "blocked", version, reason: localUpdateReason(version) }; + } + + return { status: "ready", version, launcherProtocol: SERVICE_LAUNCHER_PROTOCOL }; +} + +export function decodeServicePreflightResult(value: unknown): ServicePreflightResult | undefined { + if (typeof value !== "object" || value === null) { + return undefined; + } + const record = value as Record; + if ( + record.status === "ready" && + record.launcherProtocol === SERVICE_LAUNCHER_PROTOCOL && + typeof record.version === "string" + ) { + return { + status: "ready", + version: record.version, + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + }; + } + if ( + record.status === "blocked" && + typeof record.version === "string" && + typeof record.reason === "string" + ) { + return { status: "blocked", version: record.version, reason: record.reason }; + } + return undefined; +} diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts new file mode 100644 index 00000000000..921bc1447ed --- /dev/null +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -0,0 +1,226 @@ +import type { ServerSelfUpdateOutcome } from "@t3tools/contracts"; + +export const SERVICE_LAUNCHER_PROTOCOL = 1 as const; +export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; +export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs"; +export const SERVICE_STATE_FILE = "service-state.json"; + +export interface PendingServiceUpdate { + readonly id: string; + readonly fromVersion: string; + readonly targetVersion: string; + readonly status: "pending"; +} + +export type ServiceUpdateRecord = PendingServiceUpdate | ServerSelfUpdateOutcome; + +export interface ServiceState { + readonly protocol: typeof SERVICE_LAUNCHER_PROTOCOL; + readonly activeVersion: string; + readonly update?: ServiceUpdateRecord; +} + +/** Context is copied from launcher-owned state when a child is spawned. */ +export interface ServiceLauncherContext { + readonly protocol: typeof SERVICE_LAUNCHER_PROTOCOL; + readonly childVersion: string; + readonly update?: ServiceUpdateRecord; +} + +export type ServiceLauncherChildMessage = + | { + readonly type: "request-update"; + readonly targetVersion: string; + } + | { + readonly type: "prepared"; + readonly updateId: string; + }; + +export type ServiceLauncherParentMessage = + | { + readonly type: "update-accepted"; + readonly updateId: string; + } + | { + readonly type: "update-rejected"; + readonly reason: string; + } + | { + readonly type: "committed"; + readonly updateId: string; + }; + +const SEMVER_NUMBER = "(?:0|[1-9]\\d*)"; +const SEMVER_PRERELEASE = `(?:${SEMVER_NUMBER}|[0-9]*[A-Za-z-][0-9A-Za-z-]*)`; +const EXACT_SERVICE_VERSION = new RegExp( + `^${SEMVER_NUMBER}\\.${SEMVER_NUMBER}\\.${SEMVER_NUMBER}(?:-${SEMVER_PRERELEASE}(?:\\.${SEMVER_PRERELEASE})*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$`, +); + +/** Accepts exact SemVer only: never dist-tags or ranges passed to npm or filesystem paths. */ +export const isExactServiceVersion = (version: string): boolean => + EXACT_SERVICE_VERSION.test(version); + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +export function decodeServiceUpdate(value: unknown): ServiceUpdateRecord | undefined { + if (!isRecord(value)) return undefined; + const { id, fromVersion, targetVersion, status } = value; + if ( + typeof id !== "string" || + id.trim() === "" || + typeof fromVersion !== "string" || + !isExactServiceVersion(fromVersion) || + typeof targetVersion !== "string" || + !isExactServiceVersion(targetVersion) + ) { + return undefined; + } + if (status === "pending") { + return { id, fromVersion, targetVersion, status }; + } + if ( + (status === "committed" || status === "rolled-back" || status === "failed") && + (value.reason === undefined || (typeof value.reason === "string" && value.reason.trim() !== "")) + ) { + return { + id, + fromVersion, + targetVersion, + status, + ...(typeof value.reason === "string" ? { reason: value.reason } : {}), + }; + } + return undefined; +} + +/** SemVer precedence for exact versions. Build metadata is ignored. */ +export function compareExactServiceVersions(left: string, right: string): number { + const parse = (version: string) => { + const withoutBuild = version.split("+", 1)[0] ?? version; + const separator = withoutBuild.indexOf("-"); + const core = separator === -1 ? withoutBuild : withoutBuild.slice(0, separator); + const prerelease = separator === -1 ? undefined : withoutBuild.slice(separator + 1); + const [major = "0", minor = "0", patch = "0"] = core.split("."); + return { + core: [BigInt(major), BigInt(minor), BigInt(patch)] as const, + prerelease: prerelease?.split(".") ?? [], + }; + }; + const a = parse(left); + const b = parse(right); + for (let index = 0; index < 3; index += 1) { + const x = a.core[index] ?? 0n; + const y = b.core[index] ?? 0n; + if (x !== y) return x < y ? -1 : 1; + } + if (a.prerelease.length === 0 || b.prerelease.length === 0) { + return a.prerelease.length === b.prerelease.length ? 0 : a.prerelease.length === 0 ? 1 : -1; + } + const count = Math.max(a.prerelease.length, b.prerelease.length); + for (let index = 0; index < count; index += 1) { + const x = a.prerelease[index]; + const y = b.prerelease[index]; + if (x === undefined || y === undefined) return x === undefined ? -1 : 1; + if (x === y) continue; + const xNumeric = /^\d+$/.test(x); + const yNumeric = /^\d+$/.test(y); + if (xNumeric && yNumeric) return BigInt(x) < BigInt(y) ? -1 : 1; + if (xNumeric !== yNumeric) return xNumeric ? -1 : 1; + return x < y ? -1 : 1; + } + return 0; +} + +export function decodeServiceState(value: unknown): ServiceState | undefined { + if (!isRecord(value)) return undefined; + const update = value.update === undefined ? undefined : decodeServiceUpdate(value.update); + if ( + value.protocol !== SERVICE_LAUNCHER_PROTOCOL || + typeof value.activeVersion !== "string" || + !isExactServiceVersion(value.activeVersion) || + (value.update !== undefined && update === undefined) || + (update !== undefined && + compareExactServiceVersions(update.targetVersion, update.fromVersion) <= 0) || + (update?.status === "pending" && update.fromVersion !== value.activeVersion) || + (update?.status === "committed" && update.targetVersion !== value.activeVersion) || + ((update?.status === "rolled-back" || update?.status === "failed") && + update.fromVersion !== value.activeVersion) + ) { + return undefined; + } + return { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: value.activeVersion, + ...(update === undefined ? {} : { update }), + }; +} + +export function parseServiceState(value: string): ServiceState | undefined { + try { + return decodeServiceState(JSON.parse(value) as unknown); + } catch { + return undefined; + } +} + +export function decodeServiceLauncherContext(value: string): ServiceLauncherContext | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(value) as unknown; + } catch { + return undefined; + } + if ( + !isRecord(parsed) || + parsed.protocol !== SERVICE_LAUNCHER_PROTOCOL || + typeof parsed.childVersion !== "string" || + !isExactServiceVersion(parsed.childVersion) + ) { + return undefined; + } + const update = parsed.update === undefined ? undefined : decodeServiceUpdate(parsed.update); + if (parsed.update !== undefined && update === undefined) return undefined; + const selectedVersion = + update?.status === "pending" || update?.status === "committed" + ? update.targetVersion + : update === undefined + ? parsed.childVersion + : update.fromVersion; + if (parsed.childVersion !== selectedVersion) { + return undefined; + } + return { + protocol: SERVICE_LAUNCHER_PROTOCOL, + childVersion: parsed.childVersion, + ...(update === undefined ? {} : { update }), + }; +} + +export function decodeServiceLauncherChildMessage( + value: unknown, +): ServiceLauncherChildMessage | undefined { + if (!isRecord(value)) return undefined; + if (value.type === "request-update" && typeof value.targetVersion === "string") { + return { type: value.type, targetVersion: value.targetVersion }; + } + return value.type === "prepared" && typeof value.updateId === "string" + ? { type: value.type, updateId: value.updateId } + : undefined; +} + +export function decodeServiceLauncherParentMessage( + value: unknown, +): ServiceLauncherParentMessage | undefined { + if (!isRecord(value)) return undefined; + if (value.type === "update-rejected" && typeof value.reason === "string") { + return { type: value.type, reason: value.reason }; + } + if (value.type === "update-accepted" && typeof value.updateId === "string") { + return { type: value.type, updateId: value.updateId }; + } + return value.type === "committed" && typeof value.updateId === "string" + ? { type: value.type, updateId: value.updateId } + : undefined; +} diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index 2290450e7d3..a14f89fd031 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -10,6 +10,7 @@ import * as Schema from "effect/Schema"; import packageJson from "../../package.json" with { type: "json" }; import { resolveServerSelfUpdateCapability } from "../cloud/selfUpdate.ts"; +import { resolveServiceLauncherMode } from "../cloud/serviceLauncherClient.ts"; import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; import { resolveServerEnvironmentLabel } from "./ServerEnvironmentLabel.ts"; @@ -125,8 +126,10 @@ export const make = Effect.gen(function* () { const environmentId = EnvironmentId.make(environmentIdRaw); const cwdBaseName = path.basename(serverConfig.cwd).trim(); const label = yield* resolveServerEnvironmentLabel({ cwdBaseName }); - const serverSelfUpdate = yield* resolveServerSelfUpdateCapability({ + const launcher = yield* resolveServiceLauncherMode(); + const serverSelfUpdate = resolveServerSelfUpdateCapability({ desktopManaged: serverConfig.mode === "desktop", + launcherManaged: launcher.managed, }); const descriptor: ExecutionEnvironmentDescriptor = { @@ -144,9 +147,7 @@ export const make = Effect.gen(function* () { threadSnooze: true, threadTitleRegeneration: true, ...(serverSelfUpdate === null ? {} : { serverSelfUpdate }), - ...(serverSelfUpdate === "boot-service" || serverSelfUpdate === "respawn" - ? { serverSelfUpdateProgress: true } - : {}), + ...(serverSelfUpdate === "boot-service" ? { serverSelfUpdateProgress: true } : {}), }, }; diff --git a/apps/server/src/orchestration/Layers/CheckpointReactor.ts b/apps/server/src/orchestration/Layers/CheckpointReactor.ts index 54cbe3eb5bf..95adee0cf7f 100644 --- a/apps/server/src/orchestration/Layers/CheckpointReactor.ts +++ b/apps/server/src/orchestration/Layers/CheckpointReactor.ts @@ -29,6 +29,7 @@ import { import * as CheckpointStore from "../../checkpointing/CheckpointStore.ts"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; import { CheckpointReactor, type CheckpointReactorShape } from "../Services/CheckpointReactor.ts"; +import { forkParked } from "../../serverActivation.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; import { RuntimeReceiptBus } from "../Services/RuntimeReceiptBus.ts"; @@ -912,7 +913,7 @@ const make = Effect.gen(function* () { const worker = yield* makeDrainableWorker(processInputSafely); const start: CheckpointReactorShape["start"] = Effect.fn("start")(function* () { - yield* Effect.forkScoped( + yield* forkParked( Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { if ( event.type !== "thread.turn-start-requested" && @@ -926,7 +927,7 @@ const make = Effect.gen(function* () { }), ); - yield* Effect.forkScoped( + yield* forkParked( Stream.runForEach(providerService.streamEvents, (event) => { if (event.type !== "turn.started" && event.type !== "turn.completed") { return Effect.void; diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index 6ddc9f18cb3..1135fd579e7 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -38,6 +38,7 @@ import { ProviderCommandReactor, type ProviderCommandReactorShape, } from "../Services/ProviderCommandReactor.ts"; +import { forkParked, ServerActivation } from "../../serverActivation.ts"; import { resolveSourceControlWriterModelSelection, ServerSettingsService, @@ -899,19 +900,25 @@ const make = Effect.gen(function* () { ...(input.title !== undefined ? { title: input.title } : {}), }); }); - const clearInterruptedThreadTitleRegenerations = Effect.fn( - "clearInterruptedThreadTitleRegenerations", + const findInterruptedThreadTitleRegenerations = Effect.fn( + "findInterruptedThreadTitleRegenerations", )(function* () { const readModel = yield* projectionSnapshotQuery.getCommandReadModel(); + return readModel.threads.flatMap((thread) => { + const requestId = thread.titleRegeneration?.requestId; + return requestId === undefined ? [] : [{ threadId: thread.id, requestId }]; + }); + }); + const clearInterruptedThreadTitleRegenerations = Effect.fn( + "clearInterruptedThreadTitleRegenerations", + )(function* ( + interrupted: ReadonlyArray<{ readonly threadId: ThreadId; readonly requestId: CommandId }>, + ) { yield* Effect.forEach( - readModel.threads, - (thread) => { - const requestId = thread.titleRegeneration?.requestId; - if (requestId === undefined) { - return Effect.void; - } + interrupted, + ({ threadId, requestId }) => { return dispatchThreadTitleRegenerationCompletion({ - threadId: thread.id, + threadId, requestId, }).pipe( Effect.catchCause((cause) => { @@ -921,7 +928,7 @@ const make = Effect.gen(function* () { return Effect.logWarning( "provider command reactor failed to clear interrupted title regeneration", { - threadId: thread.id, + threadId, cause: Cause.pretty(cause), }, ); @@ -1304,7 +1311,7 @@ const make = Effect.gen(function* () { processDomainEvent(event).pipe( Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) { - return Effect.failCause(cause); + return Effect.interrupt; } return Effect.logWarning("provider command reactor failed to process event", { eventType: event.type, @@ -1316,6 +1323,17 @@ const make = Effect.gen(function* () { const worker = yield* makeDrainableWorker(processDomainEventSafely); const start: ProviderCommandReactorShape["start"] = Effect.fn("start")(function* () { + const interruptedTitleRegenerations = yield* findInterruptedThreadTitleRegenerations().pipe( + Effect.catchCause((cause) => { + if (Cause.hasInterruptsOnly(cause)) { + return Effect.interrupt; + } + return Effect.logWarning( + "provider command reactor failed to find interrupted title regenerations", + { cause: Cause.pretty(cause) }, + ).pipe(Effect.as([])); + }), + ); const processEvent = Effect.fn("processEvent")(function* (event: OrchestrationEvent) { if ( (event.type === "thread.meta-updated" && event.payload.regenerateTitle === true) || @@ -1330,14 +1348,14 @@ const make = Effect.gen(function* () { } }); - yield* Effect.forkScoped( - Stream.runForEach(orchestrationEngine.streamDomainEvents, processEvent), - ); + yield* forkParked(Stream.runForEach(orchestrationEngine.streamDomainEvents, processEvent)); // The domain event stream is hot, so work pending before this reactor // starts cannot be resumed. Correlated completions only clear the request // captured here, leaving any newer request untouched. - yield* clearInterruptedThreadTitleRegenerations().pipe( + const clearInterrupted = clearInterruptedThreadTitleRegenerations( + interruptedTitleRegenerations, + ).pipe( Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) { return Effect.interrupt; @@ -1350,6 +1368,12 @@ const make = Effect.gen(function* () { ); }), ); + const activation = yield* ServerActivation; + if (activation === undefined) { + yield* clearInterrupted; + } else { + yield* forkParked(clearInterrupted); + } }); return { diff --git a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts index a8a51b30260..c8d619270d3 100644 --- a/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts +++ b/apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts @@ -37,6 +37,7 @@ import { ProviderRuntimeIngestionService, type ProviderRuntimeIngestionShape, } from "../Services/ProviderRuntimeIngestion.ts"; +import { forkParked } from "../../serverActivation.ts"; import { ServerSettingsService } from "../../serverSettings.ts"; const providerTurnKey = (threadId: ThreadId, turnId: TurnId) => `${threadId}:${turnId}`; @@ -1804,12 +1805,12 @@ const make = Effect.gen(function* () { const start: ProviderRuntimeIngestionShape["start"] = () => Effect.gen(function* () { - yield* Effect.forkScoped( + yield* forkParked( Stream.runForEach(providerService.streamEvents, (event) => worker.enqueue({ source: "runtime", event }), ), ); - yield* Effect.forkScoped( + yield* forkParked( Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { if (event.type !== "thread.turn-start-requested") { return Effect.void; diff --git a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts index 7d8a24069a3..a026f5ad81b 100644 --- a/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts +++ b/apps/server/src/orchestration/Layers/ThreadDeletionReactor.ts @@ -12,6 +12,7 @@ import { ThreadDeletionReactor, type ThreadDeletionReactorShape, } from "../Services/ThreadDeletionReactor.ts"; +import { forkParked } from "../../serverActivation.ts"; type ThreadDeletedEvent = Extract; @@ -80,7 +81,7 @@ const make = Effect.gen(function* () { const worker = yield* makeDrainableWorker(processThreadDeletedSafely); const start: ThreadDeletionReactorShape["start"] = Effect.fn("start")(function* () { - yield* Effect.forkScoped( + yield* forkParked( Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { if (event.type !== "thread.deleted") { return Effect.void; diff --git a/apps/server/src/persistence/Layers/Sqlite.ts b/apps/server/src/persistence/Layers/Sqlite.ts index dfd338b7159..3cfdb5c5c76 100644 --- a/apps/server/src/persistence/Layers/Sqlite.ts +++ b/apps/server/src/persistence/Layers/Sqlite.ts @@ -7,6 +7,7 @@ import type { SqlError } from "effect/unstable/sql/SqlError"; import { runMigrations } from "../Migrations.ts"; import { ServerConfig } from "../../config.ts"; +import * as ServiceLauncherClient from "../../cloud/serviceLauncherClient.ts"; type RuntimeSqliteLayerConfig = { readonly filename: string; @@ -30,24 +31,28 @@ const makeRuntimeSqliteLayer = Effect.fn("makeRuntimeSqliteLayer")(function* ( return clientModule.layer(config); }, Layer.unwrap); -const setup = Layer.effectDiscard( - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - yield* sql`PRAGMA journal_mode = WAL;`; - yield* sql`PRAGMA foreign_keys = ON;`; - yield* runMigrations(); - }), -); +const setup = (trial: boolean) => + Layer.effectDiscard( + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`PRAGMA foreign_keys = ON;`; + if (!trial) { + yield* sql`PRAGMA journal_mode = WAL;`; + yield* runMigrations(); + } + }), + ); export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(function* ( dbPath: string, + options?: { readonly trial?: boolean }, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true }); return Layer.provideMerge( - setup, + setup(options?.trial === true), makeRuntimeSqliteLayer({ filename: dbPath, spanAttributes: { @@ -59,10 +64,14 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")( }, Layer.unwrap); export const SqlitePersistenceMemory = Layer.provideMerge( - setup, + setup(false), makeRuntimeSqliteLayer({ filename: ":memory:" }), ); export const layerConfig = Layer.unwrap( - Effect.map(Effect.service(ServerConfig), ({ dbPath }) => makeSqlitePersistenceLive(dbPath)), + Effect.gen(function* () { + const { dbPath } = yield* ServerConfig; + const launcher = yield* ServiceLauncherClient.resolveServiceLauncherMode(); + return makeSqlitePersistenceLive(dbPath, { trial: launcher.trial }); + }), ); diff --git a/apps/server/src/persistence/Migrations.ts b/apps/server/src/persistence/Migrations.ts index 95cb6b17f84..b24aeb503b5 100644 --- a/apps/server/src/persistence/Migrations.ts +++ b/apps/server/src/persistence/Migrations.ts @@ -9,8 +9,8 @@ */ import * as Migrator from "effect/unstable/sql/Migrator"; -import * as Layer from "effect/Layer"; import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; // Import all migrations statically import Migration0001 from "./Migrations/001_OrchestrationEvents.ts"; @@ -97,6 +97,8 @@ export const migrationEntries = [ [35, "ProjectionThreadTitleRegeneration", Migration0035], ] as const; +export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const); + export const makeMigrationLoader = (throughId?: number) => Migrator.fromRecord( Object.fromEntries( diff --git a/apps/server/src/provider/Layers/ProviderSessionReaper.ts b/apps/server/src/provider/Layers/ProviderSessionReaper.ts index ca396b40596..8eccd52fb2c 100644 --- a/apps/server/src/provider/Layers/ProviderSessionReaper.ts +++ b/apps/server/src/provider/Layers/ProviderSessionReaper.ts @@ -11,6 +11,7 @@ import { ProviderSessionReaper, type ProviderSessionReaperShape, } from "../Services/ProviderSessionReaper.ts"; +import { forkParked } from "../../serverActivation.ts"; import { ProviderService } from "../Services/ProviderService.ts"; const DEFAULT_INACTIVITY_THRESHOLD_MS = 30 * 60 * 1000; @@ -105,7 +106,7 @@ const makeProviderSessionReaper = (options?: ProviderSessionReaperLiveOptions) = const start: ProviderSessionReaperShape["start"] = () => Effect.gen(function* () { - yield* Effect.forkScoped( + yield* forkParked( sweep.pipe( Effect.catch((error: unknown) => Effect.logWarning("provider.session.reaper.sweep-failed", { diff --git a/apps/server/src/relay/AgentAwarenessRelay.ts b/apps/server/src/relay/AgentAwarenessRelay.ts index 58de98f1ca1..2a4de7eda91 100644 --- a/apps/server/src/relay/AgentAwarenessRelay.ts +++ b/apps/server/src/relay/AgentAwarenessRelay.ts @@ -44,6 +44,7 @@ import { getOrCreateEnvironmentKeyPairFromSecretStore } from "../cloud/environme import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as OrchestrationEngine from "../orchestration/Services/OrchestrationEngine.ts"; import * as ProjectionSnapshotQuery from "../orchestration/Services/ProjectionSnapshotQuery.ts"; +import { forkParked } from "../serverActivation.ts"; export class AgentAwarenessRelay extends Context.Service< AgentAwarenessRelay, @@ -599,12 +600,12 @@ export const make = Effect.gen(function* () { }); break; } - yield* Effect.forkScoped( + yield* forkParked( Effect.sleep("1 second").pipe( Effect.andThen(publishActiveThreadsOnceWhenConfigured(startupState !== "enabled")), ), ); - yield* Effect.forkScoped( + yield* forkParked( Stream.runForEach(orchestrationEngine.streamDomainEvents, (event) => { const threadId = eventThreadId(event); if (threadId === null) { diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 569e8a51c37..74d3fd2d594 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -89,6 +89,7 @@ import * as ProviderRegistry from "./provider/Services/ProviderRegistry.ts"; import { makeManualOnlyProviderMaintenanceCapabilities } from "./provider/providerMaintenance.ts"; import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts"; import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts"; +import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts"; import * as ServerSettings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; import * as PreviewManager from "./preview/Manager.ts"; @@ -550,10 +551,13 @@ const buildAppUnderTest = (options?: { ), ); - const servedRoutesLayer = HttpRouter.serve(makeRoutesLayer, { - disableListenLog: true, - disableLogger: true, - }).pipe( + const servedRoutesLayer = HttpRouter.serve( + makeRoutesLayer.pipe(Layer.provide(ServiceLauncherClient.layer)), + { + disableListenLog: true, + disableLogger: true, + }, + ).pipe( Layer.provide( Layer.mock(Keybindings.Keybindings)({ loadConfigState: Effect.succeed({ @@ -1318,6 +1322,39 @@ const getWsServerUrl = ( }); it.layer(NodeServices.layer)("server router seam", (it) => { + it.effect("parks HTTP ingress until command readiness", () => + Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const staticDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-router-gate-" }); + yield* fileSystem.writeFileString(path.join(staticDir, "index.html"), "ready"); + const entered = yield* Deferred.make(); + const ready = yield* Deferred.make(); + const completed = yield* Deferred.make(); + + yield* buildAppUnderTest({ + config: { staticDir }, + layers: { + serverRuntimeStartup: { + awaitCommandReady: Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Deferred.await(ready)), + ), + }, + }, + }); + const request = yield* HttpClient.get("/").pipe( + Effect.tap(() => Deferred.succeed(completed, undefined)), + Effect.forkChild, + ); + yield* Deferred.await(entered); + assert.isFalse(yield* Deferred.isDone(completed)); + + yield* Deferred.succeed(ready, undefined); + assert.equal((yield* Fiber.join(request)).status, 200); + assert.isTrue(yield* Deferred.isDone(completed)); + }).pipe(Effect.provide(NodeHttpServer.layerTest)), + ); + it.effect("serves static index content for GET / when staticDir is configured", () => Effect.gen(function* () { const fileSystem = yield* FileSystem.FileSystem; diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 853bb0b1101..05657af6d48 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -1,5 +1,6 @@ import { EnvironmentHttpApi } from "@t3tools/contracts"; import * as Duration from "effect/Duration"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schedule from "effect/Schedule"; @@ -91,6 +92,7 @@ import * as CloudManagedEndpointRuntime from "./cloud/ManagedEndpointRuntime.ts" import * as CloudCliTokenManager from "./cloud/CliTokenManager.ts"; import * as CloudCliState from "./cloud/CliState.ts"; import * as ServerSelfUpdate from "./cloud/selfUpdate.ts"; +import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts"; import * as ProcessDiagnostics from "./diagnostics/ProcessDiagnostics.ts"; import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts"; import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts"; @@ -109,6 +111,7 @@ import { orchestrationHttpApiLayer } from "./orchestration/http.ts"; import * as NetService from "@t3tools/shared/Net"; import * as RelayClient from "@t3tools/shared/relayClient"; import { disableTailscaleServe, ensureTailscaleServe } from "@t3tools/tailscale"; +import { forkParked, ServerActivation } from "./serverActivation.ts"; // Effect's default preemptive shutdown waits 20s before finalizing request scopes. // T3's primary transport is long-lived WebSocket RPC, whose Effect scope finalizer @@ -396,8 +399,12 @@ const RuntimeDependenciesLive = RuntimeCoreDependenciesLive.pipe( Layer.provide(NetService.layer), ); -const RuntimeServicesLive = ServerRuntimeStartup.layer.pipe( - Layer.provideMerge(RuntimeDependenciesLive), +const commandReadinessLayer = HttpRouter.middleware( + (httpEffect) => + Effect.flatMap(ServerRuntimeStartup.ServerRuntimeStartup, (startup) => + startup.awaitCommandReady.pipe(Effect.orDie, Effect.andThen(httpEffect)), + ), + { global: true }, ); export const makeRoutesLayer = Layer.mergeAll( @@ -418,6 +425,7 @@ export const makeRoutesLayer = Layer.mergeAll( ).pipe( Layer.provide(PreviewAutomationBroker.layer), Layer.provide(ServerSelfUpdate.layer), + Layer.provide(commandReadinessLayer), Layer.provide(browserApiCorsLayer), Layer.provide(httpCompressionLayer), ); @@ -425,6 +433,14 @@ export const makeRoutesLayer = Layer.mergeAll( export const makeServerLayer = Layer.unwrap( Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; + const activation = yield* Deferred.make(); + const awaitActivation = Deferred.await(activation); + const activationLayer = Layer.succeed(ServerActivation, awaitActivation); + const runtimeStateParked = yield* Deferred.make(); + const tailscaleParked = yield* Deferred.make(); + const cloudLinkParked = yield* Deferred.make(); + const routesReady = yield* Deferred.make(); + const launcherLayer = ServiceLauncherClient.layer; yield* fixPath(); @@ -438,6 +454,8 @@ export const makeServerLayer = Layer.unwrap( const runtimeStateLayer = Layer.effectDiscard( Effect.acquireRelease( Effect.gen(function* () { + yield* Deferred.succeed(runtimeStateParked, undefined).pipe(Effect.orDie); + yield* awaitActivation; const server = yield* HttpServer.HttpServer; const address = server.address; if (typeof address === "string" || !("port" in address)) { @@ -451,15 +469,26 @@ export const makeServerLayer = Layer.unwrap( yield* persistServerRuntimeState({ path: config.serverRuntimeStatePath, state, - }); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Failed to persist server runtime state", { cause }), + ), + ); }), - () => clearPersistedServerRuntimeState(config.serverRuntimeStatePath), + () => + clearPersistedServerRuntimeState(config.serverRuntimeStatePath).pipe( + Effect.catchCause((cause) => + Effect.logWarning("Failed to clear server runtime state", { cause }), + ), + ), ), ); const tailscaleServeLayer = config.tailscaleServeEnabled ? Layer.effectDiscard( Effect.acquireRelease( Effect.gen(function* () { + yield* Deferred.succeed(tailscaleParked, undefined).pipe(Effect.orDie); + yield* awaitActivation; const server = yield* HttpServer.HttpServer; const address = server.address; if (typeof address === "string" || !("port" in address)) { @@ -509,68 +538,80 @@ export const makeServerLayer = Layer.unwrap( : Layer.empty; const cloudDesiredLinkReconcileLayer = Layer.effectDiscard( Effect.gen(function* () { - if (!hasCloudPublicConfig) return; - // Idle Cloudflare tunnels are billed, so a stopping server releases its - // tunnel; the persisted desired link brings one back — same hostname, - // fresh tunnel — when the environment starts again. Registered even - // when no link is desired yet: a client can link a running server, and - // that tunnel needs the same disposal on shutdown. - yield* Effect.addFinalizer(() => - releaseManagedTunnelOnShutdown().pipe( - Effect.timeout("10 seconds"), - Effect.tap((released) => - released ? Effect.logInfo("Released the managed tunnel on shutdown") : Effect.void, - ), - Effect.catchCause((cause) => - Effect.logWarning( - "Failed to release the managed tunnel on shutdown; the next link reuses it", - { cause }, - ), - ), - Effect.asVoid, - ), - ); - if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; - const server = yield* HttpServer.HttpServer; - const address = server.address; - if (typeof address === "string" || !("port" in address)) return; - yield* Effect.forkScoped( - Effect.sleep("250 millis").pipe( - Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)), - // On reboot this races NIC/DNS bring-up, so back off exponentially - // (capped at 30s) instead of burning all retries in a second. - // Bounded overall so a permanently broken setup still surfaces the - // warning below. Bad-request/unauthorized/conflict are - // deterministic failures (malformed origin, not linked yet, linked - // to a different cloud account) that no amount of retrying - // converges. - Effect.retry({ - while: (error) => - error._tag !== "EnvironmentHttpBadRequestError" && - error._tag !== "EnvironmentHttpUnauthorizedError" && - error._tag !== "EnvironmentHttpConflictError", - schedule: Schedule.exponential("1 second").pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.seconds(30))), + if (!hasCloudPublicConfig) { + yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); + return; + } + yield* forkParked( + Effect.gen(function* () { + // Only an activated runtime owns the tunnel cleanup finalizer. + yield* Effect.addFinalizer(() => + releaseManagedTunnelOnShutdown().pipe( + Effect.timeout("10 seconds"), + Effect.tap((released) => + released + ? Effect.logInfo("Released the managed tunnel on shutdown") + : Effect.void, ), - Schedule.upTo({ duration: "10 minutes" }), + Effect.catchCause((cause) => + Effect.logWarning( + "Failed to release the managed tunnel on shutdown; the next link reuses it", + { cause }, + ), + ), + Effect.asVoid, ), - }), - Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), - Effect.catch((cause) => - Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { - cause, + ); + if (!(yield* CloudCliState.readCliDesiredCloudLink)) return; + const server = yield* HttpServer.HttpServer; + const address = server.address; + if (typeof address === "string" || !("port" in address)) return; + yield* Effect.sleep("250 millis").pipe( + Effect.andThen(reconcileDesiredCloudLink(`http://127.0.0.1:${address.port}`)), + Effect.retry({ + while: (error) => + error._tag !== "EnvironmentHttpBadRequestError" && + error._tag !== "EnvironmentHttpUnauthorizedError" && + error._tag !== "EnvironmentHttpConflictError", + schedule: Schedule.exponential("1 second").pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(30))), + ), + Schedule.upTo({ duration: "10 minutes" }), + ), }), - ), - ), + Effect.tap(() => Effect.logInfo("T3 Connect desired link reconciled on startup")), + Effect.catch((cause) => + Effect.logWarning("Failed to reconcile T3 Connect desired link on startup", { + cause, + }), + ), + ); + }), ); + yield* Deferred.succeed(cloudLinkParked, undefined).pipe(Effect.orDie); }), ); + const runtimeServicesLive = ServerRuntimeStartup.layerWithOptions({ + activate: Deferred.succeed(activation, undefined).pipe(Effect.asVoid), + abort: (error) => Deferred.die(activation, error).pipe(Effect.asVoid), + awaitAuxiliaryParked: Effect.all( + [ + Deferred.await(runtimeStateParked), + Deferred.await(cloudLinkParked), + Deferred.await(routesReady), + ...(config.tailscaleServeEnabled ? [Deferred.await(tailscaleParked)] : []), + ], + { concurrency: "unbounded" }, + ).pipe(Effect.asVoid), + }).pipe(Layer.provideMerge(RuntimeDependenciesLive), Layer.provide(launcherLayer)); + + const routesLayer = HttpRouter.serve(makeRoutesLayer.pipe(Layer.provide(launcherLayer)), { + disableLogger: !config.logWebSocketEvents, + }).pipe(Layer.tap(() => Deferred.succeed(routesReady, undefined).pipe(Effect.orDie))); const serverApplicationLayer = Layer.mergeAll( - HttpRouter.serve(makeRoutesLayer, { - disableLogger: !config.logWebSocketEvents, - }), + routesLayer, httpListeningLayer, runtimeStateLayer, tailscaleServeLayer, @@ -578,7 +619,8 @@ export const makeServerLayer = Layer.unwrap( ); return serverApplicationLayer.pipe( - Layer.provideMerge(RuntimeServicesLive), + Layer.provideMerge(runtimeServicesLive), + Layer.provide(activationLayer), Layer.provideMerge(serverRelayBrokerTracingLayer), Layer.provideMerge(HttpResponseCompressionLive), Layer.provideMerge(HttpServerLive), @@ -590,5 +632,5 @@ export const makeServerLayer = Layer.unwrap( }), ); -// Important: Only `ServerConfig` should be provided by the CLI layer!!! Don't let other requirements leak into the launch layer. +// The CLI supplies configuration. export const runServer = Layer.launch(makeServerLayer); diff --git a/apps/server/src/serverActivation.test.ts b/apps/server/src/serverActivation.test.ts new file mode 100644 index 00000000000..a4f942a95b5 --- /dev/null +++ b/apps/server/src/serverActivation.test.ts @@ -0,0 +1,23 @@ +import { expect, it } from "@effect/vitest"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; + +import { forkParked, ServerActivation } from "./serverActivation.ts"; + +it.effect("proves a root is parked before returning and releases it with one gate", () => + Effect.scoped( + Effect.gen(function* () { + const activation = yield* Deferred.make(); + const ran = yield* Deferred.make(); + + yield* forkParked(Deferred.succeed(ran, undefined)).pipe( + Effect.provideService(ServerActivation, Deferred.await(activation)), + ); + expect(yield* Deferred.isDone(ran)).toBe(false); + + yield* Deferred.succeed(activation, undefined); + yield* Deferred.await(ran); + expect(yield* Deferred.isDone(ran)).toBe(true); + }), + ), +); diff --git a/apps/server/src/serverActivation.ts b/apps/server/src/serverActivation.ts new file mode 100644 index 00000000000..c068d55e7e7 --- /dev/null +++ b/apps/server/src/serverActivation.ts @@ -0,0 +1,26 @@ +import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import type * as Scope from "effect/Scope"; + +export class ServerActivation extends Context.Reference | undefined>( + "t3/serverActivation", + { defaultValue: () => undefined }, +) {} + +/** Forks a long-running root before commit and proves it is parked at the activation boundary. */ +export const forkParked = ( + effect: Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const activation = yield* ServerActivation; + if (activation === undefined) { + yield* Effect.forkScoped(effect); + return; + } + const parked = yield* Deferred.make(); + yield* Effect.forkScoped( + Deferred.succeed(parked, undefined).pipe(Effect.andThen(activation), Effect.andThen(effect)), + ); + yield* Deferred.await(parked); + }); diff --git a/apps/server/src/serverRuntimeStartup.ts b/apps/server/src/serverRuntimeStartup.ts index b52b577c5b5..5db2b75556e 100644 --- a/apps/server/src/serverRuntimeStartup.ts +++ b/apps/server/src/serverRuntimeStartup.ts @@ -34,6 +34,8 @@ import * as AnalyticsService from "./telemetry/AnalyticsService.ts"; import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts"; import * as ProviderSessionReaper from "./provider/Services/ProviderSessionReaper.ts"; +import { forkParked } from "./serverActivation.ts"; +import * as ServiceLauncherClient from "./cloud/serviceLauncherClient.ts"; import { formatHeadlessServeOutput, formatHostForUrl, @@ -288,151 +290,161 @@ const runStartupPhase = (phase: string, effect: Effect.Effect) Effect.withSpan(`server.startup.${phase}`), ); -export const make = Effect.gen(function* () { - const serverConfig = yield* ServerConfig.ServerConfig; - const keybindings = yield* Keybindings.Keybindings; - const orchestrationReactor = yield* OrchestrationReactor.OrchestrationReactor; - const providerSessionReaper = yield* ProviderSessionReaper.ProviderSessionReaper; - const lifecycleEvents = yield* ServerLifecycleEvents.ServerLifecycleEvents; - const serverSettings = yield* ServerSettings.ServerSettingsService; - const serverEnvironment = yield* ServerEnvironment.ServerEnvironment; - const crypto = yield* Crypto.Crypto; +interface StartupOptions { + readonly activate?: Effect.Effect; + readonly awaitAuxiliaryParked?: Effect.Effect; + readonly abort?: (error: ServerRuntimeStartupError) => Effect.Effect; +} - const commandGate = yield* makeCommandGate; - const httpListening = yield* Deferred.make(); - const reactorScope = yield* Scope.make("sequential"); - - yield* Effect.addFinalizer(() => Scope.close(reactorScope, Exit.void)); - - const startup = Effect.gen(function* () { - yield* Effect.logDebug("startup phase: starting keybindings runtime"); - yield* runStartupPhase( - "keybindings.start", - keybindings.start.pipe( - Effect.catch((error) => - Effect.logWarning("failed to start keybindings runtime", { - path: error.configPath, - detail: error.detail, - cause: error.cause, - }), +export const make = (options?: StartupOptions) => + Effect.gen(function* () { + const serverConfig = yield* ServerConfig.ServerConfig; + const keybindings = yield* Keybindings.Keybindings; + const orchestrationReactor = yield* OrchestrationReactor.OrchestrationReactor; + const providerSessionReaper = yield* ProviderSessionReaper.ProviderSessionReaper; + const lifecycleEvents = yield* ServerLifecycleEvents.ServerLifecycleEvents; + const serverSettings = yield* ServerSettings.ServerSettingsService; + const serverEnvironment = yield* ServerEnvironment.ServerEnvironment; + const crypto = yield* Crypto.Crypto; + const launcher = yield* ServiceLauncherClient.ServiceLauncherClient; + + const commandGate = yield* makeCommandGate; + const httpListening = yield* Deferred.make(); + const reactorScope = yield* Scope.make("sequential"); + + yield* Effect.addFinalizer(() => Scope.close(reactorScope, Exit.void)); + + const startup = Effect.gen(function* () { + yield* Effect.logDebug("startup phase: starting keybindings runtime"); + yield* runStartupPhase( + "keybindings.start", + keybindings.start.pipe( + Effect.catch((error) => + Effect.logWarning("failed to start keybindings runtime", { + path: error.configPath, + detail: error.detail, + cause: error.cause, + }), + ), ), - Effect.forkScoped, - ), - ); + ); - yield* Effect.logDebug("startup phase: starting server settings runtime"); - yield* runStartupPhase( - "settings.start", - serverSettings.start.pipe( - Effect.catch((error) => - Effect.logWarning("failed to start server settings runtime", { - path: error.settingsPath, - operation: error.operation, - providerInstanceId: error.providerInstanceId, - environmentVariable: error.environmentVariable, - cause: error.cause, - }), + yield* Effect.logDebug("startup phase: starting server settings runtime"); + yield* runStartupPhase( + "settings.start", + serverSettings.start.pipe( + Effect.catch((error) => + Effect.logWarning("failed to start server settings runtime", { + path: error.settingsPath, + operation: error.operation, + providerInstanceId: error.providerInstanceId, + environmentVariable: error.environmentVariable, + cause: error.cause, + }), + ), ), - Effect.forkScoped, - ), - ); + ); - yield* Effect.logDebug("startup phase: starting orchestration reactors"); - yield* runStartupPhase( - "reactors.start", - Effect.gen(function* () { - yield* orchestrationReactor.start().pipe(Scope.provide(reactorScope)); - yield* providerSessionReaper.start().pipe(Scope.provide(reactorScope)); - }), - ); + yield* Effect.logDebug("startup phase: parking orchestration roots at activation"); + yield* runStartupPhase( + "reactors.start", + Effect.gen(function* () { + yield* orchestrationReactor.start().pipe(Scope.provide(reactorScope)); + yield* providerSessionReaper.start().pipe(Scope.provide(reactorScope)); + }), + ); - const welcomeBase = yield* resolveWelcomeBase; - const environment = yield* serverEnvironment.getDescriptor; - yield* Effect.logDebug("startup phase: preparing welcome payload"); - yield* Effect.logDebug("startup phase: publishing welcome event", { - environmentId: environment.environmentId, - cwd: welcomeBase.cwd, - projectName: welcomeBase.projectName, - }); - yield* runStartupPhase( - "welcome.publish", - lifecycleEvents.publish({ - version: 1, - type: "welcome", - payload: { - environment, - ...welcomeBase, - }, - }), - ); + const welcomeBase = yield* resolveWelcomeBase; + const environment = yield* serverEnvironment.getDescriptor; + yield* Effect.logDebug("startup phase: preparing welcome payload"); + + if (serverConfig.autoBootstrapProjectFromCwd) { + yield* forkParked( + runStartupPhase( + "welcome.autobootstrap", + Effect.gen(function* () { + const bootstrapTargets = yield* resolveAutoBootstrapWelcomeTargets.pipe( + Effect.provideService(Crypto.Crypto, crypto), + ); + if (!bootstrapTargets.bootstrapProjectId && !bootstrapTargets.bootstrapThreadId) { + return; + } + + yield* Effect.logDebug("startup phase: publishing bootstrapped welcome event", { + environmentId: environment.environmentId, + cwd: welcomeBase.cwd, + projectName: welcomeBase.projectName, + bootstrapProjectId: bootstrapTargets.bootstrapProjectId, + bootstrapThreadId: bootstrapTargets.bootstrapThreadId, + }); + yield* lifecycleEvents.publish({ + version: 1, + type: "welcome", + payload: { + environment, + ...welcomeBase, + ...bootstrapTargets, + }, + }); + }).pipe( + Effect.catch((cause) => + Effect.logWarning("startup auto-bootstrap welcome failed", { + cause, + }), + ), + ), + ), + ); + } - if (serverConfig.autoBootstrapProjectFromCwd) { - yield* Effect.forkScoped( - runStartupPhase( - "welcome.autobootstrap", - Effect.gen(function* () { - const bootstrapTargets = yield* resolveAutoBootstrapWelcomeTargets.pipe( - Effect.provideService(Crypto.Crypto, crypto), + yield* forkParked( + Effect.gen(function* () { + yield* Effect.logDebug("startup phase: recording startup heartbeat"); + yield* recordStartupHeartbeat.pipe( + Effect.annotateSpans({ "startup.phase": "heartbeat.record" }), + Effect.withSpan("server.startup.heartbeat.record"), + Effect.ignoreCause({ log: true }), + ); + if (serverConfig.startupPresentation === "headless") { + const accessInfo = yield* issueHeadlessServeAccessInfo(); + yield* runStartupPhase( + "headless.output", + Console.log(formatHeadlessServeOutput(accessInfo)), ); - if (!bootstrapTargets.bootstrapProjectId && !bootstrapTargets.bootstrapThreadId) { - return; + } else { + const startupBrowserTarget = yield* resolveStartupBrowserTarget; + if (serverConfig.mode !== "desktop") { + yield* Effect.logInfo( + "Authentication required. Open T3 Code using the pairing URL.", + ).pipe(Effect.annotateLogs({ pairingUrl: startupBrowserTarget })); } + yield* runStartupPhase("browser.open", maybeOpenBrowser(startupBrowserTarget)); + } + }), + ); - yield* Effect.logDebug("startup phase: publishing bootstrapped welcome event", { - environmentId: environment.environmentId, - cwd: welcomeBase.cwd, - projectName: welcomeBase.projectName, - bootstrapProjectId: bootstrapTargets.bootstrapProjectId, - bootstrapThreadId: bootstrapTargets.bootstrapThreadId, - }); - yield* lifecycleEvents.publish({ - version: 1, - type: "welcome", - payload: { - environment, - ...welcomeBase, - ...bootstrapTargets, - }, - }); - }).pipe( - Effect.catch((cause) => - Effect.logWarning("startup auto-bootstrap welcome failed", { - cause, - }), - ), - ), - ), + yield* Effect.logDebug("startup phase: waiting for http listener"); + yield* runStartupPhase("http.wait", Deferred.await(httpListening)); + yield* runStartupPhase( + "auxiliary-roots.parked", + options?.awaitAuxiliaryParked ?? Effect.void, ); - } - }).pipe( - Effect.annotateSpans({ - "server.mode": serverConfig.mode, - "server.port": serverConfig.port, - "server.host": serverConfig.host ?? "default", - }), - Effect.withSpan("server.startup", { kind: "server", root: true }), - ); - yield* Effect.forkScoped( - Effect.gen(function* () { - const startupExit = yield* Effect.exit(startup); - if (Exit.isFailure(startupExit)) { - const error = new ServerRuntimeStartupError({ - mode: serverConfig.mode, - host: serverConfig.host ?? null, - port: serverConfig.port, - cause: startupExit.cause, - }); - yield* Effect.logError("server runtime startup failed", { cause: startupExit.cause }); - yield* commandGate.failCommandReady(error); - return; - } + // This is the prepared boundary. Every dependency has been acquired and + // every runtime root has confirmed that it is parked before this request. + const updateOutcome = yield* launcher.prepareTrial; + yield* runStartupPhase( + "welcome.publish", + lifecycleEvents.publish({ + version: 1, + type: "welcome", + payload: { environment, ...welcomeBase }, + }), + ); + yield* options?.activate ?? Effect.void; yield* Effect.logDebug("Accepting commands"); yield* commandGate.signalCommandReady; - yield* Effect.logDebug("startup phase: waiting for http listener"); - yield* runStartupPhase("http.wait", Deferred.await(httpListening)); - yield* Effect.logDebug("startup phase: publishing ready event"); yield* runStartupPhase( "ready.publish", lifecycleEvents.publish({ @@ -440,39 +452,49 @@ export const make = Effect.gen(function* () { type: "ready", payload: { at: DateTime.formatIso(yield* DateTime.now), - environment: yield* serverEnvironment.getDescriptor, + environment, + ...(updateOutcome === undefined ? {} : { updateOutcome }), }, }), ); - - yield* Effect.logDebug("startup phase: recording startup heartbeat"); - yield* launchStartupHeartbeat; - if (serverConfig.startupPresentation === "headless") { - yield* Effect.logDebug("startup phase: headless access info"); - const accessInfo = yield* issueHeadlessServeAccessInfo(); - yield* runStartupPhase( - "headless.output", - Console.log(formatHeadlessServeOutput(accessInfo)), - ); - } else { - yield* Effect.logDebug("startup phase: browser open check"); - const startupBrowserTarget = yield* resolveStartupBrowserTarget; - if (serverConfig.mode !== "desktop") { - yield* Effect.logInfo( - "Authentication required. Open T3 Code using the pairing URL.", - ).pipe(Effect.annotateLogs({ pairingUrl: startupBrowserTarget })); - } - yield* runStartupPhase("browser.open", maybeOpenBrowser(startupBrowserTarget)); - } yield* Effect.logDebug("startup phase: complete"); - }), - ); + }).pipe( + Effect.annotateSpans({ + "server.mode": serverConfig.mode, + "server.port": serverConfig.port, + "server.host": serverConfig.host ?? "default", + }), + Effect.withSpan("server.startup", { kind: "server", root: true }), + ); - return { - awaitCommandReady: commandGate.awaitCommandReady, - markHttpListening: Deferred.succeed(httpListening, undefined), - enqueueCommand: commandGate.enqueueCommand, - } satisfies ServerRuntimeStartup["Service"]; -}); + yield* Effect.forkScoped( + Effect.exit(startup).pipe( + Effect.flatMap((startupExit) => { + if (Exit.isSuccess(startupExit)) return Effect.void; + const error = new ServerRuntimeStartupError({ + mode: serverConfig.mode, + host: serverConfig.host ?? null, + port: serverConfig.port, + cause: startupExit.cause, + }); + return Effect.logError("server runtime startup failed", { + cause: startupExit.cause, + }).pipe( + Effect.andThen(commandGate.failCommandReady(error)), + Effect.andThen(options?.abort?.(error) ?? Effect.void), + ); + }), + ), + ); + + return { + awaitCommandReady: commandGate.awaitCommandReady, + markHttpListening: Deferred.succeed(httpListening, undefined), + enqueueCommand: commandGate.enqueueCommand, + } satisfies ServerRuntimeStartup["Service"]; + }); + +export const layerWithOptions = (options?: StartupOptions) => + Layer.effect(ServerRuntimeStartup, make(options)); -export const layer = Layer.effect(ServerRuntimeStartup, make); +export const layer = layerWithOptions(); diff --git a/apps/server/src/service-launcher.ts b/apps/server/src/service-launcher.ts new file mode 100644 index 00000000000..10521245162 --- /dev/null +++ b/apps/server/src/service-launcher.ts @@ -0,0 +1 @@ +import "./serviceLauncher.ts"; diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts new file mode 100644 index 00000000000..21f3618d512 --- /dev/null +++ b/apps/server/src/serviceLauncher.test.ts @@ -0,0 +1,199 @@ +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { assert, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; + +import { Launcher, readServiceState, writeServiceState } from "./serviceLauncher.ts"; +import { + compareExactServiceVersions, + decodeServiceState, + isExactServiceVersion, +} from "./cloud/serviceProtocol.ts"; + +it("accepts only exact semantic versions", () => { + for (const version of ["0.0.0", "1.2.3", "1.2.3-alpha.1", "1.2.3-0", "1.2.3+001"]) { + assert.isTrue(isExactServiceVersion(version), version); + } + for (const version of ["latest", "01.2.3", "1.2.3-01", "1.2.3-alpha..1", "1.2.3+."]) { + assert.isFalse(isExactServiceVersion(version), version); + } +}); + +it("orders exact semantic versions without treating build metadata as precedence", () => { + assert.equal(compareExactServiceVersions("1.2.3", "1.2.3"), 0); + assert.equal(compareExactServiceVersions("1.2.4", "1.2.3"), 1); + assert.equal(compareExactServiceVersions("2.0.0-alpha.1", "2.0.0-alpha.2"), -1); + assert.equal(compareExactServiceVersions("2.0.0-alpha.2", "2.0.0-alpha.beta"), -1); + assert.equal(compareExactServiceVersions("2.0.0-alpha-beta", "2.0.0-alpha-alpha"), 1); + assert.equal(compareExactServiceVersions("2.0.0", "2.0.0-rc.1"), 1); + assert.equal(compareExactServiceVersions("2.0.0+one", "2.0.0+two"), 0); +}); + +it("rejects contradictory service state", () => { + assert.isUndefined( + decodeServiceState({ + protocol: 1, + activeVersion: "0.0.31", + update: { + id: "update-1", + fromVersion: "0.0.30", + targetVersion: "0.0.32", + status: "pending", + }, + }), + ); + + assert.isUndefined( + decodeServiceState({ + protocol: 1, + activeVersion: "1.0.0", + update: { + id: "update-2", + fromVersion: "1.0.0", + targetVersion: "0.9.0", + status: "pending", + }, + }), + ); +}); + +it.layer(NodeServices.layer)("service state persistence", (it) => { + it.effect("durably replaces and strictly reads one state document", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-test-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const state = { + protocol: 1, + activeVersion: "0.0.31", + } as const; + + yield* Effect.promise(() => writeServiceState(statePath, state)); + assert.deepEqual(yield* Effect.promise(() => readServiceState(statePath)), state); + }), + ); + + it.effect("serializes shutdown with launcher recovery", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-stop-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const versionDir = path.join(root, "runtime", "versions", "1.0.0"); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, "setInterval(() => {}, 1_000);\n"); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: 1, + activeVersion: "1.0.0", + }), + ); + + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + const running = launcher.run(); + yield* Effect.promise(() => launcher.stop("SIGTERM")); + yield* Effect.promise(() => running); + }), + ); + + it.effect("commits only after the trial reports prepared", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const childSource = ` +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +if (context.update?.status === "pending") { + process.send({ type: "prepared", updateId: context.update.id }); + process.on("message", (message) => { + if (message.type === "committed") process.exit(0); + }); +} else if (context.update === undefined) { + process.send({ type: "request-update", targetVersion: "1.1.0" }); + setInterval(() => {}, 1_000); +} else { + process.exit(0); +} +`; + for (const version of ["1.0.0", "1.1.0"]) { + const versionDir = path.join(root, "runtime", "versions", version); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, childSource); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), `${version}\n`); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: 1, + activeVersion: "1.0.0", + }), + ); + + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.1.0"); + assert.equal(state.update?.status, "committed"); + }), + ); + + it.effect("rolls back a trial that reports the wrong update ID", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-rollback-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const childSource = ` +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +if (context.update?.status === "pending") { + process.send({ type: "prepared", updateId: "wrong-update" }); +} else if (context.update === undefined) { + process.send({ type: "request-update", targetVersion: "1.1.0" }); + setInterval(() => {}, 1_000); +} else { + process.exit(0); +} +`; + for (const version of ["1.0.0", "1.1.0"]) { + const versionDir = path.join(root, "runtime", "versions", version); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, childSource); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), `${version}\n`); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: 1, + activeVersion: "1.0.0", + }), + ); + + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "rolled-back"); + assert.equal( + state.update?.status === "rolled-back" ? state.update.reason : undefined, + "invalid-prepared", + ); + }), + ); +}); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts new file mode 100644 index 00000000000..d7ca3357802 --- /dev/null +++ b/apps/server/src/serviceLauncher.ts @@ -0,0 +1,458 @@ +// @effect-diagnostics nodeBuiltinImport:off +// @effect-diagnostics globalDate:off +// @effect-diagnostics globalTimers:off +// This file is shipped as a standalone bundle and copied to a stable path by +// `t3 service update`. Keep runtime imports limited to Node built-ins. +import * as NodeChildProcess from "node:child_process"; +import * as NodeCrypto from "node:crypto"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; + +import type { + PendingServiceUpdate, + ServiceLauncherChildMessage, + ServiceLauncherContext, + ServiceLauncherParentMessage, + ServiceState, + ServiceUpdateRecord, +} from "./cloud/serviceProtocol.ts"; +import { + compareExactServiceVersions, + decodeServiceLauncherChildMessage, + isExactServiceVersion, + parseServiceState, + SERVICE_LAUNCHER_CONTEXT_ENV, + SERVICE_LAUNCHER_PROTOCOL, + SERVICE_STATE_FILE, +} from "./cloud/serviceProtocol.ts"; + +const HANDOFF_DELAY_MS = 2_000; +const PREPARED_TIMEOUT_MS = 120_000; +const TERMINATE_GRACE_MS = 5_000; + +type TerminalStatus = "committed" | "rolled-back" | "failed"; +type ChildRole = "active" | "trial"; + +interface ManagedChild { + readonly version: string; + role: ChildRole; + readonly process: NodeChildProcess.ChildProcess; +} + +const runtimePaths = (baseDir: string, version: string) => { + const versionDir = NodePath.join(baseDir, "runtime", "versions", version); + return { + versionDir, + entryPath: NodePath.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"), + sentinelPath: NodePath.join(versionDir, ".install-complete"), + }; +}; + +export async function readServiceState(filePath: string): Promise { + const contents = await NodeFSP.readFile(filePath, "utf8"); + const state = parseServiceState(contents); + if (state === undefined) throw new Error("Service state is invalid or unsupported."); + return state; +} + +/** Durable same-directory replacement used for every runtime state transition. */ +export async function writeServiceState(filePath: string, state: ServiceState): Promise { + const directory = NodePath.dirname(filePath); + await NodeFSP.mkdir(directory, { recursive: true, mode: 0o700 }); + const tempPath = NodePath.join( + directory, + `.${NodePath.basename(filePath)}.${process.pid}.${NodeCrypto.randomUUID()}`, + ); + let handle: NodeFSP.FileHandle | undefined; + try { + handle = await NodeFSP.open(tempPath, "wx", 0o600); + await handle.writeFile(`${JSON.stringify(state, null, 2)}\n`, "utf8"); + await handle.sync(); + await handle.close(); + handle = undefined; + await NodeFSP.rename(tempPath, filePath); + const directoryHandle = await NodeFSP.open(directory, "r"); + try { + await directoryHandle.sync(); + } finally { + await directoryHandle.close(); + } + } finally { + await handle?.close().catch(() => undefined); + await NodeFSP.rm(tempPath, { force: true }).catch(() => undefined); + } +} + +async function runtimeExists(baseDir: string, version: string): Promise { + const paths = runtimePaths(baseDir, version); + try { + const [entry, sentinel] = await Promise.all([ + NodeFSP.stat(paths.entryPath), + NodeFSP.readFile(paths.sentinelPath, "utf8"), + ]); + return entry.isFile() && sentinel.trim() === version; + } catch { + return false; + } +} + +function terminalUpdate(input: { + readonly pending: PendingServiceUpdate; + readonly status: S; + readonly reason?: string; +}): Exclude & { readonly status: S } { + return { + id: input.pending.id, + fromVersion: input.pending.fromVersion, + targetVersion: input.pending.targetVersion, + status: input.status, + ...(input.reason === undefined ? {} : { reason: input.reason }), + }; +} + +function sendMessage( + child: NodeChildProcess.ChildProcess, + message: ServiceLauncherParentMessage, +): Promise { + return new Promise((resolve, reject) => { + if (!child.connected || child.send === undefined) { + reject(new Error("service child IPC is disconnected.")); + return; + } + child.send(message, (error) => (error === null ? resolve() : reject(error))); + }); +} + +function waitForExit(child: NodeChildProcess.ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(); + return new Promise((resolve) => child.once("exit", () => resolve())); +} + +async function terminateChild( + child: NodeChildProcess.ChildProcess, + signal: NodeJS.Signals = "SIGTERM", +): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + child.kill(signal); + const force = setTimeout(() => child.kill("SIGKILL"), TERMINATE_GRACE_MS); + try { + await waitForExit(child); + } finally { + clearTimeout(force); + } +} + +export class Launcher { + readonly #baseDir: string; + readonly #statePath: string; + #state: ServiceState; + #child: ManagedChild | null = null; + #timer: NodeJS.Timeout | undefined; + #transitions: Promise = Promise.resolve(); + #stopping = false; + #done = false; + readonly #completion = Promise.withResolvers(); + + constructor(baseDir: string, state: ServiceState) { + this.#baseDir = baseDir; + this.#statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); + this.#state = state; + } + + async run(): Promise { + const onSigterm = () => void this.stop("SIGTERM"); + const onSigint = () => void this.stop("SIGINT"); + process.once("SIGTERM", onSigterm); + process.once("SIGINT", onSigint); + try { + this.#enqueue(() => this.#recover()); + await this.#completion.promise; + } finally { + process.off("SIGTERM", onSigterm); + process.off("SIGINT", onSigint); + } + } + + #enqueue(transition: () => Promise): void { + this.#transitions = this.#transitions + .then(transition, transition) + .catch((cause: unknown) => + this.#fatal(cause instanceof Error ? cause : new Error(String(cause))), + ); + } + + async #fatal(error: Error): Promise { + if (this.#done) return; + this.#done = true; + this.#stopping = true; + this.#clearTimer(); + const child = this.#child?.process; + this.#child = null; + if (child !== undefined) await terminateChild(child); + this.#completion.reject(error); + } + + async stop(signal: NodeJS.Signals): Promise { + if (this.#stopping) { + await this.#completion.promise.catch(() => undefined); + return; + } + this.#stopping = true; + this.#clearTimer(); + this.#enqueue(async () => { + const child = this.#child?.process; + this.#child = null; + if (child !== undefined) await terminateChild(child, signal); + this.#done = true; + this.#completion.resolve(); + }); + await this.#completion.promise.catch(() => undefined); + } + + #clearTimer(): void { + clearTimeout(this.#timer); + this.#timer = undefined; + } + + async #recover(): Promise { + const update = this.#state.update; + if (update?.status !== "pending") { + await this.#startChild(this.#state.activeVersion, "active", update); + return; + } + if (!(await runtimeExists(this.#baseDir, update.targetVersion))) { + await this.#returnToPrevious(update, "failed", "target-runtime-missing"); + return; + } + await this.#startTrial(update); + } + + async #startTrial(pending: PendingServiceUpdate): Promise { + try { + await this.#startChild(pending.targetVersion, "trial", pending); + } catch { + await this.#returnToPrevious(pending, "failed", "candidate-start-failed"); + } + } + + async #startChild(version: string, role: ChildRole, update?: ServiceUpdateRecord): Promise { + if (this.#stopping) return; + if (!(await runtimeExists(this.#baseDir, version))) { + throw new Error(`Selected t3@${version} runtime is missing or incomplete.`); + } + if (this.#stopping) return; + const paths = runtimePaths(this.#baseDir, version); + const context: ServiceLauncherContext = { + protocol: SERVICE_LAUNCHER_PROTOCOL, + childVersion: version, + ...(update === undefined ? {} : { update }), + }; + const child = NodeChildProcess.spawn(process.execPath, [paths.entryPath, "serve"], { + env: { ...process.env, [SERVICE_LAUNCHER_CONTEXT_ENV]: JSON.stringify(context) }, + stdio: ["inherit", "inherit", "inherit", "ipc"], + }); + await new Promise((resolve, reject) => { + const onError = (error: Error) => reject(error); + child.once("error", onError); + child.once("spawn", () => { + child.removeListener("error", onError); + child.on("error", (error) => this.#enqueue(() => Promise.reject(error))); + resolve(); + }); + }); + if (this.#stopping) { + await terminateChild(child); + return; + } + + const managed: ManagedChild = { + version, + role, + process: child, + }; + this.#child = managed; + child.on("message", (value) => { + const message = decodeServiceLauncherChildMessage(value); + if (message !== undefined) this.#enqueue(() => this.#handleMessage(managed, message)); + }); + child.once("exit", (code, signal) => + this.#enqueue(() => this.#handleExit(managed, code, signal)), + ); + + if (role === "trial") { + this.#timer = setTimeout( + () => this.#enqueue(() => this.#handlePreparedTimeout(managed)), + PREPARED_TIMEOUT_MS, + ); + } + } + + async #handleMessage(child: ManagedChild, message: ServiceLauncherChildMessage): Promise { + if (this.#child !== child || this.#stopping) return; + if (message.type === "request-update") { + await this.#handleUpdateRequest(child, message); + return; + } + await this.#handlePrepared(child, message.updateId); + } + + async #handleUpdateRequest( + child: ManagedChild, + message: Extract, + ): Promise { + const reject = (reason: string) => + sendMessage(child.process, { type: "update-rejected", reason }); + if (child.role !== "active") { + await reject("Only the active server can request an update."); + return; + } + if (child.version !== this.#state.activeVersion) { + await reject("The requesting server is not the selected active version."); + return; + } + if (this.#state.update?.status === "pending") { + await reject("Another server update is already pending."); + return; + } + if (!isExactServiceVersion(message.targetVersion)) { + await reject("The requested target is not an exact version."); + return; + } + if (compareExactServiceVersions(message.targetVersion, child.version) <= 0) { + await reject("Remote updates must select a newer server version."); + return; + } + if (!(await runtimeExists(this.#baseDir, message.targetVersion))) { + await reject("The requested target runtime is missing or incomplete."); + return; + } + + const pending: PendingServiceUpdate = { + id: NodeCrypto.randomUUID(), + fromVersion: child.version, + targetVersion: message.targetVersion, + status: "pending", + }; + const next: ServiceState = { ...this.#state, update: pending }; + await writeServiceState(this.#statePath, next); + this.#state = next; + await sendMessage(child.process, { type: "update-accepted", updateId: pending.id }); + this.#timer = setTimeout(() => this.#enqueue(() => this.#beginTrial(child)), HANDOFF_DELAY_MS); + } + + async #beginTrial(child: ManagedChild): Promise { + const pending = this.#state.update; + if (this.#child !== child || child.role !== "active" || pending?.status !== "pending") { + return; + } + this.#timer = undefined; + this.#child = null; + await terminateChild(child.process); + await this.#startTrial(pending); + } + + async #handlePrepared(child: ManagedChild, updateId: string): Promise { + const pending = this.#state.update; + if ( + child.role !== "trial" || + pending?.status !== "pending" || + pending.id !== updateId || + pending.targetVersion !== child.version + ) { + if (child.role === "trial" && pending?.status === "pending") { + await this.#returnToPrevious(pending, "rolled-back", "invalid-prepared", child); + return; + } + throw new Error("Trial child reported prepared for an unexpected update."); + } + this.#clearTimer(); + const committed = terminalUpdate({ pending, status: "committed" }); + const next: ServiceState = { + ...this.#state, + activeVersion: pending.targetVersion, + update: committed, + }; + await writeServiceState(this.#statePath, next); + this.#state = next; + child.role = "active"; + await sendMessage(child.process, { type: "committed", updateId: committed.id }); + } + + async #handlePreparedTimeout(child: ManagedChild): Promise { + const pending = this.#state.update; + if (this.#child !== child || child.role !== "trial" || pending?.status !== "pending") { + return; + } + this.#timer = undefined; + await this.#returnToPrevious(pending, "rolled-back", "prepared-timeout", child); + } + + async #handleExit( + child: ManagedChild, + code: number | null, + signal: NodeJS.Signals | null, + ): Promise { + if (this.#child !== child || this.#stopping) return; + this.#child = null; + if (child.role === "trial") { + this.#clearTimer(); + const pending = this.#state.update; + if (pending?.status !== "pending") { + throw new Error("Trial child exited without matching pending state."); + } + await this.#returnToPrevious( + pending, + "rolled-back", + `candidate-exited:${String(code ?? signal ?? "unknown")}`, + ); + return; + } + + this.#clearTimer(); + const pending = this.#state.update; + if (pending?.status === "pending") { + await this.#startTrial(pending); + return; + } + throw new Error(`Active child exited unexpectedly (${String(code ?? signal ?? "unknown")}).`); + } + + async #returnToPrevious( + pending: PendingServiceUpdate, + status: "rolled-back" | "failed", + reason: string, + child?: ManagedChild, + ): Promise { + const outcome = terminalUpdate({ pending, status, reason }); + const next: ServiceState = { + ...this.#state, + activeVersion: pending.fromVersion, + update: outcome, + }; + await writeServiceState(this.#statePath, next); + this.#state = next; + if (child !== undefined) { + this.#child = null; + await terminateChild(child.process); + } + await this.#startChild(next.activeVersion, "active", outcome); + } +} + +async function main(): Promise { + const baseDir = process.env.T3CODE_HOME?.trim(); + if (baseDir === undefined || baseDir === "") { + throw new Error("T3CODE_HOME is required by the T3 Code service launcher."); + } + const statePath = NodePath.join(baseDir, "runtime", SERVICE_STATE_FILE); + const state = await readServiceState(statePath); + await new Launcher(baseDir, state).run(); +} + +if (import.meta.main) { + main().catch((cause: unknown) => { + const error = cause instanceof Error ? cause : new Error(String(cause)); + process.stderr.write(`[service-launcher] ${error.message}\n`); + process.exitCode = 1; + }); +} diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index 1e017737148..f0a0034c3ee 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -2,139 +2,87 @@ > For maintainers. Using T3 Code? See [docs/user](../user/). -T3 Code can update a connected server to the exact version of the client that detected version -drift. This path exists primarily for remote environments, where the user may not have a terminal -open on the server machine. - -The feature has three boundaries: - -- the server advertises whether and how it can be replaced; -- the client chooses the matching user action; -- the server installs and verifies the replacement before handing off the process. - -## Detection and Presentation - -`ExecutionEnvironmentDescriptor` includes the server version and an optional -`capabilities.serverSelfUpdate` value. Progress-capable servers also advertise -`capabilities.serverSelfUpdateProgress`. The client compares the server version with `APP_VERSION` -after loading server config. - -The optional capability is intentionally backward compatible. An older server does not know about -the field, so a missing value means the client must offer a manual relaunch instead of sending an -unknown RPC. - -The shared `ServerUpdateAction` is rendered in both user-facing version-drift surfaces: - -- the conversation banner in `ChatView`; -- primary and saved environment rows in **Settings** → **Connections**. - -Both surfaces target the client's exact version. When the reconnected server reports that version, -the mismatch and action disappear. - -The operation state lives in `packages/client-runtime`, keyed by environment. Both web surfaces read -the same `downloading`, `installing`, or `resuming` state, so route changes do not own or cancel the -operation. - -## Capability Selection - -The server resolves its capability once at startup and publishes it in the environment descriptor. - -| Advertised value | Process shape | Client behavior | -| ----------------- | --------------------------------------------------------------------------------------------- | --------------------------------------------------------------------- | -| `boot-service` | Linux server running under the T3-managed systemd user service | Call the update RPC; the service unit is replaced and restarted. | -| `respawn` | Published npm CLI running in the foreground on macOS or Linux | Call the update RPC; the process hands off to a detached replacement. | -| `desktop-managed` | Backend supervised by the desktop app | Tell the user to update the desktop app on the server machine. | -| absent | Older server, development checkout, Windows foreground process, or an unrecognized supervisor | Offer the exact manual relaunch command. | - -Desktop ownership takes precedence over process-shape detection. A desktop-managed backend must -never spawn a second CLI server beside the app-owned process. Likewise, a process launched by an -unrecognized systemd unit does not claim the foreground respawn path because its supervisor could -bring the old version back. - -## Update Flow - -```mermaid -flowchart TD - A[Client detects different versions] --> B{Advertised update path} - B -->|desktop-managed| C[Update desktop app on server machine] - B -->|missing| D[Copy exact manual relaunch command] - B -->|boot-service or respawn| E{Progress capability} - E -->|present| F[server.updateServerWithProgress] - E -->|missing| G[server.updateServer fallback] - F --> H[Install exact t3 version in pinned runtime] - G --> H - H --> I[Run version preflight] - I -->|bad code or version| J[Remove candidate runtime and keep current server] - I -->|cannot run preflight| J2[Keep candidate and current server] - I -->|passes| K{Handoff method} - K -->|boot-service| L[Rewrite and restart T3 systemd unit] - K -->|respawn| M[Start delayed replacement and exit current process] - L --> N[Reconnect with fresh backoff] - M --> N - N --> O[Replacement publishes ready at target version] -``` +Remote server updates use one stable systemd launcher. Foreground CLI processes do not self-update, +and a running server never edits its systemd unit or durable service state. + +## Ownership + +The service files under `/runtime` are: + +- `service-launcher.mjs`, the stable process selected by systemd; +- `service-state.json`, the launcher's durable selection state; +- `versions/`, immutable exact-version npm installs. + +The launcher is the only runtime writer of `service-state.json`. `t3 service install` and +`t3 service update` may replace the launcher and state while the unit is stopped. Server children +only communicate with the launcher over their inherited IPC channel. -Both update RPCs require the environment's `orchestration:operate` authorization scope. Their -payload accepts only an exact npm version, including an exact prerelease version; dist-tags such as -`latest` and `nightly` are rejected. The unary `server.updateServer` method remains available so a -new client can still repair skew with an older server. +The state contains one active version and, at most, one update record: -The update service permits one update at a time. It installs `t3@` under -`/runtime/versions/` and writes an install-complete sentinel only after npm exits -successfully. Boot-service setup and self-update share the same process-wide installation lock, so -they cannot mutate a pinned runtime concurrently. +- `pending A → B` selects B as a retryable trial; +- `committed A → B` selects B for ordinary restarts; +- `rolled-back A → B` or `failed A → B` selects A; +- invalid state fails closed so systemd cannot guess at a runtime. -Before any restart, the current Node executable runs the replacement with `--version`. A failed -install, failed preflight, or wrong reported version leaves the current server running. +Every write uses same-directory replacement plus file and directory fsync. -Candidate cleanup is narrower than "any failed preflight". The candidate runtime is removed only when -the preflight process actually completes and reports a bad exit code or the wrong version: that is -the case where a completed npm install produced an unusable tree, so retrying the same version must -perform a clean install rather than reuse it. If the preflight cannot run at all, for example a spawn -error or the `PREFLIGHT_TIMEOUT` elapsing, the update fails before reaching cleanup and the candidate -directory is left in place. +## Remote Update -## Host Service Lifecycle +1. The active server installs `t3@` into a unique staging directory. +2. The target runs `__service-preflight` against the active database in read-only mode. +3. The staging directory is renamed to its immutable version path only after preflight succeeds. +4. The active child sends `request-update`. The launcher validates the child and target, writes + pending state, generates the update ID, then replies `update-accepted`. +5. After a short response-flush grace period, the launcher stops the active child. +6. The launcher starts the target as a trial and gives it the pending update over IPC. +7. The trial acquires dependencies, binds HTTP, starts every long-running root fiber, and verifies + that each root is parked at the activation gate. It does not run migrations. +8. The trial sends `prepared`. The launcher durably commits B before replying `committed`. +9. The child opens the existing activation gate, accepts commands, and publishes lifecycle ready + with the terminal update outcome. -The systemd user service is a host lifecycle concern, not a T3 Connect resource. The standalone -`t3 service install`, `uninstall`, `update`, and `status` commands own it. Install and update both -reconcile the unit through `BootService`; running `npx t3@latest service update` therefore pins and -activates the latest CLI release without requiring a connected client. +Post-commit startup does not call service `start`, `initialize`, `connect`, `load`, or `acquire` +operations. It only opens prepared gates and publishes prepared lifecycle state. -The `t3 connect` onboarding flow may offer service installation, but it calls the same reconciliation -operation as `t3 service install`. Connect logout only disables cloud access and clears its -authorization; it does not uninstall the host service. +The launcher serializes child exits, IPC messages, and timers. A trial must report prepared within +120 seconds. If the trial exits or times out before prepared, the launcher records rollback and +starts A. After commit, B is active and normal systemd restart policy applies. -## Process Handoff +## Migration Boundary + +Remote update preflight requires exact equality between the database's applied `(id, name)` rows +and the target release's migration manifest. A target with a missing, additional, renamed, or +unknown migration is blocked. Remote updates never migrate or downgrade a database. + +This deliberately means any release containing a migration requires a local service update: + +```sh +npx t3@ service update +``` -For `boot-service`, the server atomically rewrites the T3-managed user unit to point at the verified -runtime and reloads systemd. It acknowledges the handoff, then restarts the unit after the same -short grace period used by foreground respawn. A rejected deferred restart restores the previous -unit and is logged by the still-running process. +The local command stops the unit, selects the new launcher and exact runtime, then restarts the +service. Its normal server startup may run migrations. -For `respawn`, the server starts a detached, delayed replacement that replays the original CLI -arguments. It then acknowledges the request and schedules the current process to exit. The delays -give the acknowledgement time to cross direct or relayed connections before the socket closes. +## Client Correlation -Progress-capable servers emit `downloading` before installing the pinned runtime and `installing` -before preflight and handoff. A terminal stream event acknowledges that restart is scheduled. The -client then enters `resuming`, waits for the replacement lifecycle stream to publish `ready` with -the target version, and only then completes the operation. It watches for the intentional -disconnect's first backoff state and requests one fresh retry, which clears historical backoff debt -without adding a separate reconnect loop. +The update acknowledgement includes the launcher-generated update ID. After reconnecting, clients +wait for a lifecycle ready event carrying that same ID. `committed` completes the operation only +when the ready server is the target version. `rolled-back` and `failed` end it immediately with the +recorded reason. Older servers without an ID retain version-only reconnect behavior. -## Release Invariant +## Capability and Compatibility -The exact client version must exist as the `t3` npm package before a client carrying that version is -published. The release workflow therefore makes the GitHub release depend on CLI publication, and -the hosted web deployment depends on that release. See [Release Checklist](../operations/release.md#server-self-update-release-invariant). +The existing additive RPC and lifecycle schemas remain compatible with older clients. New servers +advertise remote self-update only when they have valid launcher context and a live IPC channel. +Desktop-managed servers direct the user to update the desktop app. Other process shapes provide a +manual command; the old detached foreground respawn path no longer exists. ## Source Map -- Capability contract: `packages/contracts/src/environment.ts` -- Update RPC contract: `packages/contracts/src/server.ts` and `packages/contracts/src/rpc.ts` -- Capability detection and handoff: `apps/server/src/cloud/selfUpdate.ts` -- Host service commands: `apps/server/src/cli/service.ts` -- Pinned runtime installation: `apps/server/src/cloud/pinnedRuntime.ts` -- Client version comparison: `apps/web/src/versionSkew.ts` -- Shared update action: `apps/web/src/components/ServerUpdateAction.tsx` +- Launcher and state machine: `apps/server/src/serviceLauncher.ts` +- IPC and durable state types: `apps/server/src/cloud/serviceProtocol.ts` +- Child IPC adapter: `apps/server/src/cloud/serviceLauncherClient.ts` +- Staging and preflight: `apps/server/src/cloud/pinnedRuntime.ts` and `servicePreflight.ts` +- Service installation: `apps/server/src/cloud/bootService.ts` +- Activation boundary: `apps/server/src/serverRuntimeStartup.ts` and `serverActivation.ts` +- Client outcome correlation: `packages/client-runtime/src/state/server.ts` diff --git a/docs/operations/release.md b/docs/operations/release.md index 9b33e94cc60..89c4006bafd 100644 --- a/docs/operations/release.md +++ b/docs/operations/release.md @@ -187,8 +187,10 @@ the **Update server** action targeting a package version that does not exist yet For a release smoke test, confirm `npm view t3@ version` returns the expected version, then connect the new client to a server on the previous version and verify that the update action -reconnects to the matching server. Test one automatic path and the manual or desktop-managed -guidance when those environments are available. +reconnects to the matching server. Use releases with identical migration manifests for the +automatic path. When the manifest changed, verify that the remote action stops before restart and +shows the exact local `npx t3@ service update` command. Also test the manual or +desktop-managed guidance when those environments are available. ## Desktop auto-update notes diff --git a/docs/user/background-service.md b/docs/user/background-service.md index 8f1f68dc065..0ab5b2013c7 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -31,6 +31,10 @@ npx t3@latest service uninstall Updating restarts T3 Code briefly. Let active agent work and terminal commands finish first. +The systemd unit runs a small stable launcher. Exact T3 Code versions are installed separately, so +a failed remote candidate can return to the previous version without rewriting the unit. Releases +that change the database must be installed with the local `service update` command above. + ## Using It with T3 Connect T3 Connect may offer to install the service during setup so the host stays reachable after you log diff --git a/docs/user/updating.md b/docs/user/updating.md index 8e1fac81854..a0cc0e5d1e0 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -24,13 +24,17 @@ The update does not remove saved threads, settings, or project files. | Action | What to do | | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Update server** | Select the button and leave T3 Code open. It prepares the matching version, restarts the server, and reconnects automatically. This can take several minutes. | +| **Update server** | Available for the T3 Code Linux background service. Select the button and leave T3 Code open while it prepares, tests, restarts, and reconnects. | | **Update the desktop app** | Open the T3 Code desktop app on the machine that runs the server and install the app update there. Reopen it if needed. | | **Copy update command** | Copy the command, open a terminal on the server machine, stop the current T3 Code server, and relaunch it with the copied command and any startup options you normally use. | The available action depends on how that server was started. T3 Code does not update connected servers silently in the background. +If the requested version includes a database update, remote installation stops before restart and +asks you to run the exact `npx t3@ service update` command on the server machine. This is +an intentional rollback-safety boundary. + After selecting **Update server**, the warning becomes a three-step progress rail: **Download**, **Install**, and **Resume**. The same progress appears in the conversation and in Connections, so navigating between them does not lose the update. A failed step remains visible @@ -56,8 +60,8 @@ commands. ## After the Update Keep the web or desktop app open while the server restarts. The update completes only after the -replacement server reports the requested version and is ready to accept commands. The warning and -progress rail then disappear. +service launcher reports that exact update committed and the replacement server is ready to accept +commands. A rollback is reported immediately instead of waiting for a generic reconnect timeout. If a step fails: diff --git a/packages/client-runtime/src/state/server.test.ts b/packages/client-runtime/src/state/server.test.ts index 12925a99867..d764b729fc3 100644 --- a/packages/client-runtime/src/state/server.test.ts +++ b/packages/client-runtime/src/state/server.test.ts @@ -29,11 +29,13 @@ import { applyServerConfigProjection, makeEnvironmentServerConfigState, isLegacyUpdateHandoffLoss, + matchesServerUpdateReadyEvent, projectServerWelcome, resolveServerConfigValue, resolveServerUpdateProgressResult, serverUpdateStateForProgressEvent, serverUpdateStateForServerVersion, + validateServerUpdateReadyEvent, } from "./server.ts"; const CONFIG = { @@ -173,6 +175,40 @@ describe("server state projection", () => { expect(serverUpdateStateForServerVersion(failed, "0.0.31")).toEqual({ status: "idle" }); }); + it.effect("correlates launcher outcomes and fails immediately after rollback", () => + Effect.gen(function* () { + const result = { + targetVersion: "0.0.31", + method: "boot-service" as const, + updateId: "update-1", + }; + const ready = (status: "committed" | "rolled-back") => + ({ + version: 1 as const, + sequence: 1, + type: "ready" as const, + payload: { + at: "2026-08-01T00:00:00.000Z", + environment: { serverVersion: status === "committed" ? "0.0.31" : "0.0.30" }, + updateOutcome: { + id: "update-1", + fromVersion: "0.0.30", + targetVersion: "0.0.31", + status, + ...(status === "rolled-back" ? { reason: "prepared-timeout" } : {}), + }, + }, + }) as Parameters[1]; + + expect(matchesServerUpdateReadyEvent(result, ready("committed"))).toBe(true); + yield* validateServerUpdateReadyEvent(result, ready("committed")); + const rollback = yield* Effect.flip( + validateServerUpdateReadyEvent(result, ready("rolled-back")), + ); + expect(rollback.message).toBe("prepared-timeout"); + }), + ); + it("applies every config category to the projected snapshot", () => { const snapshot = applyServerConfigProjection(Option.none(), { version: 1, diff --git a/packages/client-runtime/src/state/server.ts b/packages/client-runtime/src/state/server.ts index edd1893f739..12639c4ed7b 100644 --- a/packages/client-runtime/src/state/server.ts +++ b/packages/client-runtime/src/state/server.ts @@ -3,6 +3,7 @@ import { type ServerConfig, type ServerConfigStreamEvent, type ServerLifecycleWelcomePayload, + type ServerLifecycleStreamReadyEvent, type ServerSelfUpdateProgressEvent, type ServerSelfUpdateResult, WS_METHODS, @@ -97,6 +98,57 @@ export class ServerUpdateProgressIncompleteError extends Schema.TaggedErrorClass } } +export class ServerUpdateTerminalError extends Schema.TaggedErrorClass()( + "ServerUpdateTerminalError", + { + targetVersion: Schema.String, + status: Schema.Literals(["committed", "rolled-back", "failed"]), + reason: Schema.optional(Schema.String), + }, +) { + override get message(): string { + return this.reason ?? `The t3@${this.targetVersion} update ${this.status}.`; + } +} + +// Covers the 120-second trial deadline and a final restart of the previous +// version when the trial rolls back. +const SERVER_UPDATE_RESUME_TIMEOUT = Duration.minutes(4); + +export function matchesServerUpdateReadyEvent( + result: ServerSelfUpdateResult, + event: ServerLifecycleStreamReadyEvent, +): boolean { + return result.updateId === undefined + ? event.payload.environment.serverVersion === result.targetVersion + : event.payload.updateOutcome?.id === result.updateId; +} + +export function validateServerUpdateReadyEvent( + result: ServerSelfUpdateResult, + event: ServerLifecycleStreamReadyEvent, +): Effect.Effect { + if (result.updateId === undefined) return Effect.void; + const outcome = event.payload.updateOutcome; + if ( + outcome?.id === result.updateId && + outcome.status === "committed" && + outcome.targetVersion === result.targetVersion && + event.payload.environment.serverVersion === result.targetVersion + ) { + return Effect.void; + } + return Effect.fail( + new ServerUpdateTerminalError({ + targetVersion: result.targetVersion, + status: outcome?.status ?? "failed", + reason: + outcome?.reason ?? + "The service launcher resumed without committing the requested server version.", + }), + ); +} + export function serverUpdateStateForProgressEvent( fromVersion: string, targetVersion: string, @@ -554,11 +606,11 @@ export function createServerEnvironmentAtoms( .followStream(target.environmentId, subscribe(WS_METHODS.subscribeServerLifecycle, {})) .pipe( Stream.filter( - (event) => - event.type === "ready" && event.payload.environment.serverVersion === targetVersion, + (event): event is ServerLifecycleStreamReadyEvent => + event.type === "ready" && matchesServerUpdateReadyEvent(result, event), ), Stream.runHead, - Effect.timeoutOption(Duration.seconds(120)), + Effect.timeoutOption(SERVER_UPDATE_RESUME_TIMEOUT), Effect.map(Option.flatten), ); if (Option.isNone(resumed)) { @@ -567,6 +619,7 @@ export function createServerEnvironmentAtoms( targetVersion, }); } + yield* validateServerUpdateReadyEvent(result, resumed.value); atomRegistry.set(stateAtom, IDLE_SERVER_UPDATE_STATE); return result; diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 5d4238994fb..b49860c6388 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -20,9 +20,9 @@ export const ExecutionEnvironmentPlatform = Schema.Struct({ }); export type ExecutionEnvironmentPlatform = typeof ExecutionEnvironmentPlatform.Type; -/** How a server can replace itself with another version when asked over RPC: - "boot-service" rewrites the systemd user unit and restarts it; "respawn" - installs the target version and respawns the foreground process. */ +/** How a server can replace itself with another version when asked over RPC. + New servers only advertise the stable launcher-backed "boot-service" path; + "respawn" remains decodable for compatibility with older servers. */ export const ServerSelfUpdateMethod = Schema.Literals(["boot-service", "respawn"]); export type ServerSelfUpdateMethod = typeof ServerSelfUpdateMethod.Type; diff --git a/packages/contracts/src/server.ts b/packages/contracts/src/server.ts index 334116794f8..bb139c9782d 100644 --- a/packages/contracts/src/server.ts +++ b/packages/contracts/src/server.ts @@ -521,9 +521,21 @@ export const ServerConfigStreamEvent = Schema.Union([ ]); export type ServerConfigStreamEvent = typeof ServerConfigStreamEvent.Type; +/** Terminal selection recorded by the service launcher for one update. */ +export const ServerSelfUpdateOutcome = Schema.Struct({ + id: TrimmedNonEmptyString, + fromVersion: TrimmedNonEmptyString, + targetVersion: TrimmedNonEmptyString, + status: Schema.Literals(["committed", "rolled-back", "failed"]), + reason: Schema.optionalKey(TrimmedNonEmptyString), +}); +export type ServerSelfUpdateOutcome = typeof ServerSelfUpdateOutcome.Type; + export const ServerLifecycleReadyPayload = Schema.Struct({ at: IsoDateTime, environment: ExecutionEnvironmentDescriptor, + /** Present when this process resumed a launcher-managed update. */ + updateOutcome: Schema.optionalKey(ServerSelfUpdateOutcome), }); export type ServerLifecycleReadyPayload = typeof ServerLifecycleReadyPayload.Type; @@ -594,6 +606,8 @@ export type ServerSelfUpdateInput = typeof ServerSelfUpdateInput.Type; export const ServerSelfUpdateResult = Schema.Struct({ targetVersion: TrimmedNonEmptyString, method: ServerSelfUpdateMethod, + /** Launcher-generated correlation ID. Absent when talking to older servers. */ + updateId: Schema.optionalKey(TrimmedNonEmptyString), }); export type ServerSelfUpdateResult = typeof ServerSelfUpdateResult.Type;