diff --git a/apps/server/src/cli/connect.test.ts b/apps/server/src/cli/connect.test.ts index cc52a50fe36..1e0c88c24e8 100644 --- a/apps/server/src/cli/connect.test.ts +++ b/apps/server/src/cli/connect.test.ts @@ -10,6 +10,7 @@ import * as Option from "effect/Option"; import * as References from "effect/References"; import * as Terminal from "effect/Terminal"; +import * as BootService from "../cloud/bootService.ts"; import { acquireRelayClientForLink, formatHeadlessAuthorizationPrompt, @@ -63,6 +64,15 @@ it.effect("treats cancelling optional background setup as a successful skip", () }), ); +it.effect("keeps a successful connection when a remote service update is pending", () => + Effect.gen(function* () { + const result = yield* recoverServiceOnboardingOffer( + Effect.fail(new BootService.BootServiceUpdatePendingError()), + ); + assert.isFalse(result); + }), +); + it.effect("does not install the relay client when the user declines the managed download", () => Effect.gen(function* () { let installCalls = 0; diff --git a/apps/server/src/cli/service.ts b/apps/server/src/cli/service.ts index bd846eeee34..d55b270f183 100644 --- a/apps/server/src/cli/service.ts +++ b/apps/server/src/cli/service.ts @@ -185,6 +185,8 @@ export const recoverServiceOnboardingOffer = ( Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), BootServiceInstallError: (error) => Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), + BootServiceUpdatePendingError: (error) => + Console.warn(`Background setup did not finish: ${error.message}`).pipe(Effect.as(false)), }), ); diff --git a/apps/server/src/cloud/bootService.test.ts b/apps/server/src/cloud/bootService.test.ts index 9af69eb1792..a9ae3f49f70 100644 --- a/apps/server/src/cloud/bootService.test.ts +++ b/apps/server/src/cloud/bootService.test.ts @@ -15,7 +15,11 @@ import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawne import * as ProcessRunner from "../processRunner.ts"; import * as BootService from "./bootService.ts"; import { pinnedRuntimePaths } from "./pinnedRuntime.ts"; -import { parseServiceState } from "./serviceProtocol.ts"; +import { + parseServiceState, + SERVICE_LAUNCHER_PROTOCOL, + serviceStateHasPendingUpdate, +} from "./serviceProtocol.ts"; it("keeps systemd pinned to the stable launcher rather than a versioned server", () => { const unit = BootService.renderBootServiceUnit({ @@ -97,15 +101,24 @@ it.layer(NodeServices.layer)("boot service install", (it) => { const plan = yield* service.install; expect(parseServiceState(yield* fs.readFileString(statePath))).toEqual({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.2.3", }); expect(yield* fs.readFileString(plan.launcherPath)).toBe("export {};\n"); expect((yield* service.status).current).toBe(true); - yield* fs.writeFileString( - statePath, - '{"protocol":1,"activeVersion":"1.2.3","update":{"id":"u","fromVersion":"1.2.3","targetVersion":"1.2.4","status":"pending"}}', - ); + // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned test document. + const pendingState = JSON.stringify({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.2.3", + update: { + id: "u", + fromVersion: "1.2.3", + targetVersion: "1.2.4", + dbPath: "/tmp/state.sqlite", + status: "pending", + }, + }); + yield* fs.writeFileString(statePath, pendingState); expect((yield* service.status).current).toBe(false); expect(yield* service.uninstall).toBe(true); expect((yield* service.status).installed).toBe(false); @@ -141,6 +154,33 @@ it.layer(NodeServices.layer)("boot service install", (it) => { }), ); + it.effect("restarts without overwriting a pending remote update", () => + Effect.gen(function* () { + const { service, fs, statePath, commands } = yield* makeHarness(); + yield* service.install; + // @effect-diagnostics-next-line preferSchemaOverJson:off - fixed launcher-owned test document. + const pendingState = JSON.stringify({ + protocol: SERVICE_LAUNCHER_PROTOCOL - 1, + activeVersion: "1.2.3", + update: { + id: "remote-update", + fromVersion: "1.2.3", + targetVersion: "1.2.4", + status: "pending", + }, + }); + yield* fs.writeFileString(statePath, pendingState); + commands.length = 0; + + expect((yield* service.install.pipe(Effect.flip))._tag).toBe("BootServiceUpdatePendingError"); + expect(serviceStateHasPendingUpdate(yield* fs.readFileString(statePath))).toBe(true); + expect(commands.filter((command) => command.startsWith("systemctl "))).toEqual([ + "systemctl --user stop t3code.service", + "systemctl --user restart t3code.service", + ]); + }), + ); + it.effect("fails closed off Linux", () => Effect.gen(function* () { const { service } = yield* makeHarness("darwin"); diff --git a/apps/server/src/cloud/bootService.ts b/apps/server/src/cloud/bootService.ts index 9a8481b11b5..ec110cb8b6a 100644 --- a/apps/server/src/cloud/bootService.ts +++ b/apps/server/src/cloud/bootService.ts @@ -21,6 +21,7 @@ import { SERVICE_LAUNCHER_PROTOCOL, SERVICE_STATE_FILE, parseServiceState, + serviceStateHasPendingUpdate, type ServiceState, } from "./serviceProtocol.ts"; @@ -110,10 +111,20 @@ export class BootServiceInstallError extends Schema.TaggedErrorClass()( + "BootServiceUpdatePendingError", + {}, +) { + override get message(): string { + return "A remote server update is still pending. Wait for it to finish, then retry."; + } +} + export type BootServiceError = | BootServiceUnsupportedError | BootServiceCommandError - | BootServiceInstallError; + | BootServiceInstallError + | BootServiceUpdatePendingError; export interface BootServiceStatus { readonly supported: boolean; @@ -288,6 +299,15 @@ export const make = Effect.fn("cloud.boot_service.make")(function* (input: { } yield* Effect.gen(function* () { + if (installed) { + const previousStateText = yield* fs.readFileString(statePath).pipe(Effect.option); + if ( + Option.isSome(previousStateText) && + serviceStateHasPendingUpdate(previousStateText.value) + ) { + return yield* new BootServiceUpdatePendingError(); + } + } yield* fs .makeDirectory(unitDir, { recursive: true }) .pipe(Effect.mapError((cause) => new BootServiceInstallError({ cause }))); diff --git a/apps/server/src/cloud/selfUpdate.test.ts b/apps/server/src/cloud/selfUpdate.test.ts index 6fe1d5a4a3c..276ee037773 100644 --- a/apps/server/src/cloud/selfUpdate.test.ts +++ b/apps/server/src/cloud/selfUpdate.test.ts @@ -11,6 +11,7 @@ import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawne import * as ServerConfig from "../config.ts"; import * as ProcessRunner from "../processRunner.ts"; import * as ServiceLauncherClient from "./serviceLauncherClient.ts"; +import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; import * as ServerSelfUpdate from "./selfUpdate.ts"; interface HarnessOptions { @@ -50,7 +51,11 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( const result = options.preflight === "blocked" ? { status: "blocked", version: "1.1.0", reason: "local update required" } - : { status: "ready", version: "1.1.0", launcherProtocol: 1 }; + : { + status: "ready", + version: "1.1.0", + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + }; return { // @effect-diagnostics-next-line preferSchemaOverJson:off - fake child-process stdout. stdout: JSON.stringify(result), @@ -64,7 +69,6 @@ const makeHarness = Effect.fn("test.make_self_update_harness")(function* ( }); const launcher = ServiceLauncherClient.ServiceLauncherClient.of({ managed: options.managed ?? true, - trial: false, requestUpdate: options.requestUpdate ?? (() => diff --git a/apps/server/src/cloud/selfUpdate.ts b/apps/server/src/cloud/selfUpdate.ts index 58bb8411730..015fd557d3b 100644 --- a/apps/server/src/cloud/selfUpdate.ts +++ b/apps/server/src/cloud/selfUpdate.ts @@ -170,7 +170,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* () { yield* reportProgress("installing"); const updateId = yield* launcher - .requestUpdate({ targetVersion }) + .requestUpdate({ targetVersion, dbPath: serverConfig.dbPath }) .pipe( Effect.mapError((error) => failWith( diff --git a/apps/server/src/cloud/serviceLauncherClient.test.ts b/apps/server/src/cloud/serviceLauncherClient.test.ts index 6b9e926ff8d..eab0935419a 100644 --- a/apps/server/src/cloud/serviceLauncherClient.test.ts +++ b/apps/server/src/cloud/serviceLauncherClient.test.ts @@ -5,6 +5,7 @@ import * as Fiber from "effect/Fiber"; import { SERVICE_LAUNCHER_CONTEXT_ENV, + SERVICE_LAUNCHER_PROTOCOL, type ServiceLauncherChildMessage, type ServiceLauncherParentMessage, } from "./serviceProtocol.ts"; @@ -53,10 +54,11 @@ it.effect("waits for the launcher to durably commit the trial update ID", () => id: "update-1", fromVersion: "1.0.0", targetVersion: "1.1.0", + dbPath: "/tmp/state.sqlite", status: "pending" as const, }; const host = new FakeLauncherProcess({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, childVersion: "1.1.0", update: pending, }); @@ -79,13 +81,14 @@ it.effect("waits for the launcher to durably commit the trial update ID", () => it.effect("returns the launcher-generated ID only after update acceptance", () => Effect.gen(function* () { const host = new FakeLauncherProcess({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, childVersion: "1.0.0", }); const client = yield* makeClient(host, "1.0.0"); - const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), { - startImmediately: true, - }); + const requested = yield* Effect.forkChild( + client.requestUpdate({ targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }), + { startImmediately: true }, + ); yield* Effect.yieldNow; host.emit({ type: "update-accepted", @@ -97,11 +100,15 @@ it.effect("returns the launcher-generated ID only after update acceptance", () = it.effect("preserves a launcher rejection as a distinct error", () => Effect.gen(function* () { - const host = new FakeLauncherProcess({ protocol: 1, childVersion: "1.0.0" }); - const client = yield* makeClient(host, "1.0.0"); - const requested = yield* Effect.forkChild(client.requestUpdate({ targetVersion: "1.1.0" }), { - startImmediately: true, + const host = new FakeLauncherProcess({ + protocol: SERVICE_LAUNCHER_PROTOCOL, + childVersion: "1.0.0", }); + const client = yield* makeClient(host, "1.0.0"); + const requested = yield* Effect.forkChild( + client.requestUpdate({ targetVersion: "1.1.0", dbPath: "/tmp/state.sqlite" }), + { startImmediately: true }, + ); yield* Effect.yieldNow; host.emit({ type: "update-rejected", reason: "requires local update" }); expect(yield* Fiber.join(requested).pipe(Effect.flip)).toMatchObject({ @@ -115,12 +122,13 @@ it.effect("preserves a launcher rejection as a distinct error", () => it.effect("rejects contradictory trial context instead of leaving activation closed", () => Effect.gen(function* () { const host = new FakeLauncherProcess({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, childVersion: "1.1.0", update: { id: "update-1", fromVersion: "1.0.0", targetVersion: "1.2.0", + dbPath: "/tmp/state.sqlite", status: "pending", }, }); diff --git a/apps/server/src/cloud/serviceLauncherClient.ts b/apps/server/src/cloud/serviceLauncherClient.ts index 760642c29f5..970c72cee72 100644 --- a/apps/server/src/cloud/serviceLauncherClient.ts +++ b/apps/server/src/cloud/serviceLauncherClient.ts @@ -100,9 +100,9 @@ export class ServiceLauncherClient extends Context.Service< ServiceLauncherClient, { readonly managed: boolean; - readonly trial: boolean; readonly requestUpdate: (input: { readonly targetVersion: string; + readonly dbPath: string; }) => Effect.Effect; readonly prepareTrial: Effect.Effect< ServerSelfUpdateOutcome | undefined, @@ -137,8 +137,8 @@ const resolveStartup = Effect.fn("cloud.service_launcher_client.resolve_startup" export const resolveServiceLauncherMode = Effect.fn("cloud.service_launcher_client.resolve_mode")( function* () { - const { context, managed } = yield* resolveStartup(); - return { managed, trial: context?.update?.status === "pending" }; + const { managed } = yield* resolveStartup(); + return { managed }; }, ); @@ -199,7 +199,7 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o }), ); - const requestUpdate = (input: { readonly targetVersion: string }) => + const requestUpdate = (input: { readonly targetVersion: string; readonly dbPath: string }) => exchange( { type: "request-update", ...input }, (reply) => reply.type === "update-accepted" || reply.type === "update-rejected", @@ -233,14 +233,18 @@ export const make = Effect.fn("cloud.service_launcher_client.make")(function* (o if (reply.type !== "committed") { return Effect.die("service launcher returned an impossible prepared response"); } - return Effect.succeed({ ...pending, status: "committed" as const }); + return Effect.succeed({ + id: pending.id, + fromVersion: pending.fromVersion, + targetVersion: pending.targetVersion, + status: "committed" as const, + }); }), ) : Effect.succeed(outcome); return ServiceLauncherClient.of({ managed, - trial: pending !== undefined, requestUpdate, prepareTrial, }); diff --git a/apps/server/src/cloud/servicePreflight.test.ts b/apps/server/src/cloud/servicePreflight.test.ts index d2ce6db8de8..2eb2e02015d 100644 --- a/apps/server/src/cloud/servicePreflight.test.ts +++ b/apps/server/src/cloud/servicePreflight.test.ts @@ -1,47 +1,26 @@ -// @effect-diagnostics nodeBuiltinImport:off -import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; -import * as Effect from "effect/Effect"; -import * as FileSystem from "effect/FileSystem"; -import * as Path from "effect/Path"; -import * as NodeSqlite from "node:sqlite"; -import { migrationManifest } from "../persistence/Migrations.ts"; import { runServicePreflight } from "./servicePreflight.ts"; +import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; -it.layer(NodeServices.layer)("service update preflight", (it) => { - it.effect("requires exact migration-manifest equality without mutating the database", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-preflight-test-" }); - const databasePath = path.join(root, "state.sqlite"); - const database = new NodeSqlite.DatabaseSync(databasePath); - database.exec("CREATE TABLE effect_sql_migrations (migration_id INTEGER, name TEXT)"); - const insert = database.prepare( - "INSERT INTO effect_sql_migrations (migration_id, name) VALUES (?, ?)", - ); - for (const [id, name] of migrationManifest) insert.run(id, name); - database.close(); - - expect(runServicePreflight({ databasePath, launcherProtocol: 1, version: "1.2.3" })).toEqual({ - status: "ready", - version: "1.2.3", - launcherProtocol: 1, - }); +it("requires the database-snapshot launcher protocol", () => { + expect( + runServicePreflight({ + databasePath: "/missing/state.sqlite", + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL - 1, + version: "1.2.3", + }), + ).toMatchObject({ status: "blocked", version: "1.2.3" }); - const changed = new NodeSqlite.DatabaseSync(databasePath); - changed.exec("DELETE FROM effect_sql_migrations WHERE migration_id = 35"); - changed.close(); - const blocked = runServicePreflight({ - databasePath, - launcherProtocol: 1, - version: "1.2.3", - }); - expect(blocked.status).toBe("blocked"); - if (blocked.status === "blocked") { - expect(blocked.reason).toContain("npx t3@1.2.3 service update"); - } + expect( + runServicePreflight({ + databasePath: "/missing/state.sqlite", + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + version: "1.2.3", }), - ); + ).toEqual({ + status: "ready", + version: "1.2.3", + launcherProtocol: SERVICE_LAUNCHER_PROTOCOL, + }); }); diff --git a/apps/server/src/cloud/servicePreflight.ts b/apps/server/src/cloud/servicePreflight.ts index 1843e163881..ee0f972baa3 100644 --- a/apps/server/src/cloud/servicePreflight.ts +++ b/apps/server/src/cloud/servicePreflight.ts @@ -1,7 +1,4 @@ -import * as NodeSqlite from "node:sqlite"; - import packageJson from "../../package.json" with { type: "json" }; -import { migrationManifest } from "../persistence/Migrations.ts"; import { SERVICE_LAUNCHER_PROTOCOL } from "./serviceProtocol.ts"; export type ServicePreflightResult = @@ -16,20 +13,8 @@ export type ServicePreflightResult = readonly reason: string; }; -const localUpdateReason = (version: string) => - `This version includes a database update and cannot be installed remotely. Run \`npx t3@${version} service update\` on the server machine.`; - -const isMigrationRow = ( - value: unknown, -): value is { readonly migration_id: number; readonly name: string } => - typeof value === "object" && - value !== null && - "migration_id" in value && - typeof value.migration_id === "number" && - "name" in value && - typeof value.name === "string"; - export function runServicePreflight(input: { + /** Older servers always pass this flag when invoking a staged preflight. */ readonly databasePath: string; readonly launcherProtocol: number; readonly version?: string; @@ -44,28 +29,6 @@ export function runServicePreflight(input: { }; } - try { - const database = new NodeSqlite.DatabaseSync(input.databasePath, { readOnly: true }); - try { - const rows: ReadonlyArray = database - .prepare("SELECT migration_id, name FROM effect_sql_migrations ORDER BY migration_id") - .all(); - const exact = - rows.length === migrationManifest.length && - rows.every((row, index) => { - const expected = migrationManifest[index]; - return ( - isMigrationRow(row) && row.migration_id === expected?.[0] && row.name === expected?.[1] - ); - }); - if (!exact) return { status: "blocked", version, reason: localUpdateReason(version) }; - } finally { - database.close(); - } - } catch { - return { status: "blocked", version, reason: localUpdateReason(version) }; - } - return { status: "ready", version, launcherProtocol: SERVICE_LAUNCHER_PROTOCOL }; } diff --git a/apps/server/src/cloud/serviceProtocol.ts b/apps/server/src/cloud/serviceProtocol.ts index 921bc1447ed..ebc5d15d54f 100644 --- a/apps/server/src/cloud/serviceProtocol.ts +++ b/apps/server/src/cloud/serviceProtocol.ts @@ -1,6 +1,7 @@ import type { ServerSelfUpdateOutcome } from "@t3tools/contracts"; -export const SERVICE_LAUNCHER_PROTOCOL = 1 as const; +/** Protocol 2 snapshots SQLite before trials so migrations can be rolled back safely. */ +export const SERVICE_LAUNCHER_PROTOCOL = 2 as const; export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs"; export const SERVICE_STATE_FILE = "service-state.json"; @@ -9,6 +10,7 @@ export interface PendingServiceUpdate { readonly id: string; readonly fromVersion: string; readonly targetVersion: string; + readonly dbPath: string; readonly status: "pending"; } @@ -31,6 +33,7 @@ export type ServiceLauncherChildMessage = | { readonly type: "request-update"; readonly targetVersion: string; + readonly dbPath: string; } | { readonly type: "prepared"; @@ -78,7 +81,9 @@ export function decodeServiceUpdate(value: unknown): ServiceUpdateRecord | undef return undefined; } if (status === "pending") { - return { id, fromVersion, targetVersion, status }; + return typeof value.dbPath === "string" && value.dbPath.trim() !== "" + ? { id, fromVersion, targetVersion, dbPath: value.dbPath, status } + : undefined; } if ( (status === "committed" || status === "rolled-back" || status === "failed") && @@ -165,6 +170,16 @@ export function parseServiceState(value: string): ServiceState | undefined { } } +/** Detects an in-flight update across launcher protocol versions before replacing its state. */ +export function serviceStateHasPendingUpdate(value: string): boolean { + try { + const parsed: unknown = JSON.parse(value); + return isRecord(parsed) && isRecord(parsed.update) && parsed.update.status === "pending"; + } catch { + return false; + } +} + export function decodeServiceLauncherContext(value: string): ServiceLauncherContext | undefined { let parsed: unknown; try { @@ -202,8 +217,12 @@ export function decodeServiceLauncherChildMessage( value: unknown, ): ServiceLauncherChildMessage | undefined { if (!isRecord(value)) return undefined; - if (value.type === "request-update" && typeof value.targetVersion === "string") { - return { type: value.type, targetVersion: value.targetVersion }; + if ( + value.type === "request-update" && + typeof value.targetVersion === "string" && + typeof value.dbPath === "string" + ) { + return { type: value.type, targetVersion: value.targetVersion, dbPath: value.dbPath }; } return value.type === "prepared" && typeof value.updateId === "string" ? { type: value.type, updateId: value.updateId } diff --git a/apps/server/src/persistence/Layers/Sqlite.ts b/apps/server/src/persistence/Layers/Sqlite.ts index 3cfdb5c5c76..d1e00250126 100644 --- a/apps/server/src/persistence/Layers/Sqlite.ts +++ b/apps/server/src/persistence/Layers/Sqlite.ts @@ -7,7 +7,6 @@ import type { SqlError } from "effect/unstable/sql/SqlError"; import { runMigrations } from "../Migrations.ts"; import { ServerConfig } from "../../config.ts"; -import * as ServiceLauncherClient from "../../cloud/serviceLauncherClient.ts"; type RuntimeSqliteLayerConfig = { readonly filename: string; @@ -31,28 +30,24 @@ const makeRuntimeSqliteLayer = Effect.fn("makeRuntimeSqliteLayer")(function* ( return clientModule.layer(config); }, Layer.unwrap); -const setup = (trial: boolean) => - Layer.effectDiscard( - Effect.gen(function* () { - const sql = yield* SqlClient.SqlClient; - yield* sql`PRAGMA foreign_keys = ON;`; - if (!trial) { - yield* sql`PRAGMA journal_mode = WAL;`; - yield* runMigrations(); - } - }), - ); +const setup = Layer.effectDiscard( + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + yield* sql`PRAGMA foreign_keys = ON;`; + yield* sql`PRAGMA journal_mode = WAL;`; + yield* runMigrations(); + }), +); export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")(function* ( dbPath: string, - options?: { readonly trial?: boolean }, ) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; yield* fs.makeDirectory(path.dirname(dbPath), { recursive: true }); return Layer.provideMerge( - setup(options?.trial === true), + setup, makeRuntimeSqliteLayer({ filename: dbPath, spanAttributes: { @@ -64,14 +59,13 @@ export const makeSqlitePersistenceLive = Effect.fn("makeSqlitePersistenceLive")( }, Layer.unwrap); export const SqlitePersistenceMemory = Layer.provideMerge( - setup(false), + setup, makeRuntimeSqliteLayer({ filename: ":memory:" }), ); export const layerConfig = Layer.unwrap( Effect.gen(function* () { const { dbPath } = yield* ServerConfig; - const launcher = yield* ServiceLauncherClient.resolveServiceLauncherMode(); - return makeSqlitePersistenceLive(dbPath, { trial: launcher.trial }); + return makeSqlitePersistenceLive(dbPath); }), ); diff --git a/apps/server/src/serviceLauncher.test.ts b/apps/server/src/serviceLauncher.test.ts index 21f3618d512..4562c6a6de8 100644 --- a/apps/server/src/serviceLauncher.test.ts +++ b/apps/server/src/serviceLauncher.test.ts @@ -9,6 +9,7 @@ import { compareExactServiceVersions, decodeServiceState, isExactServiceVersion, + SERVICE_LAUNCHER_PROTOCOL, } from "./cloud/serviceProtocol.ts"; it("accepts only exact semantic versions", () => { @@ -33,12 +34,13 @@ it("orders exact semantic versions without treating build metadata as precedence it("rejects contradictory service state", () => { assert.isUndefined( decodeServiceState({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "0.0.31", update: { id: "update-1", fromVersion: "0.0.30", targetVersion: "0.0.32", + dbPath: "/tmp/state.sqlite", status: "pending", }, }), @@ -46,12 +48,26 @@ it("rejects contradictory service state", () => { assert.isUndefined( decodeServiceState({ - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + update: { + id: "update-3", + fromVersion: "1.0.0", + targetVersion: "1.1.0", + status: "pending", + }, + }), + ); + + assert.isUndefined( + decodeServiceState({ + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.0.0", update: { id: "update-2", fromVersion: "1.0.0", targetVersion: "0.9.0", + dbPath: "/tmp/state.sqlite", status: "pending", }, }), @@ -66,7 +82,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-test-" }); const statePath = path.join(root, "runtime", "service-state.json"); const state = { - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "0.0.31", } as const; @@ -88,7 +104,7 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { yield* fs.writeFileString(path.join(versionDir, ".install-complete"), "1.0.0\n"); yield* Effect.promise(() => writeServiceState(statePath, { - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.0.0", }), ); @@ -106,6 +122,11 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-flow-" }); const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source. + const encodedDatabasePath = JSON.stringify(databasePath); const childSource = ` const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); if (context.update?.status === "pending") { @@ -114,7 +135,7 @@ if (context.update?.status === "pending") { if (message.type === "committed") process.exit(0); }); } else if (context.update === undefined) { - process.send({ type: "request-update", targetVersion: "1.1.0" }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); setInterval(() => {}, 1_000); } else { process.exit(0); @@ -129,7 +150,7 @@ if (context.update?.status === "pending") { } yield* Effect.promise(() => writeServiceState(statePath, { - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.0.0", }), ); @@ -154,12 +175,17 @@ if (context.update?.status === "pending") { const path = yield* Path.Path; const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-rollback-" }); const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, "before trial"); + // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source. + const encodedDatabasePath = JSON.stringify(databasePath); const childSource = ` const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); if (context.update?.status === "pending") { process.send({ type: "prepared", updateId: "wrong-update" }); } else if (context.update === undefined) { - process.send({ type: "request-update", targetVersion: "1.1.0" }); + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); setInterval(() => {}, 1_000); } else { process.exit(0); @@ -174,7 +200,7 @@ if (context.update?.status === "pending") { } yield* Effect.promise(() => writeServiceState(statePath, { - protocol: 1, + protocol: SERVICE_LAUNCHER_PROTOCOL, activeVersion: "1.0.0", }), ); @@ -196,4 +222,65 @@ if (context.update?.status === "pending") { ); }), ); + + it.effect("restores the database when a migrating trial exits", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-service-launcher-db-" }); + const statePath = path.join(root, "runtime", "service-state.json"); + const databasePath = path.join(root, "userdata", "state.sqlite"); + const original = "database before migration"; + yield* fs.makeDirectory(path.dirname(databasePath), { recursive: true }); + yield* fs.writeFileString(databasePath, original); + // @effect-diagnostics-next-line preferSchemaOverJson:off - embeds a path in fake child source. + const encodedDatabasePath = JSON.stringify(databasePath); + const childSource = ` +import { writeFileSync } from "node:fs"; +const context = JSON.parse(process.env.T3_SERVICE_LAUNCHER_CONTEXT); +if (context.update?.status === "pending") { + writeFileSync(context.update.dbPath, "database after migration"); + writeFileSync(context.update.dbPath + "-wal", "trial wal"); + writeFileSync(context.update.dbPath + "-shm", "trial shm"); + process.exit(1); +} else if (context.update === undefined) { + process.send({ type: "request-update", targetVersion: "1.1.0", dbPath: ${encodedDatabasePath} }); + setInterval(() => {}, 1_000); +} else { + process.exit(0); +} +`; + for (const version of ["1.0.0", "1.1.0"]) { + const versionDir = path.join(root, "runtime", "versions", version); + const entryPath = path.join(versionDir, "node_modules", "t3", "dist", "bin.mjs"); + yield* fs.makeDirectory(path.dirname(entryPath), { recursive: true }); + yield* fs.writeFileString(entryPath, childSource); + yield* fs.writeFileString(path.join(versionDir, ".install-complete"), `${version}\n`); + } + yield* Effect.promise(() => + writeServiceState(statePath, { + protocol: SERVICE_LAUNCHER_PROTOCOL, + activeVersion: "1.0.0", + }), + ); + + const launcher = new Launcher(root, yield* Effect.promise(() => readServiceState(statePath))); + yield* Effect.promise(() => + launcher.run().then( + () => Promise.reject(new Error("launcher unexpectedly completed")), + () => Promise.resolve(), + ), + ); + + const state = yield* Effect.promise(() => readServiceState(statePath)); + assert.equal(state.activeVersion, "1.0.0"); + assert.equal(state.update?.status, "rolled-back"); + assert.equal(yield* fs.readFileString(databasePath), original); + assert.isFalse(yield* fs.exists(`${databasePath}-wal`)); + assert.isFalse(yield* fs.exists(`${databasePath}-shm`)); + const updateId = state.update?.id; + assert.isDefined(updateId); + assert.isFalse(yield* fs.exists(path.join(root, "runtime", "db-backup", updateId))); + }), + ); }); diff --git a/apps/server/src/serviceLauncher.ts b/apps/server/src/serviceLauncher.ts index d7ca3357802..211c0138bc4 100644 --- a/apps/server/src/serviceLauncher.ts +++ b/apps/server/src/serviceLauncher.ts @@ -48,6 +48,121 @@ const runtimePaths = (baseDir: string, version: string) => { }; }; +/** SQLite persists across the main file plus its WAL and shared-memory sidecars. */ +const DB_FILE_SUFFIXES = ["", "-wal", "-shm"] as const; +const RESTORE_MARKER = ".restore-pending"; + +const databaseBackupDir = (baseDir: string, updateId: string) => + NodePath.join(baseDir, "runtime", "db-backup", updateId); + +const databaseBackupFile = (backupDir: string, suffix: (typeof DB_FILE_SUFFIXES)[number]) => + NodePath.join(backupDir, suffix === "" ? "database" : `database${suffix}`); + +async function pathExists(target: string): Promise { + try { + await NodeFSP.access(target); + return true; + } catch (cause) { + if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return false; + throw cause; + } +} + +async function syncFile(filePath: string): Promise { + const handle = await NodeFSP.open(filePath, "r"); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +async function syncDirectory(directory: string): Promise { + const handle = await NodeFSP.open(directory, "r"); + try { + await handle.sync(); + } finally { + await handle.close(); + } +} + +/** + * Snapshots the database once per update before the first trial. A completed + * backup is never overwritten because a restarted launcher may be looking at + * database writes from an earlier attempt by the same trial. + */ +async function backupDatabaseOnce(baseDir: string, pending: PendingServiceUpdate): Promise { + const backupDir = databaseBackupDir(baseDir, pending.id); + if (await pathExists(backupDir)) return; + + const stagingDir = `${backupDir}.staging`; + await NodeFSP.rm(stagingDir, { recursive: true, force: true }); + await NodeFSP.mkdir(stagingDir, { recursive: true, mode: 0o700 }); + try { + for (const suffix of DB_FILE_SUFFIXES) { + const source = `${pending.dbPath}${suffix}`; + if (suffix !== "" && !(await pathExists(source))) continue; + const destination = databaseBackupFile(stagingDir, suffix); + await NodeFSP.copyFile(source, destination); + await syncFile(destination); + } + await NodeFSP.rename(stagingDir, backupDir); + await syncDirectory(NodePath.dirname(backupDir)); + } catch (cause) { + await NodeFSP.rm(stagingDir, { recursive: true, force: true }).catch(() => undefined); + throw cause; + } +} + +const restoreMarkerPath = (baseDir: string, updateId: string) => + NodePath.join(databaseBackupDir(baseDir, updateId), RESTORE_MARKER); + +const databaseRestorePending = (baseDir: string, pending: PendingServiceUpdate) => + pathExists(restoreMarkerPath(baseDir, pending.id)); + +/** Mark rollback before changing live files so launcher recovery cannot boot a partial restore. */ +async function markDatabaseRestorePending(backupDir: string): Promise { + const markerPath = NodePath.join(backupDir, RESTORE_MARKER); + if (!(await pathExists(markerPath))) { + const handle = await NodeFSP.open(markerPath, "wx", 0o600); + try { + await handle.sync(); + } finally { + await handle.close(); + } + await syncDirectory(backupDir); + } +} + +/** Restore is retryable after any process crash while the backup directory remains. */ +async function restoreDatabaseBackup( + baseDir: string, + pending: PendingServiceUpdate, +): Promise { + const backupDir = databaseBackupDir(baseDir, pending.id); + if (!(await pathExists(backupDir))) return; + + await markDatabaseRestorePending(backupDir); + for (const suffix of DB_FILE_SUFFIXES) { + const target = `${pending.dbPath}${suffix}`; + const source = databaseBackupFile(backupDir, suffix); + if (await pathExists(source)) { + await NodeFSP.copyFile(source, target); + await syncFile(target); + } else { + await NodeFSP.rm(target, { force: true }); + } + } + await syncDirectory(NodePath.dirname(pending.dbPath)); +} + +async function discardDatabaseBackup(baseDir: string, updateId: string): Promise { + const backupDir = databaseBackupDir(baseDir, updateId); + if (!(await pathExists(backupDir))) return; + await NodeFSP.rm(backupDir, { recursive: true, force: true }); + await syncDirectory(NodePath.dirname(backupDir)); +} + export async function readServiceState(filePath: string): Promise { const contents = await NodeFSP.readFile(filePath, "utf8"); const state = parseServiceState(contents); @@ -217,9 +332,16 @@ export class Launcher { async #recover(): Promise { const update = this.#state.update; if (update?.status !== "pending") { + if (update !== undefined) { + await discardDatabaseBackup(this.#baseDir, update.id).catch(() => undefined); + } await this.#startChild(this.#state.activeVersion, "active", update); return; } + if (await databaseRestorePending(this.#baseDir, update)) { + await this.#returnToPrevious(update, "failed", "rollback-interrupted"); + return; + } if (!(await runtimeExists(this.#baseDir, update.targetVersion))) { await this.#returnToPrevious(update, "failed", "target-runtime-missing"); return; @@ -228,6 +350,13 @@ export class Launcher { } async #startTrial(pending: PendingServiceUpdate): Promise { + // The previous child is dead here, so all three SQLite files are quiescent. + try { + await backupDatabaseOnce(this.#baseDir, pending); + } catch { + await this.#returnToPrevious(pending, "failed", "db-backup-failed"); + return; + } try { await this.#startChild(pending.targetVersion, "trial", pending); } catch { @@ -322,6 +451,10 @@ export class Launcher { await reject("Remote updates must select a newer server version."); return; } + if (!NodePath.isAbsolute(message.dbPath)) { + await reject("The requested database path is not absolute."); + return; + } if (!(await runtimeExists(this.#baseDir, message.targetVersion))) { await reject("The requested target runtime is missing or incomplete."); return; @@ -331,6 +464,7 @@ export class Launcher { id: NodeCrypto.randomUUID(), fromVersion: child.version, targetVersion: message.targetVersion, + dbPath: message.dbPath, status: "pending", }; const next: ServiceState = { ...this.#state, update: pending }; @@ -375,6 +509,7 @@ export class Launcher { await writeServiceState(this.#statePath, next); this.#state = next; child.role = "active"; + await discardDatabaseBackup(this.#baseDir, committed.id).catch(() => undefined); await sendMessage(child.process, { type: "committed", updateId: committed.id }); } @@ -423,6 +558,11 @@ export class Launcher { reason: string, child?: ManagedChild, ): Promise { + if (child !== undefined) { + this.#child = null; + await terminateChild(child.process); + } + await restoreDatabaseBackup(this.#baseDir, pending); const outcome = terminalUpdate({ pending, status, reason }); const next: ServiceState = { ...this.#state, @@ -431,10 +571,7 @@ export class Launcher { }; await writeServiceState(this.#statePath, next); this.#state = next; - if (child !== undefined) { - this.#child = null; - await terminateChild(child.process); - } + await discardDatabaseBackup(this.#baseDir, pending.id).catch(() => undefined); await this.#startChild(next.activeVersion, "active", outcome); } } diff --git a/docs/internals/server-updates.md b/docs/internals/server-updates.md index f0a0034c3ee..c2ba77f6f66 100644 --- a/docs/internals/server-updates.md +++ b/docs/internals/server-updates.md @@ -29,39 +29,49 @@ Every write uses same-directory replacement plus file and directory fsync. ## Remote Update 1. The active server installs `t3@` into a unique staging directory. -2. The target runs `__service-preflight` against the active database in read-only mode. +2. The target runs `__service-preflight` and verifies that the stable launcher supports its update + protocol. 3. The staging directory is renamed to its immutable version path only after preflight succeeds. 4. The active child sends `request-update`. The launcher validates the child and target, writes pending state, generates the update ID, then replies `update-accepted`. 5. After a short response-flush grace period, the launcher stops the active child. -6. The launcher starts the target as a trial and gives it the pending update over IPC. -7. The trial acquires dependencies, binds HTTP, starts every long-running root fiber, and verifies - that each root is parked at the activation gate. It does not run migrations. -8. The trial sends `prepared`. The launcher durably commits B before replying `committed`. -9. The child opens the existing activation gate, accepts commands, and publishes lifecycle ready - with the terminal update outcome. +6. With SQLite quiescent, the launcher snapshots the database, WAL, and shared-memory files. +7. The launcher starts the target as a trial and gives it the pending update over IPC. +8. The trial runs migrations, acquires dependencies, binds HTTP, starts every long-running root + fiber, and verifies that each root is parked at the activation gate. +9. The trial sends `prepared`. The launcher durably commits B, deletes the snapshot, then replies + `committed`. +10. The child opens the existing activation gate, accepts commands, and publishes lifecycle ready + with the terminal update outcome. Post-commit startup does not call service `start`, `initialize`, `connect`, `load`, or `acquire` operations. It only opens prepared gates and publishes prepared lifecycle state. The launcher serializes child exits, IPC messages, and timers. A trial must report prepared within -120 seconds. If the trial exits or times out before prepared, the launcher records rollback and -starts A. After commit, B is active and normal systemd restart policy applies. +120 seconds. If the trial exits or times out before prepared, the launcher stops it, restores the +snapshot, records rollback, and starts A. A durable restore marker makes an interrupted restore +resume before either version can boot. After commit, B is active and normal systemd restart policy +applies. -## Migration Boundary +## Database Rollback -Remote update preflight requires exact equality between the database's applied `(id, name)` rows -and the target release's migration manifest. A target with a missing, additional, renamed, or -unknown migration is blocked. Remote updates never migrate or downgrade a database. +The launcher snapshots `state.sqlite`, `state.sqlite-wal`, and `state.sqlite-shm` after the old +server stops and before the trial starts. This makes trial migrations and writes reversible without +requiring down migrations. The snapshot is retained across launcher restarts and is removed only +after commit or after both restore and the terminal rollback state are durable. -This deliberately means any release containing a migration requires a local service update: +The protocol version is part of the safety boundary. A target that requires database snapshots is +blocked when the installed launcher is too old. Upgrade the launcher once with: ```sh npx t3@ service update ``` The local command stops the unit, selects the new launcher and exact runtime, then restarts the -service. Its normal server startup may run migrations. +service. Later releases, including releases with migrations, can use the remote trial path. + +Snapshots briefly require enough free disk for another copy of the SQLite files. Attachments and +other files under the state directory are outside this rollback boundary. ## Client Correlation diff --git a/docs/user/background-service.md b/docs/user/background-service.md index 0ab5b2013c7..299e3b641f1 100644 --- a/docs/user/background-service.md +++ b/docs/user/background-service.md @@ -30,10 +30,13 @@ npx t3@latest service uninstall ``` Updating restarts T3 Code briefly. Let active agent work and terminal commands finish first. +If a remote update is already in progress, wait for it to finish before retrying a local update. The systemd unit runs a small stable launcher. Exact T3 Code versions are installed separately, so -a failed remote candidate can return to the previous version without rewriting the unit. Releases -that change the database must be installed with the local `service update` command above. +a failed remote candidate can return to the previous version without rewriting the unit. The +launcher snapshots the database before a remote candidate starts, so database updates roll back +with the server version. An older launcher may require one local `service update` before this is +available. ## Using It with T3 Connect diff --git a/docs/user/updating.md b/docs/user/updating.md index a0cc0e5d1e0..ea2ff644cf7 100644 --- a/docs/user/updating.md +++ b/docs/user/updating.md @@ -31,9 +31,9 @@ The update does not remove saved threads, settings, or project files. The available action depends on how that server was started. T3 Code does not update connected servers silently in the background. -If the requested version includes a database update, remote installation stops before restart and -asks you to run the exact `npx t3@ service update` command on the server machine. This is -an intentional rollback-safety boundary. +An older background-service launcher may ask you to run the exact +`npx t3@ service update` command on the server machine. That one local update installs the +rollback support needed for later remote updates, including versions that change the database. After selecting **Update server**, the warning becomes a three-step progress rail: **Download**, **Install**, and **Resume**. The same progress appears in the conversation and in