From 7f69028a25be21f1882ecba14b62f387ad60cf2a Mon Sep 17 00:00:00 2001 From: Tim Smart Date: Wed, 22 Jul 2026 12:04:49 +1200 Subject: [PATCH 1/3] Recover interrupted provider turns after server restarts - Reconcile pending and interrupted turns during startup - Persist restart markers and clear them after recovery begins - Track recovery outcomes and isolate per-thread failures --- .../OrchestrationEngineHarness.integration.ts | 1 + apps/server/src/observability/Metrics.ts | 4 + .../Layers/ProviderCommandReactor.test.ts | 573 +++++++++++++++++- .../Layers/ProviderCommandReactor.ts | 442 +++++++++++++- .../src/persistence/Layers/ProjectionTurns.ts | 28 + .../persistence/Services/ProjectionTurns.ts | 9 + .../provider/Layers/ProviderService.test.ts | 104 +++- .../src/provider/Layers/ProviderService.ts | 146 ++++- .../provider/ProviderRestartRecovery.test.ts | 73 +++ .../src/provider/ProviderRestartRecovery.ts | 103 ++++ 10 files changed, 1414 insertions(+), 69 deletions(-) create mode 100644 apps/server/src/provider/ProviderRestartRecovery.test.ts create mode 100644 apps/server/src/provider/ProviderRestartRecovery.ts diff --git a/apps/server/integration/OrchestrationEngineHarness.integration.ts b/apps/server/integration/OrchestrationEngineHarness.integration.ts index ebc4f984b86..dc4a3f8df13 100644 --- a/apps/server/integration/OrchestrationEngineHarness.integration.ts +++ b/apps/server/integration/OrchestrationEngineHarness.integration.ts @@ -304,6 +304,7 @@ export const makeOrchestrationIntegrationHarness = ( ProjectionPendingApprovalRepositoryLive, checkpointStoreLayer, providerLayer, + providerSessionDirectoryLayer, RuntimeReceiptBusTest, ); const serverSettingsLayer = ServerSettingsService.layerTest(); diff --git a/apps/server/src/observability/Metrics.ts b/apps/server/src/observability/Metrics.ts index 886833d6e2c..a09179cbab2 100644 --- a/apps/server/src/observability/Metrics.ts +++ b/apps/server/src/observability/Metrics.ts @@ -50,6 +50,10 @@ export const providerTurnsTotal = Metric.counter("t3_provider_turns_total", { description: "Total provider turn lifecycle operations.", }); +export const providerTurnRecoveriesTotal = Metric.counter("t3_provider_turn_recoveries_total", { + description: "Total provider turn restart-recovery candidates and outcomes.", +}); + export const providerTurnDuration = Metric.timer("t3_provider_turn_duration", { description: "Provider turn request duration.", }); diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index c49646b7a4b..0c91efdf3c6 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -26,6 +26,7 @@ import * as Deferred from "effect/Deferred"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as ManagedRuntime from "effect/ManagedRuntime"; +import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; @@ -37,11 +38,15 @@ import { TextGenerationError } from "@t3tools/contracts"; import { ProviderAdapterRequestError } from "../../provider/Errors.ts"; import { OrchestrationEventStoreLive } from "../../persistence/Layers/OrchestrationEventStore.ts"; import { OrchestrationCommandReceiptRepositoryLive } from "../../persistence/Layers/OrchestrationCommandReceipts.ts"; -import { SqlitePersistenceMemory } from "../../persistence/Layers/Sqlite.ts"; +import { ProjectionTurnRepositoryLive } from "../../persistence/Layers/ProjectionTurns.ts"; +import { ProjectionTurnRepository } from "../../persistence/Services/ProjectionTurns.ts"; +import { makeSqlitePersistenceLive } from "../../persistence/Layers/Sqlite.ts"; import { ProviderService, type ProviderServiceShape, } from "../../provider/Services/ProviderService.ts"; +import { ProviderSessionDirectory } from "../../provider/Services/ProviderSessionDirectory.ts"; +import type { ProviderRuntimeBindingWithMetadata } from "../../provider/Services/ProviderSessionDirectory.ts"; import { makeProviderRegistryLayer } from "../../provider/testUtils/providerRegistryMock.ts"; import { TextGeneration, type TextGenerationShape } from "../../textGeneration/TextGeneration.ts"; import * as RepositoryIdentityResolver from "../../project/RepositoryIdentityResolver.ts"; @@ -51,8 +56,10 @@ import { OrchestrationProjectionSnapshotQueryLive } from "./ProjectionSnapshotQu import { providerErrorLabel, providerErrorLabelFromInstanceHint, + RESTART_RECOVERY_CONTINUATION_INSTRUCTION, ProviderCommandReactorLive, } from "./ProviderCommandReactor.ts"; +import { makeProviderRestartRecoveryMarker } from "../../provider/ProviderRestartRecovery.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; import { ProviderCommandReactor } from "../Services/ProviderCommandReactor.ts"; import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; @@ -91,7 +98,10 @@ async function waitFor( describe("ProviderCommandReactor", () => { let runtime: ManagedRuntime.ManagedRuntime< - OrchestrationEngineService | ProviderCommandReactor | ProjectionSnapshotQuery, + | OrchestrationEngineService + | ProviderCommandReactor + | ProjectionSnapshotQuery + | ProjectionTurnRepository, unknown > | null = null; let scope: Scope.Closeable | null = null; @@ -149,13 +159,21 @@ describe("ProviderCommandReactor", () => { readonly startSessionEffect?: ( session: ProviderSession, ) => Effect.Effect; + readonly deferReactorStart?: boolean; + readonly providerBindings?: ReadonlyArray; + readonly providerBindingsMap?: Map; + readonly skipDefaultSetup?: boolean; + readonly providerInstanceEnabled?: boolean; }) { const now = "2026-01-01T00:00:00.000Z"; const baseDir = input?.baseDir ?? NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3code-reactor-")); createdBaseDirs.add(baseDir); - const { stateDir } = deriveServerPathsSync(baseDir, undefined); + const { stateDir, dbPath } = deriveServerPathsSync(baseDir, undefined); createdStateDirs.add(stateDir); + const persistenceLayer = makeSqlitePersistenceLive(dbPath).pipe( + Layer.provide(NodeServices.layer), + ); const runtimeEventPubSub = Effect.runSync(PubSub.unbounded()); let nextSessionIndex = 1; const runtimeSessions: Array = []; @@ -302,6 +320,30 @@ describe("ProviderCommandReactor", () => { : {}), }, ]; + const providerBindings = + input?.providerBindingsMap ?? + new Map((input?.providerBindings ?? []).map((binding) => [binding.threadId, binding])); + const directoryUpsert = vi.fn( + (binding: Parameters[0]) => + Effect.sync(() => { + const existing = providerBindings.get(binding.threadId); + providerBindings.set(binding.threadId, { + ...existing, + ...binding, + providerInstanceId: binding.providerInstanceId, + lastSeenAt: existing?.lastSeenAt ?? now, + runtimePayload: + existing?.runtimePayload && + typeof existing.runtimePayload === "object" && + !Array.isArray(existing.runtimePayload) && + binding.runtimePayload && + typeof binding.runtimePayload === "object" && + !Array.isArray(binding.runtimePayload) + ? { ...existing.runtimePayload, ...binding.runtimePayload } + : (binding.runtimePayload ?? existing?.runtimePayload ?? null), + } as ProviderRuntimeBindingWithMetadata); + }), + ); const unsupported = () => Effect.die(new Error("Unsupported provider call in test")) as never; const service: ProviderServiceShape = { @@ -325,7 +367,7 @@ describe("ProviderCommandReactor", () => { instanceId, driverKind, displayName: undefined, - enabled: true, + enabled: input?.providerInstanceEnabled ?? true, continuationIdentity: { driverKind, continuationKey: @@ -347,16 +389,26 @@ describe("ProviderCommandReactor", () => { Layer.provide(OrchestrationEventStoreLive), Layer.provide(OrchestrationCommandReceiptRepositoryLive), Layer.provide(RepositoryIdentityResolver.layer), - Layer.provide(SqlitePersistenceMemory), + Layer.provide(persistenceLayer), ); const projectionSnapshotLayer = OrchestrationProjectionSnapshotQueryLive.pipe( Layer.provide(RepositoryIdentityResolver.layer), - Layer.provide(SqlitePersistenceMemory), + Layer.provide(persistenceLayer), ); const layer = ProviderCommandReactorLive.pipe( Layer.provideMerge(orchestrationLayer), Layer.provideMerge(projectionSnapshotLayer), Layer.provideMerge(Layer.succeed(ProviderService, service)), + Layer.provideMerge( + Layer.succeed(ProviderSessionDirectory, { + upsert: directoryUpsert, + getProvider: () => Effect.die("getProvider should not be called in this test"), + getBinding: (threadId) => + Effect.succeed(Option.fromNullishOr(providerBindings.get(threadId))), + listThreadIds: () => Effect.succeed(Array.from(providerBindings.keys())), + listBindings: () => Effect.succeed(Array.from(providerBindings.values())), + }), + ), Layer.provideMerge(makeProviderRegistryLayer(providerSnapshots as never)), Layer.provideMerge( Layer.mock(GitWorkflowService.GitWorkflowService)({ @@ -381,46 +433,61 @@ describe("ProviderCommandReactor", () => { Layer.provideMerge(ServerSettingsService.layerTest()), Layer.provideMerge(ServerConfig.layerTest(process.cwd(), baseDir)), Layer.provideMerge(NodeServices.layer), + Layer.provideMerge(persistenceLayer), + Layer.provideMerge(ProjectionTurnRepositoryLive.pipe(Layer.provide(persistenceLayer))), ); runtime = ManagedRuntime.make(layer); const engine = await runtime.runPromise(Effect.service(OrchestrationEngineService)); const snapshotQuery = await runtime.runPromise(Effect.service(ProjectionSnapshotQuery)); const reactor = await runtime.runPromise(Effect.service(ProviderCommandReactor)); + const turnRepository = await runtime.runPromise(Effect.service(ProjectionTurnRepository)); scope = await Effect.runPromise(Scope.make("sequential")); - await Effect.runPromise(reactor.start().pipe(Scope.provide(scope))); + let reactorStarted = false; + const startReactor = async () => { + if (reactorStarted) return; + reactorStarted = true; + await Effect.runPromise(reactor.start().pipe(Scope.provide(scope!))); + }; + if (input?.deferReactorStart !== true) { + await startReactor(); + } const drain = () => Effect.runPromise(reactor.drain); - await Effect.runPromise( - engine.dispatch({ - type: "project.create", - commandId: CommandId.make("cmd-project-create"), - projectId: asProjectId("project-1"), - title: "Provider Project", - workspaceRoot: "/tmp/provider-project", - defaultModelSelection: modelSelection, - createdAt: now, - }), - ); - await Effect.runPromise( - engine.dispatch({ - type: "thread.create", - commandId: CommandId.make("cmd-thread-create"), - threadId: ThreadId.make("thread-1"), - projectId: asProjectId("project-1"), - title: "Thread", - modelSelection: modelSelection, - interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, - runtimeMode: "approval-required", - branch: null, - worktreePath: null, - createdAt: now, - }), - ); + if (input?.skipDefaultSetup !== true) { + await Effect.runPromise( + engine.dispatch({ + type: "project.create", + commandId: CommandId.make("cmd-project-create"), + projectId: asProjectId("project-1"), + title: "Provider Project", + workspaceRoot: "/tmp/provider-project", + defaultModelSelection: modelSelection, + createdAt: now, + }), + ); + await Effect.runPromise( + engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-thread-create"), + threadId: ThreadId.make("thread-1"), + projectId: asProjectId("project-1"), + title: "Thread", + modelSelection: modelSelection, + interactionMode: DEFAULT_PROVIDER_INTERACTION_MODE, + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + createdAt: now, + }), + ); + } return { engine, readModel: () => Effect.runPromise(snapshotQuery.getSnapshot()), + readTurns: (threadId: ThreadId) => + Effect.runPromise(turnRepository.listByThreadId({ threadId })), startSession, sendTurn, interruptTurn, @@ -432,8 +499,11 @@ describe("ProviderCommandReactor", () => { generateBranchName, generateThreadTitle, runtimeSessions, + directoryUpsert, + providerBindings, stateDir, drain, + startReactor, }; } @@ -586,6 +656,443 @@ describe("ProviderCommandReactor", () => { expect(thread?.session?.lastError).toBeNull(); }), ); + it("replays a persisted pending turn start exactly once on startup", async () => { + const harness = await createHarness({ deferReactorStart: true }); + const now = "2026-01-01T00:00:00.000Z"; + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.interaction-mode.set", + commandId: CommandId.make("cmd-plan-before-pending"), + threadId: ThreadId.make("thread-1"), + interactionMode: "plan", + createdAt: now, + }), + ); + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-pending-before-restart"), + threadId: ThreadId.make("thread-1"), + message: { + messageId: asMessageId("pending-user-message"), + role: "user", + text: "resume this exact pending request", + attachments: [], + }, + interactionMode: "plan", + runtimeMode: "approval-required", + createdAt: now, + }), + ); + expect(harness.sendTurn).not.toHaveBeenCalled(); + + await harness.startReactor(); + await harness.drain(); + await waitFor(() => harness.sendTurn.mock.calls.length === 1); + expect(harness.sendTurn.mock.calls[0]?.[0]).toMatchObject({ + threadId: ThreadId.make("thread-1"), + input: "resume this exact pending request", + interactionMode: "plan", + }); + await harness.startReactor(); + await harness.drain(); + expect(harness.sendTurn).toHaveBeenCalledTimes(1); + }); + + it("continues an interrupted running turn without replaying its user message", async () => { + const modelSelection: ModelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-recovery", + }; + const threadId = ThreadId.make("thread-1"); + const harness = await createHarness({ + deferReactorStart: true, + threadModelSelection: modelSelection, + providerBindings: [ + { + threadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + adapterKey: "codex", + runtimeMode: "full-access", + status: "stopped", + resumeCursor: { threadId: "provider-thread-resume" }, + runtimePayload: { + cwd: "/tmp/persisted-recovery-cwd", + modelSelection, + interactionMode: "plan", + activeTurnId: null, + restartRecovery: makeProviderRestartRecoveryMarker({ + interruptedProviderTurnId: asTurnId("provider-turn-before-restart"), + shutdownAt: "2026-01-01T00:00:01.000Z", + }), + }, + lastSeenAt: "2026-01-01T00:00:01.000Z", + }, + ], + }); + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-running-before-restart"), + threadId, + message: { + messageId: asMessageId("running-user-message"), + role: "user", + text: "the original request must not be replayed", + attachments: [], + }, + modelSelection, + interactionMode: "plan", + runtimeMode: "full-access", + createdAt: "2026-01-01T00:00:00.000Z", + }), + ); + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.session.set", + commandId: CommandId.make("server:running-before-restart"), + threadId, + session: { + threadId, + status: "running", + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "full-access", + activeTurnId: asTurnId("orchestration-turn-before-restart"), + lastError: null, + updatedAt: "2026-01-01T00:00:00.500Z", + }, + createdAt: "2026-01-01T00:00:00.500Z", + }), + ); + + await harness.startReactor(); + await harness.drain(); + await waitFor(() => harness.sendTurn.mock.calls.length === 1); + + expect(harness.startSession.mock.calls[0]?.[1]).toMatchObject({ + cwd: "/tmp/persisted-recovery-cwd", + modelSelection, + resumeCursor: { threadId: "provider-thread-resume" }, + runtimeMode: "full-access", + providerInstanceId: ProviderInstanceId.make("codex"), + }); + expect(harness.sendTurn.mock.calls[0]?.[0]).toEqual({ + threadId, + input: RESTART_RECOVERY_CONTINUATION_INSTRUCTION, + attachments: [], + modelSelection, + interactionMode: "plan", + }); + const readModel = await harness.readModel(); + const thread = readModel.threads.find((entry) => entry.id === threadId); + const oldTurn = (await harness.readTurns(threadId)).find( + (turn) => turn.turnId === asTurnId("orchestration-turn-before-restart"), + ); + expect(oldTurn?.state).toBe("interrupted"); + expect(thread?.messages.map((message) => message.text)).toEqual([ + "the original request must not be replayed", + ]); + expect(harness.providerBindings.get(threadId)?.runtimePayload).toMatchObject({ + restartRecovery: null, + interactionMode: "plan", + }); + }); + + it("recovers across temporary-SQLite runtimes and does not repeat a completed recovery", async () => { + const baseDir = NodeFS.mkdtempSync( + NodePath.join(NodeOS.tmpdir(), "t3code-reactor-two-runtime-"), + ); + const threadId = ThreadId.make("thread-1"); + const modelSelection: ModelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }; + const persistedBindings = new Map(); + const first = await createHarness({ baseDir, providerBindingsMap: persistedBindings }); + await Effect.runPromise( + first.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make("cmd-two-runtime-original-turn"), + threadId, + message: { + messageId: asMessageId("two-runtime-user-message"), + role: "user", + text: "finish this after the server restarts", + attachments: [], + }, + modelSelection, + interactionMode: "default", + runtimeMode: "approval-required", + createdAt: "2026-01-01T00:00:00.000Z", + }), + ); + await waitFor(() => first.sendTurn.mock.calls.length === 1); + await Effect.runPromise( + first.engine.dispatch({ + type: "thread.session.set", + commandId: CommandId.make("server:two-runtime-running"), + threadId, + session: { + threadId, + status: "running", + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "approval-required", + activeTurnId: asTurnId("orchestration-turn-two-runtime"), + lastError: null, + updatedAt: "2026-01-01T00:00:01.000Z", + }, + createdAt: "2026-01-01T00:00:01.000Z", + }), + ); + await Effect.runPromise( + first.directoryUpsert({ + threadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "approval-required", + status: "stopped", + resumeCursor: { threadId: "durable-provider-thread" }, + runtimePayload: { + cwd: "/tmp/provider-project", + modelSelection, + interactionMode: "default", + restartRecovery: makeProviderRestartRecoveryMarker({ + interruptedProviderTurnId: asTurnId("provider-turn-two-runtime"), + shutdownAt: "2026-01-01T00:00:02.000Z", + }), + }, + }), + ); + + await Effect.runPromise(Scope.close(scope!, Exit.void)); + scope = null; + await runtime!.dispose(); + runtime = null; + + const second = await createHarness({ + baseDir, + providerBindingsMap: persistedBindings, + skipDefaultSetup: true, + }); + await second.drain(); + await waitFor(() => second.sendTurn.mock.calls.length === 1); + expect(second.startSession.mock.calls[0]?.[1]).toMatchObject({ + resumeCursor: { threadId: "durable-provider-thread" }, + cwd: "/tmp/provider-project", + modelSelection, + runtimeMode: "approval-required", + }); + expect(second.sendTurn.mock.calls[0]?.[0]).toMatchObject({ + input: RESTART_RECOVERY_CONTINUATION_INSTRUCTION, + }); + const secondReadModel = await second.readModel(); + expect( + secondReadModel.threads + .find((thread) => thread.id === threadId) + ?.messages.map((message) => message.text), + ).toEqual(["finish this after the server restarts"]); + + await Effect.runPromise( + second.directoryUpsert({ + threadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "approval-required", + status: "running", + runtimePayload: { activeTurnId: null, restartRecovery: null }, + }), + ); + await Effect.runPromise( + second.engine.dispatch({ + type: "thread.session.set", + commandId: CommandId.make("server:two-runtime-recovery-complete"), + threadId, + session: { + threadId, + status: "ready", + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "approval-required", + activeTurnId: null, + lastError: null, + updatedAt: "2026-01-01T00:00:03.000Z", + }, + createdAt: "2026-01-01T00:00:03.000Z", + }), + ); + + await Effect.runPromise(Scope.close(scope!, Exit.void)); + scope = null; + await runtime!.dispose(); + runtime = null; + + const third = await createHarness({ + baseDir, + providerBindingsMap: persistedBindings, + skipDefaultSetup: true, + }); + await third.drain(); + expect(third.sendTurn).not.toHaveBeenCalled(); + }); + + it("isolates restart recovery failures and continues unrelated threads", async () => { + const modelSelection: ModelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }; + const failedThreadId = ThreadId.make("thread-1"); + const healthyThreadId = ThreadId.make("thread-2"); + const marker = makeProviderRestartRecoveryMarker({ + interruptedProviderTurnId: asTurnId("provider-turn-before-restart"), + shutdownAt: "2026-01-01T00:00:01.000Z", + }); + const makeBinding = ( + threadId: ThreadId, + resumeCursor: unknown | null, + ): ProviderRuntimeBindingWithMetadata => ({ + threadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + adapterKey: "codex", + runtimeMode: "approval-required", + status: "stopped", + resumeCursor, + runtimePayload: { + cwd: "/tmp/provider-project", + modelSelection, + interactionMode: "default", + restartRecovery: marker, + }, + lastSeenAt: "2026-01-01T00:00:01.000Z", + }); + const harness = await createHarness({ + deferReactorStart: true, + providerBindings: [ + makeBinding(failedThreadId, null), + makeBinding(healthyThreadId, { threadId: "healthy-provider-thread" }), + ], + }); + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-create-recovery-thread-2"), + threadId: healthyThreadId, + projectId: asProjectId("project-1"), + title: "Healthy recovery", + modelSelection, + interactionMode: "default", + runtimeMode: "approval-required", + branch: null, + worktreePath: null, + createdAt: "2026-01-01T00:00:00.000Z", + }), + ); + + await harness.startReactor(); + await harness.drain(); + expect(harness.sendTurn).toHaveBeenCalledTimes(1); + expect(harness.sendTurn.mock.calls[0]?.[0]).toMatchObject({ + threadId: healthyThreadId, + input: RESTART_RECOVERY_CONTINUATION_INSTRUCTION, + }); + + const readModel = await harness.readModel(); + const failedThread = readModel.threads.find((thread) => thread.id === failedThreadId); + expect(failedThread?.session?.status).toBe("error"); + expect( + failedThread?.activities.some( + (activity) => activity.kind === "provider.turn.recovery.failed", + ), + ).toBe(true); + }); + + it("surfaces a disabled provider instance without starting recovery work", async () => { + const threadId = ThreadId.make("thread-1"); + const modelSelection: ModelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }; + const harness = await createHarness({ + deferReactorStart: true, + providerInstanceEnabled: false, + providerBindings: [ + { + threadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "approval-required", + status: "stopped", + resumeCursor: { threadId: "disabled-provider-thread" }, + runtimePayload: { + modelSelection, + restartRecovery: makeProviderRestartRecoveryMarker({ + interruptedProviderTurnId: asTurnId("disabled-provider-turn"), + shutdownAt: "2026-01-01T00:00:01.000Z", + }), + }, + lastSeenAt: "2026-01-01T00:00:01.000Z", + }, + ], + }); + + await harness.startReactor(); + await harness.drain(); + expect(harness.startSession).not.toHaveBeenCalled(); + expect(harness.sendTurn).not.toHaveBeenCalled(); + const thread = (await harness.readModel()).threads.find((entry) => entry.id === threadId); + expect(thread?.session?.status).toBe("error"); + expect( + thread?.activities.find((activity) => activity.kind === "provider.turn.recovery.failed") + ?.payload, + ).toMatchObject({ detail: expect.stringContaining("disabled") }); + }); + + it("skips archived recovery candidates", async () => { + const threadId = ThreadId.make("thread-1"); + const modelSelection: ModelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }; + const harness = await createHarness({ + deferReactorStart: true, + providerBindings: [ + { + threadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "approval-required", + status: "stopped", + resumeCursor: { threadId: "archived-provider-thread" }, + runtimePayload: { + modelSelection, + restartRecovery: makeProviderRestartRecoveryMarker({ + interruptedProviderTurnId: asTurnId("archived-provider-turn"), + shutdownAt: "2026-01-01T00:00:01.000Z", + }), + }, + lastSeenAt: "2026-01-01T00:00:01.000Z", + }, + ], + }); + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.archive", + commandId: CommandId.make("cmd-archive-before-recovery"), + threadId, + }), + ); + + await harness.startReactor(); + await harness.drain(); + expect(harness.startSession).not.toHaveBeenCalled(); + expect(harness.sendTurn).not.toHaveBeenCalled(); + expect(harness.providerBindings.get(threadId)?.runtimePayload).toMatchObject({ + restartRecovery: expect.objectContaining({ version: 1 }), + }); + }); it("generates a thread title on the first turn", async () => { const harness = await createHarness(); diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index b6bff8c766a..c534a67ad7f 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -9,6 +9,7 @@ import { type OrchestrationSession, ThreadId, type ProviderSession, + type ProviderInteractionMode, type RuntimeMode, type TurnId, } from "@t3tools/contracts"; @@ -16,6 +17,8 @@ import { isTemporaryWorktreeBranch, WORKTREE_BRANCH_PREFIX } from "@t3tools/shar import * as Cache from "effect/Cache"; import * as Cause from "effect/Cause"; import * as Crypto from "effect/Crypto"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Equal from "effect/Equal"; @@ -26,12 +29,29 @@ import * as Stream from "effect/Stream"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; import { resolveThreadWorkspaceCwd } from "../../checkpointing/Utils.ts"; -import { increment, orchestrationEventsProcessedTotal } from "../../observability/Metrics.ts"; +import { + increment, + orchestrationEventsProcessedTotal, + providerTurnRecoveriesTotal, +} from "../../observability/Metrics.ts"; import { ProviderAdapterRequestError } from "../../provider/Errors.ts"; import type { ProviderServiceError } from "../../provider/Errors.ts"; +import { + readPersistedProviderCwd, + readPersistedProviderInteractionMode, + readPersistedProviderModelSelection, + readProviderRestartRecoveryCandidate, + type ProviderRestartRecoveryCandidate, +} from "../../provider/ProviderRestartRecovery.ts"; import { TextGeneration } from "../../textGeneration/TextGeneration.ts"; import { ProviderService } from "../../provider/Services/ProviderService.ts"; import { ProviderRegistry } from "../../provider/Services/ProviderRegistry.ts"; +import { + ProviderSessionDirectory, + type ProviderRuntimeBindingWithMetadata, +} from "../../provider/Services/ProviderSessionDirectory.ts"; +import { ProjectionTurnRepository } from "../../persistence/Services/ProjectionTurns.ts"; +import { ProjectionTurnRepositoryLive } from "../../persistence/Layers/ProjectionTurns.ts"; import { OrchestrationEngineService } from "../Services/OrchestrationEngine.ts"; import { ProjectionSnapshotQuery } from "../Services/ProjectionSnapshotQuery.ts"; import { @@ -87,6 +107,10 @@ const HANDLED_TURN_START_KEY_MAX = 10_000; const HANDLED_TURN_START_KEY_TTL = Duration.minutes(30); const DEFAULT_RUNTIME_MODE: RuntimeMode = "full-access"; const DEFAULT_THREAD_TITLE = "New thread"; +const STARTUP_RECOVERY_CONCURRENCY = 4; + +export const RESTART_RECOVERY_CONTINUATION_INSTRUCTION = + "The server restarted while you were working. Inspect the conversation and current workspace state, verify which side effects from the interrupted turn already happened, and continue the unfinished work safely. Do not repeat completed work or assume an earlier tool call failed merely because its response is absent."; export function providerErrorLabel(value: string | undefined): string { const normalized = value?.trim(); @@ -190,7 +214,9 @@ const make = Effect.gen(function* () { const crypto = yield* Crypto.Crypto; const orchestrationEngine = yield* OrchestrationEngineService; const projectionSnapshotQuery = yield* ProjectionSnapshotQuery; + const projectionTurnRepository = yield* ProjectionTurnRepository; const providerService = yield* ProviderService; + const providerSessionDirectory = yield* ProviderSessionDirectory; const providerRegistry = yield* ProviderRegistry; const gitWorkflow = yield* GitWorkflowService; const vcsStatusBroadcaster = yield* VcsStatusBroadcaster; @@ -204,6 +230,8 @@ const make = Effect.gen(function* () { timeToLive: HANDLED_TURN_START_KEY_TTL, lookup: () => Effect.succeed(true), }); + const startupReconciliationDone = yield* Deferred.make(); + const recoveredThreadIds = new Set(); const hasHandledTurnStartRecently = (key: string) => Cache.getOption(handledTurnStartKeys, key).pipe( @@ -218,6 +246,7 @@ const make = Effect.gen(function* () { readonly threadId: ThreadId; readonly kind: | "provider.turn.start.failed" + | "provider.turn.recovery.failed" | "provider.turn.interrupt.failed" | "provider.approval.respond.failed" | "provider.user-input.respond.failed" @@ -1026,6 +1055,269 @@ const make = Effect.gen(function* () { }); }); + const setRecoveryFailureState = Effect.fn("setRecoveryFailureState")(function* (input: { + readonly binding: ProviderRuntimeBindingWithMetadata; + readonly detail: string; + readonly createdAt: string; + }) { + const thread = yield* resolveThread(input.binding.threadId); + if (!thread) return; + yield* setThreadSession({ + threadId: thread.id, + session: { + threadId: thread.id, + status: "error", + providerName: input.binding.provider, + ...(input.binding.providerInstanceId !== undefined + ? { providerInstanceId: input.binding.providerInstanceId } + : {}), + runtimeMode: input.binding.runtimeMode ?? thread.runtimeMode, + activeTurnId: null, + lastError: input.detail, + updatedAt: input.createdAt, + }, + createdAt: input.createdAt, + }); + }); + + const recoverInterruptedTurn = Effect.fn("recoverInterruptedTurn")(function* (input: { + readonly binding: ProviderRuntimeBindingWithMetadata; + readonly candidate: ProviderRestartRecoveryCandidate; + }) { + const { binding, candidate } = input; + if (recoveredThreadIds.has(binding.threadId)) { + yield* increment(providerTurnRecoveriesTotal, { + outcome: "skipped", + reason: "duplicate-in-boot", + provider: binding.provider, + }); + return; + } + recoveredThreadIds.add(binding.threadId); + + const thread = yield* resolveThread(binding.threadId); + if (!thread) { + yield* Effect.logInfo("provider turn restart recovery skipped", { + threadId: binding.threadId, + provider: binding.provider, + reason: "thread-missing-archived-or-deleted", + }); + yield* increment(providerTurnRecoveriesTotal, { + outcome: "skipped", + reason: "inactive-thread", + provider: binding.provider, + }); + return; + } + + const createdAt = DateTime.formatIso(yield* DateTime.now); + const recover = Effect.gen(function* () { + const runtimeMode = binding.runtimeMode ?? thread.runtimeMode; + // This lifecycle transition settles the concrete old projection row as + // interrupted before validation or replacement work can proceed. + yield* setThreadSession({ + threadId: thread.id, + session: { + threadId: thread.id, + status: "interrupted", + providerName: binding.provider, + ...(binding.providerInstanceId !== undefined + ? { providerInstanceId: binding.providerInstanceId } + : {}), + runtimeMode, + activeTurnId: null, + lastError: null, + updatedAt: createdAt, + }, + createdAt, + }); + + const providerInstanceId = binding.providerInstanceId; + if (providerInstanceId === undefined) { + return yield* new ProviderAdapterRequestError({ + provider: binding.provider, + method: "provider.turn.restart-recovery", + detail: `Persisted provider binding for thread '${binding.threadId}' has no provider instance id.`, + }); + } + if (binding.resumeCursor === null || binding.resumeCursor === undefined) { + return yield* new ProviderAdapterRequestError({ + provider: binding.provider, + method: "provider.turn.restart-recovery", + detail: `Cannot recover thread '${binding.threadId}' because no provider resume cursor is persisted.`, + }); + } + + const instanceInfo = yield* providerService.getInstanceInfo(providerInstanceId); + if (!instanceInfo.enabled) { + return yield* new ProviderAdapterRequestError({ + provider: binding.provider, + method: "provider.turn.restart-recovery", + detail: `Provider instance '${providerInstanceId}' is disabled in T3 Code settings.`, + }); + } + if (instanceInfo.driverKind !== binding.provider) { + return yield* new ProviderAdapterRequestError({ + provider: binding.provider, + method: "provider.turn.restart-recovery", + detail: `Persisted provider instance '${providerInstanceId}' now uses driver '${instanceInfo.driverKind}', not '${binding.provider}'.`, + }); + } + + const persistedModelSelection = readPersistedProviderModelSelection(binding.runtimePayload); + const modelSelection = persistedModelSelection ?? thread.modelSelection; + if (modelSelection.instanceId !== providerInstanceId) { + return yield* new ProviderAdapterRequestError({ + provider: binding.provider, + method: "provider.turn.restart-recovery", + detail: `Persisted model selection references provider instance '${modelSelection.instanceId}', but the recoverable session belongs to '${providerInstanceId}'.`, + }); + } + const interactionMode: ProviderInteractionMode = + readPersistedProviderInteractionMode(binding.runtimePayload) ?? thread.interactionMode; + const project = yield* resolveProject(thread.projectId); + const cwd = + readPersistedProviderCwd(binding.runtimePayload) ?? + resolveThreadWorkspaceCwd({ thread, projects: project ? [project] : [] }); + + const session = yield* providerService.startSession(thread.id, { + threadId: thread.id, + provider: binding.provider, + providerInstanceId, + ...(cwd !== undefined ? { cwd } : {}), + modelSelection, + resumeCursor: binding.resumeCursor, + runtimeMode, + }); + yield* setThreadSession({ + threadId: thread.id, + session: { + threadId: thread.id, + status: mapProviderSessionStatusToOrchestrationStatus(session.status), + providerName: session.provider, + providerInstanceId, + runtimeMode, + activeTurnId: null, + lastError: session.lastError ?? null, + updatedAt: session.updatedAt, + }, + createdAt, + }); + + const replacement = yield* providerService.sendTurn({ + threadId: thread.id, + input: RESTART_RECOVERY_CONTINUATION_INSTRUCTION, + attachments: [], + modelSelection, + interactionMode, + }); + + // ProviderService clears this in the accepted sendTurn transaction. The + // explicit write keeps the reconciliation invariant local and obvious. + yield* providerSessionDirectory.upsert({ + threadId: thread.id, + provider: binding.provider, + providerInstanceId, + runtimeMode, + status: "running", + ...(replacement.resumeCursor !== undefined + ? { resumeCursor: replacement.resumeCursor } + : {}), + runtimePayload: { + activeTurnId: replacement.turnId, + modelSelection, + interactionMode, + restartRecovery: null, + lastRuntimeEvent: "provider.restartRecovery.accepted", + lastRuntimeEventAt: DateTime.formatIso(yield* DateTime.now), + }, + }); + + yield* Effect.logInfo("provider turn restart recovery accepted", { + threadId: thread.id, + provider: binding.provider, + providerInstanceId, + interruptedProviderTurnId: candidate.interruptedProviderTurnId, + replacementProviderTurnId: replacement.turnId, + recoverySource: candidate.source, + }); + yield* increment(providerTurnRecoveriesTotal, { + outcome: "continued", + source: candidate.source, + provider: binding.provider, + }); + }); + + yield* recover.pipe( + Effect.catchCause((cause) => { + const detail = formatFailureDetail(cause); + const persistFailureState = + binding.providerInstanceId === undefined + ? Effect.void + : providerSessionDirectory + .upsert({ + threadId: binding.threadId, + provider: binding.provider, + providerInstanceId: binding.providerInstanceId, + ...(binding.runtimeMode !== undefined + ? { runtimeMode: binding.runtimeMode } + : {}), + status: "error", + runtimePayload: { + activeTurnId: null, + lastError: detail, + lastRuntimeEvent: "provider.restartRecovery.failed", + lastRuntimeEventAt: createdAt, + }, + }) + .pipe( + Effect.catchCause((persistenceCause) => + Effect.logWarning("provider restart recovery failure state was not persisted", { + threadId: binding.threadId, + cause: Cause.pretty(persistenceCause), + }), + ), + ); + return persistFailureState.pipe( + Effect.andThen(setRecoveryFailureState({ binding, detail, createdAt })), + Effect.andThen( + appendProviderFailureActivity({ + threadId: binding.threadId, + kind: "provider.turn.recovery.failed", + summary: "Provider turn recovery failed", + detail, + turnId: thread.latestTurn?.turnId ?? null, + createdAt, + }), + ), + Effect.andThen( + Effect.logWarning("provider turn restart recovery failed", { + threadId: binding.threadId, + provider: binding.provider, + providerInstanceId: binding.providerInstanceId, + recoverySource: candidate.source, + cause: Cause.pretty(cause), + }), + ), + Effect.andThen( + increment(providerTurnRecoveriesTotal, { + outcome: "failed", + source: candidate.source, + provider: binding.provider, + }), + ), + Effect.catchCause((reportingCause) => + Effect.logWarning("provider turn restart recovery failure reporting failed", { + threadId: binding.threadId, + cause: Cause.pretty(reportingCause), + originalCause: Cause.pretty(cause), + }), + ), + ); + }), + ); + }); + const processDomainEvent = Effect.fn("processDomainEvent")(function* ( event: ProviderIntentEvent, ) { @@ -1084,6 +1376,139 @@ const make = Effect.gen(function* () { const worker = yield* makeDrainableWorker(processDomainEventSafely); + const reconcileStartup = Effect.fn("reconcileStartup")(function* () { + const bindings = yield* providerSessionDirectory.listBindings().pipe( + Effect.catchCause((cause) => + Effect.logWarning("provider restart recovery failed to list persisted bindings", { + cause: Cause.pretty(cause), + }).pipe(Effect.as([] as ReadonlyArray)), + ), + ); + const recoveryCandidates = bindings.flatMap((binding) => { + const candidate = readProviderRestartRecoveryCandidate({ + runtimePayload: binding.runtimePayload, + status: binding.status, + lastSeenAt: binding.lastSeenAt, + }); + return candidate === undefined ? [] : [{ binding, candidate }]; + }); + const recoveryCandidateThreadIds = new Set( + recoveryCandidates.map(({ binding }) => binding.threadId), + ); + + const pendingTurnStarts = yield* projectionTurnRepository.listPendingTurnStarts().pipe( + Effect.catchCause((cause) => + Effect.logWarning("provider restart recovery failed to list pending turn starts", { + cause: Cause.pretty(cause), + }).pipe(Effect.as([])), + ), + ); + + yield* Effect.logInfo("provider restart reconciliation candidates loaded", { + interruptedTurnCandidates: recoveryCandidates.length, + pendingTurnStartCandidates: pendingTurnStarts.length, + }); + if (recoveryCandidates.length > 0) { + yield* increment( + providerTurnRecoveriesTotal, + { outcome: "candidate", recoveryKind: "interrupted-turn" }, + recoveryCandidates.length, + ); + } + + yield* Effect.forEach(recoveryCandidates, recoverInterruptedTurn, { + concurrency: STARTUP_RECOVERY_CONCURRENCY, + discard: true, + }); + + const pendingWithoutAcceptedProviderTurn = pendingTurnStarts.filter( + (pending) => !recoveryCandidateThreadIds.has(pending.threadId), + ); + if (pendingWithoutAcceptedProviderTurn.length === 0) return; + + const persistedEvents = yield* Stream.runCollect( + orchestrationEngine.readEvents(0, Number.MAX_SAFE_INTEGER), + ).pipe( + Effect.map((events) => Array.from(events)), + Effect.catchCause((cause) => + Effect.logWarning("provider restart recovery failed to read persisted turn starts", { + cause: Cause.pretty(cause), + }).pipe(Effect.as([] as ReadonlyArray)), + ), + ); + const turnStartEventsByPendingKey = new Map< + string, + Extract + >(); + for (const event of persistedEvents) { + if (event.type !== "thread.turn-start-requested") continue; + turnStartEventsByPendingKey.set( + `${event.payload.threadId}\u0000${event.payload.messageId}\u0000${event.payload.createdAt}`, + event, + ); + } + + yield* Effect.forEach( + pendingWithoutAcceptedProviderTurn, + (pending) => + Effect.gen(function* () { + const thread = yield* resolveThread(pending.threadId); + if (!thread) { + yield* Effect.logInfo("pending provider turn start reconciliation skipped", { + threadId: pending.threadId, + messageId: pending.messageId, + reason: "thread-missing-archived-or-deleted", + }); + yield* increment(providerTurnRecoveriesTotal, { + outcome: "skipped", + recoveryKind: "pending-start", + reason: "inactive-thread", + }); + return; + } + const event = turnStartEventsByPendingKey.get( + `${pending.threadId}\u0000${pending.messageId}\u0000${pending.requestedAt}`, + ); + if (event === undefined) { + yield* appendProviderFailureActivity({ + threadId: pending.threadId, + kind: "provider.turn.start.failed", + summary: "Provider turn start recovery failed", + detail: `Persisted turn start event for user message '${pending.messageId}' could not be found.`, + turnId: null, + createdAt: DateTime.formatIso(yield* DateTime.now), + }); + yield* increment(providerTurnRecoveriesTotal, { + outcome: "failed", + recoveryKind: "pending-start", + reason: "event-missing", + }); + return; + } + + yield* worker.enqueue(event); + yield* Effect.logInfo("pending provider turn start replay enqueued", { + threadId: pending.threadId, + messageId: pending.messageId, + eventId: event.eventId, + }); + yield* increment(providerTurnRecoveriesTotal, { + outcome: "replayed", + recoveryKind: "pending-start", + }); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("pending provider turn start reconciliation failed", { + threadId: pending.threadId, + messageId: pending.messageId, + cause: Cause.pretty(cause), + }), + ), + ), + { concurrency: STARTUP_RECOVERY_CONCURRENCY, discard: true }, + ); + }); + const start: ProviderCommandReactorShape["start"] = Effect.fn("start")(function* () { const processEvent = Effect.fn("processEvent")(function* (event: OrchestrationEvent) { if ( @@ -1101,12 +1526,23 @@ const make = Effect.gen(function* () { yield* Effect.forkScoped( Stream.runForEach(orchestrationEngine.streamDomainEvents, processEvent), ); + yield* reconcileStartup().pipe( + Effect.catchCause((cause) => + Effect.logWarning("provider restart reconciliation failed", { + cause: Cause.pretty(cause), + }), + ), + Effect.ensuring(Deferred.succeed(startupReconciliationDone, undefined).pipe(Effect.ignore)), + Effect.forkScoped, + ); }); return { start, - drain: worker.drain, + drain: Deferred.await(startupReconciliationDone).pipe(Effect.andThen(worker.drain)), } satisfies ProviderCommandReactorShape; }); -export const ProviderCommandReactorLive = Layer.effect(ProviderCommandReactor, make); +export const ProviderCommandReactorLive = Layer.effect(ProviderCommandReactor, make).pipe( + Layer.provide(ProjectionTurnRepositoryLive), +); diff --git a/apps/server/src/persistence/Layers/ProjectionTurns.ts b/apps/server/src/persistence/Layers/ProjectionTurns.ts index bd57a4eaa30..bc59d3ed45f 100644 --- a/apps/server/src/persistence/Layers/ProjectionTurns.ts +++ b/apps/server/src/persistence/Layers/ProjectionTurns.ts @@ -169,6 +169,26 @@ const makeProjectionTurnRepository = Effect.gen(function* () { `, }); + const listPendingProjectionTurns = SqlSchema.findAll({ + Request: Schema.Void, + Result: ProjectionPendingTurnStart, + execute: () => + sql` + SELECT + thread_id AS "threadId", + pending_message_id AS "messageId", + source_proposed_plan_thread_id AS "sourceProposedPlanThreadId", + source_proposed_plan_id AS "sourceProposedPlanId", + requested_at AS "requestedAt" + FROM projection_turns + WHERE turn_id IS NULL + AND state = 'pending' + AND pending_message_id IS NOT NULL + AND checkpoint_turn_count IS NULL + ORDER BY requested_at ASC, thread_id ASC + `, + }); + const listProjectionTurnsByThread = SqlSchema.findAll({ Request: ListProjectionTurnsByThreadInput, Result: ProjectionTurnDbRowSchema, @@ -288,6 +308,13 @@ const makeProjectionTurnRepository = Effect.gen(function* () { ), ); + const listPendingTurnStarts: ProjectionTurnRepositoryShape["listPendingTurnStarts"] = () => + listPendingProjectionTurns(undefined).pipe( + Effect.mapError( + toPersistenceSqlError("ProjectionTurnRepository.listPendingTurnStarts:query"), + ), + ); + const deletePendingTurnStartByThreadId: ProjectionTurnRepositoryShape["deletePendingTurnStartByThreadId"] = (input) => clearPendingProjectionTurnsByThread(input).pipe( @@ -341,6 +368,7 @@ const makeProjectionTurnRepository = Effect.gen(function* () { upsertByTurnId, replacePendingTurnStart, getPendingTurnStartByThreadId, + listPendingTurnStarts, deletePendingTurnStartByThreadId, listByThreadId, getByTurnId, diff --git a/apps/server/src/persistence/Services/ProjectionTurns.ts b/apps/server/src/persistence/Services/ProjectionTurns.ts index f3d5d5e4706..faf70bce24a 100644 --- a/apps/server/src/persistence/Services/ProjectionTurns.ts +++ b/apps/server/src/persistence/Services/ProjectionTurns.ts @@ -128,6 +128,15 @@ export interface ProjectionTurnRepositoryShape { input: GetProjectionPendingTurnStartInput, ) => Effect.Effect, ProjectionRepositoryError>; + /** + * Lists every pending-start placeholder across active reconciliation scope. + * Callers must still resolve the owning thread and discard archived/deleted rows. + */ + readonly listPendingTurnStarts: () => Effect.Effect< + ReadonlyArray, + ProjectionRepositoryError + >; + /** * Deletes only pending-start placeholder rows (`turnId = null`) for a thread and leaves concrete turn rows untouched. */ diff --git a/apps/server/src/provider/Layers/ProviderService.test.ts b/apps/server/src/provider/Layers/ProviderService.test.ts index ccbbce1759f..1fb1cd92c7a 100644 --- a/apps/server/src/provider/Layers/ProviderService.test.ts +++ b/apps/server/src/provider/Layers/ProviderService.test.ts @@ -58,6 +58,7 @@ import { import * as ServerSettings from "../../serverSettings.ts"; import * as AnalyticsService from "../../telemetry/AnalyticsService.ts"; import { makeAdapterRegistryMock } from "../testUtils/providerAdapterRegistryMock.ts"; +import { readProviderRestartRecoveryMarker } from "../ProviderRestartRecovery.ts"; const defaultServerSettingsLayer = ServerSettings.ServerSettingsService.layerTest(); @@ -366,6 +367,94 @@ it.effect("ProviderServiceLive catches stopAll failures during shutdown", () => }), ); +it.effect("graceful shutdown preserves recovery intent only for working sessions", () => + Effect.gen(function* () { + const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-recovery-")); + const dbPath = NodePath.join(tempDir, "runtime.sqlite"); + const persistenceLayer = makeSqlitePersistenceLive(dbPath); + const runtimeRepositoryLayer = ProviderSessionRuntime.layer.pipe( + Layer.provide(persistenceLayer), + ); + const directoryLayer = ProviderSessionDirectoryLive.pipe(Layer.provide(runtimeRepositoryLayer)); + const codex = makeFakeCodexAdapter(); + const providerLayer = makeProviderServiceLive().pipe( + Layer.provide( + Layer.succeed( + ProviderAdapterRegistry.ProviderAdapterRegistry, + makeAdapterRegistryMock({ [CODEX_DRIVER]: codex.adapter }), + ), + ), + Layer.provide(directoryLayer), + Layer.provide(defaultServerSettingsLayer), + Layer.provide(AnalyticsService.layerTest), + Layer.provide( + Layer.succeed( + ProviderEventLoggers.ProviderEventLoggers, + ProviderEventLoggers.NoOpProviderEventLoggers, + ), + ), + ); + const scope = yield* Scope.make(); + const services = yield* Layer.build( + Layer.mergeAll(providerLayer, runtimeRepositoryLayer, directoryLayer), + ).pipe(Scope.provide(scope)); + const provider = yield* ProviderService.ProviderService.pipe(Effect.provide(services)); + + const runningThreadId = asThreadId("thread-running-on-shutdown"); + const connectingThreadId = asThreadId("thread-connecting-on-shutdown"); + const readyThreadId = asThreadId("thread-ready-on-shutdown"); + const stoppedThreadId = asThreadId("thread-explicitly-stopped"); + for (const threadId of [runningThreadId, connectingThreadId, readyThreadId, stoppedThreadId]) { + yield* provider.startSession(threadId, { + threadId, + provider: CODEX_DRIVER, + providerInstanceId: codexInstanceId, + runtimeMode: "full-access", + }); + } + yield* provider.sendTurn({ + threadId: runningThreadId, + input: "keep working", + interactionMode: "plan", + }); + codex.updateSession(runningThreadId, (session) => ({ + ...session, + status: "running", + activeTurnId: asTurnId("provider-turn-running"), + })); + codex.updateSession(connectingThreadId, (session) => ({ + ...session, + status: "connecting", + })); + + yield* provider.stopSession({ threadId: stoppedThreadId }); + yield* Scope.close(scope, Exit.void); + + const rows = yield* Effect.gen(function* () { + const repository = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + return yield* repository.list(); + }).pipe(Effect.provide(runtimeRepositoryLayer)); + const byThreadId = new Map(rows.map((row) => [row.threadId, row])); + + const runningMarker = readProviderRestartRecoveryMarker( + byThreadId.get(runningThreadId)?.runtimePayload, + ); + assert.equal(runningMarker?.interruptedProviderTurnId, asTurnId("provider-turn-running")); + assert.isDefined( + readProviderRestartRecoveryMarker(byThreadId.get(connectingThreadId)?.runtimePayload), + ); + assert.isUndefined( + readProviderRestartRecoveryMarker(byThreadId.get(readyThreadId)?.runtimePayload), + ); + assert.isUndefined( + readProviderRestartRecoveryMarker(byThreadId.get(stoppedThreadId)?.runtimePayload), + ); + assert.equal(byThreadId.get(runningThreadId)?.status, "stopped"); + + NodeFS.rmSync(tempDir, { recursive: true, force: true }); + }).pipe(Effect.provide(NodeServices.layer)), +); + it.effect("ProviderServiceLive rejects new sessions for disabled providers", () => Effect.gen(function* () { const codex = makeFakeCodexAdapter(); @@ -1499,6 +1588,7 @@ fanout.layer("ProviderServiceLive fanout", (it) => { threadId: asThreadId("thread-1"), runtimeMode: "full-access", }); + yield* provider.sendTurn({ threadId: session.threadId, input: "hello" }); const eventsRef = yield* Ref.make>([]); const consumer = yield* Stream.runForEach(provider.streamEvents, (event) => @@ -1512,7 +1602,7 @@ fanout.layer("ProviderServiceLive fanout", (it) => { provider: ProviderDriverKind.make("codex"), createdAt: "2026-01-01T00:00:00.000Z", threadId: session.threadId, - turnId: asTurnId("turn-1"), + turnId: asTurnId("turn-thread-1"), status: "completed", }; @@ -1533,6 +1623,18 @@ fanout.layer("ProviderServiceLive fanout", (it) => { ), true, ); + const runtimeRepository = yield* ProviderSessionRuntime.ProviderSessionRuntimeRepository; + const persistedRuntime = yield* runtimeRepository.getByThreadId({ + threadId: session.threadId, + }); + assert.equal(Option.isSome(persistedRuntime), true); + if (Option.isSome(persistedRuntime)) { + assert.equal(persistedRuntime.value.status, "running"); + assert.deepInclude(persistedRuntime.value.runtimePayload, { + activeTurnId: null, + lastRuntimeEvent: "turn.completed", + }); + } }), ); diff --git a/apps/server/src/provider/Layers/ProviderService.ts b/apps/server/src/provider/Layers/ProviderService.ts index 2eaaeb8ce3c..461a1eb9ff7 100644 --- a/apps/server/src/provider/Layers/ProviderService.ts +++ b/apps/server/src/provider/Layers/ProviderService.ts @@ -10,7 +10,6 @@ * @module ProviderServiceLive */ import { - ModelSelection, NonNegativeInt, ThreadId, ProviderInterruptTurnInput, @@ -55,7 +54,12 @@ import * as ProviderEventLoggers from "./ProviderEventLoggers.ts"; import * as AnalyticsService from "../../telemetry/AnalyticsService.ts"; import * as McpProviderSession from "../../mcp/McpProviderSession.ts"; import * as McpSessionRegistry from "../../mcp/McpSessionRegistry.ts"; -const isModelSelection = Schema.is(ModelSelection); +import { + makeProviderRestartRecoveryMarker, + readPersistedProviderActiveTurnId, + readPersistedProviderCwd, + readPersistedProviderModelSelection, +} from "../ProviderRestartRecovery.ts"; /** * Hook for tests that want to override the canonical event logger pulled @@ -123,8 +127,10 @@ function toRuntimePayloadFromSession( session: ProviderSession, extra?: { readonly modelSelection?: unknown; + readonly interactionMode?: unknown; readonly lastRuntimeEvent?: string; readonly lastRuntimeEventAt?: string; + readonly restartRecovery?: unknown; }, ): Record { return { @@ -133,35 +139,15 @@ function toRuntimePayloadFromSession( activeTurnId: session.activeTurnId ?? null, lastError: session.lastError ?? null, ...(extra?.modelSelection !== undefined ? { modelSelection: extra.modelSelection } : {}), + ...(extra?.interactionMode !== undefined ? { interactionMode: extra.interactionMode } : {}), ...(extra?.lastRuntimeEvent !== undefined ? { lastRuntimeEvent: extra.lastRuntimeEvent } : {}), ...(extra?.lastRuntimeEventAt !== undefined ? { lastRuntimeEventAt: extra.lastRuntimeEventAt } : {}), + ...(extra?.restartRecovery !== undefined ? { restartRecovery: extra.restartRecovery } : {}), }; } -function readPersistedModelSelection( - runtimePayload: ProviderSessionDirectory.ProviderRuntimeBinding["runtimePayload"], -): ModelSelection | undefined { - if (!runtimePayload || typeof runtimePayload !== "object" || Array.isArray(runtimePayload)) { - return undefined; - } - const raw = "modelSelection" in runtimePayload ? runtimePayload.modelSelection : undefined; - return isModelSelection(raw) ? raw : undefined; -} - -function readPersistedCwd( - runtimePayload: ProviderSessionDirectory.ProviderRuntimeBinding["runtimePayload"], -): string | undefined { - if (!runtimePayload || typeof runtimePayload !== "object" || Array.isArray(runtimePayload)) { - return undefined; - } - const rawCwd = "cwd" in runtimePayload ? runtimePayload.cwd : undefined; - if (typeof rawCwd !== "string") return undefined; - const trimmed = rawCwd.trim(); - return trimmed.length > 0 ? trimmed : undefined; -} - const dieOnMissingBindingInstanceId = ( operation: string, payload: { @@ -238,6 +224,73 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( Effect.asVoid, ); + const persistRuntimeEventState = Effect.fn("persistRuntimeEventState")(function* ( + event: ProviderRuntimeEvent, + ) { + const binding = Option.getOrUndefined(yield* directory.getBinding(event.threadId)); + if (!binding || event.providerInstanceId === undefined) return; + if (binding.providerInstanceId !== event.providerInstanceId) { + yield* Effect.logWarning("provider runtime event ignored for stale persisted instance", { + threadId: event.threadId, + eventType: event.type, + eventProviderInstanceId: event.providerInstanceId, + bindingProviderInstanceId: binding.providerInstanceId, + }); + return; + } + + const persistedActiveTurnId = readPersistedProviderActiveTurnId(binding.runtimePayload); + const lifecycle = (() => { + switch (event.type) { + case "turn.started": + return event.turnId === undefined + ? { status: "running" as const } + : { status: "running" as const, activeTurnId: event.turnId }; + case "turn.completed": + case "turn.aborted": + if ( + persistedActiveTurnId !== undefined && + (event.turnId === undefined || event.turnId !== persistedActiveTurnId) + ) { + return undefined; + } + return { status: "running" as const, activeTurnId: null }; + case "session.exited": + return { status: "stopped" as const, activeTurnId: null }; + case "session.state.changed": + switch (event.payload.state) { + case "starting": + return { status: "starting" as const }; + case "error": + return { status: "error" as const, activeTurnId: null }; + case "stopped": + return { status: "stopped" as const, activeTurnId: null }; + case "ready": + case "waiting": + return { status: "running" as const, activeTurnId: null }; + case "running": + return { status: "running" as const }; + } + default: + return undefined; + } + })(); + if (lifecycle === undefined) return; + + yield* directory.upsert({ + threadId: event.threadId, + provider: binding.provider, + providerInstanceId: event.providerInstanceId, + ...(binding.runtimeMode !== undefined ? { runtimeMode: binding.runtimeMode } : {}), + status: lifecycle.status, + runtimePayload: { + ...(lifecycle.activeTurnId !== undefined ? { activeTurnId: lifecycle.activeTurnId } : {}), + lastRuntimeEvent: event.type, + lastRuntimeEventAt: event.createdAt, + }, + }); + }); + const requireBindingInstanceId = ( operation: string, payload: { @@ -261,8 +314,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( threadId: ThreadId, extra?: { readonly modelSelection?: unknown; + readonly interactionMode?: unknown; readonly lastRuntimeEvent?: string; readonly lastRuntimeEventAt?: string; + readonly restartRecovery?: unknown; }, ) => Effect.gen(function* () { @@ -293,7 +348,20 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( increment(providerRuntimeEventsTotal, { provider: canonicalEvent.provider, eventType: canonicalEvent.type, - }).pipe(Effect.andThen(publishRuntimeEvent(canonicalEvent))), + }).pipe( + Effect.andThen( + persistRuntimeEventState(canonicalEvent).pipe( + Effect.catchCause((cause) => + Effect.logWarning("failed to persist provider runtime lifecycle event", { + threadId: canonicalEvent.threadId, + eventType: canonicalEvent.type, + cause, + }), + ), + ), + ), + Effect.andThen(publishRuntimeEvent(canonicalEvent)), + ), ), ); @@ -394,8 +462,10 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ); } - const persistedCwd = readPersistedCwd(input.binding.runtimePayload); - const persistedModelSelection = readPersistedModelSelection(input.binding.runtimePayload); + const persistedCwd = readPersistedProviderCwd(input.binding.runtimePayload); + const persistedModelSelection = readPersistedProviderModelSelection( + input.binding.runtimePayload, + ); yield* prepareMcpSession(input.binding.threadId, bindingInstanceId); const resumed = yield* adapter @@ -568,7 +638,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( const effectiveCwd = input.cwd ?? (persistedBinding?.providerInstanceId === resolvedInstanceId - ? readPersistedCwd(persistedBinding.runtimePayload) + ? readPersistedProviderCwd(persistedBinding.runtimePayload) : undefined); yield* Effect.annotateCurrentSpan({ "provider.kind": resolvedProvider, @@ -688,7 +758,11 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ...(turn.resumeCursor !== undefined ? { resumeCursor: turn.resumeCursor } : {}), runtimePayload: { ...(input.modelSelection !== undefined ? { modelSelection: input.modelSelection } : {}), + ...(input.interactionMode !== undefined + ? { interactionMode: input.interactionMode } + : {}), activeTurnId: turn.turnId, + restartRecovery: null, lastRuntimeEvent: "provider.sendTurn", lastRuntimeEventAt: yield* nowIso, }, @@ -857,6 +931,7 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( status: "stopped", runtimePayload: { activeTurnId: null, + restartRecovery: null, }, }); yield* analytics.record("provider.session.stopped", { @@ -1022,12 +1097,19 @@ const makeProviderService = Effect.fn("makeProviderService")(function* ( ), ).pipe(Effect.map((sessionsByAdapter) => sessionsByAdapter.flatMap((sessions) => sessions))); yield* Effect.forEach(activeSessions, (session) => - Effect.flatMap(nowIso, (lastRuntimeEventAt) => - upsertSessionBinding(session, session.threadId, { + Effect.flatMap(nowIso, (lastRuntimeEventAt) => { + const wasWorking = session.status === "connecting" || session.status === "running"; + return upsertSessionBinding(session, session.threadId, { lastRuntimeEvent: "provider.stopAll", lastRuntimeEventAt, - }), - ), + restartRecovery: wasWorking + ? makeProviderRestartRecoveryMarker({ + interruptedProviderTurnId: session.activeTurnId, + shutdownAt: lastRuntimeEventAt, + }) + : null, + }); + }), ).pipe(Effect.asVoid); yield* Effect.forEach(currentAdapters, ([, adapter]) => adapter.stopAll()).pipe(Effect.asVoid); yield* McpSessionRegistry.revokeAllActiveMcpCredentials(); diff --git a/apps/server/src/provider/ProviderRestartRecovery.test.ts b/apps/server/src/provider/ProviderRestartRecovery.test.ts new file mode 100644 index 00000000000..c7952f547f2 --- /dev/null +++ b/apps/server/src/provider/ProviderRestartRecovery.test.ts @@ -0,0 +1,73 @@ +import { ModelSelection, ProviderInstanceId, TurnId } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { + makeProviderRestartRecoveryMarker, + readPersistedProviderCwd, + readPersistedProviderInteractionMode, + readPersistedProviderModelSelection, + readProviderRestartRecoveryCandidate, +} from "./ProviderRestartRecovery.ts"; + +describe("ProviderRestartRecovery", () => { + it("reads typed recovery metadata and persisted restart settings", () => { + const modelSelection: ModelSelection = { + instanceId: ProviderInstanceId.make("codex-work"), + model: "gpt-5.4", + options: [{ id: "reasoningEffort", value: "high" }], + }; + const marker = makeProviderRestartRecoveryMarker({ + interruptedProviderTurnId: TurnId.make("turn-interrupted"), + shutdownAt: "2026-07-22T00:00:00.000Z", + }); + const runtimePayload = { + cwd: " /tmp/project ", + modelSelection, + interactionMode: "plan", + restartRecovery: marker, + }; + + expect( + readProviderRestartRecoveryCandidate({ + runtimePayload, + status: "stopped", + lastSeenAt: "2026-07-22T00:00:01.000Z", + }), + ).toEqual({ ...marker, source: "marker" }); + expect(readPersistedProviderCwd(runtimePayload)).toBe("/tmp/project"); + expect(readPersistedProviderModelSelection(runtimePayload)).toEqual(modelSelection); + expect(readPersistedProviderInteractionMode(runtimePayload)).toBe("plan"); + }); + + it("recognizes crash-style legacy running rows with an active turn", () => { + expect( + readProviderRestartRecoveryCandidate({ + runtimePayload: { activeTurnId: TurnId.make("turn-before-crash") }, + status: "running", + lastSeenAt: "2026-07-22T00:00:00.000Z", + }), + ).toEqual({ + version: 1, + interruptedProviderTurnId: TurnId.make("turn-before-crash"), + shutdownAt: "2026-07-22T00:00:00.000Z", + source: "legacy-active-turn", + }); + }); + + it("does not recover idle, stopped, or malformed legacy rows", () => { + expect( + readProviderRestartRecoveryCandidate({ + runtimePayload: { activeTurnId: TurnId.make("turn-ready") }, + status: "stopped", + lastSeenAt: "2026-07-22T00:00:00.000Z", + }), + ).toBeUndefined(); + expect( + readProviderRestartRecoveryCandidate({ + runtimePayload: { activeTurnId: null }, + status: "running", + lastSeenAt: "2026-07-22T00:00:00.000Z", + }), + ).toBeUndefined(); + }); +}); diff --git a/apps/server/src/provider/ProviderRestartRecovery.ts b/apps/server/src/provider/ProviderRestartRecovery.ts new file mode 100644 index 00000000000..9aa79efe582 --- /dev/null +++ b/apps/server/src/provider/ProviderRestartRecovery.ts @@ -0,0 +1,103 @@ +import { + IsoDateTime, + ModelSelection, + ProviderInteractionMode, + TurnId, + type ProviderSessionRuntimeStatus, +} from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; + +export const PROVIDER_RESTART_RECOVERY_PAYLOAD_KEY = "restartRecovery"; + +export const ProviderRestartRecoveryMarker = Schema.Struct({ + version: Schema.Literal(1), + interruptedProviderTurnId: Schema.NullOr(TurnId), + shutdownAt: IsoDateTime, +}); +export type ProviderRestartRecoveryMarker = typeof ProviderRestartRecoveryMarker.Type; + +export interface ProviderRestartRecoveryCandidate extends ProviderRestartRecoveryMarker { + readonly source: "marker" | "legacy-active-turn"; +} + +const isProviderRestartRecoveryMarker = Schema.is(ProviderRestartRecoveryMarker); +const isModelSelection = Schema.is(ModelSelection); +const isProviderInteractionMode = Schema.is(ProviderInteractionMode); +const isTurnId = Schema.is(TurnId); + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +export function makeProviderRestartRecoveryMarker(input: { + readonly interruptedProviderTurnId: TurnId | null | undefined; + readonly shutdownAt: string; +}): ProviderRestartRecoveryMarker { + return { + version: 1, + interruptedProviderTurnId: input.interruptedProviderTurnId ?? null, + shutdownAt: IsoDateTime.make(input.shutdownAt), + }; +} + +export function readProviderRestartRecoveryMarker( + runtimePayload: unknown, +): ProviderRestartRecoveryMarker | undefined { + if (!isRecord(runtimePayload)) return undefined; + const marker = runtimePayload[PROVIDER_RESTART_RECOVERY_PAYLOAD_KEY]; + return isProviderRestartRecoveryMarker(marker) ? marker : undefined; +} + +export function readProviderRestartRecoveryCandidate(input: { + readonly runtimePayload: unknown; + readonly status: ProviderSessionRuntimeStatus | undefined; + readonly lastSeenAt: string; +}): ProviderRestartRecoveryCandidate | undefined { + const marker = readProviderRestartRecoveryMarker(input.runtimePayload); + if (marker !== undefined) { + return { ...marker, source: "marker" }; + } + if (input.status !== "starting" && input.status !== "running") { + return undefined; + } + if (!isRecord(input.runtimePayload)) return undefined; + const activeTurnId = input.runtimePayload.activeTurnId; + if (!isTurnId(activeTurnId)) return undefined; + return { + version: 1, + interruptedProviderTurnId: activeTurnId, + shutdownAt: IsoDateTime.make(input.lastSeenAt), + source: "legacy-active-turn", + }; +} + +export function readPersistedProviderCwd(runtimePayload: unknown): string | undefined { + if (!isRecord(runtimePayload)) return undefined; + const cwd = runtimePayload.cwd; + if (typeof cwd !== "string") return undefined; + const trimmed = cwd.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +export function readPersistedProviderModelSelection( + runtimePayload: unknown, +): ModelSelection | undefined { + if (!isRecord(runtimePayload)) return undefined; + return isModelSelection(runtimePayload.modelSelection) + ? runtimePayload.modelSelection + : undefined; +} + +export function readPersistedProviderInteractionMode( + runtimePayload: unknown, +): ProviderInteractionMode | undefined { + if (!isRecord(runtimePayload)) return undefined; + return isProviderInteractionMode(runtimePayload.interactionMode) + ? runtimePayload.interactionMode + : undefined; +} + +export function readPersistedProviderActiveTurnId(runtimePayload: unknown): TurnId | undefined { + if (!isRecord(runtimePayload)) return undefined; + return isTurnId(runtimePayload.activeTurnId) ? runtimePayload.activeTurnId : undefined; +} From 1d52bce1376766d804ef884d7d50b8b6d1b48cf7 Mon Sep 17 00:00:00 2001 From: Tim Smart Date: Wed, 22 Jul 2026 16:21:22 +1200 Subject: [PATCH 2/3] Keep Codex event forwarding alive for the session - Scope the event-forwarding fiber to the session lifecycle - Cover events emitted after the start-session caller exits --- .../src/provider/Layers/CodexAdapter.test.ts | 44 +++++++++++++++++++ .../src/provider/Layers/CodexAdapter.ts | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/apps/server/src/provider/Layers/CodexAdapter.test.ts b/apps/server/src/provider/Layers/CodexAdapter.test.ts index 39f6250d921..e6046e35e97 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.test.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.test.ts @@ -585,6 +585,50 @@ function startLifecycleRuntime() { } lifecycleLayer("CodexAdapterLive lifecycle", (it) => { + it.effect("keeps forwarding events after the start-session caller exits", () => + Effect.gen(function* () { + const adapter = yield* CodexAdapter; + const threadId = asThreadId("thread-short-lived-start-caller"); + const startCaller = yield* adapter + .startSession({ + provider: ProviderDriverKind.make("codex"), + threadId, + runtimeMode: "full-access", + }) + .pipe(Effect.forkChild); + yield* Fiber.join(startCaller); + + const runtime = lifecycleRuntimeFactory.lastRuntime; + NodeAssert.ok(runtime); + const firstEventFiber = yield* Stream.runHead(adapter.streamEvents).pipe(Effect.forkChild); + + yield* runtime.emit({ + id: asEventId("evt-after-start-caller-exited"), + kind: "notification", + provider: ProviderDriverKind.make("codex"), + createdAt: "2026-01-01T00:00:00.000Z", + method: "turn/started", + threadId, + turnId: asTurnId("turn-after-start-caller-exited"), + payload: { + threadId: "provider-thread-1", + turn: { + id: "turn-after-start-caller-exited", + status: "inProgress", + items: [], + error: null, + }, + }, + } satisfies ProviderEvent); + + const firstEvent = yield* Fiber.join(firstEventFiber).pipe(Effect.timeout("1 second")); + NodeAssert.equal(firstEvent._tag, "Some"); + if (firstEvent._tag !== "Some") return; + NodeAssert.equal(firstEvent.value.type, "turn.started"); + NodeAssert.equal(firstEvent.value.turnId, "turn-after-start-caller-exited"); + }), + ); + it.effect("maps completed agent message items to canonical item.completed events", () => Effect.gen(function* () { const { adapter, runtime } = yield* startLifecycleRuntime(); diff --git a/apps/server/src/provider/Layers/CodexAdapter.ts b/apps/server/src/provider/Layers/CodexAdapter.ts index 130eace6151..9a46a8243ce 100644 --- a/apps/server/src/provider/Layers/CodexAdapter.ts +++ b/apps/server/src/provider/Layers/CodexAdapter.ts @@ -1485,7 +1485,7 @@ export const makeCodexAdapter = Effect.fn("makeCodexAdapter")(function* ( } yield* Queue.offerAll(runtimeEventQueue, runtimeEvents); }), - ).pipe(Effect.forkChild); + ).pipe(Effect.forkIn(sessionScope)); const started = yield* runtime.start().pipe( Effect.mapError( From b181832560177250b90bbfe07b0882c9e5b93493 Mon Sep 17 00:00:00 2001 From: Tim Smart Date: Wed, 22 Jul 2026 16:37:32 +1200 Subject: [PATCH 3/3] Skip stale restart recovery for settled turns - Clear stale recovery bindings when the projected turn is already settled - Preserve pending turn-start recovery unless restart recovery actually begins --- .../Layers/ProviderCommandReactor.test.ts | 141 ++++++++++++++++++ .../Layers/ProviderCommandReactor.ts | 63 +++++++- 2 files changed, 196 insertions(+), 8 deletions(-) diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts index 0c91efdf3c6..dca0d2188f4 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.test.ts @@ -800,6 +800,147 @@ describe("ProviderCommandReactor", () => { }); }); + it("does not resume settled turns from stale recovery bindings", async () => { + const modelSelection: ModelSelection = { + instanceId: ProviderInstanceId.make("codex"), + model: "gpt-5-codex", + }; + const markerThreadId = ThreadId.make("thread-1"); + const legacyThreadId = ThreadId.make("thread-settled-legacy"); + const markerTurnId = asTurnId("turn-settled-marker"); + const legacyTurnId = asTurnId("turn-settled-legacy"); + const harness = await createHarness({ + deferReactorStart: true, + providerBindings: [ + { + threadId: markerThreadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "full-access", + status: "stopped", + resumeCursor: { threadId: "provider-thread-marker" }, + runtimePayload: { + modelSelection, + restartRecovery: makeProviderRestartRecoveryMarker({ + interruptedProviderTurnId: markerTurnId, + shutdownAt: "2026-01-01T00:00:02.000Z", + }), + }, + lastSeenAt: "2026-01-01T00:00:02.000Z", + }, + { + threadId: legacyThreadId, + provider: ProviderDriverKind.make("codex"), + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "full-access", + status: "running", + resumeCursor: { threadId: "provider-thread-legacy" }, + runtimePayload: { + modelSelection, + activeTurnId: legacyTurnId, + }, + lastSeenAt: "2026-01-01T00:00:02.000Z", + }, + ], + }); + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.create", + commandId: CommandId.make("cmd-create-settled-legacy-thread"), + threadId: legacyThreadId, + projectId: asProjectId("project-1"), + title: "Settled legacy thread", + modelSelection, + interactionMode: "default", + runtimeMode: "full-access", + branch: null, + worktreePath: null, + createdAt: "2026-01-01T00:00:00.000Z", + }), + ); + + const settleTurn = async (threadId: ThreadId, turnId: TurnId, suffix: string) => { + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.turn.start", + commandId: CommandId.make(`cmd-settled-turn-${suffix}`), + threadId, + message: { + messageId: asMessageId(`message-settled-${suffix}`), + role: "user", + text: "already finished", + attachments: [], + }, + modelSelection, + interactionMode: "default", + runtimeMode: "full-access", + createdAt: "2026-01-01T00:00:00.500Z", + }), + ); + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.session.set", + commandId: CommandId.make(`server:settled-running-${suffix}`), + threadId, + session: { + threadId, + status: "running", + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "full-access", + activeTurnId: turnId, + lastError: null, + updatedAt: "2026-01-01T00:00:01.000Z", + }, + createdAt: "2026-01-01T00:00:01.000Z", + }), + ); + await Effect.runPromise( + harness.engine.dispatch({ + type: "thread.session.set", + commandId: CommandId.make(`server:settled-ready-${suffix}`), + threadId, + session: { + threadId, + status: "ready", + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "full-access", + activeTurnId: null, + lastError: null, + updatedAt: "2026-01-01T00:00:01.500Z", + }, + createdAt: "2026-01-01T00:00:01.500Z", + }), + ); + }; + + await settleTurn(markerThreadId, markerTurnId, "marker"); + await settleTurn(legacyThreadId, legacyTurnId, "legacy"); + expect( + (await harness.readTurns(markerThreadId)).find((turn) => turn.turnId === markerTurnId)?.state, + ).toBe("completed"); + expect( + (await harness.readTurns(legacyThreadId)).find((turn) => turn.turnId === legacyTurnId)?.state, + ).toBe("completed"); + + await harness.startReactor(); + await harness.drain(); + + expect(harness.startSession).not.toHaveBeenCalled(); + expect(harness.sendTurn).not.toHaveBeenCalled(); + for (const threadId of [markerThreadId, legacyThreadId]) { + expect(harness.providerBindings.get(threadId)).toMatchObject({ + status: "stopped", + runtimePayload: { + activeTurnId: null, + restartRecovery: null, + lastRuntimeEvent: "provider.restartRecovery.skipped", + }, + }); + } + }); + it("recovers across temporary-SQLite runtimes and does not repeat a completed recovery", async () => { const baseDir = NodeFS.mkdtempSync( NodePath.join(NodeOS.tmpdir(), "t3code-reactor-two-runtime-"), diff --git a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts index c534a67ad7f..28bbdaadfe4 100644 --- a/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts +++ b/apps/server/src/orchestration/Layers/ProviderCommandReactor.ts @@ -232,6 +232,7 @@ const make = Effect.gen(function* () { }); const startupReconciliationDone = yield* Deferred.make(); const recoveredThreadIds = new Set(); + const interruptedRecoveryThreadIds = new Set(); const hasHandledTurnStartRecently = (key: string) => Cache.getOption(handledTurnStartKeys, key).pipe( @@ -1111,6 +1112,56 @@ const make = Effect.gen(function* () { } const createdAt = DateTime.formatIso(yield* DateTime.now); + const projectedTurns = yield* projectionTurnRepository.listByThreadId({ + threadId: binding.threadId, + }); + const projectedCandidateTurn = + candidate.interruptedProviderTurnId === null + ? undefined + : projectedTurns.find((turn) => turn.turnId === candidate.interruptedProviderTurnId); + const latestProjectedTurn = projectedTurns.findLast((turn) => turn.turnId !== null); + const projectedRecoveryTurn = projectedCandidateTurn ?? latestProjectedTurn; + if (projectedRecoveryTurn?.state === "completed" || projectedRecoveryTurn?.state === "error") { + yield* providerSessionDirectory + .upsert({ + threadId: binding.threadId, + provider: binding.provider, + ...(binding.providerInstanceId !== undefined + ? { providerInstanceId: binding.providerInstanceId } + : {}), + ...(binding.runtimeMode !== undefined ? { runtimeMode: binding.runtimeMode } : {}), + status: "stopped", + runtimePayload: { + activeTurnId: null, + restartRecovery: null, + lastRuntimeEvent: "provider.restartRecovery.skipped", + lastRuntimeEventAt: createdAt, + }, + }) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning("stale provider restart recovery intent was not cleared", { + threadId: binding.threadId, + cause: Cause.pretty(cause), + }), + ), + ); + yield* Effect.logInfo("provider turn restart recovery skipped", { + threadId: binding.threadId, + provider: binding.provider, + reason: "projected-turn-settled", + projectedTurnId: projectedRecoveryTurn.turnId, + projectedTurnState: projectedRecoveryTurn.state, + }); + yield* increment(providerTurnRecoveriesTotal, { + outcome: "skipped", + reason: "projected-turn-settled", + provider: binding.provider, + }); + return; + } + interruptedRecoveryThreadIds.add(thread.id); + const recover = Effect.gen(function* () { const runtimeMode = binding.runtimeMode ?? thread.runtimeMode; // This lifecycle transition settles the concrete old projection row as @@ -1392,10 +1443,6 @@ const make = Effect.gen(function* () { }); return candidate === undefined ? [] : [{ binding, candidate }]; }); - const recoveryCandidateThreadIds = new Set( - recoveryCandidates.map(({ binding }) => binding.threadId), - ); - const pendingTurnStarts = yield* projectionTurnRepository.listPendingTurnStarts().pipe( Effect.catchCause((cause) => Effect.logWarning("provider restart recovery failed to list pending turn starts", { @@ -1421,10 +1468,10 @@ const make = Effect.gen(function* () { discard: true, }); - const pendingWithoutAcceptedProviderTurn = pendingTurnStarts.filter( - (pending) => !recoveryCandidateThreadIds.has(pending.threadId), + const pendingWithoutInterruptedRecovery = pendingTurnStarts.filter( + (pending) => !interruptedRecoveryThreadIds.has(pending.threadId), ); - if (pendingWithoutAcceptedProviderTurn.length === 0) return; + if (pendingWithoutInterruptedRecovery.length === 0) return; const persistedEvents = yield* Stream.runCollect( orchestrationEngine.readEvents(0, Number.MAX_SAFE_INTEGER), @@ -1449,7 +1496,7 @@ const make = Effect.gen(function* () { } yield* Effect.forEach( - pendingWithoutAcceptedProviderTurn, + pendingWithoutInterruptedRecovery, (pending) => Effect.gen(function* () { const thread = yield* resolveThread(pending.threadId);