diff --git a/apps/server/src/cli/config.test.ts b/apps/server/src/cli/config.test.ts index 349dd42a61d..754bdd8a16a 100644 --- a/apps/server/src/cli/config.test.ts +++ b/apps/server/src/cli/config.test.ts @@ -125,7 +125,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => { tailscaleServeEnabled: false, tailscaleServePort: 443, }); - assert.equal(resolved.stateDir, join(baseDir, "userdata")); + assert.equal(resolved.stateDir, join(baseDir, "userdata-v2")); }), ); @@ -195,7 +195,7 @@ it.layer(NodeServices.layer)("cli config resolution", (it) => { tailscaleServeEnabled: true, tailscaleServePort: 8443, }); - assert.equal(resolved.dbPath, join(baseDir, "userdata", "state.sqlite")); + assert.equal(resolved.dbPath, join(baseDir, "userdata-v2", "state.sqlite")); }), ); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index a7fd973994d..2e49fcdf7f8 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -1122,14 +1122,19 @@ describe("ClaudeAdapterV2 background wake turns", () => { uuid: "00000000-0000-4000-8000-000000000101", session_id: WAKE_NATIVE_SESSION, }); - const makeResultFrame = (input: { readonly uuid: string; readonly result: string }) => + const makeResultFrame = (input: { + readonly uuid: string; + readonly result: string; + readonly numTurns?: number; + readonly origin?: { readonly kind: "task-notification" }; + }) => claudeSdkFrame({ type: "result", subtype: "success", duration_ms: 10, duration_api_ms: 10, is_error: false, - num_turns: 1, + num_turns: input.numTurns ?? 1, result: input.result, stop_reason: "end_turn", total_cost_usd: 0, @@ -1143,6 +1148,7 @@ describe("ClaudeAdapterV2 background wake turns", () => { permission_denials: [], uuid: input.uuid, session_id: WAKE_NATIVE_SESSION, + ...(input.origin === undefined ? {} : { origin: input.origin }), }); const turnOneResult = makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000102", @@ -1161,6 +1167,15 @@ describe("ClaudeAdapterV2 background wake turns", () => { const wakeResult = makeResultFrame({ uuid: "00000000-0000-4000-8000-000000000104", result: WAKE_RESULT_TEXT, + origin: { kind: "task-notification" }, + }); + const STALE_TASK_NOTIFICATION_RESULT_TEXT = + "Stale task-notification origin text that must not appear."; + const staleTaskNotificationResult = makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000106", + result: STALE_TASK_NOTIFICATION_RESULT_TEXT, + numTurns: 0, + origin: { kind: "task-notification" }, }); const awaitUntil = (predicate: () => boolean, label: string): Effect.Effect => @@ -1441,6 +1456,150 @@ describe("ClaudeAdapterV2 background wake turns", () => { ), ); + it.effect("ignores a live task-notification origin result during a normal user turn", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + const probeAssistantText = "Probe after stale task-notification result."; + const recoveryAssistantText = "Recovered after the interrupt; continuing."; + const staleResultText = STALE_TASK_NOTIFICATION_RESULT_TEXT; + const hasMessageText = (text: string) => + harness.events.some( + (event) => event.type === "message.updated" && event.message.text === text, + ); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-stale-notif-1"), + text: "Continue after interrupt.", + attachments: [], + }), + ); + yield* awaitUntil(() => harness.offeredMessages.length === 1, "recovery prompt offered"); + + // Live interleaving seen after interrupt recovery: a stale stopped + // task_notification and its task-notification-origin result arrive + // before the real root assistant stream. + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "system", + subtype: "task_notification", + task_id: "task-stale-stopped", + status: "stopped", + output_file: "/tmp/task-stale-stopped.log", + summary: "", + uuid: "00000000-0000-4000-8000-000000000107", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* Queue.offer(harness.sdkMessages, staleTaskNotificationResult); + // Queue-ordered probe: once this assistant text is emitted, the stale + // origin result ahead of it has been consumed. + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "assistant", + message: { + role: "assistant", + content: [{ type: "text", text: probeAssistantText }], + }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-00000000010a", + session_id: WAKE_NATIVE_SESSION, + }), + ); + + yield* awaitUntil( + () => hasMessageText(probeAssistantText), + "probe assistant after stale task-notification result", + ); + assert.lengthOf(harness.terminalEvents(), 0); + assert.isFalse(hasMessageText(staleResultText)); + + yield* Queue.offer( + harness.sdkMessages, + claudeSdkFrame({ + type: "assistant", + message: { + role: "assistant", + content: [{ type: "text", text: recoveryAssistantText }], + }, + parent_tool_use_id: null, + uuid: "00000000-0000-4000-8000-000000000108", + session_id: WAKE_NATIVE_SESSION, + }), + ); + yield* Queue.offer( + harness.sdkMessages, + makeResultFrame({ + uuid: "00000000-0000-4000-8000-000000000109", + result: recoveryAssistantText, + }), + ); + + yield* awaitUntil(() => harness.terminalEvents().length === 1, "user turn terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "completed"); + assert.isTrue(hasMessageText(recoveryAssistantText)); + assert.isFalse(hasMessageText(staleResultText)); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("terminalizes a continuation turn from a task-notification origin wake result", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeWakeHarness; + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-notif-origin-2a"), + text: "Run the build in the background.", + attachments: [], + }), + ); + yield* Queue.offer(harness.sdkMessages, wakeTaskStarted); + yield* Queue.offer(harness.sdkMessages, turnOneResult); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "first turn terminal"); + yield* Queue.offer(harness.sdkMessages, wakeNotification); + yield* Queue.offer(harness.sdkMessages, wakeResult); + yield* awaitUntil(() => harness.continuationRequests.length === 1, "continuation request"); + + yield* harness.runtime.startTurn( + makeClaudeTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-claude-notif-origin-2b"), + text: "Background task completed.", + attachments: [], + providerTurnOrdinal: 2, + messageCreatedBy: "agent", + messageCreationSource: "provider", + }), + ); + + yield* awaitUntil(() => harness.terminalEvents().length === 2, "continuation terminal"); + assert.equal(harness.terminalEvents()[1]?.status, "completed"); + assert.lengthOf(harness.offeredMessages, 1); + assert.isTrue( + harness.events.some( + (event) => event.type === "message.updated" && event.message.text === WAKE_RESULT_TEXT, + ), + ); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + it.effect("clears the pending task when the wake notification carries no summary", () => Effect.scoped( Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 6fe806d6f8c..773587149c3 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -1765,6 +1765,19 @@ function isClaudeActiveSteeringAbortResult(message: SDKResultMessage): boolean { return message.terminal_reason === "aborted_streaming"; } +function isClaudeProviderContinuationTurn(input: ProviderAdapterV2TurnInput): boolean { + return input.message.createdBy === "agent" && input.message.creationSource === "provider"; +} + +function isClaudeTaskNotificationOriginResult(message: SDKMessage): message is SDKResultMessage & { + readonly origin: Extract< + NonNullable, + { readonly kind: "task-notification" } + >; +} { + return message.type === "result" && message.origin?.kind === "task-notification"; +} + function providerFailureFromResult( message: SDKResultMessage, ): OrchestrationV2ProviderFailure | null { @@ -3266,6 +3279,18 @@ export function makeClaudeAdapterV2( return; } + // Task-notification-origin results can interleave during a normal + // user turn (for example a stale background stop after interrupt + // recovery). They must not finalize that turn or supply fallback + // assistant text. Provider continuation turns still consume them + // when draining buffered wake messages. + if ( + isClaudeTaskNotificationOriginResult(message) && + !isClaudeProviderContinuationTurn(context.input) + ) { + return; + } + // An is_error result's text is the error message; it belongs on the // terminal-failure item, not on a synthetic assistant message. const resultText = @@ -3549,9 +3574,7 @@ export function makeClaudeAdapterV2( // produced instead of prompting it again: drain the buffered wake // messages into this turn and let any still-streaming messages // follow live. The continuation prompt text never reaches the CLI. - const isContinuationTurn = - turnInput.message.createdBy === "agent" && - turnInput.message.creationSource === "provider"; + const isContinuationTurn = isClaudeProviderContinuationTurn(turnInput); const userMessage = isContinuationTurn ? null : yield* makeClaudeUserMessageWithAttachments({ diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index e7832640d95..5e4ab164d6c 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -26,6 +26,7 @@ import * as CodexReplay from "effect-codex-app-server/replay"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; @@ -186,6 +187,48 @@ describe("CodexAdapterV2 assistant message streaming", () => { ]); }), ); + + it.effect("can discard an empty completion without emitting an assistant update", () => + Effect.gen(function* () { + const updates = yield* Ref.make>([]); + const coalescer = yield* makeCodexAgentMessageDeltaCoalescer({ + flushIntervalMs: 50, + emit: (update) => Ref.update(updates, (current) => [...current, update]), + }); + + yield* coalescer.append({ turnId: "turn-1", itemId: "message-1", delta: "" }); + yield* Effect.yieldNow; + yield* TestClock.adjust("50 millis"); + yield* Effect.yieldNow; + const completedText = yield* coalescer.complete({ + turnId: "turn-1", + itemId: "message-1", + finalText: "", + emitEmpty: false, + }); + + assert.equal(completedText, ""); + assert.deepEqual(yield* Ref.get(updates), []); + + yield* coalescer.append({ turnId: "turn-1", itemId: "message-2", delta: "buffered" }); + assert.equal( + yield* coalescer.complete({ + turnId: "turn-1", + itemId: "message-2", + emitEmpty: false, + }), + "buffered", + ); + assert.deepEqual(yield* Ref.get(updates), [ + { + turnId: "turn-1", + itemId: "message-2", + text: "buffered", + completed: true, + }, + ]); + }), + ); }); describe("CodexAdapterV2 runtime policy", () => { @@ -704,8 +747,10 @@ function makeCodexTestTurnInput(input: { function makeCodexReplayTurn(input: { readonly id: string; - readonly status: "inProgress" | "completed"; + readonly status: "inProgress" | "completed" | "interrupted" | "failed"; }): Record { + const terminal = + input.status === "completed" || input.status === "interrupted" || input.status === "failed"; return { id: input.id, items: [], @@ -713,7 +758,7 @@ function makeCodexReplayTurn(input: { status: input.status, error: null, startedAt: 1782622440, - completedAt: input.status === "completed" ? 1782622450 : null, + completedAt: terminal ? 1782622450 : null, durationMs: null, }; } @@ -939,6 +984,480 @@ describe("CodexAdapterV2 post-settle continuation", () => { }; }); + const assistantMessages = (events: ReadonlyArray) => + events.filter( + (event): event is Extract => + event.type === "message.updated" && event.message.role === "assistant", + ); + + const finalAnswerTranscript = ( + scenario: string, + answers: ReadonlyArray<{ + readonly id: string; + readonly text: string; + readonly phase?: "commentary" | "final_answer" | null; + readonly omitPhase?: boolean; + readonly streamed?: boolean; + readonly completionDelayMs?: number; + }>, + ) => { + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const prompt = "Reply with the requested recovery marker."; + return makeCodexReplayTranscript({ + scenario, + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt }), + ...answers.flatMap( + (answer, index): ReadonlyArray => { + const phase = answer.omitPhase + ? {} + : { phase: answer.phase === undefined ? ("final_answer" as const) : answer.phase }; + const completed: CodexReplay.CodexAppServerReplayEntry = { + type: "emit_inbound", + label: `item/completed/${answer.id}`, + ...(answer.completionDelayMs === undefined + ? {} + : { afterMs: answer.completionDelayMs }), + frame: { + method: "item/completed", + params: { + item: { + type: "agentMessage", + id: answer.id, + text: answer.text, + ...phase, + memoryCitation: null, + }, + threadId: nativeThreadId, + turnId: nativeTurnId, + completedAtMs: 1782622441000 + index, + }, + }, + }; + if (!answer.streamed) { + return [completed]; + } + return [ + { + type: "emit_inbound", + label: `item/started/${answer.id}`, + frame: { + method: "item/started", + params: { + item: { + type: "agentMessage", + id: answer.id, + text: "", + ...phase, + memoryCitation: null, + }, + threadId: nativeThreadId, + turnId: nativeTurnId, + startedAtMs: 1782622440500 + index, + }, + }, + }, + { + type: "emit_inbound", + label: `item/agentMessage/delta/${answer.id}`, + frame: { + method: "item/agentMessage/delta", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + itemId: answer.id, + delta: answer.text, + }, + }, + }, + completed, + ]; + }, + ), + { + type: "emit_inbound", + label: "turn/completed", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "completed" }), + }, + }, + }, + ], + }); + }; + + it.effect("suppresses a trailing empty final answer after a non-empty final answer", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = finalAnswerTranscript("codex-redundant-empty-final", [ + { id: "answer-non-empty", text: "CODEX_RECOVERY_OK" }, + { id: "answer-empty", text: "" }, + ]); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-redundant-empty-final"), + text: "Reply with the requested recovery marker.", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + + assert.deepEqual( + assistantMessages(harness.events).map((event) => event.message.text), + ["CODEX_RECOVERY_OK"], + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("suppresses a later streamed duplicate final answer", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = finalAnswerTranscript("codex-streamed-duplicate-final", [ + { id: "answer-original", text: "CODEX_RECOVERY_OK" }, + { + id: "answer-duplicate", + text: "CODEX_RECOVERY_OK", + streamed: true, + completionDelayMs: 100, + }, + ]); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-streamed-duplicate-final"), + text: "Reply with the requested recovery marker.", + }), + ); + yield* awaitUntil(() => assistantMessages(harness.events).length === 1, "original answer"); + yield* Effect.yieldNow; + yield* TestClock.adjust("50 millis"); + yield* Effect.yieldNow; + + assert.deepEqual( + assistantMessages(harness.events).map((event) => event.message.text), + ["CODEX_RECOVERY_OK"], + ); + + yield* TestClock.adjust("50 millis"); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + assert.deepEqual( + assistantMessages(harness.events).map((event) => event.message.text), + ["CODEX_RECOVERY_OK"], + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("buffers an overlapping later final stream until duplicate detection", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-overlapping-duplicate-final"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const answerItem = (id: string, text: string) => ({ + type: "agentMessage" as const, + id, + text, + phase: "final_answer" as const, + memoryCitation: null, + }); + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId, + prompt: "Reply with the requested recovery marker.", + }), + ...["answer-overlap-original", "answer-overlap-duplicate"].flatMap( + (itemId, index): ReadonlyArray => [ + { + type: "emit_inbound", + label: `item/started/${itemId}`, + frame: { + method: "item/started", + params: { + item: answerItem(itemId, ""), + threadId: nativeThreadId, + turnId: nativeTurnId, + startedAtMs: 1782622440500 + index, + }, + }, + }, + { + type: "emit_inbound", + label: `item/agentMessage/delta/${itemId}`, + frame: { + method: "item/agentMessage/delta", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + itemId, + delta: "CODEX_RECOVERY_OK", + }, + }, + }, + ], + ), + { + type: "emit_inbound", + label: "item/completed/answer-overlap-original", + afterMs: 100, + frame: { + method: "item/completed", + params: { + item: answerItem("answer-overlap-original", "CODEX_RECOVERY_OK"), + threadId: nativeThreadId, + turnId: nativeTurnId, + completedAtMs: 1782622441000, + }, + }, + }, + { + type: "emit_inbound", + label: "item/completed/answer-overlap-duplicate", + frame: { + method: "item/completed", + params: { + item: answerItem("answer-overlap-duplicate", "CODEX_RECOVERY_OK"), + threadId: nativeThreadId, + turnId: nativeTurnId, + completedAtMs: 1782622441001, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/completed", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "completed" }), + }, + }, + }, + ], + }); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-overlapping-duplicate-final"), + text: "Reply with the requested recovery marker.", + }), + ); + yield* Effect.yieldNow; + yield* TestClock.adjust("50 millis"); + yield* Effect.yieldNow; + + assert.equal( + new Set(assistantMessages(harness.events).map((event) => event.message.id)).size, + 1, + ); + + yield* TestClock.adjust("50 millis"); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + assert.equal( + new Set(assistantMessages(harness.events).map((event) => event.message.id)).size, + 1, + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("preserves a sole empty final answer", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = finalAnswerTranscript("codex-sole-empty-final", [ + { id: "answer-empty", text: "" }, + ]); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-sole-empty-final"), + text: "Reply with the requested recovery marker.", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + + assert.deepEqual( + assistantMessages(harness.events).map((event) => event.message.text), + [""], + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("suppresses a second empty final answer", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = finalAnswerTranscript("codex-duplicate-empty-final", [ + { id: "answer-empty-original", text: "" }, + { id: "answer-empty-duplicate", text: "" }, + ]); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-duplicate-empty-final"), + text: "Reply with the requested recovery marker.", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + + assert.deepEqual( + assistantMessages(harness.events).map((event) => event.message.text), + [""], + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("preserves an empty final answer when only commentary preceded it", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = finalAnswerTranscript("codex-commentary-then-empty-final", [ + { id: "answer-commentary", text: "Working on it.", phase: "commentary" }, + { id: "answer-empty", text: "" }, + ]); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-commentary-then-empty-final"), + text: "Reply with the requested recovery marker.", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + + assert.deepEqual( + assistantMessages(harness.events).map((event) => event.message.text), + ["Working on it.", ""], + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("suppresses an empty final answer after a non-empty unknown-phase answer", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = finalAnswerTranscript("codex-unknown-non-empty-then-empty-final", [ + { id: "answer-non-empty", text: "CODEX_RECOVERY_OK", phase: null }, + { id: "answer-empty", text: "" }, + ]); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-unknown-non-empty-then-empty-final"), + text: "Reply with the requested recovery marker.", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + + assert.deepEqual( + assistantMessages(harness.events).map((event) => event.message.text), + ["CODEX_RECOVERY_OK"], + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("suppresses a trailing empty answer with an omitted phase", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = finalAnswerTranscript("codex-final-then-empty-unknown", [ + { id: "answer-non-empty", text: "CODEX_RECOVERY_OK" }, + { id: "answer-empty", text: "", omitPhase: true }, + ]); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-final-then-empty-unknown"), + text: "Reply with the requested recovery marker.", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + + assert.deepEqual( + assistantMessages(harness.events).map((event) => event.message.text), + ["CODEX_RECOVERY_OK"], + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("keeps a later non-empty final answer after an initial empty final answer", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = finalAnswerTranscript("codex-empty-then-non-empty-final", [ + { id: "answer-empty", text: "" }, + { id: "answer-non-empty", text: "CODEX_RECOVERY_OK" }, + ]); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-empty-then-non-empty-final"), + text: "Reply with the requested recovery marker.", + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + + assert.deepEqual( + assistantMessages(harness.events).map((event) => event.message.text), + ["", "CODEX_RECOVERY_OK"], + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + const BG_SCENARIO = "codex-bg-exec-wake"; const BG_NATIVE_THREAD = "native-codex-bg-thread"; const BG_NATIVE_TURN = "native-codex-bg-turn"; @@ -1197,23 +1716,1410 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); - const RESUME_SCENARIO = "codex-resume-subagent"; - const RESUME_NATIVE_THREAD = "native-codex-resume-thread"; - const RESUME_NATIVE_TURN = "native-codex-resume-root-turn"; - const RESUME_CHILD_THREAD = "native-codex-resume-child-thread"; - const RESUME_CHILD_TURN_1 = "native-codex-resume-child-turn-1"; - const RESUME_CHILD_TURN_2 = "native-codex-resume-child-turn-2"; - const RESUME_PROMPT = "Spawn a sub-agent, nudge it, and reply NUDGED."; + const INTERRUPT_SCENARIO = "codex-interrupt-mid-command"; + const INTERRUPT_NATIVE_THREAD = "native-codex-interrupt-thread"; + const INTERRUPT_NATIVE_TURN = "native-codex-interrupt-turn"; + const INTERRUPT_COMMAND_ITEM = "exec-codex-interrupt-command"; + const INTERRUPT_COMMAND_ITEM_TWO = "exec-codex-interrupt-command-two"; + const INTERRUPT_CHILD_COMMAND_ITEM = "exec-codex-interrupt-child-command"; + const INTERRUPT_CHILD_TIMEOUT_BOUNDARY_ITEM = "exec-codex-interrupt-child-timeout-boundary"; + const INTERRUPT_CHILD_NATIVE_THREAD = "native-codex-interrupt-child-thread"; + const INTERRUPT_CHILD_NATIVE_TURN = "native-codex-interrupt-child-turn"; + const INTERRUPT_LATE_CHILD_NATIVE_TURN = "native-codex-interrupt-late-child-turn"; + const INTERRUPT_LATE_CHILD_2_NATIVE_TURN = "native-codex-interrupt-late-child-2-turn"; + const INTERRUPT_TIMEOUT_BOUNDARY_ITEM = "exec-codex-interrupt-timeout-boundary"; + const INTERRUPT_TIMEOUT_LATE_ITEM = "exec-codex-interrupt-timeout-late"; + const INTERRUPT_COMMAND = "bash -c 'sleep 30; echo SHOULD_NOT_FINISH_CMD_INTERRUPT_FIXTURE'"; + const INTERRUPT_COMMAND_TWO = "bash -c 'sleep 20; echo SECOND_COMMAND'"; + const INTERRUPT_PROMPT = "Run a long foreground command and wait until interrupted."; + + const interruptCommandItem = (status: "inProgress" | "completed"): Record => ({ + type: "commandExecution", + id: INTERRUPT_COMMAND_ITEM, + command: INTERRUPT_COMMAND, + cwd: "/workspace", + processId: "57680", + source: "unifiedExecStartup", + status, + commandActions: [{ type: "unknown", command: INTERRUPT_COMMAND }], + aggregatedOutput: status === "completed" ? "SHOULD_NOT_FINISH_CMD_INTERRUPT_FIXTURE\n" : null, + exitCode: status === "completed" ? 0 : null, + durationMs: status === "completed" ? 30_000 : null, + }); - const childAgentMessage = (input: { - readonly id: string; - readonly text: string; - readonly turnId: string; - readonly completedAtMs: number; - readonly afterMs?: number; - }): CodexReplay.CodexAppServerReplayEntry => ({ - type: "emit_inbound", - label: `item/completed/${input.id}`, + const interruptMidCommandTranscript = makeCodexReplayTranscript({ + scenario: INTERRUPT_SCENARIO, + entries: [ + ...codexReplayPreamble({ + nativeThreadId: INTERRUPT_NATIVE_THREAD, + nativeTurnId: INTERRUPT_NATIVE_TURN, + prompt: INTERRUPT_PROMPT, + }), + { + type: "emit_inbound", + label: "item/started/command", + frame: { + method: "item/started", + params: { + item: interruptCommandItem("inProgress"), + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + startedAtMs: 1782622440500, + }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt", + frame: { + id: 4, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt", + frame: { id: 4, result: {} }, + }, + { + type: "emit_inbound", + label: "item/started/command-two-after-interrupt-response", + frame: { + method: "item/started", + params: { + item: { + ...interruptCommandItem("inProgress"), + id: INTERRUPT_COMMAND_ITEM_TWO, + command: INTERRUPT_COMMAND_TWO, + processId: "57681", + commandActions: [{ type: "unknown", command: INTERRUPT_COMMAND_TWO }], + }, + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + startedAtMs: 1782622440600, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/completed", + frame: { + method: "turn/completed", + params: { + threadId: INTERRUPT_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_NATIVE_TURN, + status: "interrupted", + }), + }, + }, + }, + { + type: "expect_outbound", + label: "thread/backgroundTerminals/terminate/one", + frame: { + id: 5, + method: "thread/backgroundTerminals/terminate", + params: { threadId: INTERRUPT_NATIVE_THREAD, processId: "57680" }, + }, + }, + { + type: "emit_inbound", + label: "thread/backgroundTerminals/terminate/one", + frame: { id: 5, result: { terminated: false } }, + }, + { + type: "expect_outbound", + label: "thread/backgroundTerminals/list/after-false", + frame: { + id: 6, + method: "thread/backgroundTerminals/list", + params: { threadId: INTERRUPT_NATIVE_THREAD }, + }, + }, + { + type: "emit_inbound", + label: "thread/backgroundTerminals/list/after-false", + frame: { id: 6, result: { data: [], nextCursor: null } }, + }, + { + type: "expect_outbound", + label: "thread/backgroundTerminals/terminate/two", + frame: { + id: 7, + method: "thread/backgroundTerminals/terminate", + params: { threadId: INTERRUPT_NATIVE_THREAD, processId: "57681" }, + }, + }, + { + type: "emit_inbound", + label: "thread/backgroundTerminals/terminate/two", + frame: { id: 7, result: { terminated: true } }, + }, + { + type: "emit_inbound", + label: "item/completed/command-late", + afterMs: 30_000, + frame: { + method: "item/completed", + params: { + item: interruptCommandItem("completed"), + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + completedAtMs: 1782622465500, + }, + }, + }, + ], + }); + + it.effect("contains commands that start before and after the interrupt response", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeCodexReplayHarness(interruptMidCommandTranscript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-interrupt-mid-command"), + text: INTERRUPT_PROMPT, + }), + ); + + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.status === "running", + ), + "running command item", + ); + + const providerTurnId = harness.events.find( + (event): event is Extract => + event.type === "provider_turn.updated", + )?.providerTurn.id; + assert.isDefined(providerTurnId); + + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }); + + yield* awaitUntil(() => harness.terminalEvents().length === 1, "interrupted terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "interrupted"); + + const terminalIndex = harness.events.findIndex((event) => event.type === "turn.terminal"); + assert.isAtLeast(terminalIndex, 0); + + let lastCommandBeforeTerminal: + | Extract + | undefined; + for (let index = 0; index < terminalIndex; index++) { + const event = harness.events[index]; + if (event?.type === "turn_item.updated" && event.turnItem.type === "command_execution") { + lastCommandBeforeTerminal = event; + } + } + assert.isDefined(lastCommandBeforeTerminal); + assert.equal(lastCommandBeforeTerminal.turnItem.status, "interrupted"); + assert.isNotNull(lastCommandBeforeTerminal.turnItem.completedAt); + + const interruptedCommandsBeforeTerminal = harness.events + .slice(0, terminalIndex) + .flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.status === "interrupted" + ? [event.turnItem.input] + : [], + ) + .sort(); + assert.deepEqual( + interruptedCommandsBeforeTerminal, + [INTERRUPT_COMMAND, INTERRUPT_COMMAND_TWO].sort(), + ); + + const interruptedCommandIndex = harness.events.findIndex( + (event, index) => + index < terminalIndex && + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.status === "interrupted", + ); + assert.isAbove( + terminalIndex, + interruptedCommandIndex, + "command terminalization must precede turn.terminal", + ); + + assert.isFalse(yield* harness.hasPendingBackgroundWork); + assert.lengthOf(harness.continuationRequests, 0); + + // Late provider item/completed after interrupt must not revive the card + // or request a background-command wake continuation. + yield* TestClock.adjust("30 seconds"); + for (let attempt = 0; attempt < 100; attempt++) { + yield* Effect.yieldNow; + } + assert.lengthOf(harness.continuationRequests, 0); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + assert.lengthOf(harness.terminalEvents(), 1); + + const postTerminalCommandUpdates = harness.events.filter( + (event, index) => + index > terminalIndex && + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution", + ); + assert.lengthOf( + postTerminalCommandUpdates, + 0, + "late item/completed after interrupt must not project", + ); + + const commandUpdates = harness.events.filter( + (event): event is Extract => + event.type === "turn_item.updated" && event.turnItem.type === "command_execution", + ); + assert.isAtLeast(commandUpdates.length, 2, "start + interrupt terminalization"); + assert.equal(commandUpdates[commandUpdates.length - 1]?.turnItem.status, "interrupted"); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + const interruptSubagentCommandTranscript = makeCodexReplayTranscript({ + scenario: "codex-interrupt-subagent-command", + entries: [ + ...codexReplayPreamble({ + nativeThreadId: INTERRUPT_NATIVE_THREAD, + nativeTurnId: INTERRUPT_NATIVE_TURN, + prompt: INTERRUPT_PROMPT, + }), + { + type: "emit_inbound", + label: "item/completed/subAgentActivity-started", + frame: { + method: "item/completed", + params: { + item: { + type: "subAgentActivity", + id: "call-codex-interrupt-subagent", + kind: "started", + agentThreadId: INTERRUPT_CHILD_NATIVE_THREAD, + agentPath: "/root/stop_hold", + }, + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + completedAtMs: 1782622441000, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/started/child", + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_CHILD_NATIVE_TURN, + status: "inProgress", + }), + }, + }, + }, + { + type: "emit_inbound", + label: "item/started/child-command", + frame: { + method: "item/started", + params: { + item: { + ...interruptCommandItem("inProgress"), + id: INTERRUPT_CHILD_COMMAND_ITEM, + processId: "57682", + }, + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + startedAtMs: 1782622441500, + }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/root", + frame: { id: 4, result: {} }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/child", + frame: { id: 5, result: {} }, + }, + { + type: "expect_outbound", + label: "thread/backgroundTerminals/terminate/child", + frame: { + id: 6, + method: "thread/backgroundTerminals/terminate", + params: { threadId: INTERRUPT_CHILD_NATIVE_THREAD, processId: "57682" }, + }, + }, + { + type: "emit_inbound", + label: "thread/backgroundTerminals/terminate/child", + frame: { id: 6, result: { terminated: true } }, + }, + { + type: "emit_inbound", + label: "turn/completed/root", + frame: { + method: "turn/completed", + params: { + threadId: INTERRUPT_NATIVE_THREAD, + turn: makeCodexReplayTurn({ id: INTERRUPT_NATIVE_TURN, status: "interrupted" }), + }, + }, + }, + { + type: "emit_inbound", + label: "turn/completed/child-completed-race", + frame: { + method: "turn/completed", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_CHILD_NATIVE_TURN, + status: "completed", + }), + }, + }, + }, + { type: "runtime_exit", status: "success" }, + ], + }); + + const assertChildProviderTerminalBeforeRoot = ( + events: ReadonlyArray, + rootThreadId: ThreadId, + ) => { + const terminalIndex = events.findIndex((event) => event.type === "turn.terminal"); + const childProviderTurnIndex = events.findIndex( + (event) => + event.type === "provider_turn.updated" && + event.threadId !== rootThreadId && + event.providerTurn.status === "interrupted", + ); + const childProviderThreadIndex = events.findIndex( + (event) => + event.type === "provider_thread.updated" && + event.providerThread.appThreadId !== rootThreadId && + event.providerThread.status === "idle", + ); + assert.isAtLeast(childProviderTurnIndex, 0, "child provider turn must terminalize"); + assert.isAtLeast(childProviderThreadIndex, 0, "child provider thread must become idle"); + assert.isAbove( + terminalIndex, + childProviderTurnIndex, + "child provider turn must terminalize before the root run", + ); + assert.isAbove( + terminalIndex, + childProviderThreadIndex, + "child provider thread must become idle before the root run", + ); + }; + + it.effect("contains descendant commands and keeps Stop authoritative", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeCodexReplayHarness(interruptSubagentCommandTranscript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-interrupt-subagent-command"), + text: INTERRUPT_PROMPT, + }), + ); + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.nativeItemRef?.nativeId === INTERRUPT_CHILD_COMMAND_ITEM && + event.turnItem.status === "running", + ), + "running child command item", + ); + const providerTurnId = harness.events.find( + (event): event is Extract => + event.type === "provider_turn.updated" && event.threadId === harness.threadId, + )?.providerTurn.id; + assert.isDefined(providerTurnId); + + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }); + + yield* awaitUntil(() => harness.terminalEvents().length === 1, "interrupted root terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "interrupted"); + const childCommandUpdates = harness.events.filter( + (event): event is Extract => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.nativeItemRef?.nativeId === INTERRUPT_CHILD_COMMAND_ITEM, + ); + assert.equal(childCommandUpdates.at(-1)?.turnItem.status, "interrupted"); + assert.equal(harness.subagentUpdates().at(-1)?.subagent.status, "interrupted"); + assertChildProviderTerminalBeforeRoot(harness.events, harness.threadId); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + const childInterruptResponseIndex = interruptSubagentCommandTranscript.entries.findIndex( + (entry) => entry.type === "emit_inbound" && entry.label === "turn/interrupt/child", + ); + const rootInterruptResponseIndex = interruptSubagentCommandTranscript.entries.findIndex( + (entry) => entry.type === "emit_inbound" && entry.label === "turn/interrupt/root", + ); + const interruptSubagentRequestFailureTranscript = makeCodexReplayTranscript({ + scenario: "codex-interrupt-subagent-request-failure", + entries: [ + ...interruptSubagentCommandTranscript.entries.slice(0, rootInterruptResponseIndex + 1), + { + type: "emit_inbound", + label: "turn/completed/root-before-child-interrupt-failure", + frame: { + method: "turn/completed", + params: { + threadId: INTERRUPT_NATIVE_THREAD, + turn: makeCodexReplayTurn({ id: INTERRUPT_NATIVE_TURN, status: "interrupted" }), + }, + }, + }, + ...interruptSubagentCommandTranscript.entries.slice( + rootInterruptResponseIndex + 1, + childInterruptResponseIndex, + ), + { + type: "emit_inbound", + label: "turn/interrupt/child", + frame: { + id: 5, + error: { code: -32_000, message: "child interrupt request failed" }, + }, + }, + { type: "runtime_exit", status: "success" }, + ], + }); + + it.effect("terminalizes descendants before the root when an interrupt request fails", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeCodexReplayHarness(interruptSubagentRequestFailureTranscript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-interrupt-subagent-request-failure"), + text: INTERRUPT_PROMPT, + }), + ); + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.nativeItemRef?.nativeId === INTERRUPT_CHILD_COMMAND_ITEM && + event.turnItem.status === "running", + ), + "running child command item", + ); + const providerTurnId = harness.events.find( + (event): event is Extract => + event.type === "provider_turn.updated" && event.threadId === harness.threadId, + )?.providerTurn.id; + assert.isDefined(providerTurnId); + + const interruptExit = yield* harness.runtime + .interruptTurn({ providerThread: harness.providerThread, providerTurnId }) + .pipe(Effect.exit); + + assert.equal(interruptExit._tag, "Failure"); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "interrupted root terminal"); + assertChildProviderTerminalBeforeRoot(harness.events, harness.threadId); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + const childTerminationResponseIndex = interruptSubagentCommandTranscript.entries.findIndex( + (entry) => + entry.type === "emit_inbound" && entry.label === "thread/backgroundTerminals/terminate/child", + ); + const interruptSubagentTimeoutTranscript = makeCodexReplayTranscript({ + scenario: "codex-interrupt-subagent-timeout", + entries: [ + ...interruptSubagentCommandTranscript.entries.slice(0, childTerminationResponseIndex + 1), + { + type: "emit_inbound", + label: "item/started/child-timeout-boundary", + frame: { + method: "item/started", + params: { + item: { + ...interruptCommandItem("inProgress"), + id: INTERRUPT_CHILD_TIMEOUT_BOUNDARY_ITEM, + processId: null, + }, + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + startedAtMs: 1782622441600, + }, + }, + }, + ], + }); + + it.effect("terminalizes timed-out descendants before the root", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeCodexReplayHarness(interruptSubagentTimeoutTranscript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-interrupt-subagent-timeout"), + text: INTERRUPT_PROMPT, + }), + ); + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.nativeItemRef?.nativeId === INTERRUPT_CHILD_COMMAND_ITEM && + event.turnItem.status === "running", + ), + "running child command item", + ); + const providerTurnId = harness.events.find( + (event): event is Extract => + event.type === "provider_turn.updated" && event.threadId === harness.threadId, + )?.providerTurn.id; + assert.isDefined(providerTurnId); + + const interruptFiber = yield* harness.runtime + .interruptTurn({ providerThread: harness.providerThread, providerTurnId }) + .pipe(Effect.forkScoped); + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.nativeItemRef?.nativeId === INTERRUPT_CHILD_TIMEOUT_BOUNDARY_ITEM && + event.turnItem.status === "running", + ), + "child timeout boundary item", + ); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + yield* awaitUntil(() => harness.terminalEvents().length === 1, "interrupted root terminal"); + assertChildProviderTerminalBeforeRoot(harness.events, harness.threadId); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + const rootCompletionIndex = interruptSubagentCommandTranscript.entries.findIndex( + (entry) => entry.type === "emit_inbound" && entry.label === "turn/completed/root", + ); + const interruptLateSubagentTurnTranscript = makeCodexReplayTranscript({ + scenario: "codex-interrupt-late-subagent-turn", + entries: [ + ...interruptSubagentCommandTranscript.entries.slice(0, rootCompletionIndex), + { + type: "emit_inbound", + label: "turn/started/late-child", + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_LATE_CHILD_NATIVE_TURN, + status: "inProgress", + }), + }, + }, + }, + ...interruptSubagentCommandTranscript.entries.slice(rootCompletionIndex, -1), + { + type: "expect_outbound", + label: "turn/interrupt/late-child", + frame: { + id: 7, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/late-child", + frame: { id: 7, result: {} }, + }, + { type: "runtime_exit", status: "success" }, + ], + }); + + it.effect("interrupts descendants that start after the initial Stop snapshot", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeCodexReplayHarness(interruptLateSubagentTurnTranscript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-interrupt-late-subagent-turn"), + text: INTERRUPT_PROMPT, + }), + ); + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.nativeItemRef?.nativeId === INTERRUPT_CHILD_COMMAND_ITEM && + event.turnItem.status === "running", + ), + "running child command item", + ); + const providerTurnId = harness.events.find( + (event): event is Extract => + event.type === "provider_turn.updated" && event.threadId === harness.threadId, + )?.providerTurn.id; + assert.isDefined(providerTurnId); + + const interruptFiber = yield* harness.runtime + .interruptTurn({ providerThread: harness.providerThread, providerTurnId }) + .pipe(Effect.forkScoped); + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ), + "late child provider turn", + ); + yield* Fiber.join(interruptFiber); + + yield* awaitUntil(() => harness.terminalEvents().length === 1, "interrupted root terminal"); + const lateChildUpdates = harness.events.filter( + (event): event is Extract => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ); + assert.equal(lateChildUpdates.at(-1)?.providerTurn.status, "interrupted"); + assertChildProviderTerminalBeforeRoot(harness.events, harness.threadId); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + const interruptRescanLateSubagentTurnTranscript = makeCodexReplayTranscript({ + scenario: "codex-interrupt-rescan-late-subagent-turn", + entries: [ + ...interruptSubagentCommandTranscript.entries.slice(0, childTerminationResponseIndex + 1), + { + type: "emit_inbound", + label: "turn/started/late-child-1", + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_LATE_CHILD_NATIVE_TURN, + status: "inProgress", + }), + }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/late-child-1", + frame: { + id: 7, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/started/late-child-2", + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_LATE_CHILD_2_NATIVE_TURN, + status: "inProgress", + }), + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/late-child-1", + frame: { id: 7, result: {} }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/late-child-2", + frame: { + id: 8, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_2_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/late-child-2", + frame: { id: 8, result: {} }, + }, + { type: "runtime_exit", status: "success" }, + ], + }); + + it.effect("interrupts descendants discovered only by the final interrupt rescan", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeCodexReplayHarness(interruptRescanLateSubagentTurnTranscript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-interrupt-rescan-late-subagent-turn"), + text: INTERRUPT_PROMPT, + }), + ); + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.nativeItemRef?.nativeId === INTERRUPT_CHILD_COMMAND_ITEM && + event.turnItem.status === "running", + ), + "running child command item", + ); + const providerTurnId = harness.events.find( + (event): event is Extract => + event.type === "provider_turn.updated" && event.threadId === harness.threadId, + )?.providerTurn.id; + assert.isDefined(providerTurnId); + + const interruptFiber = yield* harness.runtime + .interruptTurn({ providerThread: harness.providerThread, providerTurnId }) + .pipe(Effect.forkScoped); + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ), + "late child 1 provider turn", + ); + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + + yield* awaitUntil(() => harness.terminalEvents().length === 1, "interrupted root terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "interrupted"); + + const rootTerminalIndex = harness.events.findIndex( + (event) => event.type === "turn.terminal", + ); + const lateChild1InterruptedIndex = harness.events.findIndex( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN && + event.providerTurn.status === "interrupted", + ); + const lateChild2InterruptedIndex = harness.events.findIndex( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_2_NATIVE_TURN && + event.providerTurn.status === "interrupted", + ); + assert.isAtLeast( + lateChild1InterruptedIndex, + 0, + "late child 1 must terminalize interrupted", + ); + assert.isAtLeast( + lateChild2InterruptedIndex, + 0, + "late child 2 must terminalize interrupted", + ); + assert.isAbove( + rootTerminalIndex, + lateChild1InterruptedIndex, + "late child 1 must terminalize before the root run", + ); + assert.isAbove( + rootTerminalIndex, + lateChild2InterruptedIndex, + "late child 2 must terminalize before the root run", + ); + assertChildProviderTerminalBeforeRoot(harness.events, harness.threadId); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + const interruptTimeoutTranscript = makeCodexReplayTranscript({ + scenario: "codex-interrupt-timeout", + entries: [ + ...codexReplayPreamble({ + nativeThreadId: INTERRUPT_NATIVE_THREAD, + nativeTurnId: INTERRUPT_NATIVE_TURN, + prompt: INTERRUPT_PROMPT, + }), + { + type: "emit_inbound", + label: "item/started/command", + frame: { + method: "item/started", + params: { + item: interruptCommandItem("inProgress"), + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + startedAtMs: 1782622440500, + }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt", + frame: { + id: 4, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt", + frame: { id: 4, result: {} }, + }, + { + type: "emit_inbound", + label: "item/started/command-two-after-interrupt-response", + frame: { + method: "item/started", + params: { + item: { + ...interruptCommandItem("inProgress"), + id: INTERRUPT_COMMAND_ITEM_TWO, + command: INTERRUPT_COMMAND_TWO, + processId: "57681", + commandActions: [{ type: "unknown", command: INTERRUPT_COMMAND_TWO }], + }, + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + startedAtMs: 1782622440600, + }, + }, + }, + { + type: "expect_outbound", + label: "thread/backgroundTerminals/terminate/one", + frame: { + id: 5, + method: "thread/backgroundTerminals/terminate", + params: { threadId: INTERRUPT_NATIVE_THREAD, processId: "57680" }, + }, + }, + { + type: "emit_inbound", + label: "thread/backgroundTerminals/terminate/one", + frame: { id: 5, result: { terminated: true } }, + }, + { + type: "emit_inbound", + label: "item/started/command-at-timeout-boundary", + afterMs: 9_999, + frame: { + method: "item/started", + params: { + item: { + ...interruptCommandItem("inProgress"), + id: INTERRUPT_TIMEOUT_BOUNDARY_ITEM, + command: "echo TIMEOUT_BOUNDARY", + processId: null, + commandActions: [{ type: "unknown", command: "echo TIMEOUT_BOUNDARY" }], + }, + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + startedAtMs: 1782622450500, + }, + }, + }, + { + type: "expect_outbound", + label: "thread/backgroundTerminals/terminate/two", + frame: { + id: 6, + method: "thread/backgroundTerminals/terminate", + params: { threadId: INTERRUPT_NATIVE_THREAD, processId: "57681" }, + }, + }, + { + type: "emit_inbound", + label: "thread/backgroundTerminals/terminate/two", + frame: { id: 6, result: { terminated: true } }, + }, + { + type: "emit_inbound", + label: "turn/completed/late", + afterMs: 20_000, + frame: { + method: "turn/completed", + params: { + threadId: INTERRUPT_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_NATIVE_TURN, + status: "interrupted", + }), + }, + }, + }, + { + type: "emit_inbound", + label: "item/started/command-after-timeout", + frame: { + method: "item/started", + params: { + item: { + ...interruptCommandItem("inProgress"), + id: INTERRUPT_TIMEOUT_LATE_ITEM, + command: "echo LATE_AFTER_TIMEOUT", + processId: null, + commandActions: [{ type: "unknown", command: "echo LATE_AFTER_TIMEOUT" }], + }, + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + startedAtMs: 1782622470500, + }, + }, + }, + { + type: "emit_inbound", + label: "item/completed/command-late", + frame: { + method: "item/completed", + params: { + item: interruptCommandItem("completed"), + threadId: INTERRUPT_NATIVE_THREAD, + turnId: INTERRUPT_NATIVE_TURN, + completedAtMs: 1782622465500, + }, + }, + }, + { type: "runtime_exit", status: "success" }, + ], + }); + + it.effect("bounds interrupt settlement and drops late completion events", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeCodexReplayHarness(interruptTimeoutTranscript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-interrupt-timeout"), + text: INTERRUPT_PROMPT, + }), + ); + yield* awaitUntil( + () => + harness.events.filter( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.status === "running", + ).length === 1, + "running command item", + ); + const providerTurnId = harness.events.find( + (event): event is Extract => + event.type === "provider_turn.updated", + )?.providerTurn.id; + assert.isDefined(providerTurnId); + + const interruptFiber = yield* harness.runtime + .interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }) + .pipe(Effect.forkScoped); + yield* awaitUntil( + () => + harness.events.filter( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.status === "running", + ).length === 2, + "post-interrupt running command item", + ); + + yield* TestClock.adjust("10 seconds"); + yield* Fiber.join(interruptFiber); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "timeout terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "interrupted"); + const terminalProviderTurnsBeforeLateEvents = harness.events.filter( + (event) => + event.type === "provider_turn.updated" && event.providerTurn.status === "interrupted", + ); + assert.lengthOf(terminalProviderTurnsBeforeLateEvents, 1); + + const commandUpdatesBeforeLateEvents = harness.events.filter( + (event): event is Extract => + event.type === "turn_item.updated" && event.turnItem.type === "command_execution", + ); + const terminalCommands = commandUpdatesBeforeLateEvents.filter( + (event) => + event.turnItem.status === "interrupted" && + event.turnItem.nativeItemRef?.nativeId !== INTERRUPT_TIMEOUT_BOUNDARY_ITEM, + ); + assert.lengthOf(terminalCommands, 2); + const boundaryUpdates = commandUpdatesBeforeLateEvents.filter( + (event) => event.turnItem.nativeItemRef?.nativeId === INTERRUPT_TIMEOUT_BOUNDARY_ITEM, + ); + const lastBoundaryUpdate = boundaryUpdates.at(-1); + assert.isDefined(lastBoundaryUpdate); + assert.equal(lastBoundaryUpdate.turnItem.status, "interrupted"); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + + yield* TestClock.adjust("20 seconds"); + for (let attempt = 0; attempt < 100; attempt++) { + yield* Effect.yieldNow; + } + assert.lengthOf(harness.terminalEvents(), 1); + assert.lengthOf( + harness.events.filter( + (event) => + event.type === "provider_turn.updated" && event.providerTurn.status === "interrupted", + ), + terminalProviderTurnsBeforeLateEvents.length, + "late completion must not duplicate provider-turn finalization", + ); + assert.lengthOf( + harness.events.filter( + (event) => + event.type === "turn_item.updated" && event.turnItem.type === "command_execution", + ), + commandUpdatesBeforeLateEvents.length, + "late starts and completions must not project after timeout", + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + const interruptTerminationFailureTranscript = makeCodexReplayTranscript({ + scenario: "codex-interrupt-termination-failure", + entries: [ + ...interruptMidCommandTranscript.entries + .filter( + (entry) => entry.type === "runtime_exit" || entry.label !== "item/completed/command-late", + ) + .map((entry) => + entry.type === "emit_inbound" && + entry.label === "thread/backgroundTerminals/list/after-false" + ? { + ...entry, + frame: { + id: 6, + result: { + data: [{ processId: "57680" }], + nextCursor: null, + }, + }, + } + : entry, + ), + { + type: "expect_outbound", + label: "thread/backgroundTerminals/terminate/one-retry", + frame: { + id: 8, + method: "thread/backgroundTerminals/terminate", + params: { threadId: INTERRUPT_NATIVE_THREAD, processId: "57680" }, + }, + }, + { + type: "emit_inbound", + label: "thread/backgroundTerminals/terminate/one-retry", + frame: { id: 8, result: { terminated: false } }, + }, + { + type: "expect_outbound", + label: "thread/backgroundTerminals/list/after-false-retry", + frame: { + id: 9, + method: "thread/backgroundTerminals/list", + params: { threadId: INTERRUPT_NATIVE_THREAD }, + }, + }, + { + type: "emit_inbound", + label: "thread/backgroundTerminals/list/after-false-retry", + frame: { + id: 9, + result: { data: [{ processId: "57680" }], nextCursor: null }, + }, + }, + { type: "runtime_exit", status: "success" }, + ], + }); + + it.effect("attempts every terminal and cleans up tracking when termination fails", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeCodexReplayHarness(interruptTerminationFailureTranscript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-interrupt-termination-failure"), + text: INTERRUPT_PROMPT, + }), + ); + yield* awaitUntil( + () => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.status === "running", + ), + "running command item", + ); + const providerTurnId = harness.events.find( + (event): event is Extract => + event.type === "provider_turn.updated", + )?.providerTurn.id; + assert.isDefined(providerTurnId); + + const interruptExit = yield* harness.runtime + .interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }) + .pipe(Effect.exit); + + assert.equal(interruptExit._tag, "Failure"); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "interrupted terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "interrupted"); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + const FAILED_SCENARIO = "codex-failed-mid-command"; + const FAILED_NATIVE_THREAD = "native-codex-failed-thread"; + const FAILED_NATIVE_TURN = "native-codex-failed-turn"; + const FAILED_COMMAND_ITEM = "exec-codex-failed-command"; + const FAILED_COMMAND = "sleep 30"; + const FAILED_PROMPT = "Run a command that will be abandoned when the turn fails."; + + const failedMidCommandTranscript = makeCodexReplayTranscript({ + scenario: FAILED_SCENARIO, + entries: [ + ...codexReplayPreamble({ + nativeThreadId: FAILED_NATIVE_THREAD, + nativeTurnId: FAILED_NATIVE_TURN, + prompt: FAILED_PROMPT, + }), + { + type: "emit_inbound", + label: "item/started/command", + frame: { + method: "item/started", + params: { + item: { + type: "commandExecution", + id: FAILED_COMMAND_ITEM, + command: FAILED_COMMAND, + cwd: "/workspace", + processId: "99", + source: "unifiedExecStartup", + status: "inProgress", + commandActions: [{ type: "unknown", command: FAILED_COMMAND }], + aggregatedOutput: null, + exitCode: null, + durationMs: null, + }, + threadId: FAILED_NATIVE_THREAD, + turnId: FAILED_NATIVE_TURN, + startedAtMs: 1782622440500, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/completed", + frame: { + method: "turn/completed", + params: { + threadId: FAILED_NATIVE_THREAD, + turn: { + ...makeCodexReplayTurn({ + id: FAILED_NATIVE_TURN, + status: "failed", + }), + error: { message: "provider failed mid-command" }, + }, + }, + }, + }, + ], + }); + + it.effect("terminalizes running command items before turn.terminal on failed turns", () => + Effect.scoped( + Effect.gen(function* () { + const harness = yield* makeCodexReplayHarness(failedMidCommandTranscript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-failed-mid-command"), + text: FAILED_PROMPT, + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "failed terminal"); + assert.equal(harness.terminalEvents()[0]?.status, "failed"); + + const terminalIndex = harness.events.findIndex((event) => event.type === "turn.terminal"); + const failedCommandIndex = harness.events.findIndex( + (event, index) => + index < terminalIndex && + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.status === "failed", + ); + assert.isAtLeast(failedCommandIndex, 0); + assert.isAbove( + terminalIndex, + failedCommandIndex, + "failed-turn command terminalization must precede turn.terminal", + ); + assert.isFalse(yield* harness.hasPendingBackgroundWork); + assert.lengthOf(harness.continuationRequests, 0); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + const RESUME_SCENARIO = "codex-resume-subagent"; + const RESUME_NATIVE_THREAD = "native-codex-resume-thread"; + const RESUME_NATIVE_TURN = "native-codex-resume-root-turn"; + const RESUME_CHILD_THREAD = "native-codex-resume-child-thread"; + const RESUME_CHILD_TURN_1 = "native-codex-resume-child-turn-1"; + const RESUME_CHILD_TURN_2 = "native-codex-resume-child-turn-2"; + const RESUME_PROMPT = "Spawn a sub-agent, nudge it, and reply NUDGED."; + + const childAgentMessage = (input: { + readonly id: string; + readonly text: string; + readonly turnId: string; + readonly completedAtMs: number; + readonly afterMs?: number; + readonly omitPhase?: boolean; + }): CodexReplay.CodexAppServerReplayEntry => ({ + type: "emit_inbound", + label: `item/completed/${input.id}`, ...(input.afterMs === undefined ? {} : { afterMs: input.afterMs }), frame: { method: "item/completed", @@ -1222,7 +3128,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { type: "agentMessage", id: input.id, text: input.text, - phase: "final_answer", + ...(input.omitPhase ? {} : { phase: "final_answer" as const }), memoryCitation: null, }, threadId: RESUME_CHILD_THREAD, @@ -1248,9 +3154,13 @@ describe("CodexAdapterV2 post-settle continuation", () => { }, }); - const childTurnCompleted = (turnId: string): CodexReplay.CodexAppServerReplayEntry => ({ + const childTurnCompleted = ( + turnId: string, + afterMs?: number, + ): CodexReplay.CodexAppServerReplayEntry => ({ type: "emit_inbound", label: `turn/completed/${turnId}`, + ...(afterMs === undefined ? {} : { afterMs }), frame: { method: "turn/completed", params: { @@ -1294,7 +3204,20 @@ describe("CodexAdapterV2 post-settle continuation", () => { turnId: RESUME_CHILD_TURN_1, completedAtMs: 1782622442000, }), - childTurnCompleted(RESUME_CHILD_TURN_1), + childAgentMessage({ + id: "child-first-answer-empty", + text: "", + turnId: RESUME_CHILD_TURN_1, + completedAtMs: 1782622442001, + omitPhase: true, + }), + childAgentMessage({ + id: "child-first-answer-duplicate", + text: "CODEX_FIRST_DONE", + turnId: RESUME_CHILD_TURN_1, + completedAtMs: 1782622442002, + }), + childTurnCompleted(RESUME_CHILD_TURN_1, 100), { type: "emit_inbound", label: "item/completed/root-answer", @@ -1337,7 +3260,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { ], }); - it.effect("re-opens a resumed subagent and hydrates its post-settle result", () => + it.effect("preserves a subagent result across a trailing empty final and resume", () => Effect.scoped( Effect.gen(function* () { const harness = yield* makeCodexReplayHarness(resumeSubagentTranscript); @@ -1352,6 +3275,16 @@ describe("CodexAdapterV2 post-settle continuation", () => { text: RESUME_PROMPT, }), ); + yield* awaitUntil( + () => + harness.subagentUpdates().some((event) => event.subagent.result === "CODEX_FIRST_DONE"), + "first subagent result", + ); + assert.lengthOf( + harness.subagentUpdates().filter((event) => event.subagent.result === "CODEX_FIRST_DONE"), + 1, + ); + yield* TestClock.adjust("100 millis"); yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); assert.equal(harness.terminalEvents()[0]?.status, "completed"); const settledUpdates = harness.subagentUpdates(); diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 15f65146179..e287e60103a 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -35,6 +35,7 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; @@ -105,6 +106,24 @@ export const CODEX_DRIVER_KIND = CODEX_PROVIDER; export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND); const DEFAULT_CODEX_SETTINGS = Schema.decodeSync(CodexSettings)({}); const CODEX_ASSISTANT_DELTA_FLUSH_INTERVAL_MS = 50; +const CodexBackgroundTerminalTerminateResponse = Schema.Struct({ + terminated: Schema.Boolean, +}); +const CodexBackgroundTerminalsListResponse = Schema.Struct({ + data: Schema.Array( + Schema.Struct({ + processId: Schema.String, + }), + ), + nextCursor: Schema.NullOr(Schema.String), +}); +type CodexBackgroundTerminalsListPage = typeof CodexBackgroundTerminalsListResponse.Type; +const decodeCodexBackgroundTerminalTerminateResponse = Schema.decodeUnknownEffect( + CodexBackgroundTerminalTerminateResponse, +); +const decodeCodexBackgroundTerminalsListResponse = Schema.decodeUnknownEffect( + CodexBackgroundTerminalsListResponse, +); const CODEX_CLIENT_INFO = { name: "t3code_desktop", title: "T3 Code Desktop", @@ -769,6 +788,21 @@ interface ActiveCodexTurnContext { readonly startedAt: DateTime.Utc; } +/** Snapshot of a still-running commandExecution item for interrupt/fail terminalization. */ +interface TrackedRunningCommandItem { + readonly id: string; + readonly command: string; + readonly aggregatedOutput?: string; + readonly processId?: string; +} + +type CodexRootTerminalEvent = Extract; + +interface DeferredCodexRootTerminal { + readonly context: ActiveCodexTurnContext; + readonly event: CodexRootTerminalEvent; +} + interface CodexSubagentThreadContext { readonly parentContext: ActiveCodexTurnContext; readonly providerThread: OrchestrationV2ProviderThread; @@ -784,6 +818,20 @@ interface CodexSubagentThreadContext { task: OrchestrationV2Subagent; } +const isDescendantCodexTurn = ( + candidate: ActiveCodexTurnContext, + ancestor: ActiveCodexTurnContext, +): boolean => { + let parent = candidate.subagent?.parentContext; + while (parent !== undefined) { + if (parent === ancestor) { + return true; + } + parent = parent.subagent?.parentContext; + } + return false; +}; + interface PendingCodexSubagentTurnStarted { readonly nativeTurnId: string; readonly startedAt: DateTime.Utc; @@ -846,6 +894,7 @@ export interface CodexAgentMessageDeltaCoalescer { readonly turnId: string; readonly itemId: string; readonly finalText?: string; + readonly emitEmpty?: boolean; }) => Effect.Effect; readonly flushTurn: (turnId: string) => Effect.Effect; } @@ -923,41 +972,45 @@ export const makeCodexAgentMessageDeltaCoalescer = Effect.fn( return { append: ({ turnId, itemId, delta }) => - Effect.uninterruptible( - Effect.gen(function* () { - const shouldSchedule = yield* flushLock.withPermit( + delta.length === 0 + ? Effect.void + : Effect.uninterruptible( Effect.gen(function* () { - yield* Ref.update(buffered, (current) => { - const key = codexAgentMessageBufferKey(turnId, itemId); - const existing = current.get(key); - const next = new Map(current); - next.set(key, { - turnId, - itemId, - text: `${existing?.text ?? ""}${delta}`, - dirty: true, - }); - return next; - }); - return yield* Ref.modify(flushScheduled, (scheduled) => [!scheduled, true]); + const shouldSchedule = yield* flushLock.withPermit( + Effect.gen(function* () { + yield* Ref.update(buffered, (current) => { + const key = codexAgentMessageBufferKey(turnId, itemId); + const existing = current.get(key); + const next = new Map(current); + next.set(key, { + turnId, + itemId, + text: `${existing?.text ?? ""}${delta}`, + dirty: true, + }); + return next; + }); + return yield* Ref.modify(flushScheduled, (scheduled) => [!scheduled, true]); + }), + ); + if (shouldSchedule) { + yield* Effect.sleep(Duration.millis(Math.max(1, input.flushIntervalMs))).pipe( + Effect.andThen(flushDirty), + Effect.interruptible, + Effect.forkIn(coalescerScope), + ); + } }), - ); - if (shouldSchedule) { - yield* Effect.sleep(Duration.millis(Math.max(1, input.flushIntervalMs))).pipe( - Effect.andThen(flushDirty), - Effect.interruptible, - Effect.forkIn(coalescerScope), - ); - } - }), - ), - complete: ({ turnId, itemId, finalText }) => + ), + complete: ({ turnId, itemId, finalText, emitEmpty = true }) => flushLock.withPermit( Effect.gen(function* () { const key = codexAgentMessageBufferKey(turnId, itemId); const existing = (yield* Ref.get(buffered)).get(key); const text = finalText && finalText.length > 0 ? finalText : (existing?.text ?? ""); - yield* input.emit({ turnId, itemId, text, completed: true }); + if (emitEmpty || text.length > 0) { + yield* input.emit({ turnId, itemId, text, completed: true }); + } yield* Ref.update(buffered, (current) => { const next = new Map(current); next.delete(key); @@ -1359,8 +1412,20 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi * keep projecting instead of being dropped. */ const settledTurns = yield* Ref.make(new Map()); - const runningCommandItemsByTurn = yield* Ref.make(new Map>()); + const runningCommandItemsByTurn = yield* Ref.make( + new Map>(), + ); + const interruptingNativeTurns = yield* Ref.make(new Set()); + const terminalizedNonCompletedNativeTurns = yield* Ref.make(new Set()); + // Keep the run event stream open until descendant provider state is + // terminal, otherwise the root terminal can strand child projections. + const deferredRootTerminals = yield* Ref.make(new Map()); const offeredContinuationItemsByTurn = yield* Ref.make(new Map>()); + const finalAnswerItemIdsByTurn = yield* Ref.make(new Map>()); + const completedFinalAnswerTextsByTurn = yield* Ref.make(new Map>()); + // Native completion and the interrupt timeout share one finalization + // path. Serialize the race so only one can publish terminal events. + const turnTerminalizationPermit = yield* Semaphore.make(1); const emitProviderEvent = (event: ProviderAdapterV2Event) => Queue.offer(events, event).pipe(Effect.asVoid); @@ -1460,11 +1525,11 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi return context === undefined ? undefined : ({ context, settled: false } as const); }); - const trackRunningCommandItem = (nativeTurnId: string, nativeItemId: string) => + const trackRunningCommandItem = (nativeTurnId: string, item: TrackedRunningCommandItem) => Ref.update(runningCommandItemsByTurn, (current) => { const updated = new Map(current); - const items = new Set(updated.get(nativeTurnId) ?? []); - items.add(nativeItemId); + const items = new Map(updated.get(nativeTurnId) ?? []); + items.set(item.id, item); updated.set(nativeTurnId, items); return updated; }); @@ -1476,7 +1541,7 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi if (items === undefined || !items.has(nativeItemId)) { return [items === undefined || items.size === 0, current] as const; } - const remaining = new Set(items); + const remaining = new Map(items); remaining.delete(nativeItemId); const updated = new Map(current); if (remaining.size === 0) { @@ -1487,6 +1552,84 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi return [remaining.size === 0, updated] as const; }); + /** + * When a turn is interrupted or failed, Codex often leaves commandExecution + * items mid-flight (no item/completed). Emit terminal turn items before + * turn.terminal so the projection never keeps a forever-running command card. + * Does not retain settled context: late completions must not wake the run. + */ + const terminalizeRunningCommandItems = ( + context: ActiveCodexTurnContext, + nativeTurnId: string, + status: "interrupted" | "failed", + completedAt: DateTime.Utc, + ) => + Effect.gen(function* () { + const items = (yield* Ref.get(runningCommandItemsByTurn)).get(nativeTurnId); + if (items === undefined || items.size === 0) { + return; + } + for (const tracked of items.values()) { + const nodeId = idAllocator.derive.nodeFromProviderItem({ + driver: CODEX_PROVIDER, + nativeItemId: tracked.id, + }); + const turnItemId = idAllocator.derive.turnItemFromProviderItem({ + driver: CODEX_PROVIDER, + nativeItemId: tracked.id, + }); + const ordinal = yield* resolveItemOrdinal(context, tracked.id); + const node: OrchestrationV2ExecutionNode = { + id: nodeId, + threadId: context.projectionThreadId, + runId: context.projectionRunId, + parentNodeId: context.itemParentNodeId, + rootNodeId: context.rootNodeId, + kind: "tool_call", + status, + countsForRun: false, + providerThreadId: context.providerThread.id, + providerTurnId: context.providerTurnId, + nativeItemRef: codexNativeItemRef(tracked.id), + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: context.startedAt, + completedAt, + }; + const turnItem: OrchestrationV2TurnItem = { + id: turnItemId, + threadId: context.projectionThreadId, + runId: context.projectionRunId, + nodeId, + providerThreadId: context.providerThread.id, + providerTurnId: context.providerTurnId, + nativeItemRef: codexNativeItemRef(tracked.id), + parentItemId: null, + ordinal, + status, + title: null, + startedAt: context.startedAt, + completedAt, + updatedAt: completedAt, + type: "command_execution", + input: tracked.command, + ...(tracked.aggregatedOutput === undefined + ? {} + : { output: tracked.aggregatedOutput }), + }; + yield* emitProviderEvent({ + type: "node.updated", + driver: CODEX_PROVIDER, + node, + }); + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CODEX_PROVIDER, + turnItem, + }); + } + }); + const resolveItemOrdinal = (context: ActiveCodexTurnContext, nativeItemId: string) => Effect.gen(function* () { const existing = (yield* Ref.get(itemOrdinals)).get(nativeItemId); @@ -1585,6 +1728,40 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi turn: PendingCodexSubagentTurnStarted, ) => Effect.gen(function* () { + const terminalizedNativeTurns = yield* Ref.get(terminalizedNonCompletedNativeTurns); + let ancestor: ActiveCodexTurnContext | undefined = subagent.parentContext; + while (ancestor !== undefined) { + if (terminalizedNativeTurns.has(ancestor.nativeTurnId)) { + const nativeThreadId = yield* getNativeThreadId(subagent.providerThread); + const interrupted = yield* client + .request("turn/interrupt", { + threadId: nativeThreadId, + turnId: turn.nativeTurnId, + }) + .pipe( + Effect.as(true), + Effect.catch((cause) => + Effect.logWarning("orchestration-v2.codex-late-subagent-interrupt-failed", { + nativeThreadId, + nativeTurnId: turn.nativeTurnId, + cause, + }).pipe(Effect.as(false)), + ), + Effect.timeoutOption("10 seconds"), + ); + if (Option.isNone(interrupted)) { + yield* Effect.logWarning( + "orchestration-v2.codex-late-subagent-interrupt-timeout", + { + nativeThreadId, + nativeTurnId: turn.nativeTurnId, + }, + ); + } + return; + } + ancestor = ancestor.subagent?.parentContext; + } const providerTurnId = idAllocator.derive.providerTurn({ driver: CODEX_PROVIDER, nativeTurnId: turn.nativeTurnId, @@ -2163,6 +2340,28 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi if (context === undefined) { return; } + const finalAnswerItem = (yield* Ref.get(finalAnswerItemIdsByTurn)) + .get(update.turnId) + ?.has(update.itemId); + if (finalAnswerItem) { + const finalAnswerItemIds = (yield* Ref.get(finalAnswerItemIdsByTurn)).get( + update.turnId, + ); + const completedTexts = + (yield* Ref.get(completedFinalAnswerTextsByTurn)).get(update.turnId) ?? + new Set(); + const firstFinalAnswerItemId = finalAnswerItemIds?.values().next().value; + const duplicateCompletion = + update.completed && + completedTexts.size > 0 && + (update.text.length === 0 || completedTexts.has(update.text)); + const deferredStreamingUpdate = + !update.completed && + (completedTexts.size > 0 || firstFinalAnswerItemId !== update.itemId); + if (deferredStreamingUpdate || duplicateCompletion) { + return; + } + } const artifacts = yield* buildAgentMessageArtifacts( context, { id: update.itemId, text: update.text }, @@ -2183,6 +2382,15 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi driver: CODEX_PROVIDER, turnItem: artifacts.turnItem, }); + if (finalAnswerItem && update.completed) { + yield* Ref.update(completedFinalAnswerTextsByTurn, (current) => { + const updated = new Map(current); + const texts = new Set(updated.get(update.turnId) ?? []); + texts.add(update.text); + updated.set(update.turnId, texts); + return updated; + }); + } }), }); @@ -2910,7 +3118,7 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi nativeTurnId: payload.turn.id, startedAt: codexTimestamp(payload.turn.startedAt), }); - }).pipe(Effect.orDie), + }).pipe(Effect.orDie, turnTerminalizationPermit.withPermits(1)), ); yield* client.handleServerNotification("item/started", (payload) => @@ -2934,9 +3142,32 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi return; } + if (payload.item.type === "agentMessage") { + if (payload.item.phase !== "commentary") { + yield* Ref.update(finalAnswerItemIdsByTurn, (current) => { + const updated = new Map(current); + const itemIds = new Set(updated.get(payload.turnId) ?? []); + itemIds.add(payload.item.id); + updated.set(payload.turnId, itemIds); + return updated; + }); + } + return; + } + if (payload.item.type === "commandExecution") { if (!codexItemStatus(payload.item.status).completed) { - yield* trackRunningCommandItem(payload.turnId, payload.item.id); + yield* trackRunningCommandItem(payload.turnId, { + id: payload.item.id, + command: payload.item.command, + ...(payload.item.aggregatedOutput === null || + payload.item.aggregatedOutput === undefined + ? {} + : { aggregatedOutput: payload.item.aggregatedOutput }), + ...(typeof payload.item.processId === "string" + ? { processId: payload.item.processId } + : {}), + }); } const artifacts = yield* buildCommandExecutionArtifacts(context, payload.item); yield* emitProviderEvent({ @@ -2986,7 +3217,7 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi driver: CODEX_PROVIDER, turnItem: artifacts.turnItem, }); - }).pipe(Effect.orDie), + }).pipe(Effect.orDie, turnTerminalizationPermit.withPermits(1)), ); yield* client.handleServerNotification("item/completed", (payload) => @@ -3055,6 +3286,22 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi updated.delete(payload.turnId); return updated; }); + yield* Ref.update(completedFinalAnswerTextsByTurn, (current) => { + if (!current.has(payload.turnId)) { + return current; + } + const updated = new Map(current); + updated.delete(payload.turnId); + return updated; + }); + yield* Ref.update(finalAnswerItemIdsByTurn, (current) => { + if (!current.has(payload.turnId)) { + return current; + } + const updated = new Map(current); + updated.delete(payload.turnId); + return updated; + }); } } return; @@ -3166,19 +3413,51 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi return; } + const finalAnswer = payload.item.phase !== "commentary"; + if (finalAnswer) { + yield* Ref.update(finalAnswerItemIdsByTurn, (current) => { + const updated = new Map(current); + const itemIds = new Set(updated.get(payload.turnId) ?? []); + itemIds.add(payload.item.id); + updated.set(payload.turnId, itemIds); + return updated; + }); + } + const completedTextsBefore = + (yield* Ref.get(completedFinalAnswerTextsByTurn)).get(payload.turnId) ?? + new Set(); const text = yield* agentMessageDeltas.complete({ turnId: payload.turnId, itemId: payload.item.id, finalText: payload.item.text, }); - if (context.subagent !== null && payload.item.phase !== "commentary") { + yield* Ref.update(finalAnswerItemIdsByTurn, (current) => { + const itemIds = current.get(payload.turnId); + if (itemIds === undefined || !itemIds.has(payload.item.id)) { + return current; + } + const updated = new Map(current); + const remainingItemIds = new Set(itemIds); + remainingItemIds.delete(payload.item.id); + if (remainingItemIds.size === 0) { + updated.delete(payload.turnId); + } else { + updated.set(payload.turnId, remainingItemIds); + } + return updated; + }); + const emitted = + !finalAnswer || + completedTextsBefore.size === 0 || + (text.length > 0 && !completedTextsBefore.has(text)); + if (emitted && context.subagent !== null && finalAnswer) { yield* emitSubagentTaskUpdate({ subagent: context.subagent, status: context.subagent.task.status, result: text, }); } - }).pipe(Effect.orDie), + }).pipe(Effect.orDie, turnTerminalizationPermit.withPermits(1)), ); yield* client.handleServerRequest("item/commandExecution/requestApproval", (payload) => @@ -3548,158 +3827,283 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi }).pipe(Effect.orDie), ); - yield* client.handleServerNotification("turn/completed", (payload) => - Effect.gen(function* () { - const context = (yield* Ref.get(activeTurns)).get(payload.turn.id); - if (context === undefined) { - return; + const makeRootTerminalEvent = Effect.fn("CodexAdapterV2.makeRootTerminalEvent")( + function* (input: { + readonly context: ActiveCodexTurnContext; + readonly status: OrchestrationV2ProviderTurn["status"]; + readonly failureMessage?: string; + }): Effect.fn.Return { + const terminalStatus = providerTurnStatusToTerminal(input.status); + if (terminalStatus === "failed") { + return { + type: "turn.terminal", + driver: CODEX_PROVIDER, + providerThreadId: input.context.providerThread.id, + providerTurnId: input.context.providerTurnId, + runOrdinal: input.context.input.runOrdinal, + failureItemOrdinal: yield* resolveItemOrdinal( + input.context, + `terminal-failure:${input.context.providerTurnId}`, + ), + status: terminalStatus, + failure: makeProviderFailure({ + message: input.failureMessage, + class: "provider_error", + }), + threadDisposition: "reusable", + }; } - yield* agentMessageDeltas.flushTurn(payload.turn.id); - const completedAt = codexTimestamp(payload.turn.completedAt); - const status = mapCodexTurnStatus(payload.turn.status); - yield* emitProviderEvent({ - type: "provider_turn.updated", + return { + type: "turn.terminal", driver: CODEX_PROVIDER, - threadId: context.projectionThreadId, - providerTurn: { - id: context.providerTurnId, - providerThreadId: context.providerThread.id, - nodeId: context.providerNodeId, - runAttemptId: context.subagent === null ? context.input.attemptId : null, - nativeTurnRef: { - driver: CODEX_PROVIDER, - nativeId: payload.turn.id, - strength: "strong", - }, - ordinal: context.providerTurnOrdinal, - status, - startedAt: context.startedAt, - completedAt, - }, - }); - if (context.subagent !== null) { - yield* emitProviderEvent({ - type: "node.updated", - driver: CODEX_PROVIDER, - node: { - id: context.providerNodeId, - threadId: context.projectionThreadId, - runId: null, - parentNodeId: null, - rootNodeId: context.rootNodeId, - kind: "root_turn", - status, - countsForRun: false, - providerThreadId: context.providerThread.id, - providerTurnId: context.providerTurnId, - nativeItemRef: context.subagent.task.nativeTaskRef, - runtimeRequestId: null, - checkpointScopeId: null, - startedAt: context.providerNodeStartedAt, - completedAt, - }, - }); - yield* emitProviderEvent({ - type: "provider_thread.updated", - driver: CODEX_PROVIDER, - providerThread: { - ...context.providerThread, - status: "idle", - updatedAt: completedAt, - }, + providerThreadId: input.context.providerThread.id, + providerTurnId: input.context.providerTurnId, + runOrdinal: input.context.input.runOrdinal, + status: terminalStatus, + failure: null, + threadDisposition: "reusable", + }; + }, + ); + + const emitOrDeferRootTerminal = Effect.fn("CodexAdapterV2.emitOrDeferRootTerminal")( + function* (input: { + readonly context: ActiveCodexTurnContext; + readonly nativeTurnId: string; + readonly status: OrchestrationV2ProviderTurn["status"]; + readonly failureMessage?: string; + }) { + const event = yield* makeRootTerminalEvent(input); + const hasActiveDescendants = Array.from((yield* Ref.get(activeTurns)).values()).some( + (candidate) => isDescendantCodexTurn(candidate, input.context), + ); + if (event.status !== "completed" && hasActiveDescendants) { + yield* Ref.update(deferredRootTerminals, (current) => { + const updated = new Map(current); + updated.set(input.nativeTurnId, { context: input.context, event }); + return updated; }); + return; + } + yield* emitProviderEvent(event); + }, + ); + + const flushReadyRootTerminals = Effect.fn("CodexAdapterV2.flushReadyRootTerminals")( + function* () { + const activeTurnContexts = Array.from((yield* Ref.get(activeTurns)).values()); + const readyEvents = yield* Ref.modify(deferredRootTerminals, (current) => { + const updated = new Map(current); + const ready: Array = []; + for (const [nativeTurnId, deferred] of current) { + if ( + !activeTurnContexts.some((candidate) => + isDescendantCodexTurn(candidate, deferred.context), + ) + ) { + updated.delete(nativeTurnId); + ready.push(deferred.event); + } + } + return [ready, updated] as const; + }); + for (const event of readyEvents) { + yield* emitProviderEvent(event); + } + }, + ); + + const finalizeCodexTurn = (input: { + readonly context: ActiveCodexTurnContext; + readonly nativeTurnId: string; + readonly status: OrchestrationV2ProviderTurn["status"]; + readonly completedAt: DateTime.Utc; + readonly failureMessage?: string; + }) => + turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + const current = (yield* Ref.get(activeTurns)).get(input.nativeTurnId); + if (current !== input.context) { + return false; + } + if (input.status !== "completed") { + yield* Ref.update(terminalizedNonCompletedNativeTurns, (current) => { + const updated = new Set(current); + updated.add(input.nativeTurnId); + return updated; + }); + } + yield* agentMessageDeltas.flushTurn(input.nativeTurnId); yield* emitProviderEvent({ - type: "node.updated", + type: "provider_turn.updated", driver: CODEX_PROVIDER, - node: { - id: context.subagent.subagentNodeId, - threadId: context.subagent.parentContext.projectionThreadId, - runId: context.subagent.parentContext.projectionRunId, - parentNodeId: context.subagent.parentContext.itemParentNodeId, - rootNodeId: context.subagent.parentContext.rootNodeId, - kind: "subagent", - status, - countsForRun: false, - providerThreadId: context.providerThread.id, - providerTurnId: context.subagent.parentContext.providerTurnId, - nativeItemRef: context.subagent.task.nativeTaskRef, - runtimeRequestId: null, - checkpointScopeId: null, - startedAt: context.subagent.startedAt, - completedAt, + threadId: input.context.projectionThreadId, + providerTurn: { + id: input.context.providerTurnId, + providerThreadId: input.context.providerThread.id, + nodeId: input.context.providerNodeId, + runAttemptId: + input.context.subagent === null ? input.context.input.attemptId : null, + nativeTurnRef: { + driver: CODEX_PROVIDER, + nativeId: input.nativeTurnId, + strength: "strong", + }, + ordinal: input.context.providerTurnOrdinal, + status: input.status, + startedAt: input.context.startedAt, + completedAt: input.completedAt, }, }); - yield* emitSubagentTaskUpdate({ - subagent: context.subagent, - status, - completedAt, - }); - } - if (context.subagent === null) { - const terminalStatus = providerTurnStatusToTerminal(status); - yield* emitProviderEvent( - terminalStatus === "failed" - ? { - type: "turn.terminal", - driver: CODEX_PROVIDER, - providerThreadId: context.providerThread.id, - providerTurnId: context.providerTurnId, - runOrdinal: context.input.runOrdinal, - failureItemOrdinal: yield* resolveItemOrdinal( - context, - `terminal-failure:${context.providerTurnId}`, - ), - status: terminalStatus, - failure: makeProviderFailure({ - message: payload.turn.error?.message, - class: "provider_error", - }), - threadDisposition: "reusable", - } - : { - type: "turn.terminal", - driver: CODEX_PROVIDER, - providerThreadId: context.providerThread.id, - providerTurnId: context.providerTurnId, - runOrdinal: context.input.runOrdinal, - status: terminalStatus, - failure: null, - threadDisposition: "reusable", - }, + if (input.context.subagent !== null) { + yield* emitProviderEvent({ + type: "node.updated", + driver: CODEX_PROVIDER, + node: { + id: input.context.providerNodeId, + threadId: input.context.projectionThreadId, + runId: null, + parentNodeId: null, + rootNodeId: input.context.rootNodeId, + kind: "root_turn", + status: input.status, + countsForRun: false, + providerThreadId: input.context.providerThread.id, + providerTurnId: input.context.providerTurnId, + nativeItemRef: input.context.subagent.task.nativeTaskRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: input.context.providerNodeStartedAt, + completedAt: input.completedAt, + }, + }); + yield* emitProviderEvent({ + type: "provider_thread.updated", + driver: CODEX_PROVIDER, + providerThread: { + ...input.context.providerThread, + status: "idle", + updatedAt: input.completedAt, + }, + }); + yield* emitProviderEvent({ + type: "node.updated", + driver: CODEX_PROVIDER, + node: { + id: input.context.subagent.subagentNodeId, + threadId: input.context.subagent.parentContext.projectionThreadId, + runId: input.context.subagent.parentContext.projectionRunId, + parentNodeId: input.context.subagent.parentContext.itemParentNodeId, + rootNodeId: input.context.subagent.parentContext.rootNodeId, + kind: "subagent", + status: input.status, + countsForRun: false, + providerThreadId: input.context.providerThread.id, + providerTurnId: input.context.subagent.parentContext.providerTurnId, + nativeItemRef: input.context.subagent.task.nativeTaskRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: input.context.subagent.startedAt, + completedAt: input.completedAt, + }, + }); + yield* emitSubagentTaskUpdate({ + subagent: input.context.subagent, + status: input.status, + completedAt: input.completedAt, + }); + } + if (input.status === "interrupted" || input.status === "failed") { + yield* terminalizeRunningCommandItems( + input.context, + input.nativeTurnId, + input.status, + input.completedAt, + ); + } + if (input.context.subagent === null) { + yield* emitOrDeferRootTerminal(input); + } + const waiter = (yield* Ref.get(turnWaiters)).get(input.nativeTurnId); + if (waiter !== undefined) { + yield* Deferred.succeed(waiter, undefined); + } + const runningItems = (yield* Ref.get(runningCommandItemsByTurn)).get( + input.nativeTurnId, ); - } - const waiter = (yield* Ref.get(turnWaiters)).get(payload.turn.id); - if (waiter !== undefined) { - yield* Deferred.succeed(waiter, undefined); - } - const runningItems = (yield* Ref.get(runningCommandItemsByTurn)).get(payload.turn.id); - // Failed and interrupted turns intentionally drop background command - // tracking: late completions should not wake a turn the user stopped - // or that errored, and the session should not stay pinned for them. - const retainSettledContext = - status === "completed" && runningItems !== undefined && runningItems.size > 0; - if (retainSettledContext) { - yield* Ref.update(settledTurns, (current) => { + const interruptInProgress = (yield* Ref.get(interruptingNativeTurns)).has( + input.nativeTurnId, + ); + // Completed turns can retain late background command context. + // Interrupted and failed turns never wake from late item events. + const retainSettledContext = + input.status === "completed" && runningItems !== undefined && runningItems.size > 0; + if (retainSettledContext) { + yield* Ref.update(settledTurns, (current) => { + const updated = new Map(current); + updated.set(input.nativeTurnId, input.context); + return updated; + }); + } + yield* Ref.update(activeTurns, (current) => { const updated = new Map(current); - updated.set(payload.turn.id, context); + updated.delete(input.nativeTurnId); return updated; }); + yield* flushReadyRootTerminals(); + if (!retainSettledContext && !interruptInProgress) { + yield* Ref.update(runningCommandItemsByTurn, (current) => { + if (!current.has(input.nativeTurnId)) { + return current; + } + const updated = new Map(current); + updated.delete(input.nativeTurnId); + return updated; + }); + } + if (!retainSettledContext) { + yield* Ref.update(completedFinalAnswerTextsByTurn, (current) => { + if (!current.has(input.nativeTurnId)) { + return current; + } + const updated = new Map(current); + updated.delete(input.nativeTurnId); + return updated; + }); + yield* Ref.update(finalAnswerItemIdsByTurn, (current) => { + if (!current.has(input.nativeTurnId)) { + return current; + } + const updated = new Map(current); + updated.delete(input.nativeTurnId); + return updated; + }); + } + return true; + }), + ); + + yield* client.handleServerNotification("turn/completed", (payload) => + Effect.gen(function* () { + const context = (yield* Ref.get(activeTurns)).get(payload.turn.id); + if (context === undefined) { + return; } - yield* Ref.update(activeTurns, (current) => { - const updated = new Map(current); - updated.delete(payload.turn.id); - return updated; + const nativeStatus = mapCodexTurnStatus(payload.turn.status); + const status = + nativeStatus === "completed" && + (yield* Ref.get(interruptingNativeTurns)).has(payload.turn.id) + ? "interrupted" + : nativeStatus; + yield* finalizeCodexTurn({ + context, + nativeTurnId: payload.turn.id, + status, + completedAt: codexTimestamp(payload.turn.completedAt), + ...(payload.turn.error?.message === undefined + ? {} + : { failureMessage: payload.turn.error.message }), }); - if (!retainSettledContext) { - yield* Ref.update(runningCommandItemsByTurn, (current) => { - if (!current.has(payload.turn.id)) { - return current; - } - const updated = new Map(current); - updated.delete(payload.turn.id); - return updated; - }); - } }), ); @@ -3881,8 +4285,8 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi ), interruptTurn: (turnInput) => Effect.gen(function* () { - const threadId = yield* getNativeThreadId(turnInput.providerThread); - const activeTurn = Array.from((yield* Ref.get(activeTurns)).values()).find( + const activeTurnContexts = Array.from((yield* Ref.get(activeTurns)).values()); + const activeTurn = activeTurnContexts.find( (candidate) => candidate.providerTurnId === turnInput.providerTurnId, ); if (activeTurn === undefined) { @@ -3890,10 +4294,337 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi `Provider turn ${turnInput.providerTurnId} is not active and cannot be interrupted.`, ); } - yield* client.request("turn/interrupt", { - threadId, - turnId: activeTurn.nativeTurnId, + const interruptTargetContexts = [ + activeTurn, + ...activeTurnContexts.filter( + (candidate) => + candidate !== activeTurn && isDescendantCodexTurn(candidate, activeTurn), + ), + ]; + const interruptTargets: Array<{ + readonly context: ActiveCodexTurnContext; + readonly completion: Deferred.Deferred; + }> = []; + for (const context of interruptTargetContexts) { + interruptTargets.push({ context, completion: yield* Deferred.make() }); + } + yield* Ref.update(turnWaiters, (current) => { + const updated = new Map(current); + for (const target of interruptTargets) { + updated.set(target.context.nativeTurnId, target.completion); + } + return updated; + }); + yield* Ref.update(interruptingNativeTurns, (current) => { + const updated = new Set(current); + for (const target of interruptTargets) { + updated.add(target.context.nativeTurnId); + } + return updated; + }); + const registeredActiveTurns = yield* Ref.get(activeTurns); + for (const target of interruptTargets) { + if (registeredActiveTurns.get(target.context.nativeTurnId) !== target.context) { + yield* Deferred.succeed(target.completion, undefined); + } + } + + const cleanupInterruptState = Effect.gen(function* () { + yield* Ref.update(turnWaiters, (current) => { + const updated = new Map(current); + for (const target of interruptTargets) { + updated.delete(target.context.nativeTurnId); + } + return updated; + }); + yield* Ref.update(interruptingNativeTurns, (current) => { + const updated = new Set(current); + for (const target of interruptTargets) { + updated.delete(target.context.nativeTurnId); + } + return updated; + }); + yield* Ref.update(runningCommandItemsByTurn, (current) => { + const updated = new Map(current); + for (const target of interruptTargets) { + updated.delete(target.context.nativeTurnId); + } + return updated; + }); + }); + + const trackedInterruptNativeTurnIds = new Set( + interruptTargets.map((target) => target.context.nativeTurnId), + ); + const finalizeRemainingInterruptLineage = Effect.gen(function* () { + while (true) { + const newlyDiscovered: Array = []; + const activeLineage = yield* turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + const lineage = Array.from((yield* Ref.get(activeTurns)).values()).filter( + (context) => + context === activeTurn || isDescendantCodexTurn(context, activeTurn), + ); + for (const context of lineage) { + if (trackedInterruptNativeTurnIds.has(context.nativeTurnId)) { + continue; + } + const completion = yield* Deferred.make(); + interruptTargets.push({ context, completion }); + trackedInterruptNativeTurnIds.add(context.nativeTurnId); + newlyDiscovered.push(context); + yield* Ref.update(turnWaiters, (current) => { + const updated = new Map(current); + updated.set(context.nativeTurnId, completion); + return updated; + }); + yield* Ref.update(interruptingNativeTurns, (current) => { + const updated = new Set(current); + updated.add(context.nativeTurnId); + return updated; + }); + } + return lineage; + }), + ); + if (activeLineage.length === 0) { + return; + } + if (newlyDiscovered.length > 0) { + const completed = yield* Effect.forEach( + newlyDiscovered, + (context) => + Effect.flatMap(getNativeThreadId(context.providerThread), (threadId) => + client.request("turn/interrupt", { + threadId, + turnId: context.nativeTurnId, + }), + ).pipe( + Effect.catch((cause) => + Effect.logWarning( + "orchestration-v2.codex-remaining-lineage-interrupt-failed", + { + nativeTurnId: context.nativeTurnId, + providerSessionId: input.providerSessionId, + cause, + }, + ), + ), + ), + { concurrency: "unbounded", discard: true }, + ).pipe(Effect.timeoutOption("10 seconds")); + if (Option.isNone(completed)) { + yield* Effect.logWarning( + "orchestration-v2.codex-remaining-lineage-interrupt-timeout", + { + nativeTurnIds: newlyDiscovered.map((context) => context.nativeTurnId), + providerSessionId: input.providerSessionId, + }, + ); + } + } + const completedAt = yield* DateTime.now; + for (const context of activeLineage) { + yield* finalizeCodexTurn({ + context, + nativeTurnId: context.nativeTurnId, + status: "interrupted", + completedAt, + }); + } + } }); + + const interruptLateDescendants = Effect.gen(function* () { + const activeLineage = yield* turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + const lineage = Array.from((yield* Ref.get(activeTurns)).values()).filter( + (context) => + context !== activeTurn && + isDescendantCodexTurn(context, activeTurn) && + !trackedInterruptNativeTurnIds.has(context.nativeTurnId), + ); + for (const context of lineage) { + const completion = yield* Deferred.make(); + interruptTargets.push({ context, completion }); + trackedInterruptNativeTurnIds.add(context.nativeTurnId); + yield* Ref.update(turnWaiters, (current) => { + const updated = new Map(current); + updated.set(context.nativeTurnId, completion); + return updated; + }); + yield* Ref.update(interruptingNativeTurns, (current) => { + const updated = new Set(current); + updated.add(context.nativeTurnId); + return updated; + }); + } + return lineage; + }), + ); + const completed = yield* Effect.forEach( + activeLineage, + (context) => + Effect.flatMap(getNativeThreadId(context.providerThread), (threadId) => + client.request("turn/interrupt", { + threadId, + turnId: context.nativeTurnId, + }), + ), + { concurrency: "unbounded", discard: true }, + ).pipe(Effect.timeoutOption("10 seconds")); + if (Option.isNone(completed)) { + yield* Effect.logWarning( + "orchestration-v2.codex-late-descendant-interrupt-timeout", + { + nativeTurnIds: activeLineage.map((context) => context.nativeTurnId), + providerSessionId: input.providerSessionId, + }, + ); + } + }); + + yield* Effect.gen(function* () { + for (const target of interruptTargets) { + const context = target.context; + if ((yield* Ref.get(activeTurns)).get(context.nativeTurnId) !== context) { + continue; + } + yield* client.request("turn/interrupt", { + threadId: yield* getNativeThreadId(context.providerThread), + turnId: context.nativeTurnId, + }); + } + const containedTerminalKeys = new Set(); + const attemptedTerminalKeys = new Set(); + const collectTrackedTerminals = (targets: typeof interruptTargets) => + Effect.gen(function* () { + const trackedTerminals = new Map< + string, + { readonly nativeThreadId: string; readonly processId: string } + >(); + const runningItems = yield* Ref.get(runningCommandItemsByTurn); + for (const target of targets) { + const nativeThreadId = yield* getNativeThreadId( + target.context.providerThread, + ); + const items = runningItems.get(target.context.nativeTurnId); + for (const item of items?.values() ?? []) { + if (item.processId !== undefined) { + trackedTerminals.set(`${nativeThreadId}:${item.processId}`, { + nativeThreadId, + processId: item.processId, + }); + } + } + } + return trackedTerminals; + }); + const isBackgroundTerminalStillRunning = ( + nativeThreadId: string, + processId: string, + ) => + Effect.gen(function* () { + let cursor: string | null = null; + while (true) { + const response: unknown = yield* client.raw.request( + "thread/backgroundTerminals/list", + { + threadId: nativeThreadId, + ...(cursor === null ? {} : { cursor }), + }, + ); + const page: CodexBackgroundTerminalsListPage = + yield* decodeCodexBackgroundTerminalsListResponse(response); + if (page.data.some((terminal) => terminal.processId === processId)) { + return true; + } + if (page.nextCursor === null) { + return false; + } + cursor = page.nextCursor; + } + }); + const terminateTrackedTerminals = (targets: typeof interruptTargets) => + Effect.gen(function* () { + const trackedTerminals = yield* collectTrackedTerminals(targets); + const pendingTerminals = Array.from(trackedTerminals.entries()) + .filter(([key]) => !containedTerminalKeys.has(key)) + .sort( + ([leftKey], [rightKey]) => + Number(attemptedTerminalKeys.has(leftKey)) - + Number(attemptedTerminalKeys.has(rightKey)), + ); + return yield* Effect.forEach( + pendingTerminals, + ([key, { nativeThreadId, processId }]) => + Effect.gen(function* () { + attemptedTerminalKeys.add(key); + const response = yield* client.raw.request( + "thread/backgroundTerminals/terminate", + { + threadId: nativeThreadId, + processId, + }, + ); + const result = + yield* decodeCodexBackgroundTerminalTerminateResponse(response); + if ( + !result.terminated && + (yield* isBackgroundTerminalStillRunning(nativeThreadId, processId)) + ) { + return yield* toProtocolError( + `Codex background terminal ${processId} remained active after termination.`, + ); + } + containedTerminalKeys.add(key); + }).pipe( + Effect.as({ success: true as const }), + Effect.catch((error) => + Effect.succeed({ success: false as const, error }), + ), + ), + ); + }); + + const [, completed] = yield* Effect.all( + [ + terminateTrackedTerminals(interruptTargets), + Effect.forEach( + interruptTargets, + (target) => Deferred.await(target.completion), + { concurrency: "unbounded", discard: true }, + ).pipe(Effect.timeoutOption("10 seconds")), + ], + { concurrency: "unbounded" }, + ); + if (Option.isNone(completed)) { + for (const target of interruptTargets) { + const context = target.context; + if ((yield* Ref.get(activeTurns)).get(context.nativeTurnId) !== context) { + continue; + } + yield* Effect.logWarning("orchestration-v2.codex-interrupt-timeout", { + providerSessionId: input.providerSessionId, + providerThreadId: context.providerThread.id, + providerTurnId: context.providerTurnId, + nativeTurnId: context.nativeTurnId, + }); + } + } + + yield* interruptLateDescendants; + yield* finalizeRemainingInterruptLineage; + + const terminationResults = yield* terminateTrackedTerminals(interruptTargets); + const failedTermination = terminationResults.find((result) => !result.success); + if (failedTermination !== undefined && !failedTermination.success) { + return yield* Effect.fail(failedTermination.error); + } + }).pipe( + Effect.onError(() => finalizeRemainingInterruptLineage), + Effect.ensuring(cleanupInterruptState), + ); }).pipe( Effect.mapError( (cause) => diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 693a00b57dd..3b6f74bf5f7 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -22,7 +22,7 @@ import { import { IdAllocatorV2 } from "./IdAllocator.ts"; import { ProjectionStoreV2 } from "./ProjectionStore.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; -import { RunExecutionServiceV2 } from "./RunExecutionService.ts"; +import { canRouteRelatedSubagent, RunExecutionServiceV2 } from "./RunExecutionService.ts"; import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; export class ProviderTurnStartError extends Schema.TaggedErrorClass()( @@ -403,6 +403,9 @@ export const layer: Layer.Layer< if (!runningWrite.committed) { return; } + const routableSubagents = projection.subagents.filter((subagent) => + canRouteRelatedSubagent(subagent.status), + ); yield* runExecution.startRootRun({ commandId: CommandId.make(`command:effect:provider-turn.start:${run.id}`), appThread: projection.thread, @@ -414,10 +417,10 @@ export const layer: Layer.Layer< providerThread: runningProviderThread, attempt: runningAttempt, attemptId: attempt.id, - relatedThreadIds: projection.subagents.flatMap((subagent) => + relatedThreadIds: routableSubagents.flatMap((subagent) => subagent.childThreadId === null ? [] : [subagent.childThreadId], ), - relatedProviderThreadIds: projection.subagents.flatMap((subagent) => + relatedProviderThreadIds: routableSubagents.flatMap((subagent) => subagent.providerThreadId === null ? [] : [subagent.providerThreadId], ), providerTurnOrdinal: diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 2822f60851b..28f2120e599 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2,14 +2,18 @@ import { assert, it } from "@effect/vitest"; import { CheckpointScopeId, CommandId, + EventId, MessageId, NodeId, type OrchestrationV2AppThread, type OrchestrationV2CheckpointScope, + type OrchestrationV2DomainEvent, type OrchestrationV2ExecutionNode, type OrchestrationV2ProviderThread, type OrchestrationV2Run, type OrchestrationV2RunAttempt, + type OrchestrationV2Subagent, + type OrchestrationV2TurnItem, ProviderDriverKind, ProviderInstanceId, ProviderSessionId, @@ -35,6 +39,8 @@ import { IdAllocatorV2, layer as idAllocatorLayer } from "./IdAllocator.ts"; import type { ProviderAdapterV2Event, ProviderAdapterV2SessionRuntime } from "./ProviderAdapter.ts"; import { ProviderEventIngestorV2 } from "./ProviderEventIngestor.ts"; import { + canRouteRelatedSubagent, + cascadeTerminalizeRunOwnedSubagents, finalProviderThreadStatus, layer as runExecutionServiceLayer, makeProviderEventRoutingState, @@ -181,6 +187,52 @@ it("does not route a superseded attempt through a reused provider thread", () => assert.isFalse(routeProviderEvent(oldTurnEvent, newAttempt, newState)[0]); }); +it("does not carry interrupted child ownership into later attempts", () => { + assert.isFalse(canRouteRelatedSubagent("interrupted")); + assert.isFalse(canRouteRelatedSubagent("failed")); + assert.isFalse(canRouteRelatedSubagent("cancelled")); + assert.isTrue(canRouteRelatedSubagent("completed")); + assert.isTrue(canRouteRelatedSubagent("running")); + + const threadId = ThreadId.make("thread:related-child:next-attempt"); + const childThreadId = ThreadId.make("thread:related-child:interrupted"); + const identity: ProviderEventRouteIdentity = { + threadId, + runId: RunId.make("run:related-child:next-attempt"), + attemptId: RunAttemptId.make("attempt:related-child:next-attempt"), + providerThreadId: ProviderThreadId.make("provider-thread:related-child:next-attempt"), + }; + const state = makeProviderEventRoutingState({ + identity, + providerTurnId: null, + relatedThreadIds: canRouteRelatedSubagent("interrupted") ? [childThreadId] : [], + }); + const childNodeId = NodeId.make("node:related-child:interrupted"); + const lateChildNode = { + type: "node.updated", + driver, + node: { + id: childNodeId, + threadId: childThreadId, + runId: null, + parentNodeId: null, + rootNodeId: childNodeId, + kind: "root_turn", + status: "completed", + countsForRun: false, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: null, + completedAt: null, + }, + } satisfies ProviderAdapterV2Event; + + assert.isFalse(routeProviderEvent(lateChildNode, identity, state)[0]); +}); + it.effect("rechecks run ownership immediately before calling the provider", () => Effect.gen(function* () { const runExecution = yield* RunExecutionServiceV2; @@ -542,11 +594,863 @@ it.effect("does not pin ingestion on background items when the root turn is inte }), ); -it.effect("omits run_interrupt_result when a superseding attempt already owns the run", () => +it.effect( + "cascade-terminalizes run-owned subagent rows on interrupt before root finalization", + () => + Effect.gen(function* () { + const ids = backgroundScenarioIds("subagent-interrupt-cascade"); + const childThreadId = ids.childThreadId; + const unrelatedChildThreadId = ThreadId.make( + "thread:subagent-interrupt-cascade:unrelated-child", + ); + const providerInstanceId = ProviderInstanceId.make("codex"); + const written = yield* Ref.make>([]); + const ingested = yield* Ref.make>([]); + const ingestionDone = yield* Deferred.make(); + const testLayer = runExecutionServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + Layer.mock(EventSinkV2)({ + write: () => Effect.succeed([]), + writeWithEffects: (input) => + Effect.gen(function* () { + yield* Ref.update(written, (current) => [...current, ...input.events]); + return []; + }), + writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), + }), + idAllocatorLayer, + Layer.mock(ProviderEventIngestorV2)({ + ingestNormalized: (input) => + Ref.update(ingested, (current) => [...current, input.event]).pipe(Effect.as([])), + }), + ServerSettingsService.layerTest(), + ), + ), + ); + + const runningSubagent = makeRunOwnedSubagentFixture({ + ids, + providerInstanceId, + childThreadId, + driver, + status: "running", + }); + const runningTurnItem = makeRunOwnedSubagentTurnItemFixture({ + ids, + providerInstanceId, + childThreadId, + driver, + status: "running", + }); + const runningNode = makeRunOwnedSubagentNodeFixture({ + ids, + status: "running", + }); + const runningChildNode = makeRunOwnedSubagentChildNodeFixture({ + ids, + status: "running", + }); + const runningChildTurnItem = makeLinkedChildTurnItemFixture({ + ids, + driver, + type: "command_execution", + }); + const suppressedChildAssistantTurnItem = makeLinkedChildTurnItemFixture({ + ids: { + ...ids, + childItemId: TurnItemId.make("turn-item:subagent-interrupt-cascade:suppressed-assistant"), + }, + driver, + type: "assistant_message", + }); + const unrelatedChildNode = { + ...runningChildNode, + id: NodeId.make("node:subagent-interrupt-cascade:unrelated-child"), + threadId: unrelatedChildThreadId, + runId: ids.runId, + rootNodeId: NodeId.make("node:subagent-interrupt-cascade:unrelated-child"), + }; + const unrelatedChildTurnItem = { + ...runningChildTurnItem, + id: TurnItemId.make("turn-item:subagent-interrupt-cascade:unrelated-child"), + threadId: unrelatedChildThreadId, + nodeId: unrelatedChildNode.id, + }; + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make("command:subagent-interrupt-cascade"), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make("session:subagent-interrupt-cascade"), + session: { + events: Stream.empty, + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable([ + childThreadCreatedEvent(ids), + { + type: "subagent.updated", + driver, + subagent: runningSubagent, + }, + { + type: "node.updated", + driver, + node: runningNode, + }, + { + type: "node.updated", + driver, + node: runningChildNode, + }, + { + type: "node.updated", + driver, + node: unrelatedChildNode, + }, + { + type: "turn_item.updated", + driver, + turnItem: runningTurnItem, + }, + { + type: "turn_item.updated", + driver, + turnItem: runningChildTurnItem, + }, + { + type: "turn_item.updated", + driver, + turnItem: suppressedChildAssistantTurnItem, + }, + { + type: "turn_item.updated", + driver, + turnItem: unrelatedChildTurnItem, + }, + rootTerminalEvent(ids, "interrupted"), + // Late provider completion after interrupt must not be ingested. + { + type: "subagent.updated", + driver, + subagent: { + ...runningSubagent, + status: "completed" as const, + result: "should-not-apply", + completedAt: runningSubagent.updatedAt, + }, + }, + { + type: "turn_item.updated", + driver, + turnItem: { + ...runningTurnItem, + status: "completed" as const, + result: "should-not-apply", + completedAt: runningTurnItem.updatedAt, + updatedAt: runningTurnItem.updatedAt, + }, + }, + { + type: "turn_item.updated", + driver, + turnItem: { + ...runningChildTurnItem, + status: "completed" as const, + completedAt: runningChildTurnItem.updatedAt, + updatedAt: runningChildTurnItem.updatedAt, + }, + }, + { + type: "node.updated", + driver, + node: { + ...runningChildNode, + status: "completed" as const, + completedAt: runningChildNode.startedAt, + }, + }, + ] satisfies ReadonlyArray), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make("checkpoint-scope:subagent-interrupt-cascade"), + } as OrchestrationV2CheckpointScope, + providerThread: { + id: ids.providerThreadId, + driver, + } as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + relatedThreadIds: [unrelatedChildThreadId], + providerTurnOrdinal: 1, + message: { + messageId: MessageId.make("message:subagent-interrupt-cascade:user"), + text: "Spawn a subagent then stop.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(testLayer)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish"); + + const events = yield* Ref.get(written); + const subagentEvents = events.flatMap((event) => + event.type === "subagent.updated" ? [event] : [], + ); + const turnItemEvents = events.flatMap((event) => { + if (event.type !== "turn-item.updated" || event.payload.type !== "subagent") { + return []; + } + return [ + { + ...event, + payload: event.payload, + }, + ]; + }); + const nodeEvents = events.flatMap((event) => + event.type === "node.updated" && + event.payload.status === "interrupted" && + (event.payload.id === runningNode.id || event.payload.id === runningChildNode.id) + ? [event] + : [], + ); + const runUpdatedIndex = events.findIndex((event) => event.type === "run.updated"); + assert.isAtLeast(runUpdatedIndex, 0, "root run.updated must be written"); + + assert.lengthOf(subagentEvents, 1); + const terminalSubagent = subagentEvents[0]; + assert.isDefined(terminalSubagent); + assert.equal(terminalSubagent.payload.status, "interrupted"); + assert.equal(terminalSubagent.payload.childThreadId, childThreadId); + assert.equal(terminalSubagent.payload.result, null); + assert.isNotNull(terminalSubagent.payload.completedAt); + + assert.lengthOf(turnItemEvents, 1); + const terminalTurnItem = turnItemEvents[0]; + assert.isDefined(terminalTurnItem); + assert.equal(terminalTurnItem.payload.status, "interrupted"); + assert.equal(terminalTurnItem.payload.childThreadId, childThreadId); + assert.equal(terminalTurnItem.payload.result, null); + + const terminalChildTurnItems = events.flatMap((event) => + event.type === "turn-item.updated" && + event.payload.id === runningChildTurnItem.id && + event.payload.status === "interrupted" + ? [event] + : [], + ); + assert.lengthOf(terminalChildTurnItems, 1); + const terminalChildTurnItem = terminalChildTurnItems[0]; + assert.isDefined(terminalChildTurnItem); + assert.equal(terminalChildTurnItem.threadId, childThreadId); + assert.equal(terminalChildTurnItem.payload.threadId, childThreadId); + assert.equal(terminalChildTurnItem.payload.runId, null); + assert.equal(terminalChildTurnItem.payload.type, "command_execution"); + assert.isFalse( + events.some( + (event) => + event.type === "turn-item.updated" && + event.payload.id === unrelatedChildTurnItem.id && + event.payload.status === "interrupted", + ), + "related but unlinked child turn items must not cascade", + ); + assert.isFalse( + events.some( + (event) => + event.type === "turn-item.updated" && + event.payload.id === suppressedChildAssistantTurnItem.id, + ), + "suppressed streaming child items must not be created by the cascade", + ); + + assert.lengthOf(nodeEvents, 2); + assert.isTrue( + nodeEvents.some( + (event) => event.payload.id === runningNode.id && event.payload.kind === "subagent", + ), + ); + assert.isFalse( + events.some( + (event) => + event.type === "node.updated" && + event.payload.id === unrelatedChildNode.id && + event.payload.status === "interrupted", + ), + "owned child threads without a live run-owned subagent link must not cascade", + ); + assert.isTrue( + nodeEvents.some( + (event) => + event.payload.id === runningChildNode.id && + event.runId === (runningChildNode.runId ?? ids.runId) && + event.payload.threadId === childThreadId && + event.payload.kind === "root_turn", + ), + ); + + const cascadeIndexes = events.flatMap((event, index) => { + if (event.type === "subagent.updated" && event.payload.status === "interrupted") { + return [index]; + } + if ( + event.type === "turn-item.updated" && + event.payload.status === "interrupted" && + (event.payload.type === "subagent" || event.payload.id === runningChildTurnItem.id) + ) { + return [index]; + } + if (event.type === "node.updated" && event.payload.status === "interrupted") { + return event.payload.id === runningNode.id || event.payload.id === runningChildNode.id + ? [index] + : []; + } + return []; + }); + assert.isTrue( + cascadeIndexes.every((index) => index < runUpdatedIndex), + "subagent cascade must precede run.updated", + ); + assert.isFalse( + events.some((event) => { + if (event.type === "subagent.updated") { + return event.payload.status === "completed"; + } + if (event.type === "turn-item.updated" && event.payload.type === "subagent") { + return event.payload.status === "completed"; + } + return false; + }), + "late provider completion must not reopen cascaded subagent rows", + ); + assert.isFalse( + (yield* Ref.get(ingested)).some( + (event) => + (event.type === "subagent.updated" && event.subagent.status === "completed") || + (event.type === "turn_item.updated" && event.turnItem.status === "completed") || + (event.type === "node.updated" && event.node.status === "completed"), + ), + "late provider completion must not be ingested after interrupt", + ); + }), +); + +it.effect( + "cascades linked child-thread nodes after run-owned subagent and turn-item terminalize", + () => + Effect.gen(function* () { + const ids = backgroundScenarioIds("subagent-link-survives-terminal"); + const childThreadId = ids.childThreadId; + const unrelatedChildThreadId = ThreadId.make( + "thread:subagent-link-survives-terminal:unrelated-child", + ); + const providerInstanceId = ProviderInstanceId.make("codex"); + const written = yield* Ref.make>([]); + const ingestionDone = yield* Deferred.make(); + const testLayer = runExecutionServiceLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CheckpointServiceV2)({ captureBaseline: () => Effect.void }), + Layer.mock(EventSinkV2)({ + write: () => Effect.succeed([]), + writeWithEffects: (input) => + Effect.gen(function* () { + yield* Ref.update(written, (current) => [...current, ...input.events]); + return []; + }), + writeIfRunCurrent: () => Effect.succeed({ committed: true, storedEvents: [] }), + }), + idAllocatorLayer, + Layer.mock(ProviderEventIngestorV2)({ + ingestNormalized: () => Effect.succeed([]), + }), + ServerSettingsService.layerTest(), + ), + ), + ); + + const runningSubagent = makeRunOwnedSubagentFixture({ + ids, + providerInstanceId, + childThreadId, + driver, + status: "running", + }); + const runningTurnItem = makeRunOwnedSubagentTurnItemFixture({ + ids, + providerInstanceId, + childThreadId, + driver, + status: "running", + }); + const runningNode = makeRunOwnedSubagentNodeFixture({ + ids, + status: "running", + }); + const runningChildNode = makeRunOwnedSubagentChildNodeFixture({ + ids, + status: "running", + }); + const unrelatedChildNode = { + ...runningChildNode, + id: NodeId.make("node:subagent-link-survives-terminal:unrelated-child"), + threadId: unrelatedChildThreadId, + runId: ids.runId, + rootNodeId: NodeId.make("node:subagent-link-survives-terminal:unrelated-child"), + }; + const completedAt = runningSubagent.updatedAt; + + yield* Effect.gen(function* () { + const runExecution = yield* RunExecutionServiceV2; + yield* runExecution.startRootRun({ + commandId: CommandId.make("command:subagent-link-survives-terminal"), + appThread: { id: ids.threadId } as OrchestrationV2AppThread, + providerSessionId: ProviderSessionId.make("session:subagent-link-survives-terminal"), + session: { + events: Stream.empty, + subscribeEvents: Effect.succeed({ + events: Stream.fromIterable([ + childThreadCreatedEvent(ids), + { + type: "subagent.updated", + driver, + subagent: runningSubagent, + }, + { + type: "node.updated", + driver, + node: runningNode, + }, + { + type: "node.updated", + driver, + node: runningChildNode, + }, + { + type: "node.updated", + driver, + node: unrelatedChildNode, + }, + { + type: "turn_item.updated", + driver, + turnItem: runningTurnItem, + }, + // Subagent + turn-item settle before root interrupt; linkage + // must still prove the open child-thread node is cascadeable. + { + type: "subagent.updated", + driver, + subagent: { + ...runningSubagent, + status: "completed" as const, + result: "subagent finished first", + completedAt, + }, + }, + { + type: "turn_item.updated", + driver, + turnItem: { + ...runningTurnItem, + status: "completed" as const, + result: "turn item finished first", + completedAt, + updatedAt: completedAt, + }, + }, + { + type: "node.updated", + driver, + node: { + ...runningNode, + status: "completed" as const, + completedAt, + }, + }, + rootTerminalEvent(ids, "interrupted"), + ] satisfies ReadonlyArray), + close: Deferred.succeed(ingestionDone, undefined), + }), + startTurn: () => Effect.void, + } as unknown as ProviderAdapterV2SessionRuntime, + run: { + id: ids.runId, + threadId: ids.threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + rootNode: { id: ids.rootNodeId } as OrchestrationV2ExecutionNode, + checkpointScope: { + id: CheckpointScopeId.make("checkpoint-scope:subagent-link-survives-terminal"), + } as OrchestrationV2CheckpointScope, + providerThread: { + id: ids.providerThreadId, + driver, + } as OrchestrationV2ProviderThread, + attempt: { + id: ids.attemptId, + providerTurnId: ids.rootProviderTurnId, + } as OrchestrationV2RunAttempt, + attemptId: ids.attemptId, + relatedThreadIds: [unrelatedChildThreadId], + providerTurnOrdinal: 1, + message: { + messageId: MessageId.make("message:subagent-link-survives-terminal:user"), + text: "Subagent settles before root interrupt.", + attachments: [], + createdBy: "user", + creationSource: "web", + }, + modelSelection: { instanceId: providerInstanceId, model: "gpt-5.4" }, + runtimePolicy: { + runtimeMode: "full-access", + interactionMode: "default", + cwd: process.cwd(), + approvalPolicy: "never", + sandboxPolicy: { + type: "readOnly", + access: { type: "fullAccess" }, + networkAccess: false, + }, + }, + }); + }).pipe(Effect.provide(testLayer)); + + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish"); + + const events = yield* Ref.get(written); + const runUpdatedIndex = events.findIndex((event) => event.type === "run.updated"); + assert.isAtLeast(runUpdatedIndex, 0, "root run.updated must be written"); + + const cascadedChildNodeEvents = events.flatMap((event, index) => + event.type === "node.updated" && + event.payload.id === runningChildNode.id && + event.payload.status === "interrupted" + ? [{ event, index }] + : [], + ); + assert.lengthOf( + cascadedChildNodeEvents, + 1, + "open linked child-thread node must cascade after subagent/turn-item terminalize", + ); + const cascadedChild = cascadedChildNodeEvents[0]; + assert.isDefined(cascadedChild); + assert.isTrue( + cascadedChild.index < runUpdatedIndex, + "child-thread cascade must precede run.updated", + ); + assert.equal(cascadedChild.event.payload.threadId, childThreadId); + assert.equal(cascadedChild.event.payload.kind, "root_turn"); + assert.isFalse( + events.some( + (event) => + event.type === "node.updated" && + event.payload.id === unrelatedChildNode.id && + event.payload.status === "interrupted", + ), + "related but unlinked child threads must not cascade", + ); + assert.isFalse( + events.some( + (event) => event.type === "subagent.updated" && event.payload.status === "interrupted", + ), + "already-terminal subagent rows must not be re-cascaded", + ); + assert.isFalse( + events.some( + (event) => + event.type === "turn-item.updated" && + event.payload.type === "subagent" && + event.payload.status === "interrupted", + ), + "already-terminal subagent turn items must not be re-cascaded", + ); + }), +); + +it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child projections", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + let nextId = 0; + const allocateEventId = () => + Effect.sync(() => EventId.make(`event:cascade-helper:${nextId++}`)); + + for (const driverKind of [ + ProviderDriverKind.make("claudeAgent"), + ProviderDriverKind.make("codex"), + ] as const) { + const runId = RunId.make(`run:cascade-helper:${driverKind}`); + const threadId = ThreadId.make(`thread:cascade-helper:${driverKind}`); + const childThreadId = ThreadId.make(`thread:cascade-helper:${driverKind}:child`); + const subagentId = NodeId.make(`node:cascade-helper:${driverKind}:subagent`); + const childNodeId = NodeId.make(`node:cascade-helper:${driverKind}:child-root`); + const providerInstanceId = ProviderInstanceId.make(String(driverKind)); + const terminalStatus = driverKind === "claudeAgent" ? "failed" : "cancelled"; + const subagent: OrchestrationV2Subagent = { + id: subagentId, + threadId, + runId, + parentNodeId: NodeId.make(`node:cascade-helper:${driverKind}:root`), + origin: "provider_native", + createdBy: "agent", + driver: driverKind, + providerInstanceId, + providerThreadId: null, + childThreadId, + nativeTaskRef: { + driver: driverKind, + nativeId: `native-${driverKind}`, + strength: "strong", + }, + prompt: "hold", + title: "hold", + model: null, + status: "running", + progress: "partial progress", + result: "partial result", + startedAt: now, + completedAt: null, + updatedAt: now, + }; + const turnItem = { + id: TurnItemId.make(`turn-item:cascade-helper:${driverKind}`), + threadId, + runId, + nodeId: subagentId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: subagent.nativeTaskRef, + parentItemId: null, + ordinal: 3, + status: "running" as const, + title: "hold", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "subagent" as const, + subagentId, + origin: "provider_native" as const, + driver: driverKind, + providerInstanceId, + childThreadId, + prompt: "hold", + progress: "partial progress", + result: "partial result", + } satisfies Extract; + const node: OrchestrationV2ExecutionNode = { + id: subagentId, + threadId, + runId, + parentNodeId: NodeId.make(`node:cascade-helper:${driverKind}:root`), + rootNodeId: NodeId.make(`node:cascade-helper:${driverKind}:root`), + kind: "subagent", + status: "running", + countsForRun: false, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: subagent.nativeTaskRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: null, + }; + const openChildNode: OrchestrationV2ExecutionNode = { + id: childNodeId, + threadId: childThreadId, + runId: null, + parentNodeId: null, + rootNodeId: childNodeId, + kind: "root_turn", + status: "running", + countsForRun: false, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: null, + }; + const childTurnItem: OrchestrationV2TurnItem = + driverKind === "claudeAgent" + ? { + id: TurnItemId.make(`turn-item:cascade-helper:${driverKind}:child-reasoning`), + threadId: childThreadId, + runId: null, + nodeId: childNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "running", + title: "Working", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "reasoning", + text: "partial progress", + streaming: true, + } + : { + id: TurnItemId.make(`turn-item:cascade-helper:${driverKind}:child-command`), + threadId: childThreadId, + runId: null, + nodeId: childNodeId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "running", + title: "sleep 300", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "command_execution", + input: "sleep 300", + }; + + const events = yield* cascadeTerminalizeRunOwnedSubagents({ + run: { + id: runId, + threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + open: { + subagents: new Map([[subagentId, subagent]]), + turnItems: new Map([[subagentId, turnItem]]), + childTurnItems: new Map(), + nodes: new Map([[subagentId, node]]), + linkedChildThreadIds: new Set([childThreadId]), + }, + status: terminalStatus, + completedAt: now, + allocateEventId, + }); + + assert.equal(events.length, 3, `${driverKind}: subagent + node + turn item`); + const terminalSubagent = events.find((event) => event.type === "subagent.updated"); + assert.isDefined(terminalSubagent); + if (terminalSubagent?.type !== "subagent.updated") { + assert.fail("expected subagent.updated event"); + return; + } + assert.equal(terminalSubagent.payload.status, terminalStatus); + assert.equal(terminalSubagent.payload.childThreadId, childThreadId); + assert.equal(terminalSubagent.payload.progress, "partial progress"); + assert.equal(terminalSubagent.payload.result, "partial result"); + assert.equal(terminalSubagent.payload.driver, driverKind); + + const terminalItem = events.find( + (event) => event.type === "turn-item.updated" && event.payload.type === "subagent", + ); + assert.isDefined(terminalItem); + if (terminalItem?.type !== "turn-item.updated" || terminalItem.payload.type !== "subagent") { + assert.fail("expected subagent turn-item.updated event"); + return; + } + assert.equal(terminalItem.payload.status, terminalStatus); + assert.equal(terminalItem.payload.childThreadId, childThreadId); + assert.equal(terminalItem.payload.progress, "partial progress"); + assert.equal(terminalItem.payload.result, "partial result"); + + // Shared cascade path: after subagent/turn-item rows are gone, only the + // preserved linkage may prove an open child-thread node is cascadeable. + const afterTerminalLinkEvents = yield* cascadeTerminalizeRunOwnedSubagents({ + run: { + id: runId, + threadId, + ordinal: 1, + providerInstanceId, + } as OrchestrationV2Run, + open: { + subagents: new Map(), + turnItems: new Map(), + childTurnItems: new Map([[childTurnItem.id, childTurnItem]]), + nodes: new Map([[childNodeId, openChildNode]]), + linkedChildThreadIds: new Set([childThreadId]), + }, + status: terminalStatus, + completedAt: now, + allocateEventId, + }); + assert.equal( + afterTerminalLinkEvents.length, + 2, + `${driverKind}: linked child node and turn item cascade after link rows terminalize`, + ); + const cascadedChild = afterTerminalLinkEvents.find((event) => event.type === "node.updated"); + assert.isDefined(cascadedChild); + if (cascadedChild?.type !== "node.updated") { + assert.fail("expected node.updated for linked child thread"); + return; + } + assert.equal(cascadedChild.payload.id, childNodeId); + assert.equal(cascadedChild.payload.threadId, childThreadId); + assert.equal(cascadedChild.payload.status, terminalStatus); + assert.equal(cascadedChild.payload.kind, "root_turn"); + const cascadedChildTurnItem = afterTerminalLinkEvents.find( + (event) => event.type === "turn-item.updated", + ); + assert.isDefined(cascadedChildTurnItem); + if (cascadedChildTurnItem?.type !== "turn-item.updated") { + assert.fail("expected turn-item.updated for linked child thread"); + return; + } + assert.equal(cascadedChildTurnItem.threadId, childThreadId); + assert.equal(cascadedChildTurnItem.payload.id, childTurnItem.id); + assert.equal(cascadedChildTurnItem.payload.status, terminalStatus); + assert.equal(cascadedChildTurnItem.payload.runId, null); + assert.equal(cascadedChildTurnItem.payload.type, childTurnItem.type); + if (cascadedChildTurnItem.payload.type === "reasoning") { + assert.isFalse(cascadedChildTurnItem.payload.streaming); + } + } + }), +); + +it.effect("omits interrupt results and subagent cascade for a superseded attempt", () => Effect.gen(function* () { const written = yield* captureInterruptTerminalTurnItems({ key: "steer-supersede", shouldFinalizeRun: () => Effect.succeed(false), + seedOpenSubagent: true, }); assert.deepEqual( written.map((item) => item.type), @@ -609,10 +1513,18 @@ function captureInterruptTerminalTurnItems(input: { readonly key: string; readonly shouldFinalizeRun: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; + readonly seedOpenSubagent?: boolean; }) { return Effect.gen(function* () { const ids = backgroundScenarioIds(input.key); const providerInstanceId = ProviderInstanceId.make("codex"); + const runningSubagent = makeRunOwnedSubagentFixture({ + ids, + providerInstanceId, + childThreadId: ids.childThreadId, + driver, + status: "running", + }); const writtenItems = yield* Ref.make< ReadonlyArray<{ readonly type: string; readonly parentItemId: string | null }> >([]); @@ -668,7 +1580,30 @@ function captureInterruptTerminalTurnItems(input: { session: { events: Stream.empty, subscribeEvents: Effect.succeed({ - events: Stream.fromIterable([rootTerminalEvent(ids, "interrupted")]), + events: Stream.fromIterable([ + ...(input.seedOpenSubagent + ? [ + { type: "subagent.updated", driver, subagent: runningSubagent } as const, + { + type: "node.updated", + driver, + node: makeRunOwnedSubagentNodeFixture({ ids, status: "running" }), + } as const, + { + type: "turn_item.updated", + driver, + turnItem: makeRunOwnedSubagentTurnItemFixture({ + ids, + providerInstanceId, + childThreadId: ids.childThreadId, + driver, + status: "running", + }), + } as const, + ] + : []), + rootTerminalEvent(ids, "interrupted"), + ] satisfies ReadonlyArray), close: Deferred.succeed(ingestionDone, undefined), }), startTurn: () => Effect.void, @@ -831,6 +1766,177 @@ function subagentEvent( } as ProviderAdapterV2Event; } +function makeRunOwnedSubagentFixture(input: { + readonly ids: BackgroundScenarioIds; + readonly providerInstanceId: ProviderInstanceId; + readonly childThreadId: ThreadId; + readonly driver: typeof driver; + readonly status: "running" | "interrupted"; +}): OrchestrationV2Subagent { + const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z"); + return { + id: input.ids.subagentNodeId, + threadId: input.ids.threadId, + runId: input.ids.runId, + parentNodeId: input.ids.rootNodeId, + origin: "provider_native", + createdBy: "agent", + driver: input.driver, + providerInstanceId: input.providerInstanceId, + providerThreadId: null, + childThreadId: input.childThreadId, + nativeTaskRef: { + driver: input.driver, + nativeId: `task:${input.ids.subagentNodeId}`, + strength: "strong", + }, + prompt: "hold", + title: "Live-test subagent hold", + model: null, + status: input.status, + result: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }; +} + +function makeRunOwnedSubagentTurnItemFixture(input: { + readonly ids: BackgroundScenarioIds; + readonly providerInstanceId: ProviderInstanceId; + readonly childThreadId: ThreadId; + readonly driver: typeof driver; + readonly status: "running" | "interrupted"; +}): Extract { + const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z"); + return { + id: input.ids.itemId, + threadId: input.ids.threadId, + runId: input.ids.runId, + nodeId: input.ids.subagentNodeId, + providerThreadId: input.ids.providerThreadId, + providerTurnId: input.ids.rootProviderTurnId, + nativeItemRef: { + driver: input.driver, + nativeId: `task:${input.ids.subagentNodeId}`, + strength: "strong", + }, + parentItemId: null, + ordinal: 3, + status: input.status, + title: "Live-test subagent hold", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "subagent", + subagentId: input.ids.subagentNodeId, + origin: "provider_native", + driver: input.driver, + providerInstanceId: input.providerInstanceId, + childThreadId: input.childThreadId, + prompt: "hold", + result: null, + }; +} + +function makeLinkedChildTurnItemFixture(input: { + readonly ids: BackgroundScenarioIds; + readonly driver: typeof driver; + readonly type: "assistant_message" | "command_execution" | "reasoning"; +}): OrchestrationV2TurnItem { + const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z"); + const base = { + id: input.ids.childItemId, + threadId: input.ids.childThreadId, + runId: null, + nodeId: NodeId.make(`${input.ids.subagentNodeId}:child-root`), + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "running" as const, + title: "Working", + startedAt: now, + completedAt: null, + updatedAt: now, + }; + if (input.type === "assistant_message") { + return { + ...base, + type: "assistant_message", + messageId: MessageId.make(`${input.ids.childItemId}:message`), + text: "partial response", + streaming: true, + }; + } + if (input.type === "reasoning") { + return { + ...base, + type: "reasoning", + text: "partial progress", + streaming: true, + }; + } + return { + ...base, + type: "command_execution", + input: "sleep 300", + }; +} + +function makeRunOwnedSubagentNodeFixture(input: { + readonly ids: BackgroundScenarioIds; + readonly status: "running" | "interrupted"; +}): OrchestrationV2ExecutionNode { + const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z"); + return { + id: input.ids.subagentNodeId, + threadId: input.ids.threadId, + runId: input.ids.runId, + parentNodeId: input.ids.rootNodeId, + rootNodeId: input.ids.rootNodeId, + kind: "subagent", + status: input.status, + countsForRun: false, + providerThreadId: input.ids.providerThreadId, + providerTurnId: input.ids.rootProviderTurnId, + nativeItemRef: { + driver, + nativeId: `task:${input.ids.subagentNodeId}`, + strength: "strong", + }, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: null, + }; +} + +function makeRunOwnedSubagentChildNodeFixture(input: { + readonly ids: BackgroundScenarioIds; + readonly status: "running" | "interrupted"; +}): OrchestrationV2ExecutionNode { + const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z"); + return { + id: NodeId.make(`${input.ids.subagentNodeId}:child-root`), + threadId: input.ids.childThreadId, + runId: null, + parentNodeId: null, + rootNodeId: NodeId.make(`${input.ids.subagentNodeId}:child-root`), + kind: "root_turn", + status: input.status, + countsForRun: false, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: null, + }; +} + function rootTerminalEvent( ids: BackgroundScenarioIds, status: "completed" | "interrupted", diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index c833fdf7b36..a3d72b065b7 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1,9 +1,11 @@ import { CommandId, + type EventId, type ModelSelection, type NodeId, type OrchestrationV2AppThread, type OrchestrationV2CheckpointScope, + type OrchestrationV2DomainEvent, type OrchestrationV2ExecutionNode, type OrchestrationV2ProviderFailure, type OrchestrationV2ProviderThread, @@ -17,6 +19,7 @@ import { type ProviderTurnId, type RunAttemptId, type ThreadId, + type TurnItemId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Cause from "effect/Cause"; @@ -31,7 +34,11 @@ import * as Stream from "effect/Stream"; import { ServerSettingsService } from "../serverSettings.ts"; import { CheckpointServiceV2 } from "./CheckpointService.ts"; import { EventSinkV2 } from "./EventSink.ts"; -import { IdAllocatorV2, type IdAllocatorV2Shape } from "./IdAllocator.ts"; +import { + IdAllocatorV2, + type IdAllocatorV2AllocationError, + type IdAllocatorV2Shape, +} from "./IdAllocator.ts"; import type { ProviderAdapterV2Event, ProviderAdapterV2RuntimePolicy, @@ -94,6 +101,170 @@ function isTerminalTurnItemStatus(status: OrchestrationV2TurnItem["status"]): bo ); } +type SubagentTurnItem = Extract; + +type OpenRunOwnedSubagentProjection = { + readonly subagents: ReadonlyMap; + readonly turnItems: ReadonlyMap; + readonly childTurnItems: ReadonlyMap; + readonly nodes: ReadonlyMap; + /** Child threads once linked by a root-run subagent row; kept for cascade. */ + readonly linkedChildThreadIds: ReadonlySet; +}; + +type RunOwnedSubagentTerminalStatus = Extract< + OrchestrationV2Subagent["status"], + "interrupted" | "failed" | "cancelled" +>; + +function isOpenExecutionNodeStatus(status: OrchestrationV2ExecutionNode["status"]): boolean { + return status === "pending" || status === "running" || status === "waiting"; +} + +function isRunOwnedSubagentTerminalStatus( + status: ProviderTerminalEvent["status"], +): status is RunOwnedSubagentTerminalStatus { + return status === "interrupted" || status === "failed" || status === "cancelled"; +} + +export function canRouteRelatedSubagent(status: OrchestrationV2Subagent["status"]): boolean { + return status !== "interrupted" && status !== "failed" && status !== "cancelled"; +} + +function emptyOpenRunOwnedSubagentProjection(): OpenRunOwnedSubagentProjection { + return { + subagents: new Map(), + turnItems: new Map(), + childTurnItems: new Map(), + nodes: new Map(), + linkedChildThreadIds: new Set(), + }; +} + +function withLinkedChildThreadId( + current: OpenRunOwnedSubagentProjection, + childThreadId: ThreadId | null, +): OpenRunOwnedSubagentProjection { + if (childThreadId === null || current.linkedChildThreadIds.has(childThreadId)) { + return current; + } + const linkedChildThreadIds = new Set(current.linkedChildThreadIds); + linkedChildThreadIds.add(childThreadId); + return { ...current, linkedChildThreadIds }; +} + +export function cascadeTerminalizeRunOwnedSubagents(input: { + readonly run: OrchestrationV2Run; + readonly open: OpenRunOwnedSubagentProjection; + readonly status: RunOwnedSubagentTerminalStatus; + readonly completedAt: DateTime.Utc; + readonly allocateEventId: () => Effect.Effect; +}): Effect.Effect, IdAllocatorV2AllocationError> { + return Effect.gen(function* () { + const events: Array = []; + // Prefer lifetime linkage over currently-open rows: subagent/turn-item + // snapshots may terminalize before the linked child-thread node settles. + const childThreadIds = new Set(input.open.linkedChildThreadIds); + for (const item of [...input.open.subagents.values(), ...input.open.turnItems.values()]) { + if (item.childThreadId !== null) { + childThreadIds.add(item.childThreadId); + } + } + const keys = new Set([ + ...input.open.subagents.keys(), + ...input.open.turnItems.keys(), + ...input.open.nodes.keys(), + ]); + for (const key of keys) { + const subagent = input.open.subagents.get(key); + if (subagent !== undefined && !isTerminalSubagentStatus(subagent.status)) { + events.push({ + id: yield* input.allocateEventId(), + type: "subagent.updated", + threadId: subagent.threadId, + runId: input.run.id, + nodeId: subagent.id, + driver: subagent.driver, + providerInstanceId: subagent.providerInstanceId, + occurredAt: input.completedAt, + payload: { + ...subagent, + status: input.status, + completedAt: input.completedAt, + updatedAt: input.completedAt, + }, + }); + } + const node = input.open.nodes.get(key); + if ( + node !== undefined && + ((node.threadId === input.run.threadId && node.runId === input.run.id) || + childThreadIds.has(node.threadId)) && + isOpenExecutionNodeStatus(node.status) + ) { + events.push({ + id: yield* input.allocateEventId(), + type: "node.updated", + threadId: node.threadId, + runId: node.runId ?? input.run.id, + nodeId: node.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: input.completedAt, + payload: { + ...node, + status: input.status, + completedAt: input.completedAt, + }, + }); + } + const turnItem = input.open.turnItems.get(key); + if ( + turnItem !== undefined && + turnItem.runId === input.run.id && + !isTerminalTurnItemStatus(turnItem.status) + ) { + events.push({ + id: yield* input.allocateEventId(), + type: "turn-item.updated", + threadId: turnItem.threadId, + runId: input.run.id, + ...(turnItem.nodeId === null ? {} : { nodeId: turnItem.nodeId }), + providerInstanceId: input.run.providerInstanceId, + occurredAt: input.completedAt, + payload: { + ...turnItem, + status: input.status, + completedAt: input.completedAt, + updatedAt: input.completedAt, + }, + }); + } + } + for (const turnItem of input.open.childTurnItems.values()) { + if (!childThreadIds.has(turnItem.threadId) || isTerminalTurnItemStatus(turnItem.status)) { + continue; + } + events.push({ + id: yield* input.allocateEventId(), + type: "turn-item.updated", + threadId: turnItem.threadId, + runId: turnItem.runId ?? input.run.id, + ...(turnItem.nodeId === null ? {} : { nodeId: turnItem.nodeId }), + providerInstanceId: input.run.providerInstanceId, + occurredAt: input.completedAt, + payload: { + ...turnItem, + ...("streaming" in turnItem ? { streaming: false } : {}), + status: input.status, + completedAt: input.completedAt, + updatedAt: input.completedAt, + }, + }); + } + return events; + }); +} + export function finalProviderThreadStatus( disposition: ProviderTerminalEvent["threadDisposition"], ): OrchestrationV2ProviderThread["status"] { @@ -325,6 +496,7 @@ export const layer: Layer.Layer< readonly attempt: OrchestrationV2RunAttempt; readonly shouldFinalizeRun?: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; + readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection; readonly terminal: ProviderTerminalEvent; readonly failureItemPersisted: boolean; }) => @@ -372,6 +544,23 @@ export const layer: Layer.Layer< } return; } + const allocateEventId = () => idAllocator.allocate.event({ threadId: input.run.threadId }); + const open = input.openRunOwnedSubagents ?? emptyOpenRunOwnedSubagentProjection(); + const hasOpenSubagentProjection = + open.subagents.size > 0 || + open.turnItems.size > 0 || + open.childTurnItems.size > 0 || + open.nodes.size > 0; + const cascadedSubagentEvents = + isRunOwnedSubagentTerminalStatus(input.terminal.status) && hasOpenSubagentProjection + ? yield* cascadeTerminalizeRunOwnedSubagents({ + run: input.run, + open, + status: input.terminal.status, + completedAt, + allocateEventId, + }) + : []; const persistedStatus = input.terminal.status === "completed" ? "waiting" : input.terminal.status; const finalizedRun: OrchestrationV2Run = { @@ -390,11 +579,9 @@ export const layer: Layer.Layer< status: finalProviderThreadStatus(input.terminal.threadDisposition), updatedAt: completedAt, }; - const runEventId = yield* idAllocator.allocate.event({ threadId: input.run.threadId }); - const nodeEventId = yield* idAllocator.allocate.event({ threadId: input.run.threadId }); - const providerThreadEventId = yield* idAllocator.allocate.event({ - threadId: input.run.threadId, - }); + const runEventId = yield* allocateEventId(); + const nodeEventId = yield* allocateEventId(); + const providerThreadEventId = yield* allocateEventId(); const checkpointCaptureCommandId = CommandId.make( `command:effect:checkpoint.capture:${input.run.id}`, ); @@ -415,11 +602,14 @@ export const layer: Layer.Layer< ] : [], events: [ + // Terminalize open run-owned subagent rows before the root run + // settles so projections never keep a forever-running subagent card. + ...cascadedSubagentEvents, ...(finalizedAttempt === null ? [] : [ { - id: yield* idAllocator.allocate.event({ threadId: input.run.threadId }), + id: yield* allocateEventId(), type: "run-attempt.updated" as const, threadId: input.run.threadId, runId: input.run.id, @@ -432,7 +622,7 @@ export const layer: Layer.Layer< ...(input.terminal.status === "interrupted" ? [ { - id: yield* idAllocator.allocate.event({ threadId: input.run.threadId }), + id: yield* allocateEventId(), type: "turn-item.updated" as const, threadId: input.run.threadId, runId: input.run.id, @@ -452,7 +642,7 @@ export const layer: Layer.Layer< ...(input.terminal.status === "failed" && !input.failureItemPersisted ? [ { - id: yield* idAllocator.allocate.event({ threadId: input.run.threadId }), + id: yield* allocateEventId(), type: "turn-item.updated" as const, threadId: input.run.threadId, runId: input.run.id, @@ -588,12 +778,14 @@ export const layer: Layer.Layer< const activeBackgroundTurnItems = yield* Ref.make< ReadonlySet >(new Set()); + const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection()); const finalizeRootRun = (terminal: ProviderTerminalEvent) => Effect.gen(function* () { if (yield* Ref.get(rootRunFinalized)) { return; } const providerThread = yield* Ref.get(latestProviderThread); + const openSubagents = yield* Ref.get(openRunOwnedSubagents); yield* writeFinalRunEvents({ run: input.run, rootNode: input.rootNode, @@ -608,6 +800,7 @@ export const layer: Layer.Layer< : { hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, }), + openRunOwnedSubagents: openSubagents, terminal, failureItemPersisted: terminal.status === "failed", }).pipe( @@ -615,9 +808,12 @@ export const layer: Layer.Layer< (cause) => new RunExecutionIngestError({ runId: input.run.id, cause }), ), ); + if (isRunOwnedSubagentTerminalStatus(terminal.status)) { + yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection()); + } yield* Ref.set(rootRunFinalized, true); }); - const trackChildLifecycle = (event: ProviderAdapterV2Event) => + const trackChildLifecycle = (event: ProviderAdapterV2Event, deliverable: boolean) => Effect.gen(function* () { const routing = yield* Ref.get(eventRouting); if (event.type === "provider_turn.updated") { @@ -652,16 +848,51 @@ export const layer: Layer.Layer< return next; }); } + // Snapshot run-owned subagents for interrupt cascade. + // Preserve childThreadId linkage for the root-run lifetime even + // after the subagent row terminalizes, so open child-thread + // nodes can still be proven linked on a later root interrupt. + if (belongsToRootRun) { + yield* Ref.update(openRunOwnedSubagents, (current) => { + const withLink = withLinkedChildThreadId(current, event.subagent.childThreadId); + const subagents = new Map(withLink.subagents); + if (isTerminalSubagentStatus(event.subagent.status)) { + subagents.delete(event.subagent.id); + } else { + subagents.set(event.subagent.id, event.subagent); + } + return { ...withLink, subagents }; + }); + } } - if ( - event.type === "turn_item.updated" && - backgroundCapableTurnItemTypes.has(event.turnItem.type) - ) { + if (event.type === "node.updated") { + const belongsToRootSubagent = + event.node.kind === "subagent" && event.node.runId === input.run.id; + const belongsToOwnedChildThread = + event.node.threadId !== input.run.threadId && + routing.ownedThreadIds.has(event.node.threadId); + if (!belongsToRootSubagent && !belongsToOwnedChildThread) { + return; + } + yield* Ref.update(openRunOwnedSubagents, (current) => { + const nodes = new Map(current.nodes); + if (isOpenExecutionNodeStatus(event.node.status)) { + nodes.set(event.node.id, event.node); + } else { + nodes.delete(event.node.id); + } + return { ...current, nodes }; + }); + } + if (event.type === "turn_item.updated") { const belongsToRootRun = event.turnItem.runId === input.run.id; const belongsToOwnedChildThread = event.turnItem.threadId !== input.run.threadId && routing.ownedThreadIds.has(event.turnItem.threadId); - if (belongsToRootRun || belongsToOwnedChildThread) { + if ( + backgroundCapableTurnItemTypes.has(event.turnItem.type) && + (belongsToRootRun || belongsToOwnedChildThread) + ) { yield* Ref.update(activeBackgroundTurnItems, (current) => { const next = new Set(current); if (isTerminalTurnItemStatus(event.turnItem.status)) { @@ -672,12 +903,40 @@ export const layer: Layer.Layer< return next; }); } + if (belongsToOwnedChildThread && deliverable) { + yield* Ref.update(openRunOwnedSubagents, (current) => { + const childTurnItems = new Map(current.childTurnItems); + if (isTerminalTurnItemStatus(event.turnItem.status)) { + childTurnItems.delete(event.turnItem.id); + } else { + childTurnItems.set(event.turnItem.id, event.turnItem); + } + return { ...current, childTurnItems }; + }); + } + if (belongsToRootRun && event.turnItem.type === "subagent") { + const subagentItem = event.turnItem; + yield* Ref.update(openRunOwnedSubagents, (current) => { + const withLink = withLinkedChildThreadId(current, subagentItem.childThreadId); + const turnItems = new Map(withLink.turnItems); + if (isTerminalTurnItemStatus(subagentItem.status)) { + turnItems.delete(subagentItem.subagentId); + } else { + turnItems.set(subagentItem.subagentId, subagentItem); + } + return { ...withLink, turnItems }; + }); + } } }); const shouldStopProviderEventIngestion = Effect.gen(function* () { if (!(yield* Ref.get(rootTerminalSeen))) { return false; } + const terminal = yield* Ref.get(terminalEvent); + if (terminal !== null && terminal.status !== "completed") { + return true; + } const childProviderTurns = yield* Ref.get(activeChildProviderTurns); if (childProviderTurns.size > 0) { return false; @@ -694,12 +953,8 @@ export const layer: Layer.Layer< // rather than pinning the stream open. Assumes adapters emit an // item's non-terminal event before the root terminal; an item // first seen after the terminal is not pinned. - const terminal = yield* Ref.get(terminalEvent); - if (terminal !== null && terminal.status === "completed") { - const backgroundItems = yield* Ref.get(activeBackgroundTurnItems); - return backgroundItems.size === 0; - } - return true; + const backgroundItems = yield* Ref.get(activeBackgroundTurnItems); + return backgroundItems.size === 0; }); const eventSubscription = input.session.subscribeEvents === undefined @@ -712,7 +967,8 @@ export const layer: Layer.Layer< Stream.tap((event) => Effect.gen(function* () { let storedEventCount = 0; - if (shouldDeliverProviderEvent(event, assistantStreamingEnabled)) { + const shouldDeliver = shouldDeliverProviderEvent(event, assistantStreamingEnabled); + if (shouldDeliver) { const storedEvents = yield* providerEventIngestor.ingestNormalized({ providerSessionId: input.providerSessionId, providerInstanceId: input.run.providerInstanceId, @@ -752,7 +1008,7 @@ export const layer: Layer.Layer< yield* Ref.set(rootTerminalSeen, true); yield* finalizeRootRun(event); } - yield* trackChildLifecycle(event); + yield* trackChildLifecycle(event, shouldDeliver); }), ), Stream.takeUntilEffect(() => shouldStopProviderEventIngestion), @@ -781,30 +1037,35 @@ export const layer: Layer.Layer< Effect.flatMap((providerThread) => Ref.get(latestTurnItemOrdinal).pipe( Effect.flatMap((latestItemOrdinal) => - writeFinalRunEvents({ - run: input.run, - rootNode: input.rootNode, - checkpointScope: input.checkpointScope, - providerThread, - attempt: input.attempt, - ...(input.shouldFinalizeRun === undefined - ? {} - : { shouldFinalizeRun: input.shouldFinalizeRun }), - ...(input.hasUnpairedRunInterruptRequest === undefined - ? {} - : { - hasUnpairedRunInterruptRequest: - input.hasUnpairedRunInterruptRequest, - }), - terminal: makeFailedTerminalEvent( - makeProviderFailure({ - cause: Cause.squash(cause), - class: "unknown", + Ref.get(openRunOwnedSubagents).pipe( + Effect.flatMap((openSubagents) => + writeFinalRunEvents({ + run: input.run, + rootNode: input.rootNode, + checkpointScope: input.checkpointScope, + providerThread, + attempt: input.attempt, + ...(input.shouldFinalizeRun === undefined + ? {} + : { shouldFinalizeRun: input.shouldFinalizeRun }), + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: + input.hasUnpairedRunInterruptRequest, + }), + openRunOwnedSubagents: openSubagents, + terminal: makeFailedTerminalEvent( + makeProviderFailure({ + cause: Cause.squash(cause), + class: "unknown", + }), + latestItemOrdinal + 1, + ), + failureItemPersisted: false, }), - latestItemOrdinal + 1, ), - failureItemPersisted: false, - }), + ), ), ), ), @@ -858,30 +1119,35 @@ export const layer: Layer.Layer< Effect.flatMap((providerThread) => Ref.get(latestTurnItemOrdinal).pipe( Effect.flatMap((latestItemOrdinal) => - writeFinalRunEvents({ - run: input.run, - rootNode: input.rootNode, - checkpointScope: input.checkpointScope, - providerThread, - attempt: input.attempt, - ...(input.shouldFinalizeRun === undefined - ? {} - : { shouldFinalizeRun: input.shouldFinalizeRun }), - ...(input.hasUnpairedRunInterruptRequest === undefined - ? {} - : { - hasUnpairedRunInterruptRequest: - input.hasUnpairedRunInterruptRequest, - }), - terminal: makeFailedTerminalEvent( - makeProviderFailure({ - cause: Cause.squash(cause), - class: "provider_error", + Ref.get(openRunOwnedSubagents).pipe( + Effect.flatMap((openSubagents) => + writeFinalRunEvents({ + run: input.run, + rootNode: input.rootNode, + checkpointScope: input.checkpointScope, + providerThread, + attempt: input.attempt, + ...(input.shouldFinalizeRun === undefined + ? {} + : { shouldFinalizeRun: input.shouldFinalizeRun }), + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: + input.hasUnpairedRunInterruptRequest, + }), + openRunOwnedSubagents: openSubagents, + terminal: makeFailedTerminalEvent( + makeProviderFailure({ + cause: Cause.squash(cause), + class: "provider_error", + }), + latestItemOrdinal + 1, + ), + failureItemPersisted: false, }), - latestItemOrdinal + 1, ), - failureItemPersisted: false, - }), + ), ), ), ), diff --git a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts index ed66bd30844..9c0b279332d 100644 --- a/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts @@ -228,7 +228,14 @@ const scenarioExpectations = { approvalRequestCount: 0, }, turn_interrupt_mid_tool: { - outgoing: ["initialize", "initialized", "thread/start", "turn/start", "turn/interrupt"], + outgoing: [ + "initialize", + "initialized", + "thread/start", + "turn/start", + "turn/interrupt", + "thread/backgroundTerminals/terminate", + ], incoming: ["turn/started", "item/started", "turn/completed"], turnStartCount: 1, turnCompletedCount: 1, diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/tool_call_read_only_on_request/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/tool_call_read_only_on_request/claude_transcript.ndjson index e9b0b981eec..033f08ff9ba 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/tool_call_read_only_on_request/claude_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/tool_call_read_only_on_request/claude_transcript.ndjson @@ -1,5 +1,5 @@ {"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"tool_call_read_only_on_request","metadata":{"prompts":["Create or overwrite .codex-probe-write-action.txt with exactly this text: codex app-server approval fixture. Use a local shell command or file edit only, then briefly report what happened. Do not read package metadata, use GitHub, use web, or use MCP."],"model":"claude-sonnet-4-6","nativeSessionId":"2b73817d-f7ab-41a7-a533-31b13b698b0d","queryMode":"streaming","tools":"claude_code","permissionMode":"default","enablePermissionCallback":true,"generatedBy":"recordClaudeAgentSdkReplayTranscript"}} -{"type":"expect_outbound","label":"query.open","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"default","sessionId":"2b73817d-f7ab-41a7-a533-31b13b698b0d"}}} +{"type":"expect_outbound","label":"query.open","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":["Read","Glob","Grep"],"permissionMode":"default","allowedTools":["Read","Glob","Grep"],"sessionId":"2b73817d-f7ab-41a7-a533-31b13b698b0d"}}} {"type":"expect_outbound","label":"prompt.offer:1","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Create or overwrite .codex-probe-write-action.txt with exactly this text: codex app-server approval fixture. Use a local shell command or file edit only, then briefly report what happened. Do not read package metadata, use GitHub, use web, or use MCP."},"parent_tool_use_id":null}}} {"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_started","hook_id":"ae3f0d68-becd-4b08-871b-f857a9ca1f90","hook_name":"SessionStart:startup","hook_event":"SessionStart","uuid":"1d299042-40fc-44af-8459-0a80baa7b7de","session_id":"2b73817d-f7ab-41a7-a533-31b13b698b0d"}} {"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_response","hook_id":"ae3f0d68-becd-4b08-871b-f857a9ca1f90","hook_name":"SessionStart:startup","hook_event":"SessionStart","output":"","stdout":"","stderr":"","exit_code":0,"outcome":"success","uuid":"5c410b64-6b03-49c4-9991-a8e624d0fe12","session_id":"2b73817d-f7ab-41a7-a533-31b13b698b0d"}} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/tool_call_restricted_granular/claude_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/tool_call_restricted_granular/claude_transcript.ndjson index 348a77c2bbd..abd246cea4e 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/tool_call_restricted_granular/claude_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/tool_call_restricted_granular/claude_transcript.ndjson @@ -1,5 +1,5 @@ {"type":"transcript_start","provider":"claudeAgent","protocol":"claude-agent-sdk.query","version":"0.2.111","scenario":"tool_call_restricted_granular","metadata":{"prompts":["Create or overwrite .codex-probe-write-action.txt with exactly this text: codex app-server approval fixture. Use a local shell command or file edit only, then briefly report what happened. Do not read package metadata, use GitHub, use web, or use MCP."],"model":"claude-sonnet-4-6","nativeSessionId":"ffd3df75-f3cf-433d-bb67-1dd1e9db316b","queryMode":"streaming","tools":"claude_code","permissionMode":"default","enablePermissionCallback":true,"generatedBy":"recordClaudeAgentSdkReplayTranscript"}} -{"type":"expect_outbound","label":"query.open","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":{"type":"preset","preset":"claude_code"},"permissionMode":"default","sessionId":"ffd3df75-f3cf-433d-bb67-1dd1e9db316b"}}} +{"type":"expect_outbound","label":"query.open","frame":{"type":"query.open","options":{"model":"claude-sonnet-4-6","tools":["Read","Glob","Grep"],"permissionMode":"default","sessionId":"ffd3df75-f3cf-433d-bb67-1dd1e9db316b"}}} {"type":"expect_outbound","label":"prompt.offer:1","frame":{"type":"prompt.offer","message":{"type":"user","message":{"role":"user","content":"Create or overwrite .codex-probe-write-action.txt with exactly this text: codex app-server approval fixture. Use a local shell command or file edit only, then briefly report what happened. Do not read package metadata, use GitHub, use web, or use MCP."},"parent_tool_use_id":null}}} {"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_started","hook_id":"ba333d12-6a9a-4a5e-9cb5-2d6ce18cd0f8","hook_name":"SessionStart:startup","hook_event":"SessionStart","uuid":"544c0cee-2195-49a9-ab30-088c3d3d5e83","session_id":"ffd3df75-f3cf-433d-bb67-1dd1e9db316b"}} {"type":"emit_inbound","label":"system","frame":{"type":"system","subtype":"hook_response","hook_id":"ba333d12-6a9a-4a5e-9cb5-2d6ce18cd0f8","hook_name":"SessionStart:startup","hook_event":"SessionStart","output":"","stdout":"","stderr":"","exit_code":0,"outcome":"success","uuid":"09419836-e234-4231-a15d-927acb1fbdb1","session_id":"ffd3df75-f3cf-433d-bb67-1dd1e9db316b"}} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/turn_interrupt_mid_tool/codex_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/turn_interrupt_mid_tool/codex_output.ts index f3f98b53104..1f5260878d6 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/turn_interrupt_mid_tool/codex_output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/turn_interrupt_mid_tool/codex_output.ts @@ -38,6 +38,16 @@ function isCommandExecutionStartedFrame(frame: unknown): boolean { ); } +function frameParams(frame: unknown): Record | undefined { + if (typeof frame !== "object" || frame === null || !("params" in frame)) { + return undefined; + } + const params = (frame as { readonly params?: unknown }).params; + return typeof params === "object" && params !== null + ? (params as Record) + : undefined; +} + function assertCodexInterruptAfterCommandExecution(transcript: ProviderReplayTranscript) { const commandIndex = transcript.entries.findIndex( (entry) => entry.type === "emit_inbound" && isCommandExecutionStartedFrame(entry.frame), @@ -45,12 +55,40 @@ function assertCodexInterruptAfterCommandExecution(transcript: ProviderReplayTra const interruptIndex = transcript.entries.findIndex( (entry) => entry.type === "expect_outbound" && protocolMethod(entry.frame) === "turn/interrupt", ); + const completedIndex = transcript.entries.findIndex( + (entry) => entry.type === "emit_inbound" && protocolMethod(entry.frame) === "turn/completed", + ); + const terminateIndex = transcript.entries.findIndex( + (entry) => + entry.type === "expect_outbound" && + protocolMethod(entry.frame) === "thread/backgroundTerminals/terminate", + ); assert.isAtLeast(commandIndex, 0, "Codex interrupt fixture must record command execution start"); assert.isAbove( interruptIndex, commandIndex, "Codex interrupt must be issued after command execution starts in replay", ); + assert.isAtLeast( + completedIndex, + 0, + "Codex interrupt fixture must record turn/completed before terminal cleanup", + ); + assert.isAtLeast( + terminateIndex, + 0, + "Codex interrupt fixture must record background terminal terminate", + ); + assert.isAbove( + terminateIndex, + completedIndex, + "This recorded Codex interrupt fixture terminates background terminals after turn/completed", + ); + const terminateEntry = transcript.entries[terminateIndex]; + const terminateParams = + terminateEntry?.type === "expect_outbound" ? frameParams(terminateEntry.frame) : undefined; + assert.equal(terminateParams?.threadId, "019e03b8-9e5c-7b32-88ab-f742a29b75b8"); + assert.equal(terminateParams?.processId, "4275"); } export function assertTurnInterruptMidToolCodexOutput( @@ -80,7 +118,10 @@ export function assertTurnInterruptMidToolCodexOutput( assert.isDefined(commandItem); assert.isDefined(interruptRequest); assert.isDefined(interruptResult); - assert.include(["running", "completed", "failed"], commandItem.status); + // Interrupted Codex turns must terminalize mid-flight commandExecution items so + // the projected card is never left running forever after turn.terminal. + assert.equal(commandItem.status, "interrupted"); + assert.isNotNull(commandItem.completedAt); assert.include(commandItem.input, "node -e"); assert.equal(interruptRequest.status, "completed"); assert.equal(interruptResult.status, "interrupted"); @@ -91,4 +132,15 @@ export function assertTurnInterruptMidToolCodexOutput( ); assert.equal(projection.providerThreads[0]?.status, "idle"); assert.include(["interrupted", "cancelled"], projection.providerTurns[0]?.status); + + const runningCommands = projection.turnItems.filter( + (item) => item.type === "command_execution" && item.status === "running", + ); + assert.lengthOf(runningCommands, 0, "interrupted turn must not leave running command items"); + + const toolNodes = projection.nodes.filter((node) => node.kind === "tool_call"); + for (const node of toolNodes) { + assert.notEqual(node.status, "running", "interrupted turn tool nodes must be terminal"); + assert.isNotNull(node.completedAt); + } } diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/turn_interrupt_mid_tool/codex_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/turn_interrupt_mid_tool/codex_transcript.ndjson index bf3036c580c..dbb35e044eb 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/turn_interrupt_mid_tool/codex_transcript.ndjson +++ b/apps/server/src/orchestration-v2/testkit/fixtures/turn_interrupt_mid_tool/codex_transcript.ndjson @@ -30,4 +30,6 @@ {"type":"emit_inbound","label":"turn/interrupt","frame":{"id":4,"result":{}}} {"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019e03b8-9e5c-7b32-88ab-f742a29b75b8","status":{"type":"idle"}}}} {"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"019e03b8-9e5c-7b32-88ab-f742a29b75b8","turn":{"id":"019e03b8-9f34-7bb1-8267-ad79b7e2e69a","items":[],"status":"interrupted","error":null,"startedAt":1778178891,"completedAt":1778178895,"durationMs":3792}}}} +{"type":"expect_outbound","label":"thread/backgroundTerminals/terminate","frame":{"id":5,"method":"thread/backgroundTerminals/terminate","params":{"threadId":"019e03b8-9e5c-7b32-88ab-f742a29b75b8","processId":"4275"}}} +{"type":"emit_inbound","label":"thread/backgroundTerminals/terminate","frame":{"id":5,"result":{"terminated":true}}} {"type":"runtime_exit","status":"success"}