diff --git a/apps/server/scripts/record-codex-app-server-replay-fixture.ts b/apps/server/scripts/record-codex-app-server-replay-fixture.ts index 026703d1115..1dd2fd2e396 100644 --- a/apps/server/scripts/record-codex-app-server-replay-fixture.ts +++ b/apps/server/scripts/record-codex-app-server-replay-fixture.ts @@ -27,6 +27,10 @@ import { SUBAGENT_CONTINUE_PARENT_PROMPT, SUBAGENT_CONTINUE_PROMPT, SUBAGENT_PROMPT, + SUBAGENT_REUSE_AFTER_IDLE_PROMPT, + SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT, + SUBAGENT_REUSE_AFTER_INTERRUPT_PROMPT, + SUBAGENT_REUSE_AFTER_INTERRUPT_RESUME_PROMPT, THREAD_ROLLBACK_AFTER_PROMPT, THREAD_ROLLBACK_FIRST_PROMPT, THREAD_ROLLBACK_SECOND_PROMPT, @@ -71,6 +75,8 @@ const SCENARIO_NAMES = [ "tool_call_restricted_granular", "subagent", "subagent_continue", + "subagent_reuse_after_interrupt", + "subagent_reuse_after_idle", "multi_turn", "provider_thread_resume", "todo_list", @@ -443,6 +449,61 @@ function scenarios(): ReadonlyArray { }, ], }, + { + name: "subagent_reuse_after_interrupt", + fileName: "subagent_reuse_after_interrupt.ndjson", + description: + "A root turn spawns a native Codex subagent and is interrupted while it works, then a later root turn re-activates the same child thread.", + runs: [ + { + name: "reuse-after-interrupt", + description: + "Interrupt the spawning turn mid-subagent, then message the same subagent from a new root turn.", + turnDefaults: { + approvalPolicy: "never", + sandboxPolicy: workspaceWriteSandbox(), + }, + steps: [ + { + type: "interruptedTurn", + label: "spawn-subagent-then-interrupt", + prompt: SUBAGENT_REUSE_AFTER_INTERRUPT_PROMPT, + interruptAfterCommandExecutionStarted: true, + }, + { + type: "turn", + label: "reuse-interrupted-subagent", + prompt: SUBAGENT_REUSE_AFTER_INTERRUPT_RESUME_PROMPT, + }, + ], + }, + ], + }, + { + name: "subagent_reuse_after_idle", + fileName: "subagent_reuse_after_idle.ndjson", + description: + "A root turn spawns a native Codex subagent, then a later root turn re-activates it. Replay advances past the session idle timeout between turns so the adapter registry is empty on reuse.", + runs: [ + { + name: "reuse-after-idle", + description: + "Second root turn messages the same subagent after the session would have been reaped.", + steps: [ + { + type: "turn", + label: "spawn-subagent", + prompt: SUBAGENT_REUSE_AFTER_IDLE_PROMPT, + }, + { + type: "turn", + label: "reuse-subagent-after-idle", + prompt: SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT, + }, + ], + }, + ], + }, { name: "multi_turn", fileName: "multi_turn.ndjson", @@ -1070,7 +1131,7 @@ function installReplayHandlers({ readonly client: CodexClient.CodexAppServerClient["Service"]; readonly startTurn: (turnId: string) => Effect.Effect; readonly completeTurn: (turnId: string) => Effect.Effect; - readonly startCommandExecution: (turnId: string) => Effect.Effect; + readonly startCommandExecution: () => Effect.Effect; readonly beforeApprovalResponse: () => Effect.Effect; }) { return Effect.all( @@ -1135,12 +1196,12 @@ function installReplayHandlers({ ), client.handleServerNotification("item/completed", (payload) => isRecord(payload.item) && payload.item.type === "commandExecution" - ? startCommandExecution(payload.turnId).pipe(Effect.ignore) + ? startCommandExecution().pipe(Effect.ignore) : Effect.void, ), client.handleServerNotification("item/started", (payload) => isRecord(payload.item) && payload.item.type === "commandExecution" - ? startCommandExecution(payload.turnId).pipe(Effect.ignore) + ? startCommandExecution().pipe(Effect.ignore) : Effect.void, ), ], @@ -1160,7 +1221,7 @@ function runReplaySession({ return Effect.gen(function* () { const startedTurns = new Map>(); const completedTurns = new Map>(); - const startedCommandExecutions = new Map>(); + let commandExecutionGate: Deferred.Deferred | undefined; let approvalGate: Deferred.Deferred | undefined; const getStarted = (turnId: string) => { const existing = startedTurns.get(turnId); @@ -1180,23 +1241,14 @@ function runReplaySession({ Effect.tap((deferred) => Effect.sync(() => completedTurns.set(turnId, deferred))), ); }; - const getCommandExecutionStarted = (turnId: string) => { - const existing = startedCommandExecutions.get(turnId); - if (existing) { - return Effect.succeed(existing); - } - return Deferred.make().pipe( - Effect.tap((deferred) => Effect.sync(() => startedCommandExecutions.set(turnId, deferred))), - ); - }; const startTurn = (turnId: string) => getStarted(turnId).pipe(Effect.flatMap((deferred) => Deferred.succeed(deferred, void 0))); const completeTurn = (turnId: string) => getCompletion(turnId).pipe(Effect.flatMap((deferred) => Deferred.succeed(deferred, void 0))); - const startCommandExecution = (turnId: string) => - getCommandExecutionStarted(turnId).pipe( - Effect.flatMap((deferred) => Deferred.succeed(deferred, void 0)), - ); + const startCommandExecution = () => + commandExecutionGate === undefined + ? Effect.void + : Deferred.succeed(commandExecutionGate, void 0).pipe(Effect.ignore); const beforeApprovalResponse = () => approvalGate ? Deferred.await(approvalGate) : Effect.void; @@ -1237,6 +1289,12 @@ function runReplaySession({ if (step.type === "steeredTurn") { approvalGate = yield* Deferred.make(); } + if ( + step.type === "interruptedTurn" && + step.interruptAfterCommandExecutionStarted === true + ) { + commandExecutionGate = yield* Deferred.make(); + } const turn = yield* client.request("turn/start", turnParams); const turnId = getTurnId(turn); @@ -1258,8 +1316,11 @@ function runReplaySession({ if (step.type === "interruptedTurn") { if (step.interruptAfterCommandExecutionStarted === true) { - const commandExecutionStarted = yield* getCommandExecutionStarted(turnId); - yield* Deferred.await(commandExecutionStarted); + if (commandExecutionGate === undefined) { + throw new Error("Command execution interrupt gate was not initialized."); + } + yield* Deferred.await(commandExecutionGate); + commandExecutionGate = undefined; } else { yield* Effect.sleep(`${step.interruptAfterMs ?? 1_500} millis`); } diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index c3768fffcb7..bff31aa0a26 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -17,6 +17,7 @@ import { RunAttemptId, RunId, ThreadId, + TurnItemId, } from "@t3tools/contracts"; import { assert, describe, it } from "@effect/vitest"; import { HostProcessEnvironment, HostProcessPlatform } from "@t3tools/shared/hostProcess"; @@ -41,6 +42,7 @@ import { ProviderAdapterOpenSessionError, ProviderAdapterV2RuntimePolicy, type ProviderAdapterV2Event, + type ProviderAdapterV2ExistingSubagent, type ProviderAdapterV2TurnInput, } from "../ProviderAdapter.ts"; import type { ProviderContinuationRequest } from "../ProviderContinuationRequests.ts"; @@ -3431,4 +3433,295 @@ describe("CodexAdapterV2 post-settle continuation", () => { }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), ), ); + + const RESTART_SCENARIO = "codex-restart-rehydrate"; + const RESTART_NATIVE_THREAD = "native-codex-restart-thread"; + const RESTART_NATIVE_TURN = "native-codex-restart-root-turn"; + const RESTART_CHILD_THREAD = "native-codex-restart-child-thread"; + const RESTART_CHILD_TURN = "native-codex-restart-child-turn-2"; + const RESTART_TOOL_CALL = "call-codex-restart-spawn"; + const RESTART_PROMPT = "Nudge the agent from before the restart."; + const RESTART_SUBAGENT_ID = NodeId.make("node-codex-restart-subagent"); + + const restartChildFrame = (input: { + readonly label: string; + readonly method: string; + readonly params: Record; + }): CodexReplay.CodexAppServerReplayEntry => ({ + type: "emit_inbound", + label: input.label, + frame: { method: input.method, params: input.params }, + }); + + const restartTranscript = makeCodexReplayTranscript({ + scenario: RESTART_SCENARIO, + entries: [ + ...codexReplayPreamble({ + nativeThreadId: RESTART_NATIVE_THREAD, + nativeTurnId: RESTART_NATIVE_TURN, + prompt: RESTART_PROMPT, + }), + // No spawn frame: this process never saw the agent start. It only sees + // the pre-existing native thread come back to life. + restartChildFrame({ + label: "turn/started/child", + method: "turn/started", + params: { + threadId: RESTART_CHILD_THREAD, + turn: makeCodexReplayTurn({ id: RESTART_CHILD_TURN, status: "inProgress" }), + }, + }), + restartChildFrame({ + label: "item/completed/child-answer", + method: "item/completed", + params: { + item: { + type: "agentMessage", + id: "child-restart-answer", + text: "CODEX_REHYDRATED_DONE", + phase: "final_answer", + memoryCitation: null, + }, + threadId: RESTART_CHILD_THREAD, + turnId: RESTART_CHILD_TURN, + completedAtMs: 1782622482000, + }, + }), + restartChildFrame({ + label: "turn/completed/child", + method: "turn/completed", + params: { + threadId: RESTART_CHILD_THREAD, + turn: makeCodexReplayTurn({ id: RESTART_CHILD_TURN, status: "completed" }), + }, + }), + restartChildFrame({ + label: "thread/tokenUsage/updated/child", + method: "thread/tokenUsage/updated", + params: { + threadId: RESTART_CHILD_THREAD, + turnId: RESTART_CHILD_TURN, + tokenUsage: { + total: { + totalTokens: 250, + inputTokens: 200, + cachedInputTokens: 0, + outputTokens: 50, + reasoningOutputTokens: 0, + }, + // Codex reports `total` per child thread, not per activation, so + // this snapshot already contains the 100 recorded before the + // restart; `last` is what this activation added. + last: { + totalTokens: 150, + inputTokens: 120, + cachedInputTokens: 0, + outputTokens: 30, + reasoningOutputTokens: 0, + }, + }, + }, + }), + restartChildFrame({ + label: "item/completed/root-answer", + method: "item/completed", + params: { + item: { + type: "agentMessage", + id: "root-answer-restart", + text: "NUDGED", + phase: "final_answer", + memoryCitation: null, + }, + threadId: RESTART_NATIVE_THREAD, + turnId: RESTART_NATIVE_TURN, + completedAtMs: 1782622483000, + }, + }), + restartChildFrame({ + label: "turn/completed/root", + method: "turn/completed", + params: { + threadId: RESTART_NATIVE_THREAD, + turn: makeCodexReplayTurn({ id: RESTART_NATIVE_TURN, status: "completed" }), + }, + }), + ], + }); + + /** + * The subagent as the projection holds it after a restart: settled at idle, + * still bound to its child thread and native item, one activation recorded. + */ + const makeRestartExistingSubagent = (input: { + readonly parentThreadId: ThreadId; + readonly parentProviderThread: OrchestrationV2ProviderThread; + readonly now: DateTime.Utc; + }): ProviderAdapterV2ExistingSubagent => { + const childThreadId = ThreadId.make("thread-codex-restart-child"); + const childProviderThread: OrchestrationV2ProviderThread = { + id: ProviderThreadId.make("provider-thread-codex-restart-child"), + driver: CODEX_DRIVER_KIND, + providerInstanceId: CODEX_DEFAULT_INSTANCE_ID, + providerSessionId: input.parentProviderThread.providerSessionId, + appThreadId: childThreadId, + ownerNodeId: null, + nativeThreadRef: { + driver: CODEX_DRIVER_KIND, + nativeId: RESTART_CHILD_THREAD, + strength: "strong", + }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: input.now, + updatedAt: input.now, + }; + return { + subagent: { + id: RESTART_SUBAGENT_ID, + threadId: input.parentThreadId, + runId: RunId.make("run-before-restart"), + parentNodeId: NodeId.make("node-before-restart-root"), + origin: "provider_native", + createdBy: "agent", + driver: CODEX_DRIVER_KIND, + providerInstanceId: CODEX_DEFAULT_INSTANCE_ID, + providerThreadId: childProviderThread.id, + childThreadId, + nativeTaskRef: { + driver: CODEX_DRIVER_KIND, + nativeId: `${RESTART_TOOL_CALL}:${RESTART_CHILD_THREAD}`, + strength: "strong", + }, + prompt: "Audit the parser.", + title: "Parser audit", + model: "gpt-5.4", + kind: "subagent", + role: { name: "general-purpose", source: "app_default" }, + status: "idle", + result: "CODEX_BEFORE_RESTART_DONE", + usage: { totalTokens: 100 }, + currentActivationId: null, + activationCount: 1, + workflow: null, + workflowMembership: null, + recentActivity: [], + startedAt: input.now, + completedAt: input.now, + updatedAt: input.now, + }, + childThread: { + ...makeCodexTestAppThread({ + threadId: childThreadId, + providerThread: childProviderThread, + now: input.now, + }), + lineage: { + parentThreadId: input.parentThreadId, + relationshipToParent: "subagent" as const, + rootThreadId: input.parentThreadId, + }, + }, + childProviderThread, + turnItemId: TurnItemId.make("turn-item-codex-restart-subagent"), + turnItemOrdinal: 2, + ordinal: 1, + }; + }; + + it.effect("reuses a projection-known subagent after the registry is lost", () => + Effect.scoped( + Effect.gen(function* () { + // A fresh adapter is exactly the post-restart state: the process-local + // registry is empty, so the only way to recognise the returning native + // thread is the projection handing it back through existingSubagents. + const harness = yield* makeCodexReplayHarness(restartTranscript); + const now = yield* DateTime.now; + const existing = makeRestartExistingSubagent({ + parentThreadId: harness.threadId, + parentProviderThread: harness.providerThread, + now, + }); + + yield* harness.runtime.startTurn({ + ...makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-restart"), + text: RESTART_PROMPT, + }), + existingSubagents: [existing], + }); + + yield* awaitUntil( + () => + harness + .subagentUpdates() + .some((event) => event.subagent.result === "CODEX_REHYDRATED_DONE"), + "rehydrated subagent result", + ); + + const updates = harness.subagentUpdates(); + // One identity, and it is the one from before the restart. + assert.deepStrictEqual( + [...new Set(updates.map((event) => event.subagent.id))], + [RESTART_SUBAGENT_ID], + "the returning agent must not be spawned as a second identity", + ); + const latest = updates.at(-1)?.subagent; + assert.equal(latest?.result, "CODEX_REHYDRATED_DONE"); + assert.equal(latest?.childThreadId, existing.childThread.id); + // Continues the prior activation rather than restarting the count. + assert.equal(latest?.activationCount, 2); + // Usage carries across the restart instead of resetting: the seeded + // 100 is not lost, and the thread-cumulative 250 is not double-counted + // on top of it. + assert.equal(latest?.usage?.totalTokens, 250); + // The new activation takes the next ordinal, not a colliding ordinal 1. + const activations = harness.subagentActivationUpdates(); + assert.deepStrictEqual( + [...new Set(activations.map((event) => event.activation.ordinal))], + [2], + ); + assert.deepStrictEqual( + [...new Set(activations.map((event) => event.activation.subagentId))], + [RESTART_SUBAGENT_ID], + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); + + it.effect("does not adopt an unknown native thread when nothing is rehydrated", () => + Effect.scoped( + Effect.gen(function* () { + // Negative control for the test above: the identical transcript with no + // existingSubagents. Without the seed the returning thread is not the + // prior agent, so the assertion above is proving the seed, not the + // transcript. + const harness = yield* makeCodexReplayHarness(restartTranscript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make("attempt-codex-restart-unseeded"), + text: RESTART_PROMPT, + }), + ); + yield* awaitUntil(() => harness.terminalEvents().length === 1, "root turn terminal"); + + assert.notInclude( + harness.subagentUpdates().map((event) => String(event.subagent.id)), + String(RESTART_SUBAGENT_ID), + ); + }).pipe(Effect.provide(Layer.merge(idAllocatorLayer, NodeServices.layer))), + ), + ); }); diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 9c31a19ece4..64cdd4f23a4 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -119,12 +119,6 @@ export const CODEX_DRIVER_KIND = CODEX_PROVIDER; export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND); const DEFAULT_CODEX_SETTINGS = Schema.decodeSync(CodexSettings)({}); const CODEX_ASSISTANT_DELTA_FLUSH_INTERVAL_MS = 50; -const isSettledCodexSubagent = (subagent: OrchestrationV2Subagent) => - subagent.status === "idle" || - subagent.status === "completed" || - subagent.status === "failed" || - subagent.status === "cancelled" || - subagent.status === "interrupted"; const CodexBackgroundTerminalTerminateResponse = Schema.Struct({ terminated: Schema.Boolean, }); @@ -839,7 +833,9 @@ interface DeferredCodexRootTerminal { } interface CodexSubagentThreadContext { - readonly parentContext: ActiveCodexTurnContext; + // Rebound whenever a later root turn re-activates this subagent, so its rows + // are attributed to the run that is actually driving it. + parentContext: ActiveCodexTurnContext; readonly providerThread: OrchestrationV2ProviderThread; readonly childThread: OrchestrationV2AppThread; readonly subagentNodeId: OrchestrationV2ExecutionNode["id"]; @@ -1502,6 +1498,74 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi const emitProviderEvent = (event: ProviderAdapterV2Event) => Queue.offer(events, event).pipe(Effect.asVoid); + /** + * Rebuild registry entries for subagents that already exist in the + * projection. The registry is process-local, so without this a session + * reap or server restart orphans every prior agent: the returning + * native thread is unrecognised and gets spawned as a new identity. + * Entries created here emit nothing — they only let a later + * re-activation resolve to the original subagent. + */ + const seedExistingSubagents = (context: ActiveCodexTurnContext) => + Effect.gen(function* () { + const existing = context.input.existingSubagents ?? []; + if (existing.length === 0) return; + yield* Ref.update(subagentThreads, (current) => { + const updated = new Map(current); + for (const entry of existing) { + const nativeThreadId = entry.childProviderThread.nativeThreadRef?.nativeId ?? null; + if (nativeThreadId === null || updated.has(nativeThreadId)) continue; + const nativeItemId = entry.subagent.nativeTaskRef?.nativeId ?? null; + if (nativeItemId === null || !nativeItemId.includes(":")) continue; + updated.set(nativeThreadId, { + parentContext: context, + providerThread: entry.childProviderThread, + childThread: entry.childThread, + subagentNodeId: entry.subagent.id, + childRootNodeId: idAllocator.derive.nodeFromProviderItem({ + driver: CODEX_PROVIDER, + nativeItemId: `${nativeItemId}:thread-root`, + }), + childThreadId: entry.childThread.id, + nativeToolCallId: nativeItemId.slice(0, nativeItemId.indexOf(":")), + ordinal: entry.ordinal, + startedAt: entry.subagent.startedAt ?? context.startedAt, + turnItemId: entry.turnItemId, + turnItemOrdinal: entry.turnItemOrdinal, + task: { + ...entry.subagent, + runId: context.projectionRunId, + parentNodeId: context.itemParentNodeId, + // A non-terminal status here is a leftover from the session + // that died; this one never drove that activation and can + // never terminalize it. Left as running it would keep + // hasPendingBackgroundWork true and pin the session open. + status: + entry.subagent.status === "pending" || + entry.subagent.status === "running" || + entry.subagent.status === "waiting" + ? "idle" + : entry.subagent.status, + currentActivationId: null, + }, + }); + } + return updated; + }); + // Turn ordinals live in the same lost registry. Restarting them at + // 1 would re-derive childRootNodeId for a different native turn, so + // continue past the activations already recorded. + yield* Ref.update(nextProviderTurnOrdinals, (current) => { + const updated = new Map(current); + for (const entry of existing) { + const key = String(entry.childProviderThread.id); + if (updated.has(key)) continue; + updated.set(key, Math.max(1, entry.subagent.activationCount)); + } + return updated; + }); + }); + const registerRootTurn = (input: { readonly turnInput: ProviderAdapterV2TurnInput; readonly nativeTurnId: string; @@ -1539,6 +1603,7 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi updated.set(input.nativeTurnId, context); return updated; }); + yield* seedExistingSubagents(context); yield* emitProviderEvent({ type: "provider_turn.updated", driver: CODEX_PROVIDER, @@ -2009,7 +2074,11 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi }) => Effect.gen(function* () { const registeredSubagents = yield* Ref.get(subagentThreads); - if (registeredSubagents.has(input.nativeThreadId)) { + const existing = registeredSubagents.get(input.nativeThreadId); + if (existing !== undefined) { + // Same agent thread referenced again by a later turn: keep the + // identity and adopt it into the run now driving it. + rebindSubagentToContext(existing, input.context); return; } @@ -2254,6 +2323,39 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi } }); + /** + * Adopt an already-registered subagent into the root turn that is + * re-activating it. The task row keeps the run it was spawned under + * otherwise, and every later row would be attributed to a closed run: + * routing drops it, and the current run stops counting the subagent as + * open work and can finalize while it is still going. + */ + const rebindSubagentToContext = ( + subagent: CodexSubagentThreadContext, + context: ActiveCodexTurnContext, + ) => { + if (subagent.parentContext.projectionRunId === context.projectionRunId) return; + subagent.parentContext = context; + subagent.task = { + ...subagent.task, + runId: context.projectionRunId, + parentNodeId: context.itemParentNodeId, + }; + }; + + const rebindSubagentsToCurrentTurn = (input: { + readonly context: ActiveCodexTurnContext; + readonly item: CodexCollabAgentToolCallItem; + }) => + Effect.gen(function* () { + if (input.item.receiverThreadIds.length === 0) return; + const registered = yield* Ref.get(subagentThreads); + for (const nativeThreadId of input.item.receiverThreadIds) { + const subagent = registered.get(nativeThreadId); + if (subagent !== undefined) rebindSubagentToContext(subagent, input.context); + } + }); + const registerSubagentActivity = (input: { readonly context: ActiveCodexTurnContext; readonly item: CodexSubAgentActivityItem; @@ -2287,6 +2389,17 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi return; } + // "interacted" is how Codex reports a root turn messaging an agent + // it did not spawn. It is the only reuse signal on this path — the + // collab tool call that accompanies it carries neither + // receiverThreadIds nor agentsStates — and it arrives before the + // child turn starts, so adopting the identity here is what lets the + // resulting activation be attributed to the run driving it. + if (input.item.kind === "interacted") { + rebindSubagentToContext(subagent, input.context); + return; + } + if (input.item.kind === "interrupted") { yield* emitSubagentTaskUpdate({ subagent, @@ -3291,7 +3404,13 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi yield* client.handleServerNotification("thread/status/changed", (payload) => Effect.gen(function* () { const subagent = (yield* Ref.get(subagentThreads)).get(payload.threadId); - if (subagent === undefined || isSettledCodexSubagent(subagent.task)) return; + // Live status applies only while an activation is in flight. Every + // resting and terminal state clears currentActivationId, so this + // single check drops trailing frames for an idle or finished + // identity while letting a re-activated one resume immediately. + if (subagent === undefined || subagent.task.currentActivationId === null) { + return; + } const status = payload.status.type === "active" ? payload.status.activeFlags.length > 0 @@ -3347,6 +3466,14 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi } } + // A collab tool call that targets an existing agent thread is a + // re-activation; adopt it into this turn before its child turn + // starts so the activation is attributed to the current run. + if (payload.item.type === "collabAgentToolCall") { + yield* rebindSubagentsToCurrentTurn({ context, item: payload.item }); + return; + } + if (payload.item.type === "subAgentActivity") { yield* registerSubagentActivity({ context, @@ -3608,6 +3735,11 @@ export function makeCodexAdapterV2(adapterOptions: CodexAdapterV2Options): Provi context, item: payload.item, }); + // Reuse arrives here, not on item/started: resumeAgent and + // sendInput only ever complete. registerSubagentThreads ignores + // them because they are not spawnAgent, so without this the + // returning agent keeps the run it was spawned under. + yield* rebindSubagentsToCurrentTurn({ context, item: payload.item }); yield* updateSubagentStates({ item: payload.item, }); diff --git a/apps/server/src/orchestration-v2/ProviderAdapter.ts b/apps/server/src/orchestration-v2/ProviderAdapter.ts index c64db5e8b6c..f2b2c148ee7 100644 --- a/apps/server/src/orchestration-v2/ProviderAdapter.ts +++ b/apps/server/src/orchestration-v2/ProviderAdapter.ts @@ -389,6 +389,21 @@ export interface ProviderAdapterV2EnsureThreadInput { readonly existingProviderThread?: OrchestrationV2ProviderThread; } +/** + * A reusable subagent that already exists in the projection, supplied so an + * adapter whose in-memory registry was lost (session reap, server restart) can + * still recognise the agent thread and re-activate the same identity instead of + * spawning a duplicate. + */ +export interface ProviderAdapterV2ExistingSubagent { + readonly subagent: OrchestrationV2Subagent; + readonly childThread: OrchestrationV2AppThread; + readonly childProviderThread: OrchestrationV2ProviderThread; + readonly turnItemId: TurnItemId; + readonly turnItemOrdinal: number; + readonly ordinal: number; +} + export interface ProviderAdapterV2TurnInput { readonly appThread: OrchestrationV2AppThread; readonly threadId: ThreadId; @@ -401,6 +416,7 @@ export interface ProviderAdapterV2TurnInput { readonly message: ProviderAdapterV2TurnMessage; readonly modelSelection: ModelSelection; readonly runtimePolicy: ProviderAdapterV2RuntimePolicy; + readonly existingSubagents?: ReadonlyArray; } export interface ProviderAdapterV2SteerInput { diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts new file mode 100644 index 00000000000..a481f23ad6b --- /dev/null +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -0,0 +1,296 @@ +import { assert, it } from "@effect/vitest"; +import { + CheckpointScopeId, + MessageId, + NodeId, + type OrchestrationV2ThreadProjection, + ProviderDriverKind, + ProviderInstanceId, + ProviderSessionId, + ProviderThreadId, + RunAttemptId, + RunId, + ThreadId, + TurnItemId, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Ref from "effect/Ref"; + +import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts"; +import { EventSinkV2 } from "./EventSink.ts"; +import { layer as idAllocatorLayer } from "./IdAllocator.ts"; +import type { ProviderAdapterV2SessionRuntime } from "./ProviderAdapter.ts"; +import { + ProjectionStoreReadError, + ProjectionStoreThreadNotFoundError, + ProjectionStoreV2, + type ProjectionStoreV2Error, +} from "./ProjectionStore.ts"; +import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; +import { + layer as providerTurnStartLayer, + ProviderTurnStartServiceV2, +} from "./ProviderTurnStartService.ts"; +import { RunExecutionServiceV2 } from "./RunExecutionService.ts"; +import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; + +const driver = ProviderDriverKind.make("codex"); +const threadId = ThreadId.make("thread:provider-turn-start"); +const childThreadId = ThreadId.make("thread:provider-turn-start:child"); +const runId = RunId.make("run:provider-turn-start"); +const rootNodeId = NodeId.make("node:provider-turn-start:root"); +const subagentId = NodeId.make("node:provider-turn-start:subagent"); +const attemptId = RunAttemptId.make("attempt:provider-turn-start"); +const providerThreadId = ProviderThreadId.make("provider-thread:provider-turn-start"); +const childProviderThreadId = ProviderThreadId.make("provider-thread:provider-turn-start:child"); +const providerSessionId = ProviderSessionId.make("provider-session:provider-turn-start"); +const providerInstanceId = ProviderInstanceId.make("codex"); +const messageId = MessageId.make("message:provider-turn-start"); +const checkpointScopeId = CheckpointScopeId.make("checkpoint-scope:provider-turn-start"); + +function makeRootProjection(now: DateTime.Utc): OrchestrationV2ThreadProjection { + return { + thread: { + id: threadId, + runtimeMode: "full-access", + interactionMode: "default", + worktreePath: "/tmp/provider-turn-start", + }, + runs: [ + { + id: runId, + threadId, + ordinal: 1, + status: "starting", + rootNodeId, + activeAttemptId: attemptId, + providerThreadId, + providerInstanceId, + userMessageId: messageId, + modelSelection: { instanceId: providerInstanceId }, + }, + ], + nodes: [ + { + id: rootNodeId, + checkpointScopeId, + }, + ], + attempts: [ + { + id: attemptId, + providerTurnId: null, + }, + ], + providerThreads: [ + { + id: providerThreadId, + providerSessionId, + nativeThreadRef: { + driver, + nativeId: "native-root-thread", + strength: "strong", + }, + ownerNodeId: rootNodeId, + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + }, + ], + messages: [ + { + id: messageId, + text: "continue the agent", + attachments: [], + createdBy: "user", + creationSource: "user", + }, + ], + checkpointScopes: [{ id: checkpointScopeId }], + contextHandoffs: [], + contextTransfers: [], + providerSessions: [], + subagents: [ + { + id: subagentId, + status: "idle", + childThreadId, + providerThreadId: childProviderThreadId, + }, + ], + turnItems: [ + { + id: TurnItemId.make("turn-item:provider-turn-start:subagent"), + type: "subagent", + subagentId, + ordinal: 1, + }, + ], + providerTurns: [], + } as unknown as OrchestrationV2ThreadProjection; +} + +function makeChildProjection(now: DateTime.Utc): OrchestrationV2ThreadProjection { + return { + thread: { id: childThreadId }, + providerThreads: [ + { + id: childProviderThreadId, + nativeThreadRef: { + driver, + nativeId: "native-child-thread", + strength: "strong", + }, + createdAt: now, + }, + ], + } as unknown as OrchestrationV2ThreadProjection; +} + +function makeTestLayer(input: { + readonly rootProjection: OrchestrationV2ThreadProjection; + readonly childProjection: Effect.Effect; + readonly existingSubagentCounts: Ref.Ref>; + readonly providerSessionOpens: Ref.Ref; + readonly runningWrites: Ref.Ref; +}) { + const session = { + driver, + providerSession: { id: providerSessionId }, + resumeThread: ({ providerThread }: { readonly providerThread: unknown }) => + Effect.succeed(providerThread), + } as unknown as ProviderAdapterV2SessionRuntime; + + return providerTurnStartLayer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(ContextHandoffServiceV2)({}), + Layer.mock(EventSinkV2)({ + writeIfRunCurrent: () => + Ref.update(input.runningWrites, (count) => count + 1).pipe( + Effect.as({ committed: true, storedEvents: [] }), + ), + }), + idAllocatorLayer, + Layer.mock(ProjectionStoreV2)({ + getThreadProjection: (requestedThreadId) => + requestedThreadId === threadId + ? Effect.succeed(input.rootProjection) + : input.childProjection, + }), + Layer.mock(ProviderSessionManagerV2)({ + open: () => + Ref.update(input.providerSessionOpens, (count) => count + 1).pipe(Effect.as(session)), + }), + Layer.mock(RunExecutionServiceV2)({ + startRootRun: (startInput) => + Ref.update(input.existingSubagentCounts, (counts) => [ + ...counts, + startInput.existingSubagents?.length ?? 0, + ]), + }), + Layer.mock(RuntimePolicyV2)({ + resolve: () => + Effect.succeed({ + runtimeMode: "full-access", + interactionMode: "default", + cwd: "/tmp/provider-turn-start", + }), + }), + ), + ), + ); +} + +it.effect("fails before provider turn start when child projection rehydration cannot be read", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const existingSubagentCounts = yield* Ref.make>([]); + const providerSessionOpens = yield* Ref.make(0); + const runningWrites = yield* Ref.make(0); + const readError = new ProjectionStoreReadError({ + threadId: childThreadId, + cause: "temporary database failure", + }); + const error = yield* Effect.gen(function* () { + const service = yield* ProviderTurnStartServiceV2; + return yield* service.start({ threadId, runId }).pipe(Effect.flip); + }).pipe( + Effect.provide( + makeTestLayer({ + rootProjection: makeRootProjection(now), + childProjection: Effect.fail(readError), + existingSubagentCounts, + providerSessionOpens, + runningWrites, + }), + ), + ); + + assert.equal(error._tag, "ProviderTurnStartError"); + assert.strictEqual(error.cause, readError); + assert.deepEqual(yield* Ref.get(existingSubagentCounts), []); + assert.equal(yield* Ref.get(providerSessionOpens), 0); + assert.equal(yield* Ref.get(runningWrites), 0); + }), +); + +it.effect("skips a missing child projection and still starts the provider turn", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const existingSubagentCounts = yield* Ref.make>([]); + const providerSessionOpens = yield* Ref.make(0); + const runningWrites = yield* Ref.make(0); + yield* Effect.gen(function* () { + const service = yield* ProviderTurnStartServiceV2; + yield* service.start({ threadId, runId }); + }).pipe( + Effect.provide( + makeTestLayer({ + rootProjection: makeRootProjection(now), + childProjection: Effect.fail( + new ProjectionStoreThreadNotFoundError({ threadId: childThreadId }), + ), + existingSubagentCounts, + providerSessionOpens, + runningWrites, + }), + ), + ); + + assert.deepEqual(yield* Ref.get(existingSubagentCounts), [0]); + assert.equal(yield* Ref.get(providerSessionOpens), 1); + assert.equal(yield* Ref.get(runningWrites), 1); + }), +); + +it.effect("rehydrates an existing child projection before starting the provider turn", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const existingSubagentCounts = yield* Ref.make>([]); + const providerSessionOpens = yield* Ref.make(0); + const runningWrites = yield* Ref.make(0); + yield* Effect.gen(function* () { + const service = yield* ProviderTurnStartServiceV2; + yield* service.start({ threadId, runId }); + }).pipe( + Effect.provide( + makeTestLayer({ + rootProjection: makeRootProjection(now), + childProjection: Effect.succeed(makeChildProjection(now)), + existingSubagentCounts, + providerSessionOpens, + runningWrites, + }), + ), + ); + + assert.deepEqual(yield* Ref.get(existingSubagentCounts), [1]); + assert.equal(yield* Ref.get(providerSessionOpens), 1); + assert.equal(yield* Ref.get(runningWrites), 1); + }), +); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 3b6f74bf5f7..dc2fbcda749 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -20,9 +20,13 @@ import { providerMessageWithContextHandoffs, } from "./ContextHandoffService.ts"; import { IdAllocatorV2 } from "./IdAllocator.ts"; -import { ProjectionStoreV2 } from "./ProjectionStore.ts"; +import { ProjectionStoreThreadNotFoundError, ProjectionStoreV2 } from "./ProjectionStore.ts"; import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts"; -import { canRouteRelatedSubagent, RunExecutionServiceV2 } from "./RunExecutionService.ts"; +import { + canReactivateSubagent, + canRouteRelatedSubagent, + RunExecutionServiceV2, +} from "./RunExecutionService.ts"; import { RuntimePolicyV2 } from "./RuntimePolicy.ts"; export class ProviderTurnStartError extends Schema.TaggedErrorClass()( @@ -125,6 +129,53 @@ export const layer: Layer.Layer< }); } const providerSessionId = providerThread.providerSessionId; + const routableSubagents = projection.subagents.filter((subagent) => + canRouteRelatedSubagent(subagent.status), + ); + // Broader than the child-thread list above: an interrupted agent is still + // resumable, so it stays re-activatable even though its thread must not + // be adopted up front. + const reactivatableSubagents = projection.subagents.filter((subagent) => + canReactivateSubagent(subagent.status), + ); + // Resolve the persisted registry before opening the provider session or + // advancing the run. Operational read failures must leave the run in + // `starting` so the durable effect can retry safely. + const existingSubagents = yield* Effect.forEach( + reactivatableSubagents, + (subagent, index) => + Effect.gen(function* () { + const childThreadId = subagent.childThreadId; + if (childThreadId === null) return []; + const turnItem = projection.turnItems.find( + (item) => item.type === "subagent" && item.subagentId === subagent.id, + ); + if (turnItem === undefined) return []; + const childProjection = yield* projectionStore + .getThreadProjection(childThreadId) + .pipe( + Effect.catchIf(Schema.is(ProjectionStoreThreadNotFoundError), () => + Effect.succeed(null), + ), + ); + if (childProjection === null) return []; + const childProviderThread = childProjection.providerThreads.find( + (candidate) => candidate.id === subagent.providerThreadId, + ); + if (childProviderThread === undefined) return []; + return [ + { + subagent, + childThread: childProjection.thread, + childProviderThread, + turnItemId: turnItem.id, + turnItemOrdinal: turnItem.ordinal, + ordinal: index + 1, + }, + ]; + }), + { concurrency: "unbounded" }, + ).pipe(Effect.map((entries) => entries.flat())); const isCurrentAttemptInStatus = ( expectedStatus: OrchestrationV2Run["status"], ): Effect.Effect => @@ -403,9 +454,6 @@ export const layer: Layer.Layer< if (!runningWrite.committed) { return; } - const routableSubagents = projection.subagents.filter((subagent) => - canRouteRelatedSubagent(subagent.status), - ); yield* runExecution.startRootRun({ commandId: CommandId.make(`command:effect:provider-turn.start:${run.id}`), appThread: projection.thread, @@ -423,6 +471,8 @@ export const layer: Layer.Layer< relatedProviderThreadIds: routableSubagents.flatMap((subagent) => subagent.providerThreadId === null ? [] : [subagent.providerThreadId], ), + relatedSubagentIds: reactivatableSubagents.map((subagent) => subagent.id), + existingSubagents, providerTurnOrdinal: Math.max( 0, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 2a1ce09d1d6..08cce151f80 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -13,6 +13,7 @@ import { type OrchestrationV2Run, type OrchestrationV2RunAttempt, type OrchestrationV2Subagent, + type OrchestrationV2SubagentActivation, type OrchestrationV2TurnItem, ProviderDriverKind, ProviderInstanceId, @@ -21,6 +22,7 @@ import { ProviderTurnId, RunAttemptId, RunId, + SubagentActivationId, ThreadId, TurnItemId, } from "@t3tools/contracts"; @@ -40,9 +42,11 @@ import { IdAllocatorV2, layer as idAllocatorLayer } from "./IdAllocator.ts"; import type { ProviderAdapterV2Event, ProviderAdapterV2SessionRuntime } from "./ProviderAdapter.ts"; import { ProviderEventIngestorV2 } from "./ProviderEventIngestor.ts"; import { + canReactivateSubagent, canRouteRelatedSubagent, cascadeTerminalizeRunOwnedSubagents, finalProviderThreadStatus, + isRunOwnedSubagentTurnItem, layer as runExecutionServiceLayer, makeProviderEventRoutingState, type ProviderEventRouteIdentity, @@ -154,6 +158,120 @@ it.effect("routes shared-runtime events only to their owning root run", () => }), ); +it.effect("routes a reused subagent's rows to the run that re-activated it", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:reuse"); + const spawnRunId = RunId.make("run:reuse:1"); + const subagentId = NodeId.make("node:reuse:subagent"); + const childThreadId = ThreadId.make("thread:reuse:child"); + // The turn that re-activates an existing subagent, two runs later. + const laterRun: ProviderEventRouteIdentity = { + threadId, + runId: RunId.make("run:reuse:2"), + attemptId: RunAttemptId.make("attempt:reuse:2"), + providerThreadId: ProviderThreadId.make("provider-thread:reuse"), + }; + + // A reusable subagent keeps the run id it was spawned under, and its row is + // keyed on the parent thread — so neither the run test nor the child-thread + // test can match. Only the identity seeded from the projection can. + const subagentEvent: ProviderAdapterV2Event = { + type: "subagent.updated", + driver, + subagent: { + ...makeRunOwnedSubagentFixture({ + ids: { + threadId, + runId: spawnRunId, + rootNodeId: NodeId.make("node:reuse:root"), + subagentNodeId: subagentId, + } as BackgroundScenarioIds, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId, + driver, + status: "running", + }), + activationCount: 2, + updatedAt: now, + }, + }; + const activationEvent: ProviderAdapterV2Event = { + type: "subagent_activation.updated", + driver, + activation: { + id: SubagentActivationId.make(`${subagentId}:activation:2`), + threadId, + subagentId, + runId: spawnRunId, + providerTurnId: null, + ordinal: 2, + status: "running", + usage: null, + startedAt: now, + completedAt: null, + updatedAt: now, + }, + }; + const turnItemEvent = { + type: "turn_item.updated", + driver, + turnItem: { + id: TurnItemId.make("turn-item:reuse:subagent"), + threadId, + runId: spawnRunId, + nodeId: subagentId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 2, + status: "running", + title: "reused agent", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "subagent", + subagentId, + origin: "provider_native", + driver, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId, + prompt: "continue", + result: null, + }, + } satisfies ProviderAdapterV2Event; + + const withoutSeed = makeProviderEventRoutingState({ + identity: laterRun, + providerTurnId: null, + }); + assert.isFalse(routeProviderEvent(subagentEvent, laterRun, withoutSeed)[0]); + assert.isFalse(routeProviderEvent(activationEvent, laterRun, withoutSeed)[0]); + assert.isFalse(routeProviderEvent(turnItemEvent, laterRun, withoutSeed)[0]); + + const seeded = makeProviderEventRoutingState({ + identity: laterRun, + providerTurnId: null, + relatedSubagentIds: [subagentId], + }); + const [subagentAccepted, afterSubagent] = routeProviderEvent(subagentEvent, laterRun, seeded); + assert.isTrue(subagentAccepted); + assert.isTrue(routeProviderEvent(activationEvent, laterRun, afterSubagent)[0]); + assert.isTrue(routeProviderEvent(turnItemEvent, laterRun, afterSubagent)[0]); + assert.isTrue( + isRunOwnedSubagentTurnItem({ + turnItem: turnItemEvent.turnItem, + runId: laterRun.runId, + ownedSubagentIds: afterSubagent.ownedSubagentIds, + }), + ); + // Accepting the row also adopts its child thread, so the agent's own + // messages and turn items route into this run for the rest of it. + assert.isTrue(afterSubagent.ownedThreadIds.has(childThreadId)); + }), +); + it("does not route a superseded attempt through a reused provider thread", () => { const threadId = ThreadId.make("thread:shared-runtime:restart"); const providerThreadId = ProviderThreadId.make("provider-thread:shared-runtime:restart"); @@ -195,6 +313,13 @@ it("does not carry interrupted child ownership into later attempts", () => { assert.isTrue(canRouteRelatedSubagent("completed")); assert.isTrue(canRouteRelatedSubagent("running")); + // Re-activation is deliberately broader: the user stopped an interrupted + // agent, they did not lose it, and its provider thread is still resumable. + assert.isTrue(canReactivateSubagent("interrupted")); + assert.isTrue(canReactivateSubagent("idle")); + assert.isFalse(canReactivateSubagent("failed")); + assert.isFalse(canReactivateSubagent("cancelled")); + const threadId = ThreadId.make("thread:related-child:next-attempt"); const childThreadId = ThreadId.make("thread:related-child:interrupted"); const identity: ProviderEventRouteIdentity = { @@ -234,6 +359,82 @@ it("does not carry interrupted child ownership into later attempts", () => { assert.isFalse(routeProviderEvent(lateChildNode, identity, state)[0]); }); +it.effect("re-activates an interrupted subagent without adopting its stale child traffic", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread:interrupted-reuse"); + const subagentId = NodeId.make("node:interrupted-reuse:subagent"); + const childThreadId = ThreadId.make("thread:interrupted-reuse:child"); + const laterRun: ProviderEventRouteIdentity = { + threadId, + runId: RunId.make("run:interrupted-reuse:2"), + attemptId: RunAttemptId.make("attempt:interrupted-reuse:2"), + providerThreadId: ProviderThreadId.make("provider-thread:interrupted-reuse"), + }; + + // What ProviderTurnStartService builds for a previously interrupted agent: + // its identity is re-activatable, but its child thread is NOT pre-owned. + const state = makeProviderEventRoutingState({ + identity: laterRun, + providerTurnId: null, + relatedThreadIds: [], + relatedSubagentIds: [subagentId], + }); + + // A late event from the interrupted child thread must still be refused — + // this is the invariant the post-interrupt recovery fix depends on. + const lateChildNode: ProviderAdapterV2Event = { + type: "node.updated", + driver, + node: { + id: NodeId.make("node:interrupted-reuse:late"), + threadId: childThreadId, + runId: null, + parentNodeId: null, + rootNodeId: NodeId.make("node:interrupted-reuse:late"), + kind: "root_turn", + status: "completed", + countsForRun: false, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: null, + completedAt: null, + }, + }; + assert.isFalse(routeProviderEvent(lateChildNode, laterRun, state)[0]); + + // A genuine re-activation of that identity is admitted, and only then is + // the child thread adopted — ownership is earned, not granted up front. + const reactivation: ProviderAdapterV2Event = { + type: "subagent.updated", + driver, + subagent: { + ...makeRunOwnedSubagentFixture({ + ids: { + threadId, + runId: RunId.make("run:interrupted-reuse:1"), + rootNodeId: NodeId.make("node:interrupted-reuse:root"), + subagentNodeId: subagentId, + } as BackgroundScenarioIds, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId, + driver, + status: "running", + }), + activationCount: 4, + updatedAt: now, + }, + }; + const [accepted, afterReactivation] = routeProviderEvent(reactivation, laterRun, state); + assert.isTrue(accepted); + assert.isTrue(afterReactivation.ownedThreadIds.has(childThreadId)); + assert.isTrue(routeProviderEvent(lateChildNode, laterRun, afterReactivation)[0]); + }), +); + it.effect("rechecks run ownership immediately before calling the provider", () => Effect.gen(function* () { const runExecution = yield* RunExecutionServiceV2; @@ -1289,6 +1490,12 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child const threadId = ThreadId.make(`thread:cascade-helper:${driverKind}`); const childThreadId = ThreadId.make(`thread:cascade-helper:${driverKind}:child`); const subagentId = NodeId.make(`node:cascade-helper:${driverKind}:subagent`); + const activationId = SubagentActivationId.make( + `node:cascade-helper:${driverKind}:subagent:activation:2`, + ); + const previousActivationId = SubagentActivationId.make( + `node:cascade-helper:${driverKind}:subagent:activation:1`, + ); const childNodeId = NodeId.make(`node:cascade-helper:${driverKind}:child-root`); const providerInstanceId = ProviderInstanceId.make(String(driverKind)); const terminalStatus = driverKind === "claudeAgent" ? "failed" : "cancelled"; @@ -1313,19 +1520,38 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child model: null, kind: "subagent", role: { name: "general-purpose", source: "app_default" }, + status: "running", + progress: "partial progress", + result: "partial result", usage: null, - currentActivationId: null, - activationCount: 1, + currentActivationId: activationId, + activationCount: 2, workflow: null, workflowMembership: null, recentActivity: [], + startedAt: now, + completedAt: null, + updatedAt: now, + }; + const activation: OrchestrationV2SubagentActivation = { + id: activationId, + threadId, + subagentId, + runId, + providerTurnId: null, + ordinal: 2, status: "running", - progress: "partial progress", - result: "partial result", + usage: { totalTokens: 120, toolUses: 2 }, startedAt: now, completedAt: null, updatedAt: now, }; + const previousActivation: OrchestrationV2SubagentActivation = { + ...activation, + id: previousActivationId, + ordinal: 1, + usage: { totalTokens: 60, toolUses: 1 }, + }; const turnItem = { id: TurnItemId.make(`turn-item:cascade-helper:${driverKind}`), threadId, @@ -1434,6 +1660,10 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child } as OrchestrationV2Run, open: { subagents: new Map([[subagentId, subagent]]), + activations: new Map([ + [previousActivation.id, previousActivation], + [activation.id, activation], + ]), turnItems: new Map([[subagentId, turnItem]]), childTurnItems: new Map(), nodes: new Map([[subagentId, node]]), @@ -1444,7 +1674,7 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child allocateEventId, }); - assert.equal(events.length, 3, `${driverKind}: subagent + node + turn item`); + assert.equal(events.length, 5, `${driverKind}: subagent + activations + node + turn item`); const terminalSubagent = events.find((event) => event.type === "subagent.updated"); assert.isDefined(terminalSubagent); if (terminalSubagent?.type !== "subagent.updated") { @@ -1456,6 +1686,25 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child assert.equal(terminalSubagent.payload.progress, "partial progress"); assert.equal(terminalSubagent.payload.result, "partial result"); assert.equal(terminalSubagent.payload.driver, driverKind); + assert.isNull(terminalSubagent.payload.currentActivationId); + + const terminalActivations = events.filter( + ( + event, + ): event is Extract => + event.type === "subagent-activation.updated", + ); + assert.deepEqual( + terminalActivations.map((event) => ({ + id: event.payload.id, + status: event.payload.status, + totalTokens: event.payload.usage?.totalTokens, + })), + [ + { id: previousActivationId, status: terminalStatus, totalTokens: 60 }, + { id: activationId, status: terminalStatus, totalTokens: 120 }, + ], + ); const terminalItem = events.find( (event) => event.type === "turn-item.updated" && event.payload.type === "subagent", @@ -1481,6 +1730,7 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child } as OrchestrationV2Run, open: { subagents: new Map(), + activations: new Map(), turnItems: new Map(), childTurnItems: new Map([[childTurnItem.id, childTurnItem]]), nodes: new Map([[childNodeId, openChildNode]]), @@ -1525,6 +1775,73 @@ it.effect("cascade helper is provider-neutral for Claude and Codex-shaped child }), ); +it.effect("terminalizes an identity-owned reused subagent turn item from its spawning run", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const runId = RunId.make("run:reused-turn-item:current"); + const spawnRunId = RunId.make("run:reused-turn-item:spawn"); + const threadId = ThreadId.make("thread:reused-turn-item"); + const subagentId = NodeId.make("node:reused-turn-item:subagent"); + const turnItem = { + id: TurnItemId.make("turn-item:reused-turn-item:subagent"), + threadId, + runId: spawnRunId, + nodeId: subagentId, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "running" as const, + title: "reused agent", + startedAt: now, + completedAt: null, + updatedAt: now, + type: "subagent" as const, + subagentId, + origin: "provider_native" as const, + driver, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId: ThreadId.make("thread:reused-turn-item:child"), + prompt: "continue", + progress: "working", + result: null, + } satisfies Extract; + + const events = yield* cascadeTerminalizeRunOwnedSubagents({ + run: { + id: runId, + threadId, + ordinal: 2, + providerInstanceId: ProviderInstanceId.make("codex"), + } as OrchestrationV2Run, + open: { + subagents: new Map(), + activations: new Map(), + turnItems: new Map([[subagentId, turnItem]]), + childTurnItems: new Map(), + nodes: new Map(), + linkedChildThreadIds: new Set(), + }, + status: "interrupted", + completedAt: now, + allocateEventId: () => Effect.succeed(EventId.make("event:reused-turn-item:cascade")), + }); + + assert.lengthOf(events, 1); + const event = events[0]; + assert.isDefined(event); + if (event.type !== "turn-item.updated" || event.payload.type !== "subagent") { + assert.fail("expected reused subagent turn-item.updated event"); + return; + } + assert.equal(event.runId, runId); + assert.equal(event.payload.runId, spawnRunId); + assert.equal(event.payload.status, "interrupted"); + assert.equal(event.payload.id, turnItem.id); + }), +); + it.effect("omits interrupt results and subagent cascade for a superseded attempt", () => Effect.gen(function* () { const written = yield* captureInterruptTerminalTurnItems({ @@ -1875,14 +2192,14 @@ function makeRunOwnedSubagentFixture(input: { model: null, kind: "subagent", role: { name: "general-purpose", source: "app_default" }, + status: input.status, + result: null, usage: null, currentActivationId: null, activationCount: 1, workflow: null, workflowMembership: null, recentActivity: [], - status: input.status, - result: null, startedAt: now, completedAt: null, updatedAt: now, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 4638466e5fb..40afea9faed 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -13,11 +13,13 @@ import { type OrchestrationV2Run, type OrchestrationV2RunAttempt, type OrchestrationV2Subagent, + type OrchestrationV2SubagentActivation, type OrchestrationV2TurnItem, type ProviderSessionId, type ProviderThreadId, type ProviderTurnId, type RunAttemptId, + type SubagentActivationId, type ThreadId, type TurnItemId, } from "@t3tools/contracts"; @@ -43,6 +45,7 @@ import { import type { ProviderAdapterV2Event, ProviderAdapterV2RuntimePolicy, + ProviderAdapterV2ExistingSubagent, ProviderAdapterV2SessionRuntime, ProviderAdapterV2TurnMessage, } from "./ProviderAdapter.ts"; @@ -53,6 +56,10 @@ export interface ProviderEventRoutingState { readonly ownedThreadIds: ReadonlySet; readonly ownedProviderThreadIds: ReadonlySet; readonly ownedProviderTurnIds: ReadonlySet; + // Subagent identities this run may observe. A reusable subagent keeps the + // run id it was spawned under, so a later run that re-activates it cannot + // recognise its rows by run or by parent thread; it matches on identity. + readonly ownedSubagentIds: ReadonlySet; readonly rootProviderTurnId: ProviderTurnId | null; } @@ -76,6 +83,7 @@ function isTerminalProviderTurnStatus(status: OrchestrationV2ProviderTurn["statu function isTerminalSubagentStatus(status: OrchestrationV2Subagent["status"]): boolean { return ( + status === "idle" || status === "completed" || status === "interrupted" || status === "failed" || @@ -106,6 +114,7 @@ type SubagentTurnItem = Extract; + readonly activations: ReadonlyMap; readonly turnItems: ReadonlyMap; readonly childTurnItems: ReadonlyMap; readonly nodes: ReadonlyMap; @@ -128,13 +137,46 @@ function isRunOwnedSubagentTerminalStatus( return status === "interrupted" || status === "failed" || status === "cancelled"; } +/** + * Whether a later run may pre-emptively own this subagent's child thread. + * + * Interrupted children are excluded on purpose: their threads can still emit + * late events, and adopting them up front lets that stale traffic land in the + * next attempt (see "preserve claude/codex post-interrupt recovery state"). + */ export function canRouteRelatedSubagent(status: OrchestrationV2Subagent["status"]): boolean { return status !== "interrupted" && status !== "failed" && status !== "cancelled"; } +/** + * Whether a later run may re-activate this subagent identity. + * + * Deliberately broader than {@link canRouteRelatedSubagent}: an interrupted + * agent is one the user stopped, not one that died, and its provider thread is + * still resumable. Only rows for this exact subagent id are admitted by + * identity, and those are emitted solely on a genuine re-activation — so the + * child thread is adopted when the agent proves it is live again rather than up + * front, and late stale traffic still has no way in. + */ +export function canReactivateSubagent(status: OrchestrationV2Subagent["status"]): boolean { + return status !== "failed" && status !== "cancelled"; +} + +export function isRunOwnedSubagentTurnItem(input: { + readonly turnItem: OrchestrationV2TurnItem; + readonly runId: OrchestrationV2Run["id"]; + readonly ownedSubagentIds: ReadonlySet; +}): boolean { + return ( + input.turnItem.runId === input.runId || + (input.turnItem.type === "subagent" && input.ownedSubagentIds.has(input.turnItem.subagentId)) + ); +} + function emptyOpenRunOwnedSubagentProjection(): OpenRunOwnedSubagentProjection { return { subagents: new Map(), + activations: new Map(), turnItems: new Map(), childTurnItems: new Map(), nodes: new Map(), @@ -191,6 +233,7 @@ export function cascadeTerminalizeRunOwnedSubagents(input: { payload: { ...subagent, status: input.status, + currentActivationId: null, completedAt: input.completedAt, updatedAt: input.completedAt, }, @@ -219,11 +262,7 @@ export function cascadeTerminalizeRunOwnedSubagents(input: { }); } const turnItem = input.open.turnItems.get(key); - if ( - turnItem !== undefined && - turnItem.runId === input.run.id && - !isTerminalTurnItemStatus(turnItem.status) - ) { + if (turnItem !== undefined && !isTerminalTurnItemStatus(turnItem.status)) { events.push({ id: yield* input.allocateEventId(), type: "turn-item.updated", @@ -241,6 +280,31 @@ export function cascadeTerminalizeRunOwnedSubagents(input: { }); } } + for (const activation of input.open.activations.values()) { + if ( + activation.status === "completed" || + activation.status === "interrupted" || + activation.status === "failed" || + activation.status === "cancelled" + ) { + continue; + } + events.push({ + id: yield* input.allocateEventId(), + type: "subagent-activation.updated", + threadId: activation.threadId, + runId: activation.runId ?? input.run.id, + nodeId: activation.subagentId, + providerInstanceId: input.run.providerInstanceId, + occurredAt: input.completedAt, + payload: { + ...activation, + status: input.status, + completedAt: input.completedAt, + updatedAt: input.completedAt, + }, + }); + } for (const turnItem of input.open.childTurnItems.values()) { if (!childThreadIds.has(turnItem.threadId) || isTerminalTurnItemStatus(turnItem.status)) { continue; @@ -277,6 +341,7 @@ export function makeProviderEventRoutingState(input: { readonly providerTurnId: ProviderTurnId | null; readonly relatedThreadIds?: ReadonlyArray; readonly relatedProviderThreadIds?: ReadonlyArray; + readonly relatedSubagentIds?: ReadonlyArray; }): ProviderEventRoutingState { return { ownedThreadIds: new Set([input.identity.threadId, ...(input.relatedThreadIds ?? [])]), @@ -286,6 +351,7 @@ export function makeProviderEventRoutingState(input: { ]), ownedProviderTurnIds: input.providerTurnId === null ? new Set() : new Set([input.providerTurnId]), + ownedSubagentIds: new Set(input.relatedSubagentIds ?? []), rootProviderTurnId: input.providerTurnId, }; } @@ -311,6 +377,28 @@ export function routeProviderEvent( ownedProviderTurnIds: new Set([...state.ownedProviderTurnIds, providerTurnId]), rootProviderTurnId: root ? providerTurnId : state.rootProviderTurnId, }); + const ownsSubagent = (subagentId: NodeId): boolean => state.ownedSubagentIds.has(subagentId); + /** + * Track the identity so every later row for the same subagent routes. + * + * The child thread is adopted only while the agent is actually live. A + * settled row is also emitted by trailing traffic — a token-usage frame or a + * collab state sweep re-emits the last known status — and adopting on those + * would hand a later run an interrupted agent's thread, letting exactly the + * stale child-thread events that post-interrupt recovery excludes back in. + */ + const addSubagent = ( + subagentId: NodeId, + childThreadId: ThreadId | null, + live: boolean, + ): ProviderEventRoutingState => ({ + ...state, + ownedSubagentIds: new Set([...state.ownedSubagentIds, subagentId]), + ownedThreadIds: + childThreadId === null || !live + ? state.ownedThreadIds + : new Set([...state.ownedThreadIds, childThreadId]), + }); switch (event.type) { case "provider_session.updated": @@ -359,14 +447,41 @@ export function routeProviderEvent( } return [true, addProviderThread(event.node.providerThreadId)]; } - case "subagent.updated": - return [ownsRun(event.subagent.runId) || ownsChildThread(event.subagent.threadId), state]; + case "subagent.updated": { + const belongs = + ownsRun(event.subagent.runId) || + ownsChildThread(event.subagent.threadId) || + ownsSubagent(event.subagent.id); + return belongs + ? [ + true, + addSubagent( + event.subagent.id, + event.subagent.childThreadId, + !isTerminalSubagentStatus(event.subagent.status), + ), + ] + : [false, state]; + } case "subagent_activation.updated": - return [ownsRun(event.activation.runId) || ownsChildThread(event.activation.threadId), state]; + return [ + ownsRun(event.activation.runId) || + ownsChildThread(event.activation.threadId) || + ownsSubagent(event.activation.subagentId), + state, + ]; case "message.updated": return [ownsRun(event.message.runId) || ownsChildThread(event.message.threadId), state]; case "turn_item.updated": - return [ownsRun(event.turnItem.runId) || ownsChildThread(event.turnItem.threadId), state]; + return [ + ownsRun(event.turnItem.runId) || + ownsChildThread(event.turnItem.threadId) || + // A reused subagent's row and its timeline item are emitted together; + // routing only the row would advance the agent while its item stayed + // frozen on the spawning run. + (event.turnItem.type === "subagent" && ownsSubagent(event.turnItem.subagentId)), + state, + ]; case "plan.updated": return [ownsRun(event.plan.runId) || ownsChildThread(event.plan.threadId), state]; case "runtime_request.updated": @@ -432,6 +547,8 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly providerTurnOrdinal: number; readonly relatedThreadIds?: ReadonlyArray; readonly relatedProviderThreadIds?: ReadonlyArray; + readonly relatedSubagentIds?: ReadonlyArray; + readonly existingSubagents?: ReadonlyArray; readonly shouldStartProviderTurn?: () => Effect.Effect; readonly shouldFinalizeRun?: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; @@ -551,6 +668,7 @@ export const layer: Layer.Layer< const open = input.openRunOwnedSubagents ?? emptyOpenRunOwnedSubagentProjection(); const hasOpenSubagentProjection = open.subagents.size > 0 || + open.activations.size > 0 || open.turnItems.size > 0 || open.childTurnItems.size > 0 || open.nodes.size > 0; @@ -772,6 +890,9 @@ export const layer: Layer.Layer< ...(input.relatedProviderThreadIds === undefined ? {} : { relatedProviderThreadIds: input.relatedProviderThreadIds }), + ...(input.relatedSubagentIds === undefined + ? {} + : { relatedSubagentIds: input.relatedSubagentIds }), }), ); const rootTerminalSeen = yield* Ref.make(false); @@ -836,7 +957,11 @@ export const layer: Layer.Layer< } } if (event.type === "subagent.updated") { - const belongsToRootRun = event.subagent.runId === input.run.id; + // A re-activated subagent is adopted by the current run, so it + // holds this run open the same way a freshly spawned one does. + const belongsToRootRun = + event.subagent.runId === input.run.id || + routing.ownedSubagentIds.has(event.subagent.id); const belongsToOwnedChildThread = event.subagent.threadId !== input.run.threadId && routing.ownedThreadIds.has(event.subagent.threadId); @@ -868,6 +993,31 @@ export const layer: Layer.Layer< }); } } + if (event.type === "subagent_activation.updated") { + const belongsToRootRun = + event.activation.runId === input.run.id || + routing.ownedSubagentIds.has(event.activation.subagentId); + const belongsToOwnedChildThread = + event.activation.threadId !== input.run.threadId && + routing.ownedThreadIds.has(event.activation.threadId); + if (!belongsToRootRun && !belongsToOwnedChildThread) { + return; + } + yield* Ref.update(openRunOwnedSubagents, (current) => { + const activations = new Map(current.activations); + if ( + event.activation.status === "completed" || + event.activation.status === "interrupted" || + event.activation.status === "failed" || + event.activation.status === "cancelled" + ) { + activations.delete(event.activation.id); + } else { + activations.set(event.activation.id, event.activation); + } + return { ...current, activations }; + }); + } if (event.type === "node.updated") { const belongsToRootSubagent = event.node.kind === "subagent" && event.node.runId === input.run.id; @@ -888,7 +1038,11 @@ export const layer: Layer.Layer< }); } if (event.type === "turn_item.updated") { - const belongsToRootRun = event.turnItem.runId === input.run.id; + const belongsToRootRun = isRunOwnedSubagentTurnItem({ + turnItem: event.turnItem, + runId: input.run.id, + ownedSubagentIds: routing.ownedSubagentIds, + }); const belongsToOwnedChildThread = event.turnItem.threadId !== input.run.threadId && routing.ownedThreadIds.has(event.turnItem.threadId); @@ -1114,6 +1268,9 @@ export const layer: Layer.Layer< message: input.message, modelSelection: input.modelSelection, runtimePolicy: input.runtimePolicy, + ...(input.existingSubagents === undefined + ? {} + : { existingSubagents: input.existingSubagents }), }) .pipe( Effect.catchCause((cause) => diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts index 04c157b99cf..afb071a6793 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/index.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/index.ts @@ -387,6 +387,14 @@ export const ORCHESTRATOR_REPLAY_FIXTURES = [ }, ], }, + // subagent_reuse_after_idle is scaffolded and recorded but not registered: + // replaying it leaves the second run running forever, so the scenario never + // reaches thread idle. Not caused by the reuse routing — the hang reproduces + // with those changes reverted, and persists now that the rebind demonstrably + // fires (subagent_continue proves that). Root cause still unknown; the + // recorded transcript uses the current Codex item shape (subAgentActivity + // started/interacted, collab `wait` with empty receiverThreadIds), whereas + // the passing subagent_continue transcript uses resumeAgent/sendInput. { name: "subagent_v2", buildInput: subagentV2Input, diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts b/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts index 5377ba675a0..5f2be9f4e8a 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/shared.ts @@ -55,6 +55,14 @@ export const SUBAGENT_CONTINUE_PROMPT = export const SUBAGENT_CONTINUE_PARENT_PROMPT = "@hooke have the same subagent reply exactly: continued subagent response"; export const SUBAGENT_CONTINUE_CHILD_PROMPT = "Reply exactly: continued subagent response"; +export const SUBAGENT_REUSE_AFTER_INTERRUPT_PROMPT = + "Spawn one subagent named reuse_probe. Have it run the local shell command `sleep 30` and then reply exactly: reuse probe first response"; +export const SUBAGENT_REUSE_AFTER_INTERRUPT_RESUME_PROMPT = + "Send another message to the same reuse_probe subagent. Do not spawn a replacement. Have it reply exactly: reuse probe resumed response"; +export const SUBAGENT_REUSE_AFTER_IDLE_PROMPT = + "Spawn one subagent named idle_probe and have it reply exactly: idle probe first response"; +export const SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT = + "Send another message to the same idle_probe subagent. Do not spawn a replacement. Have it reply exactly: idle probe resumed response"; export const TURN_INTERRUPT_PROMPT = "Do not answer immediately. First run the local shell command `sleep 30`, then respond with exactly: interrupt fixture should not finish naturally."; export const TURN_INTERRUPT_MID_TOOL_PROMPT = diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_continue/codex_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_continue/codex_output.ts index d9f3dfcd43a..46503730427 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_continue/codex_output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_continue/codex_output.ts @@ -34,10 +34,18 @@ export function assertSubagentContinueOutput( assert.lengthOf(projection.subagents, 1); const subagent = projection.subagents[0]!; - // A reusable identity rests at idle between activations rather than - // finishing outright, and carries the latest activation's result. assert.equal(subagent.status, "idle"); + // The continuation runs under a second app run. The task identity is + // reusable, so its row must carry the latest activation's result and be + // adopted by the run that drove it, not stay frozen on the spawning run. assert.equal(subagent.result, "continued subagent response"); + assert.equal(subagent.activationCount, 2); + // Adoption by the resuming run is what keeps the row routable once the + // spawning run's ingestion fiber closes; pinned to run 1 it is only visible + // for as long as that fiber happens to outlive the turn. + const latestRun = projection.runs.at(-1); + assert.isDefined(latestRun); + assert.equal(subagent.runId, latestRun.id); assert.isNotNull(subagent.childThreadId); if (subagent.childThreadId === null) { throw new Error("Continued subagent is missing its child thread"); diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_output.ts new file mode 100644 index 00000000000..00e6e146260 --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_output.ts @@ -0,0 +1,60 @@ +import { assert } from "@effect/vitest"; +import type { ProviderReplayTranscript } from "@t3tools/contracts"; + +import type { OrchestratorV2ScenarioResult } from "../../OrchestratorScenario.ts"; +import { + assertSemanticProjectionIntegrity, + assertUserMessagesInclude, + projectionFor, + SUBAGENT_REUSE_AFTER_IDLE_PROMPT, + SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT, +} from "../shared.ts"; + +export function assertSubagentReuseAfterIdleOutput( + result: OrchestratorV2ScenarioResult, + transcript: ProviderReplayTranscript, +) { + const projection = projectionFor(result, transcript.scenario); + assertSemanticProjectionIntegrity(projection); + assertUserMessagesInclude(projection, [ + SUBAGENT_REUSE_AFTER_IDLE_PROMPT, + SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT, + ]); + + // A duplicate here means the reaped registry was never rehydrated from the + // projection, so the returning agent thread read as a stranger. + assert.lengthOf( + projection.subagents, + 1, + "the session reap caused the reused subagent to be registered as a new identity", + ); + const subagent = projection.subagents[0]!; + + assert.isAtLeast( + subagent.activationCount, + 2, + "the subagent was not re-activated after the session was reaped", + ); + const activations = projection.subagentActivations.filter( + (activation) => activation.subagentId === subagent.id, + ); + assert.isAtLeast(activations.length, 2, "each activation must be recorded separately"); + + const latestRun = projection.runs.at(-1); + assert.isDefined(latestRun); + assert.equal( + subagent.runId, + latestRun.id, + "the re-activated subagent was not adopted by the run driving it", + ); + + assert.isNotNull(subagent.childThreadId); + if (subagent.childThreadId === null) { + throw new Error("Re-activated subagent is missing its child thread"); + } + const childProjection = result.projections.get(subagent.childThreadId); + assert.isDefined(childProjection); + // One child thread carrying both turns proves the original agent thread was + // resumed rather than a second one created alongside it. + assert.isAtLeast(childProjection.providerTurns.length, 2); +} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_transcript.ndjson b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_transcript.ndjson new file mode 100644 index 00000000000..8daa767bfa6 --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/codex_transcript.ndjson @@ -0,0 +1,149 @@ +{"type":"transcript_start","provider":"codex","protocol":"codex.app-server","version":"0.145.0","scenario":"subagent_reuse_after_idle","metadata":{"source":"record-codex-app-server-replay-fixture","fileName":"subagent_reuse_after_idle.ndjson","description":"A root turn spawns a native Codex subagent, then a later root turn re-activates it. Replay advances past the session idle timeout between turns so the adapter registry is empty on reuse."}} +{"type":"expect_outbound","label":"initialize","frame":{"id":1,"method":"initialize","params":{"capabilities":{"experimentalApi":true},"clientInfo":{"name":"t3code_desktop","title":"T3 Code Desktop","version":"0.1.0"}}}} +{"type":"emit_inbound","label":"initialize","frame":{"id":1,"result":{"userAgent":"t3code_desktop/0.145.0 (Mac OS 26.5.2; arm64) unknown (t3code_desktop; 0.1.0)","codexHome":"/Users/shivam/.codex","platformFamily":"unix","platformOs":"macos"}}} +{"type":"expect_outbound","label":"initialized","frame":{"method":"initialized"}} +{"type":"expect_outbound","label":"thread/start","frame":{"id":2,"method":"thread/start","params":{}}} +{"type":"emit_inbound","label":"remoteControl/status/changed","frame":{"method":"remoteControl/status/changed","params":{"status":"disabled","serverName":"shivam-macpro.local","installationId":"6a5680bf-d908-4d8f-b9f4-ee9e1d7089ae","environmentId":null},"emittedAtMs":1785137018968}} +{"type":"emit_inbound","label":"thread/start","frame":{"id":2,"result":{"thread":{"id":"019fa275-3860-7732-8f0b-6d43c27baf50","extra":null,"sessionId":"019fa275-3860-7732-8f0b-6d43c27baf50","forkedFromId":null,"parentThreadId":null,"preview":"","ephemeral":false,"historyMode":"legacy","modelProvider":"openai","createdAt":1785137019,"updatedAt":1785137019,"recencyAt":1785137019,"status":{"type":"idle"},"path":"/Users/shivam/.codex/sessions/2026/07/27/rollout-2026-07-27T12-53-38-019fa275-3860-7732-8f0b-6d43c27baf50.jsonl","cwd":"/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server","cliVersion":"0.145.0","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":null,"name":null,"turns":[]},"model":"gpt-5.6-sol","modelProvider":"openai","serviceTier":"priority","cwd":"/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server","runtimeWorkspaceRoots":["/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server"],"instructionSources":["/Users/shivam/.codex/AGENTS.md","/Users/shivam/Developer/t3/ccing/open/t3code-2/AGENTS.md"],"approvalPolicy":"never","approvalsReviewer":"user","sandbox":{"type":"dangerFullAccess"},"activePermissionProfile":null,"reasoningEffort":"high","multiAgentMode":"explicitRequestOnly"}}} +{"type":"expect_outbound","label":"turn/start","frame":{"id":3,"method":"turn/start","params":{"input":[{"text":"Spawn one subagent named idle_probe and have it reply exactly: idle probe first response","type":"text"}],"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50"}}} +{"type":"emit_inbound","label":"thread/started","frame":{"method":"thread/started","params":{"thread":{"id":"019fa275-3860-7732-8f0b-6d43c27baf50","extra":null,"sessionId":"019fa275-3860-7732-8f0b-6d43c27baf50","forkedFromId":null,"parentThreadId":null,"preview":"","ephemeral":false,"historyMode":"legacy","modelProvider":"openai","createdAt":1785137019,"updatedAt":1785137019,"recencyAt":1785137019,"status":{"type":"idle"},"path":"/Users/shivam/.codex/sessions/2026/07/27/rollout-2026-07-27T12-53-38-019fa275-3860-7732-8f0b-6d43c27baf50.jsonl","cwd":"/Users/shivam/Developer/t3/ccing/open/t3code-2/apps/server","cliVersion":"0.145.0","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":null,"name":null,"turns":[]}},"emittedAtMs":1785137019108}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"xcodebuildmcp","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137019109}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137019109}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"node_repl","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137019109}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"openaiDeveloperDocs","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137019109}} +{"type":"emit_inbound","label":"turn/start","frame":{"id":3,"result":{"turn":{"id":"019fa275-38e7-7a12-9975-235b0e73bc77","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":null,"completedAt":null,"durationMs":null}}}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","status":{"type":"active","activeFlags":[]}},"emittedAtMs":1785137019148}} +{"type":"emit_inbound","label":"turn/started","frame":{"method":"turn/started","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turn":{"id":"019fa275-38e7-7a12-9975-235b0e73bc77","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":1785137019,"completedAt":null,"durationMs":null}},"emittedAtMs":1785137019148}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"node_repl","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137019208}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"openaiDeveloperDocs","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137020206}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"xcodebuildmcp","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137020728}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"codex_apps","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137021595}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"userMessage","id":"019fa275-46aa-7841-8d4d-f6884a8e7f83","clientId":null,"content":[{"type":"text","text":"Spawn one subagent named idle_probe and have it reply exactly: idle probe first response","text_elements":[]}]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137022634},"emittedAtMs":1785137022634}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"userMessage","id":"019fa275-46aa-7841-8d4d-f6884a8e7f83","clientId":null,"content":[{"type":"text","text":"Spawn one subagent named idle_probe and have it reply exactly: idle probe first response","text_elements":[]}]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137022634},"emittedAtMs":1785137022634}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"reasoning","id":"rs_01463f7aca6c9f0e016a6707800dd4819182530528d17657f6","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137024054},"emittedAtMs":1785137024054}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"reasoning","id":"rs_01463f7aca6c9f0e016a6707800dd4819182530528d17657f6","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137024373},"emittedAtMs":1785137024373}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","text":"","phase":"commentary","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137024375},"emittedAtMs":1785137024375}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":"I"},"emittedAtMs":1785137024383}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":"’ll"},"emittedAtMs":1785137024383}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" start"},"emittedAtMs":1785137024383}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" the"},"emittedAtMs":1785137024383}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" requested"},"emittedAtMs":1785137024419}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" sub"},"emittedAtMs":1785137024421}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":"agent"},"emittedAtMs":1785137024421}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" and"},"emittedAtMs":1785137024422}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" relay"},"emittedAtMs":1785137024474}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" its"},"emittedAtMs":1785137024477}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" exact"},"emittedAtMs":1785137024482}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":" response"},"emittedAtMs":1785137024485}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","delta":"."},"emittedAtMs":1785137024485}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_01463f7aca6c9f0e016a6707805f648191b15ff18b0d9de2ad","text":"I’ll start the requested subagent and relay its exact response.","phase":"commentary","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137024609},"emittedAtMs":1785137024609}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","status":{"type":"idle"}},"emittedAtMs":1785137025213}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"subAgentActivity","id":"call_6KAd2XhIg5t3F25Hs8IUu96D","kind":"started","agentThreadId":"019fa275-5034-73b1-a29e-552df3271b27","agentPath":"/root/idle_probe"},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137025214},"emittedAtMs":1785137025214}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"xcodebuildmcp","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137025215}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"openaiDeveloperDocs","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137025215}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137025215}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"node_repl","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137025215}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","status":{"type":"active","activeFlags":[]}},"emittedAtMs":1785137025218}} +{"type":"emit_inbound","label":"turn/started","frame":{"method":"turn/started","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turn":{"id":"019fa275-50be-7e01-9ce1-a85fe3c2f250","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":1785137025,"completedAt":null,"durationMs":null}},"emittedAtMs":1785137025218}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","tokenUsage":{"total":{"totalTokens":19105,"inputTokens":19025,"cachedInputTokens":0,"cacheWriteInputTokens":0,"outputTokens":80,"reasoningOutputTokens":15},"last":{"totalTokens":19105,"inputTokens":19025,"cachedInputTokens":0,"cacheWriteInputTokens":0,"outputTokens":80,"reasoningOutputTokens":15},"modelContextWindow":258400}},"emittedAtMs":1785137025221}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137025221}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"node_repl","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137025288}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"codex_apps","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137025826}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"xcodebuildmcp","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137026337}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","name":"openaiDeveloperDocs","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137026454}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"reasoning","id":"rs_01463f7aca6c9f0e016a670783507481919c0813670bdd3e22","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137027315},"emittedAtMs":1785137027315}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"reasoning","id":"rs_01463f7aca6c9f0e016a670783507481919c0813670bdd3e22","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137027537},"emittedAtMs":1785137027537}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"collabAgentToolCall","id":"call_E7ieRlVqMfllfBpPGi5Dl1UW","tool":"wait","status":"inProgress","senderThreadId":"019fa275-3860-7732-8f0b-6d43c27baf50","receiverThreadIds":[],"prompt":null,"model":null,"reasoningEffort":null,"agentsStates":{}},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137027699},"emittedAtMs":1785137027699}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","text":"","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","startedAtMs":1785137030566},"emittedAtMs":1785137030566}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","itemId":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","delta":"idle"},"emittedAtMs":1785137030566}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","itemId":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","delta":" probe"},"emittedAtMs":1785137030578}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","itemId":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","delta":" first"},"emittedAtMs":1785137030578}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","itemId":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","delta":" response"},"emittedAtMs":1785137030590}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_058f48e256fe5748016a6707868b948191b7e425d852b9f4f2","text":"idle probe first response","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","completedAtMs":1785137030698},"emittedAtMs":1785137030698}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-50be-7e01-9ce1-a85fe3c2f250","tokenUsage":{"total":{"totalTokens":22650,"inputTokens":22642,"cachedInputTokens":16128,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"last":{"totalTokens":22650,"inputTokens":22642,"cachedInputTokens":16128,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1785137030774}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137030774}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"collabAgentToolCall","id":"call_E7ieRlVqMfllfBpPGi5Dl1UW","tool":"wait","status":"completed","senderThreadId":"019fa275-3860-7732-8f0b-6d43c27baf50","receiverThreadIds":[],"prompt":null,"model":null,"reasoningEffort":null,"agentsStates":{}},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137030823},"emittedAtMs":1785137030824}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","status":{"type":"idle"}},"emittedAtMs":1785137030824}} +{"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turn":{"id":"019fa275-50be-7e01-9ce1-a85fe3c2f250","items":[],"itemsView":"notLoaded","status":"completed","error":null,"startedAt":1785137025,"completedAt":1785137030,"durationMs":5607}},"emittedAtMs":1785137030824}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","tokenUsage":{"total":{"totalTokens":38262,"inputTokens":38152,"cachedInputTokens":18176,"cacheWriteInputTokens":0,"outputTokens":110,"reasoningOutputTokens":22},"last":{"totalTokens":19157,"inputTokens":19127,"cachedInputTokens":18176,"cacheWriteInputTokens":0,"outputTokens":30,"reasoningOutputTokens":7},"modelContextWindow":258400}},"emittedAtMs":1785137030825}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137030825}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","text":"","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","startedAtMs":1785137032765},"emittedAtMs":1785137032765}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","delta":"idle"},"emittedAtMs":1785137032767}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","delta":" probe"},"emittedAtMs":1785137032768}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","delta":" first"},"emittedAtMs":1785137032776}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","itemId":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","delta":" response"},"emittedAtMs":1785137032790}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_01463f7aca6c9f0e016a670788c3a48191b0cf4218c3aa4940","text":"idle probe first response","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","completedAtMs":1785137032874},"emittedAtMs":1785137032874}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-38e7-7a12-9975-235b0e73bc77","tokenUsage":{"total":{"totalTokens":57492,"inputTokens":57374,"cachedInputTokens":36352,"cacheWriteInputTokens":0,"outputTokens":118,"reasoningOutputTokens":22},"last":{"totalTokens":19230,"inputTokens":19222,"cachedInputTokens":18176,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1785137032924}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137032924}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","status":{"type":"idle"}},"emittedAtMs":1785137032925}} +{"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turn":{"id":"019fa275-38e7-7a12-9975-235b0e73bc77","items":[],"itemsView":"notLoaded","status":"completed","error":null,"startedAt":1785137019,"completedAt":1785137032,"durationMs":13811}},"emittedAtMs":1785137032925}} +{"type":"expect_outbound","label":"turn/start","frame":{"id":4,"method":"turn/start","params":{"input":[{"text":"Send another message to the same idle_probe subagent. Do not spawn a replacement. Have it reply exactly: idle probe resumed response","type":"text"}],"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50"}}} +{"type":"emit_inbound","label":"turn/start","frame":{"id":4,"result":{"turn":{"id":"019fa275-6edd-7d23-932f-9a8bcd630dce","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":null,"completedAt":null,"durationMs":null}}}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137032927}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"openaiDeveloperDocs","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137032927}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"node_repl","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137032927}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"xcodebuildmcp","status":"starting","error":null,"failureReason":null},"emittedAtMs":1785137032927}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","status":{"type":"active","activeFlags":[]}},"emittedAtMs":1785137032939}} +{"type":"emit_inbound","label":"turn/started","frame":{"method":"turn/started","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turn":{"id":"019fa275-6edd-7d23-932f-9a8bcd630dce","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":1785137032,"completedAt":null,"durationMs":null}},"emittedAtMs":1785137032939}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"node_repl","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137033024}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"codex_apps","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137033516}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"openaiDeveloperDocs","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137034028}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"userMessage","id":"019fa275-7333-71c1-8c45-67070525dbe9","clientId":null,"content":[{"type":"text","text":"Send another message to the same idle_probe subagent. Do not spawn a replacement. Have it reply exactly: idle probe resumed response","text_elements":[]}]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","startedAtMs":1785137034035},"emittedAtMs":1785137034035}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"userMessage","id":"019fa275-7333-71c1-8c45-67070525dbe9","clientId":null,"content":[{"type":"text","text":"Send another message to the same idle_probe subagent. Do not spawn a replacement. Have it reply exactly: idle probe resumed response","text_elements":[]}]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137034035},"emittedAtMs":1785137034035}} +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","name":"xcodebuildmcp","status":"ready","error":null,"failureReason":null},"emittedAtMs":1785137034079}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"reasoning","id":"rs_0c64722b069256f7016a67078c265c8191a2be05492bb9f2f8","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","startedAtMs":1785137036152},"emittedAtMs":1785137036152}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"reasoning","id":"rs_0c64722b069256f7016a67078c265c8191a2be05492bb9f2f8","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137036530},"emittedAtMs":1785137036530}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","text":"","phase":"commentary","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","startedAtMs":1785137036532},"emittedAtMs":1785137036532}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"I"},"emittedAtMs":1785137036535}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"’ll"},"emittedAtMs":1785137036537}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" resume"},"emittedAtMs":1785137036539}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" the"},"emittedAtMs":1785137036540}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" existing"},"emittedAtMs":1785137036543}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" `"},"emittedAtMs":1785137036544}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"idle"},"emittedAtMs":1785137036736}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"_probe"},"emittedAtMs":1785137036738}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"`"},"emittedAtMs":1785137036739}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" sub"},"emittedAtMs":1785137036739}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"agent"},"emittedAtMs":1785137036773}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"—"},"emittedAtMs":1785137036784}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"no"},"emittedAtMs":1785137036784}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" replacement"},"emittedAtMs":1785137036789}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"—and"},"emittedAtMs":1785137036833}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" relay"},"emittedAtMs":1785137036839}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" its"},"emittedAtMs":1785137036839}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" exact"},"emittedAtMs":1785137036839}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":" response"},"emittedAtMs":1785137036841}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","delta":"."},"emittedAtMs":1785137036841}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_0c64722b069256f7016a67078c87808191a6671bf5d5534c15","text":"I’ll resume the existing `idle_probe` subagent—no replacement—and relay its exact response.","phase":"commentary","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137036854},"emittedAtMs":1785137036854}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"subAgentActivity","id":"call_Eo2mTZMc8KumZl7Gc0tGySg5","kind":"interacted","agentThreadId":"019fa275-5034-73b1-a29e-552df3271b27","agentPath":"/root/idle_probe"},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137037205},"emittedAtMs":1785137037205}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","status":{"type":"active","activeFlags":[]}},"emittedAtMs":1785137037209}} +{"type":"emit_inbound","label":"turn/started","frame":{"method":"turn/started","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turn":{"id":"019fa275-7f95-7181-b5c0-e2959a15e3c1","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":1785137037,"completedAt":null,"durationMs":null}},"emittedAtMs":1785137037209}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","tokenUsage":{"total":{"totalTokens":79941,"inputTokens":79741,"cachedInputTokens":36352,"cacheWriteInputTokens":0,"outputTokens":200,"reasoningOutputTokens":35},"last":{"totalTokens":22449,"inputTokens":22367,"cachedInputTokens":0,"cacheWriteInputTokens":0,"outputTokens":82,"reasoningOutputTokens":13},"modelContextWindow":258400}},"emittedAtMs":1785137037257}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137037257}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"reasoning","id":"rs_058f48e256fe5748016a67078f36a8819195c0b103e3d41f47","summary":[],"content":[]},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","startedAtMs":1785137039233},"emittedAtMs":1785137039233}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"reasoning","id":"rs_058f48e256fe5748016a67078f36a8819195c0b103e3d41f47","summary":[],"content":[]},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","completedAtMs":1785137039357},"emittedAtMs":1785137039357}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","text":"","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","startedAtMs":1785137039360},"emittedAtMs":1785137039360}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","itemId":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","delta":"idle"},"emittedAtMs":1785137039363}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","itemId":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","delta":" probe"},"emittedAtMs":1785137039370}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","itemId":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","delta":" resumed"},"emittedAtMs":1785137039380}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","itemId":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","delta":" response"},"emittedAtMs":1785137039398}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"reasoning","id":"rs_09cd91668d0353dd016a67078f80048191b5cc53d08e8bca89","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","startedAtMs":1785137039500},"emittedAtMs":1785137039500}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_058f48e256fe5748016a67078f56148191a87fca200a919c40","text":"idle probe resumed response","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","completedAtMs":1785137039533},"emittedAtMs":1785137039533}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turnId":"019fa275-7f95-7181-b5c0-e2959a15e3c1","tokenUsage":{"total":{"totalTokens":45370,"inputTokens":45343,"cachedInputTokens":38400,"cacheWriteInputTokens":0,"outputTokens":27,"reasoningOutputTokens":9},"last":{"totalTokens":22720,"inputTokens":22701,"cachedInputTokens":22272,"cacheWriteInputTokens":0,"outputTokens":19,"reasoningOutputTokens":9},"modelContextWindow":258400}},"emittedAtMs":1785137039563}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137039563}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","status":{"type":"idle"}},"emittedAtMs":1785137039566}} +{"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"019fa275-5034-73b1-a29e-552df3271b27","turn":{"id":"019fa275-7f95-7181-b5c0-e2959a15e3c1","items":[],"itemsView":"notLoaded","status":"completed","error":null,"startedAt":1785137037,"completedAt":1785137039,"durationMs":2358}},"emittedAtMs":1785137039566}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"reasoning","id":"rs_09cd91668d0353dd016a67078f80048191b5cc53d08e8bca89","summary":[],"content":[]},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137039817},"emittedAtMs":1785137039817}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","tokenUsage":{"total":{"totalTokens":79941,"inputTokens":79741,"cachedInputTokens":36352,"cacheWriteInputTokens":0,"outputTokens":200,"reasoningOutputTokens":35},"last":{"totalTokens":22449,"inputTokens":22367,"cachedInputTokens":0,"cacheWriteInputTokens":0,"outputTokens":82,"reasoningOutputTokens":13},"modelContextWindow":258400}},"emittedAtMs":1785137039820}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634198},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137039820}} +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","text":"","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","startedAtMs":1785137041773},"emittedAtMs":1785137041773}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","delta":"idle"},"emittedAtMs":1785137041773}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","delta":" probe"},"emittedAtMs":1785137041773}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","delta":" resumed"},"emittedAtMs":1785137041797}} +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","itemId":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","delta":" response"},"emittedAtMs":1785137041805}} +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_001550b2df635fbe016a670791c46881918e3167f585df353b","text":"idle probe resumed response","phase":"final_answer","memoryCitation":null},"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","completedAtMs":1785137041975},"emittedAtMs":1785137041975}} +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turnId":"019fa275-6edd-7d23-932f-9a8bcd630dce","tokenUsage":{"total":{"totalTokens":102942,"inputTokens":102734,"cachedInputTokens":58624,"cacheWriteInputTokens":0,"outputTokens":208,"reasoningOutputTokens":35},"last":{"totalTokens":23001,"inputTokens":22993,"cachedInputTokens":22272,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1785137041989}} +{"type":"emit_inbound","label":"account/rateLimits/updated","frame":{"method":"account/rateLimits/updated","params":{"rateLimits":{"limitId":"codex","limitName":null,"primary":{"usedPercent":8,"windowDurationMins":10080,"resetsAt":1785634197},"secondary":null,"credits":{"hasCredits":false,"unlimited":false,"balance":"0"},"individualLimit":null,"spendControlReached":null,"planType":"prolite","rateLimitReachedType":null}},"emittedAtMs":1785137041989}} +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","status":{"type":"idle"}},"emittedAtMs":1785137041991}} +{"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"019fa275-3860-7732-8f0b-6d43c27baf50","turn":{"id":"019fa275-6edd-7d23-932f-9a8bcd630dce","items":[],"itemsView":"notLoaded","status":"completed","error":null,"startedAt":1785137032,"completedAt":1785137041,"durationMs":9053}},"emittedAtMs":1785137041991}} +{"type":"runtime_exit","status":"success"} diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/input.ts b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/input.ts new file mode 100644 index 00000000000..4b23f65b0df --- /dev/null +++ b/apps/server/src/orchestration-v2/testkit/fixtures/subagent_reuse_after_idle/input.ts @@ -0,0 +1,31 @@ +import { + SUBAGENT_REUSE_AFTER_IDLE_PROMPT, + SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT, + type OrchestratorFixtureInput, +} from "../shared.ts"; + +/** + * Regression for subagent reuse across a session reap. + * + * Adapters track spawned subagents in a process-local map, so the reaper + * releasing the session runtime wipes every identity it knows about. Without + * rehydrating them from the projection, the returning agent thread is + * unrecognised on the next turn and gets registered as a brand new subagent — + * the user sees a duplicate appear and the original frozen forever. + * + * Not registered yet. The recorded transcript does contain a real reuse, but + * replaying it leaves the second run running forever so the scenario never + * reaches thread idle, and that is unrelated to the reuse routing here — it + * reproduces with those changes reverted. An advance_clock step to cross the + * 30-minute idle timeout (which is what would exercise rehydration rather than + * plain reuse) fails the same way, so it is left out until the hang is + * understood. + */ +export function subagentReuseAfterIdleInput(): OrchestratorFixtureInput { + return { + steps: [ + { type: "message", text: SUBAGENT_REUSE_AFTER_IDLE_PROMPT }, + { type: "message", text: SUBAGENT_REUSE_AFTER_IDLE_RESUME_PROMPT }, + ], + }; +}