From 6a87a62a85e84e22897687fbb8aadc054fc7d9f0 Mon Sep 17 00:00:00 2001 From: atryan Date: Sun, 2 Aug 2026 03:31:24 +0000 Subject: [PATCH 1/2] fix command center startup failure status --- .../src/command-center/RunLifecycle.test.ts | 73 +++++++++++++++++++ .../server/src/command-center/RunLifecycle.ts | 47 +++++++++++- .../settings/ProviderInstanceCard.tsx | 26 +++++++ 3 files changed, 145 insertions(+), 1 deletion(-) diff --git a/apps/server/src/command-center/RunLifecycle.test.ts b/apps/server/src/command-center/RunLifecycle.test.ts index da832b6df91..3621d82850b 100644 --- a/apps/server/src/command-center/RunLifecycle.test.ts +++ b/apps/server/src/command-center/RunLifecycle.test.ts @@ -3,6 +3,9 @@ import * as NodeCrypto from "node:crypto"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import { + CommandId, + EventId, + type OrchestrationEvent, ProviderRuntimeEvent, type OrchestrationThreadShell, type ProviderSession, @@ -116,6 +119,36 @@ const startedEvent = (input: { readonly eventId: string; readonly threadId: stri createdAt: fixtureTime, }); +const failedSessionEvent = (input: { + readonly eventId: string; + readonly threadId: string; + readonly errorMessage: string; +}): OrchestrationEvent => ({ + sequence: 1, + eventId: EventId.make(input.eventId), + type: "thread.session-set", + aggregateKind: "thread", + aggregateId: ThreadId.make(input.threadId), + occurredAt: fixtureTime, + commandId: CommandId.make(`command-${input.eventId}`), + causationEventId: null, + correlationId: null, + metadata: {}, + payload: { + threadId: ThreadId.make(input.threadId), + session: { + threadId: ThreadId.make(input.threadId), + status: "error", + providerName: "codex", + providerInstanceId: ProviderInstanceId.make("codex"), + runtimeMode: "approval-required", + activeTurnId: null, + lastError: input.errorMessage, + updatedAt: fixtureTime, + }, + }, +}); + it.effect("projects provider completion, preserves the audit chain, and revokes MCP scope", () => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; @@ -248,6 +281,46 @@ it.effect("records one actionable failure and one urgent Needs You alert under r }).pipe(Effect.provide(testLayer)), ); +it.effect( + "fails a running command-center run when provider startup sets the session to error", + () => + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const persistence = yield* makeRunLifecyclePersistence; + yield* insertRun(sql, { id: "run-startup-error", threadId: "thread-startup-error" }); + + const revoked: Array = []; + const lifecycle = makeWithDependencies({ + persistence, + getThread: () => Effect.sync((): OrchestrationThreadShell | undefined => undefined), + listProviderSessions: Effect.succeed([]), + revokeThread: (threadId) => + Effect.sync(() => { + revoked.push(threadId); + }), + }); + const event = failedSessionEvent({ + eventId: "session-startup-error", + threadId: "thread-startup-error", + errorMessage: "The Windows sandbox could not be initialized.", + }); + + const first = yield* lifecycle.handleOrchestrationEvent(event); + const duplicate = yield* lifecycle.handleOrchestrationEvent(event); + const rows = yield* sql<{ readonly state: string; readonly error: string | null }>` + SELECT state, error FROM command_center_runs WHERE id = 'run-startup-error' + `; + + expect(first?.status).toBe("failed"); + expect(duplicate).toBeUndefined(); + expect(rows[0]).toEqual({ + state: "failed", + error: "The Windows sandbox could not be initialized.", + }); + expect(revoked).toEqual(["thread-startup-error"]); + }).pipe(Effect.provide(testLayer)), +); + const threadShell = (input: { readonly threadId: string; readonly state: "running" | "interrupted" | "completed" | "error"; diff --git a/apps/server/src/command-center/RunLifecycle.ts b/apps/server/src/command-center/RunLifecycle.ts index a82bd0ab844..43e059bddc6 100644 --- a/apps/server/src/command-center/RunLifecycle.ts +++ b/apps/server/src/command-center/RunLifecycle.ts @@ -1,5 +1,6 @@ import { COMMAND_CENTER_EVENT_ACTIONS, + type OrchestrationEvent, type OrchestrationThreadShell, type ProviderRuntimeEvent, type ProviderSession, @@ -15,6 +16,7 @@ import * as Stream from "effect/Stream"; import * as SqlClient from "effect/unstable/sql/SqlClient"; import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; +import * as OrchestrationEngine from "../orchestration/Services/OrchestrationEngine.ts"; import * as ProjectionSnapshotQuery from "../orchestration/Services/ProjectionSnapshotQuery.ts"; import * as ProviderService from "../provider/Services/ProviderService.ts"; import { makeCommandCenterAuditLog } from "./AuditLog.ts"; @@ -390,6 +392,9 @@ export interface RunLifecycleShape { readonly handleProviderEvent: ( event: ProviderRuntimeEvent, ) => Effect.Effect; + readonly handleOrchestrationEvent: ( + event: OrchestrationEvent, + ) => Effect.Effect; readonly reconcile: Effect.Effect, RunLifecycleError>; readonly failRun: (input: { readonly runId: string; @@ -456,6 +461,33 @@ export const makeWithDependencies = (deps: RuntimeDependencies): RunLifecycleSha return yield* revokeTerminal(transition); }); + const handleOrchestrationEvent = Effect.fn("RunLifecycle.handleOrchestrationEvent")(function* ( + event: OrchestrationEvent, + ) { + if (event.type !== "thread.session-set" || event.payload.session.status !== "error") { + return undefined; + } + const message = nonEmptyMessage( + event.payload.session.lastError, + "The provider session failed to start.", + ); + const transition = yield* deps.persistence.transition({ + threadId: event.payload.threadId, + sourceEventId: event.eventId, + status: "failed", + actorKind: "agent", + occurredAt: event.occurredAt, + error: message, + failure: { + reason: "provider-session-error", + message, + retryable: true, + }, + allowedPreviousStates: ["queued", "running"], + }); + return yield* revokeTerminal(transition); + }); + const failRun: RunLifecycleShape["failRun"] = Effect.fn("RunLifecycle.failRun")( function* (input) { const transition = yield* deps.persistence.transition({ @@ -572,13 +604,14 @@ export const makeWithDependencies = (deps: RuntimeDependencies): RunLifecycleSha return transitions; }); - return RunLifecycle.of({ handleProviderEvent, reconcile, failRun }); + return RunLifecycle.of({ handleProviderEvent, handleOrchestrationEvent, reconcile, failRun }); }; const make = Effect.gen(function* () { const persistence = yield* makeRunLifecyclePersistence; const projection = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery; const provider = yield* ProviderService.ProviderService; + const orchestration = yield* OrchestrationEngine.OrchestrationEngineService; const service = makeWithDependencies({ persistence, getThread: (threadId) => @@ -604,6 +637,18 @@ const make = Effect.gen(function* () { ), ).pipe(Effect.forkScoped); + yield* Stream.runForEach(orchestration.streamDomainEvents, (event) => + service.handleOrchestrationEvent(event).pipe( + Effect.catch((error) => + Effect.logError("command-center.run-lifecycle.orchestration-event-failed", { + eventId: event.eventId, + threadId: event.aggregateKind === "thread" ? event.aggregateId : undefined, + reason: error.reason, + }), + ), + ), + ).pipe(Effect.forkScoped); + yield* service.reconcile.pipe( Effect.tap((transitions) => transitions.length === 0 diff --git a/apps/web/src/components/settings/ProviderInstanceCard.tsx b/apps/web/src/components/settings/ProviderInstanceCard.tsx index 738283c39d4..05ade23cb60 100644 --- a/apps/web/src/components/settings/ProviderInstanceCard.tsx +++ b/apps/web/src/components/settings/ProviderInstanceCard.tsx @@ -704,6 +704,32 @@ export function ProviderInstanceCard({ ) : null} + {!versionAdvisory && onRunUpdate && updateCommand ? ( + + + {isUpdating ? ( + + ) : ( + + )} + + } + /> + + {isUpdating ? "Updating provider" : "Update or reinstall provider"} + + + ) : null} {titleTailNode} {authRowNode} From e3c0bd4b87d9013136722329c0665a862d20e355 Mon Sep 17 00:00:00 2001 From: atryan Date: Sun, 2 Aug 2026 17:05:18 -0400 Subject: [PATCH 2/2] fix(server): provide orchestration engine to run lifecycle --- apps/server/src/server.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 51fa91a33bf..711b5074669 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -412,6 +412,7 @@ const AutomationRecoveryCoordinatorLayerLive = AutomationRecoveryCoordinator.lay const RunLifecycleLayerLive = RunLifecycle.layer.pipe( Layer.provide(ProviderLayerLive), Layer.provide(OrchestrationProjectionSnapshotQueryLive), + Layer.provide(OrchestrationLayerLive), Layer.provide(PersistenceLayerLive), );