Fix stale Command Center failure status and expose Codex repair

apps/server/src/command-center/RunLifecycle.test.ts

@@ -3,6 +3,9 @@ import * as NodeCrypto from "node:crypto";
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import { expect, it } from "@effect/vitest";
 import {
+  CommandId,
+  EventId,
+  type OrchestrationEvent,
   ProviderRuntimeEvent,
   type OrchestrationThreadShell,
   type ProviderSession,
@@ -116,6 +119,36 @@ const startedEvent = (input: { readonly eventId: string; readonly threadId: stri
     createdAt: fixtureTime,
   });
 
+const failedSessionEvent = (input: {
+  readonly eventId: string;
+  readonly threadId: string;
+  readonly errorMessage: string;
+}): OrchestrationEvent => ({
+  sequence: 1,
+  eventId: EventId.make(input.eventId),
+  type: "thread.session-set",
+  aggregateKind: "thread",
+  aggregateId: ThreadId.make(input.threadId),
+  occurredAt: fixtureTime,
+  commandId: CommandId.make(`command-${input.eventId}`),
+  causationEventId: null,
+  correlationId: null,
+  metadata: {},
+  payload: {
+    threadId: ThreadId.make(input.threadId),
+    session: {
+      threadId: ThreadId.make(input.threadId),
+      status: "error",
+      providerName: "codex",
+      providerInstanceId: ProviderInstanceId.make("codex"),
+      runtimeMode: "approval-required",
+      activeTurnId: null,
+      lastError: input.errorMessage,
+      updatedAt: fixtureTime,
+    },
+  },
+});
+
 it.effect("projects provider completion, preserves the audit chain, and revokes MCP scope", () =>
   Effect.gen(function* () {
     const sql = yield* SqlClient.SqlClient;
@@ -248,6 +281,46 @@ it.effect("records one actionable failure and one urgent Needs You alert under r
   }).pipe(Effect.provide(testLayer)),
 );
 
+it.effect(
+  "fails a running command-center run when provider startup sets the session to error",
+  () =>
+    Effect.gen(function* () {
+      const sql = yield* SqlClient.SqlClient;
+      const persistence = yield* makeRunLifecyclePersistence;
+      yield* insertRun(sql, { id: "run-startup-error", threadId: "thread-startup-error" });
+
+      const revoked: Array<string> = [];
+      const lifecycle = makeWithDependencies({
+        persistence,
+        getThread: () => Effect.sync((): OrchestrationThreadShell | undefined => undefined),
+        listProviderSessions: Effect.succeed([]),
+        revokeThread: (threadId) =>
+          Effect.sync(() => {
+            revoked.push(threadId);
+          }),
+      });
+      const event = failedSessionEvent({
+        eventId: "session-startup-error",
+        threadId: "thread-startup-error",
+        errorMessage: "The Windows sandbox could not be initialized.",
+      });
+
+      const first = yield* lifecycle.handleOrchestrationEvent(event);
+      const duplicate = yield* lifecycle.handleOrchestrationEvent(event);
+      const rows = yield* sql<{ readonly state: string; readonly error: string | null }>`
+      SELECT state, error FROM command_center_runs WHERE id = 'run-startup-error'
+    `;
+
+      expect(first?.status).toBe("failed");
+      expect(duplicate).toBeUndefined();
+      expect(rows[0]).toEqual({
+        state: "failed",
+        error: "The Windows sandbox could not be initialized.",
+      });
+      expect(revoked).toEqual(["thread-startup-error"]);
+    }).pipe(Effect.provide(testLayer)),
+);
+
 const threadShell = (input: {
   readonly threadId: string;
   readonly state: "running" | "interrupted" | "completed" | "error";

apps/server/src/command-center/RunLifecycle.ts

@@ -1,5 +1,6 @@
 import {
   COMMAND_CENTER_EVENT_ACTIONS,
+  type OrchestrationEvent,
   type OrchestrationThreadShell,
   type ProviderRuntimeEvent,
   type ProviderSession,
@@ -15,6 +16,7 @@ import * as Stream from "effect/Stream";
 import * as SqlClient from "effect/unstable/sql/SqlClient";
 
 import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
+import * as OrchestrationEngine from "../orchestration/Services/OrchestrationEngine.ts";
 import * as ProjectionSnapshotQuery from "../orchestration/Services/ProjectionSnapshotQuery.ts";
 import * as ProviderService from "../provider/Services/ProviderService.ts";
 import { makeCommandCenterAuditLog } from "./AuditLog.ts";
@@ -390,6 +392,9 @@ export interface RunLifecycleShape {
   readonly handleProviderEvent: (
     event: ProviderRuntimeEvent,
   ) => Effect.Effect<RunTransition | undefined, RunLifecycleError>;
+  readonly handleOrchestrationEvent: (
+    event: OrchestrationEvent,
+  ) => Effect.Effect<RunTransition | undefined, RunLifecycleError>;
   readonly reconcile: Effect.Effect<ReadonlyArray<RunTransition>, RunLifecycleError>;
   readonly failRun: (input: {
     readonly runId: string;
@@ -456,6 +461,33 @@ export const makeWithDependencies = (deps: RuntimeDependencies): RunLifecycleSha
     return yield* revokeTerminal(transition);
   });
 
+  const handleOrchestrationEvent = Effect.fn("RunLifecycle.handleOrchestrationEvent")(function* (
+    event: OrchestrationEvent,
+  ) {
+    if (event.type !== "thread.session-set" || event.payload.session.status !== "error") {
+      return undefined;
+    }
+    const message = nonEmptyMessage(
+      event.payload.session.lastError,
+      "The provider session failed to start.",
+    );
+    const transition = yield* deps.persistence.transition({
+      threadId: event.payload.threadId,
+      sourceEventId: event.eventId,
+      status: "failed",
+      actorKind: "agent",
+      occurredAt: event.occurredAt,
+      error: message,
+      failure: {
+        reason: "provider-session-error",
+        message,
+        retryable: true,
+      },
+      allowedPreviousStates: ["queued", "running"],
+    });
+    return yield* revokeTerminal(transition);
+  });
+
   const failRun: RunLifecycleShape["failRun"] = Effect.fn("RunLifecycle.failRun")(
     function* (input) {
       const transition = yield* deps.persistence.transition({
@@ -572,13 +604,14 @@ export const makeWithDependencies = (deps: RuntimeDependencies): RunLifecycleSha
     return transitions;
   });
 
-  return RunLifecycle.of({ handleProviderEvent, reconcile, failRun });
+  return RunLifecycle.of({ handleProviderEvent, handleOrchestrationEvent, reconcile, failRun });
 };
 
 const make = Effect.gen(function* () {
   const persistence = yield* makeRunLifecyclePersistence;
   const projection = yield* ProjectionSnapshotQuery.ProjectionSnapshotQuery;
   const provider = yield* ProviderService.ProviderService;
+  const orchestration = yield* OrchestrationEngine.OrchestrationEngineService;
   const service = makeWithDependencies({
     persistence,
     getThread: (threadId) =>
@@ -604,6 +637,18 @@ const make = Effect.gen(function* () {
     ),
   ).pipe(Effect.forkScoped);
 
+  yield* Stream.runForEach(orchestration.streamDomainEvents, (event) =>
+    service.handleOrchestrationEvent(event).pipe(
+      Effect.catch((error) =>
+        Effect.logError("command-center.run-lifecycle.orchestration-event-failed", {
+          eventId: event.eventId,
+          threadId: event.aggregateKind === "thread" ? event.aggregateId : undefined,
+          reason: error.reason,
+        }),
+      ),
+    ),
+  ).pipe(Effect.forkScoped);
+
   yield* service.reconcile.pipe(
     Effect.tap((transitions) =>
       transitions.length === 0

apps/server/src/server.ts

@@ -412,6 +412,7 @@ const AutomationRecoveryCoordinatorLayerLive = AutomationRecoveryCoordinator.lay
 const RunLifecycleLayerLive = RunLifecycle.layer.pipe(
   Layer.provide(ProviderLayerLive),
   Layer.provide(OrchestrationProjectionSnapshotQueryLive),
+  Layer.provide(OrchestrationLayerLive),
   Layer.provide(PersistenceLayerLive),
 );
 

apps/web/src/components/settings/ProviderInstanceCard.tsx

@@ -704,6 +704,32 @@ export function ProviderInstanceCard({
                   </PopoverPopup>
                 </Popover>
               ) : null}
+              {!versionAdvisory && onRunUpdate && updateCommand ? (
+                <Tooltip>
+                  <TooltipTrigger
+                    render={
+                      <Button
+                        type="button"
+                        size="icon-xs"
+                        variant="ghost"
+                        className="size-5 rounded-sm p-0 text-muted-foreground hover:text-foreground"
+                        disabled={isUpdating}
+                        onClick={onRunUpdate}
+                        aria-label={`Update or reinstall ${displayName}`}
+                      >
+                        {isUpdating ? (
+                          <LoaderIcon className="size-3.5 animate-spin" />
+                        ) : (
+                          <DownloadIcon className="size-3.5" />
+                        )}
+                      </Button>
+                    }
+                  />
+                  <TooltipPopup side="top">
+                    {isUpdating ? "Updating provider" : "Update or reinstall provider"}
+                  </TooltipPopup>
+                </Tooltip>
+              ) : null}
               {titleTailNode}
             </div>
             {authRowNode}