From 0d944623fa8090cc606cb3f2e7b388f133b0e11e Mon Sep 17 00:00:00 2001 From: test Date: Tue, 9 Jun 2026 22:15:54 +0800 Subject: [PATCH 1/6] fix(daemon): tolerate transient project-service health misses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daemon health-checked each project service with a 1000ms timeout and hard-restarted it (SIGTERM/respawn) on a single failure. Under load a busy event loop can miss one ping, so healthy services were being killed in a restart loop — dropping SSE streams and breaking in-flight interaction hooks (observed as repeated 'health check failed: request timed out after 1000ms' followed by terminate/respawn). Raise the timeout to 2500ms and only restart after 3 consecutive failures (counter resets on any success), so a transient stall no longer churns the service. Co-Authored-By: Claude Opus 4.8 --- src/daemon.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/daemon.ts b/src/daemon.ts index e793c03f..09be6a66 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -23,6 +23,10 @@ const DEFAULT_DAEMON_PORT = 43190; const DEFAULT_DAEMON_HOST = "127.0.0.1"; const DAEMON_STARTUP_TIMEOUT_MS = 10_000; const PROJECT_SERVICE_STARTUP_GRACE_MS = 15_000; +const PROJECT_SERVICE_HEALTH_TIMEOUT_MS = 2_500; +// A busy event loop can miss a health ping; only restart after this many +// consecutive failures so transient stalls don't churn the service. +const PROJECT_SERVICE_HEALTH_FAILURE_THRESHOLD = 3; const PROJECT_SERVICE_TERM_GRACE_MS = 2_000; const PROJECT_SERVICE_KILL_GRACE_MS = 3_000; const PROJECT_SERVICE_EXIT_POLL_MS = 50; @@ -367,6 +371,9 @@ export class AimuxDaemon { private readonly pushThrottle = new MobilePushThrottle(); private readonly children = new Map(); private readonly projectEnsurePromises = new Map>(); + // Consecutive failed health checks per project; a single transient stall + // (event loop briefly busy) must not trigger a restart. + private readonly projectHealthFailures = new Map(); private state: DaemonState = loadDaemonState(); async start(): Promise { @@ -575,7 +582,7 @@ export class AimuxDaemon { } try { const { status, json } = await requestJson(`http://${endpoint.host}:${endpoint.port}/health`, { - timeoutMs: 1000, + timeoutMs: PROJECT_SERVICE_HEALTH_TIMEOUT_MS, }); if (status < 200 || status >= 300 || json?.ok === false) { throw new Error(json?.error || `health request failed: ${status}`); @@ -584,14 +591,24 @@ export class AimuxDaemon { if (withinStartupGrace) { return refreshExisting(); } + const failures = (this.projectHealthFailures.get(projectId) ?? 0) + 1; + this.projectHealthFailures.set(projectId, failures); log.warn("project service health check failed", "daemon", { projectId, projectRoot: resolvedRoot, pid: existing.pid, error: error instanceof Error ? error.message : String(error), + consecutiveFailures: failures, + threshold: PROJECT_SERVICE_HEALTH_FAILURE_THRESHOLD, }); + // Tolerate transient stalls; only restart after sustained failures. + if (failures < PROJECT_SERVICE_HEALTH_FAILURE_THRESHOLD) { + return refreshExisting(); + } + this.projectHealthFailures.delete(projectId); return this.replaceProjectServiceAfterExit(resolvedRoot, projectId, existing, refreshExisting); } + this.projectHealthFailures.delete(projectId); return refreshExisting(); } return this.spawnProjectService(resolvedRoot, projectId); From f60d61cfa79c593d59b378d85de16be3f7e54b88 Mon Sep 17 00:00:00 2001 From: test Date: Tue, 9 Jun 2026 22:49:10 +0800 Subject: [PATCH 2/6] fix(codex): make permission hook telemetry-only; keep native TUI primary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex's PermissionRequest hook used the same blocking long-poll as Claude (resolvePermissionRequestOutput, 115s). Unlike Claude — which still renders its native approval prompt while the hook blocks — Codex suppresses its native prompt and sits at 'Working / Running PermissionRequest hook' until the hook returns, so the only way to decide was a remote Feed card. That makes the TUI non-primary, which is unacceptable. Mirror cmux's codex behavior (PermissionRequest = telemetry): the codex hook now emits a needs_input event + sets attention (so the dashboard/Feed still shows codex needs you) and returns {} immediately, deferring to codex's native prompt. Claude's blocking/actionable path is unchanged. Co-Authored-By: Claude Opus 4.8 --- src/main.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main.ts b/src/main.ts index 537aa4b6..dbce9f74 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3506,11 +3506,6 @@ program const payload = parseCodexHookPayload(rawInput); const sessionId = opts.session.trim(); - if (action === "permission-request") { - console.log(JSON.stringify(await resolvePermissionRequestOutput(projectRoot, sessionId, payload))); - return; - } - const result: Record = { ok: true, action, sessionId }; const setActivity = async (activity: AgentActivityState) => postLiveProjectServiceJsonOrLocal(projectRoot, "/set-activity", { session: sessionId, activity }, () => @@ -3558,6 +3553,16 @@ program case "stop": await emitEvent("task_done", payload.message?.trim() || "Codex completed its turn.", "success"); break; + case "permission-request": { + // Telemetry only — never block. Codex's native TUI prompt stays the + // primary decision surface (mirrors cmux's codex behavior); we only + // surface that it needs attention. Falls through to `console.log({})`, + // which defers to the native prompt. + const { summary } = summarizeClaudePermissionRequest(payload); + await setAttention("needs_input"); + await emitEvent("needs_input", summary, "warn"); + break; + } default: throw new Error(`Unsupported codex hook action: ${action}`); } From b9239487c9a30cac958f06d2e7f58725f140945a Mon Sep 17 00:00:00 2001 From: test Date: Tue, 9 Jun 2026 23:20:42 +0800 Subject: [PATCH 3/6] feat(interactions): carry agent worktree cwd in permission payload So clients can show which project/worktree a permission prompt is from, include the hook's working dir (the worktree, or project root if none) in the interaction payload. cmux renders it as project/worktree in the Feed card header. Co-Authored-By: Claude Opus 4.8 --- src/main.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main.ts b/src/main.ts index dbce9f74..ef3be6da 100644 --- a/src/main.ts +++ b/src/main.ts @@ -403,14 +403,17 @@ function exitAfterOpen(): never { async function resolvePermissionRequestOutput( projectRoot: string, sessionId: string, - payload: { tool_name?: string; tool_input?: Record }, + payload: { tool_name?: string; tool_input?: Record; cwd?: string }, ): Promise> { try { const { toolName, input, summary } = summarizeClaudePermissionRequest(payload); + // The hook runs in the agent's working dir, which is the worktree (or the + // project root if no worktree). Carry it so clients can show project/worktree. + const cwd = (typeof payload.cwd === "string" && payload.cwd) || process.cwd(); const result = await postLiveProjectServiceJsonOrLocal( projectRoot, "/agents/interaction/request", - { session: sessionId, type: "permission", payload: { toolName, input }, summary, timeoutMs: 115_000 }, + { session: sessionId, type: "permission", payload: { toolName, input, cwd }, summary, timeoutMs: 115_000 }, () => ({}), ); if (result?.request?.status === "resolved") { From 5371d1320c72dc0624977e06cf1dbb8642183065 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 10 Jun 2026 08:51:24 +0800 Subject: [PATCH 4/6] feat(codex): surface permission prompts as read-only Feed telemetry Codex's native TUI owns the decision, but the prompt was invisible in clients. Add a non-blocking /agents/interaction/notify endpoint that emits a telemetry interaction alert (interaction.telemetry=true, with toolName/toolInputJSON and the worktree cwd) and flags attention, without registering a blocking interaction. The codex permission hook posts to it, then defers to the native prompt. cmux renders telemetry alerts as a non-actionable read-only row. Co-Authored-By: Claude Opus 4.8 --- src/main.ts | 17 +++++++++------ src/metadata-server.ts | 48 +++++++++++++++++++++++++++++++++++++++++- src/project-events.ts | 13 ++++++++++-- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/main.ts b/src/main.ts index ef3be6da..81d5a134 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3557,13 +3557,16 @@ program await emitEvent("task_done", payload.message?.trim() || "Codex completed its turn.", "success"); break; case "permission-request": { - // Telemetry only — never block. Codex's native TUI prompt stays the - // primary decision surface (mirrors cmux's codex behavior); we only - // surface that it needs attention. Falls through to `console.log({})`, - // which defers to the native prompt. - const { summary } = summarizeClaudePermissionRequest(payload); - await setAttention("needs_input"); - await emitEvent("needs_input", summary, "warn"); + // Read-only telemetry — never block. Codex's native TUI prompt stays the + // primary decision surface; we post a non-actionable Feed notice (which + // also flags attention). Falls through to `console.log({})` → native prompt. + const { toolName, input, summary } = summarizeClaudePermissionRequest(payload); + await postLiveProjectServiceJsonOrLocal( + projectRoot, + "/agents/interaction/notify", + { session: sessionId, summary, payload: { toolName, input, cwd: process.cwd() } }, + () => ({}), + ); break; } default: diff --git a/src/metadata-server.ts b/src/metadata-server.ts index 7c453d2d..6bc22958 100644 --- a/src/metadata-server.ts +++ b/src/metadata-server.ts @@ -848,7 +848,14 @@ export class MetadataServer { dedupeKey?: string; cooldownMs?: number; forceNotify?: boolean; - interaction?: { id: string; type: InteractionType; summary?: string }; + interaction?: { + id: string; + type: InteractionType; + summary?: string; + telemetry?: boolean; + toolName?: string; + toolInputJSON?: string; + }; }): void { const displayContext = this.resolveSessionAlertDisplayContext(input.sessionId, input.worktreePath); this.eventBus.publishAlert(contextualizeAlertInput(input, displayContext)); @@ -1978,6 +1985,45 @@ export class MetadataServer { return; } + if (req.method === "POST" && url.pathname === "/agents/interaction/notify") { + // Read-only telemetry (e.g. Codex, whose native TUI owns the decision): + // emit a non-actionable interaction alert and flag attention, but never + // register a blocking interaction. Returns immediately. + const body = (await readJson(req)) as { + session?: string; + summary?: string; + payload?: { toolName?: string; input?: Record; cwd?: string }; + }; + const sessionId = body.session?.trim(); + if (!sessionId) { + send(res, 400, { ok: false, error: "session is required" }); + return; + } + const toolName = body.payload?.toolName; + const input = body.payload?.input ?? {}; + const cwd = typeof body.payload?.cwd === "string" ? body.payload.cwd : undefined; + const summary = body.summary?.trim() || undefined; + this.tracker.setAttention(sessionId, "needs_input"); + this.emitAlert({ + kind: "interaction_request", + sessionId, + title: `${sessionId} needs a response`, + message: summary ?? "Agent is waiting on a permission response.", + worktreePath: cwd, + interaction: { + id: randomUUID(), + type: "permission", + summary, + telemetry: true, + toolName, + toolInputJSON: JSON.stringify(input), + }, + }); + this.options.onChange?.(); + send(res, 200, { ok: true, telemetry: true }); + return; + } + if (req.method === "POST" && url.pathname === "/agents/interaction/request") { const body = (await readJson(req)) as { session?: string; diff --git a/src/project-events.ts b/src/project-events.ts index 0c072dbf..a52dca0a 100644 --- a/src/project-events.ts +++ b/src/project-events.ts @@ -28,8 +28,17 @@ export interface AlertEvent { worktreePath?: string; dedupeKey?: string; forceNotify?: boolean; - /** Present on actionable interaction_request alerts so clients can resolve them. */ - interaction?: { id: string; type: InteractionType; summary?: string }; + /** Present on actionable interaction_request alerts so clients can resolve them. + * `telemetry: true` marks a read-only notice (e.g. Codex, whose native TUI owns + * the decision) — clients render it as a non-actionable Feed row. */ + interaction?: { + id: string; + type: InteractionType; + summary?: string; + telemetry?: boolean; + toolName?: string; + toolInputJSON?: string; + }; } export type ProjectStreamEvent = AlertEvent; From 2a8ed2d9affe43143ced1e1f4e35bdc8b761b107 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 10 Jun 2026 13:19:45 +0800 Subject: [PATCH 5/6] test(daemon): update health-check tests for the consecutive-failure threshold The health-tolerance fix replaces a service only after 3 consecutive failed health checks (not the first). Update the two replacement tests to tolerate two misses and assert replacement on the threshold-crossing third. Co-Authored-By: Claude Opus 4.8 --- src/daemon.test.ts | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/daemon.test.ts b/src/daemon.test.ts index 25216587..e0d3eaee 100644 --- a/src/daemon.test.ts +++ b/src/daemon.test.ts @@ -142,8 +142,8 @@ describe("daemon supervision", () => { expect(livePids.has(first.pid)).toBe(true); }); - it("waits for an unhealthy project service to exit before spawning a replacement", async () => { - vi.mocked(requestJson).mockRejectedValueOnce(new Error("health failed")); + it("waits for a repeatedly-unhealthy project service to exit before spawning a replacement", async () => { + vi.mocked(requestJson).mockRejectedValue(new Error("health failed")); const { AimuxDaemon } = await import("./daemon.js"); const daemon = new AimuxDaemon(); @@ -164,6 +164,12 @@ describe("daemon supervision", () => { return true; }) as typeof process.kill); + // Transient misses below the threshold are tolerated: the same service stays. + expect((await (daemon as any).ensureProject(projectRoot)).pid).toBe(first.pid); + expect((await (daemon as any).ensureProject(projectRoot)).pid).toBe(first.pid); + expect(spawnMock).toHaveBeenCalledTimes(1); + + // The threshold-crossing third failure replaces it, waiting for the old exit. const replacementPromise = (daemon as any).ensureProject(projectRoot); await new Promise((resolve) => setTimeout(resolve, 5)); expect(spawnMock).toHaveBeenCalledTimes(1); @@ -174,9 +180,7 @@ describe("daemon supervision", () => { }); it("serializes concurrent unhealthy project ensures into one replacement spawn", async () => { - vi.mocked(requestJson) - .mockRejectedValueOnce(new Error("health failed")) - .mockRejectedValueOnce(new Error("health failed")); + vi.mocked(requestJson).mockRejectedValue(new Error("health failed")); const { AimuxDaemon } = await import("./daemon.js"); const daemon = new AimuxDaemon(); @@ -185,6 +189,12 @@ describe("daemon supervision", () => { writeMetadataEndpointFor(first.pid); (daemon as any).state.projects[first.projectId].startedAt = new Date(Date.now() - 60_000).toISOString(); + // Two consecutive misses keep the failure counter just below the threshold. + expect((await (daemon as any).ensureProject(projectRoot)).pid).toBe(first.pid); + expect((await (daemon as any).ensureProject(projectRoot)).pid).toBe(first.pid); + + // Two concurrent ensures dedupe into one threshold-crossing health check, + // producing a single replacement spawn shared by both callers. const [second, third] = await Promise.all([ (daemon as any).ensureProject(projectRoot), (daemon as any).ensureProject(projectRoot), From 4fea99c01280e53f27a8f47e4e44054bebd24551 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 10 Jun 2026 13:29:02 +0800 Subject: [PATCH 6/6] fix(review): address CodeRabbit + review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - daemon: reset projectHealthFailures in spawnProjectService so a fresh service instance never inherits the previous pid's failure debt (covers all spawn/ replace paths); also clear on stopProject. Keep the debt if termination fails. - codex hook: make the /notify telemetry post best-effort (.catch) so a transport failure never breaks the hook — it always defers to the native prompt. - /notify endpoint: tolerate malformed JSON (readJson catch) and reject/ignore a non-object payload.input, matching the /request handler's validation. Co-Authored-By: Claude Opus 4.8 --- src/daemon.ts | 7 ++++++- src/main.ts | 4 +++- src/metadata-server.ts | 11 ++++++----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/daemon.ts b/src/daemon.ts index 09be6a66..e2913270 100644 --- a/src/daemon.ts +++ b/src/daemon.ts @@ -489,6 +489,9 @@ export class AimuxDaemon { } private spawnProjectService(projectRoot: string, projectId: string): ProjectServiceState { + // A fresh service instance starts with a clean health-failure slate, so a + // new pid never inherits the previous instance's accumulated failure debt. + this.projectHealthFailures.delete(projectId); const stdio = loggingChildStdio(getProjectServiceStdioLogPathFor(projectRoot)); let child: ChildProcess; try { @@ -605,7 +608,8 @@ export class AimuxDaemon { if (failures < PROJECT_SERVICE_HEALTH_FAILURE_THRESHOLD) { return refreshExisting(); } - this.projectHealthFailures.delete(projectId); + // The counter is reset by spawnProjectService once a replacement + // actually starts; if termination fails we keep the debt and retry. return this.replaceProjectServiceAfterExit(resolvedRoot, projectId, existing, refreshExisting); } this.projectHealthFailures.delete(projectId); @@ -703,6 +707,7 @@ export class AimuxDaemon { process.kill(existing.pid, "SIGTERM"); } catch {} delete this.state.projects[projectId]; + this.projectHealthFailures.delete(projectId); if (this.children.get(projectId)?.pid === existing.pid) { this.children.delete(projectId); } diff --git a/src/main.ts b/src/main.ts index 81d5a134..dcfeb733 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3561,12 +3561,14 @@ program // primary decision surface; we post a non-actionable Feed notice (which // also flags attention). Falls through to `console.log({})` → native prompt. const { toolName, input, summary } = summarizeClaudePermissionRequest(payload); + // Best-effort: a telemetry transport failure must never break the hook — + // it always falls through to `console.log({})` and the native prompt. await postLiveProjectServiceJsonOrLocal( projectRoot, "/agents/interaction/notify", { session: sessionId, summary, payload: { toolName, input, cwd: process.cwd() } }, () => ({}), - ); + ).catch(() => undefined); break; } default: diff --git a/src/metadata-server.ts b/src/metadata-server.ts index 6bc22958..2089d0e1 100644 --- a/src/metadata-server.ts +++ b/src/metadata-server.ts @@ -1989,18 +1989,19 @@ export class MetadataServer { // Read-only telemetry (e.g. Codex, whose native TUI owns the decision): // emit a non-actionable interaction alert and flag attention, but never // register a blocking interaction. Returns immediately. - const body = (await readJson(req)) as { + const body = (await readJson(req).catch(() => null)) as { session?: string; summary?: string; payload?: { toolName?: string; input?: Record; cwd?: string }; - }; - const sessionId = body.session?.trim(); - if (!sessionId) { + } | null; + const sessionId = body?.session?.trim(); + if (!body || !sessionId) { send(res, 400, { ok: false, error: "session is required" }); return; } const toolName = body.payload?.toolName; - const input = body.payload?.input ?? {}; + const rawInput = body.payload?.input; + const input = rawInput && typeof rawInput === "object" && !Array.isArray(rawInput) ? rawInput : {}; const cwd = typeof body.payload?.cwd === "string" ? body.payload.cwd : undefined; const summary = body.summary?.trim() || undefined; this.tracker.setAttention(sessionId, "needs_input");