Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions src/daemon.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,8 @@ describe("daemon supervision", () => {
expect(livePids.has(first.pid)).toBe(true);
});

it("waits for an unhealthy project service to exit before spawning a replacement", async () => {
vi.mocked(requestJson).mockRejectedValueOnce(new Error("health failed"));
it("waits for a repeatedly-unhealthy project service to exit before spawning a replacement", async () => {
vi.mocked(requestJson).mockRejectedValue(new Error("health failed"));
const { AimuxDaemon } = await import("./daemon.js");

const daemon = new AimuxDaemon();
Expand All @@ -164,6 +164,12 @@ describe("daemon supervision", () => {
return true;
}) as typeof process.kill);

// Transient misses below the threshold are tolerated: the same service stays.
expect((await (daemon as any).ensureProject(projectRoot)).pid).toBe(first.pid);
expect((await (daemon as any).ensureProject(projectRoot)).pid).toBe(first.pid);
expect(spawnMock).toHaveBeenCalledTimes(1);

// The threshold-crossing third failure replaces it, waiting for the old exit.
const replacementPromise = (daemon as any).ensureProject(projectRoot);
await new Promise((resolve) => setTimeout(resolve, 5));
expect(spawnMock).toHaveBeenCalledTimes(1);
Expand All @@ -174,9 +180,7 @@ describe("daemon supervision", () => {
});

it("serializes concurrent unhealthy project ensures into one replacement spawn", async () => {
vi.mocked(requestJson)
.mockRejectedValueOnce(new Error("health failed"))
.mockRejectedValueOnce(new Error("health failed"));
vi.mocked(requestJson).mockRejectedValue(new Error("health failed"));
const { AimuxDaemon } = await import("./daemon.js");

const daemon = new AimuxDaemon();
Expand All @@ -185,6 +189,12 @@ describe("daemon supervision", () => {
writeMetadataEndpointFor(first.pid);
(daemon as any).state.projects[first.projectId].startedAt = new Date(Date.now() - 60_000).toISOString();

// Two consecutive misses keep the failure counter just below the threshold.
expect((await (daemon as any).ensureProject(projectRoot)).pid).toBe(first.pid);
expect((await (daemon as any).ensureProject(projectRoot)).pid).toBe(first.pid);

// Two concurrent ensures dedupe into one threshold-crossing health check,
// producing a single replacement spawn shared by both callers.
const [second, third] = await Promise.all([
(daemon as any).ensureProject(projectRoot),
(daemon as any).ensureProject(projectRoot),
Expand Down
24 changes: 23 additions & 1 deletion src/daemon.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ const DEFAULT_DAEMON_PORT = 43190;
const DEFAULT_DAEMON_HOST = "127.0.0.1";
const DAEMON_STARTUP_TIMEOUT_MS = 10_000;
const PROJECT_SERVICE_STARTUP_GRACE_MS = 15_000;
const PROJECT_SERVICE_HEALTH_TIMEOUT_MS = 2_500;
// A busy event loop can miss a health ping; only restart after this many
// consecutive failures so transient stalls don't churn the service.
const PROJECT_SERVICE_HEALTH_FAILURE_THRESHOLD = 3;
const PROJECT_SERVICE_TERM_GRACE_MS = 2_000;
const PROJECT_SERVICE_KILL_GRACE_MS = 3_000;
const PROJECT_SERVICE_EXIT_POLL_MS = 50;
Expand Down Expand Up @@ -367,6 +371,9 @@ export class AimuxDaemon {
private readonly pushThrottle = new MobilePushThrottle();
private readonly children = new Map<string, ChildProcess>();
private readonly projectEnsurePromises = new Map<string, Promise<ProjectServiceState>>();
// Consecutive failed health checks per project; a single transient stall
// (event loop briefly busy) must not trigger a restart.
private readonly projectHealthFailures = new Map<string, number>();
private state: DaemonState = loadDaemonState();

async start(): Promise<void> {
Expand Down Expand Up @@ -482,6 +489,9 @@ export class AimuxDaemon {
}

private spawnProjectService(projectRoot: string, projectId: string): ProjectServiceState {
// A fresh service instance starts with a clean health-failure slate, so a
// new pid never inherits the previous instance's accumulated failure debt.
this.projectHealthFailures.delete(projectId);
const stdio = loggingChildStdio(getProjectServiceStdioLogPathFor(projectRoot));
let child: ChildProcess;
try {
Expand Down Expand Up @@ -575,7 +585,7 @@ export class AimuxDaemon {
}
try {
const { status, json } = await requestJson(`http://${endpoint.host}:${endpoint.port}/health`, {
timeoutMs: 1000,
timeoutMs: PROJECT_SERVICE_HEALTH_TIMEOUT_MS,
});
if (status < 200 || status >= 300 || json?.ok === false) {
throw new Error(json?.error || `health request failed: ${status}`);
Expand All @@ -584,14 +594,25 @@ export class AimuxDaemon {
if (withinStartupGrace) {
return refreshExisting();
}
const failures = (this.projectHealthFailures.get(projectId) ?? 0) + 1;
this.projectHealthFailures.set(projectId, failures);
log.warn("project service health check failed", "daemon", {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
projectId,
projectRoot: resolvedRoot,
pid: existing.pid,
error: error instanceof Error ? error.message : String(error),
consecutiveFailures: failures,
threshold: PROJECT_SERVICE_HEALTH_FAILURE_THRESHOLD,
});
// Tolerate transient stalls; only restart after sustained failures.
if (failures < PROJECT_SERVICE_HEALTH_FAILURE_THRESHOLD) {
return refreshExisting();
}
// The counter is reset by spawnProjectService once a replacement
// actually starts; if termination fails we keep the debt and retry.
return this.replaceProjectServiceAfterExit(resolvedRoot, projectId, existing, refreshExisting);
}
this.projectHealthFailures.delete(projectId);
return refreshExisting();
}
return this.spawnProjectService(resolvedRoot, projectId);
Expand Down Expand Up @@ -686,6 +707,7 @@ export class AimuxDaemon {
process.kill(existing.pid, "SIGTERM");
} catch {}
delete this.state.projects[projectId];
this.projectHealthFailures.delete(projectId);
if (this.children.get(projectId)?.pid === existing.pid) {
this.children.delete(projectId);
}
Expand Down
27 changes: 20 additions & 7 deletions src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -403,14 +403,17 @@ function exitAfterOpen(): never {
async function resolvePermissionRequestOutput(
projectRoot: string,
sessionId: string,
payload: { tool_name?: string; tool_input?: Record<string, unknown> },
payload: { tool_name?: string; tool_input?: Record<string, unknown>; cwd?: string },
): Promise<Record<string, unknown>> {
try {
const { toolName, input, summary } = summarizeClaudePermissionRequest(payload);
// The hook runs in the agent's working dir, which is the worktree (or the
// project root if no worktree). Carry it so clients can show project/worktree.
const cwd = (typeof payload.cwd === "string" && payload.cwd) || process.cwd();
const result = await postLiveProjectServiceJsonOrLocal(
projectRoot,
"/agents/interaction/request",
{ session: sessionId, type: "permission", payload: { toolName, input }, summary, timeoutMs: 115_000 },
{ session: sessionId, type: "permission", payload: { toolName, input, cwd }, summary, timeoutMs: 115_000 },
() => ({}),
);
if (result?.request?.status === "resolved") {
Expand Down Expand Up @@ -3506,11 +3509,6 @@ program
const payload = parseCodexHookPayload(rawInput);
const sessionId = opts.session.trim();

if (action === "permission-request") {
console.log(JSON.stringify(await resolvePermissionRequestOutput(projectRoot, sessionId, payload)));
return;
}

const result: Record<string, unknown> = { ok: true, action, sessionId };
const setActivity = async (activity: AgentActivityState) =>
postLiveProjectServiceJsonOrLocal(projectRoot, "/set-activity", { session: sessionId, activity }, () =>
Expand Down Expand Up @@ -3558,6 +3556,21 @@ program
case "stop":
await emitEvent("task_done", payload.message?.trim() || "Codex completed its turn.", "success");
break;
case "permission-request": {
// Read-only telemetry — never block. Codex's native TUI prompt stays the
// primary decision surface; we post a non-actionable Feed notice (which
// also flags attention). Falls through to `console.log({})` → native prompt.
const { toolName, input, summary } = summarizeClaudePermissionRequest(payload);
// Best-effort: a telemetry transport failure must never break the hook —
// it always falls through to `console.log({})` and the native prompt.
await postLiveProjectServiceJsonOrLocal(
projectRoot,
"/agents/interaction/notify",
{ session: sessionId, summary, payload: { toolName, input, cwd: process.cwd() } },
() => ({}),
).catch(() => undefined);
break;
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
default:
throw new Error(`Unsupported codex hook action: ${action}`);
}
Expand Down
49 changes: 48 additions & 1 deletion src/metadata-server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -848,7 +848,14 @@ export class MetadataServer {
dedupeKey?: string;
cooldownMs?: number;
forceNotify?: boolean;
interaction?: { id: string; type: InteractionType; summary?: string };
interaction?: {
id: string;
type: InteractionType;
summary?: string;
telemetry?: boolean;
toolName?: string;
toolInputJSON?: string;
};
}): void {
const displayContext = this.resolveSessionAlertDisplayContext(input.sessionId, input.worktreePath);
this.eventBus.publishAlert(contextualizeAlertInput(input, displayContext));
Expand Down Expand Up @@ -1978,6 +1985,46 @@ export class MetadataServer {
return;
}

if (req.method === "POST" && url.pathname === "/agents/interaction/notify") {
// Read-only telemetry (e.g. Codex, whose native TUI owns the decision):
// emit a non-actionable interaction alert and flag attention, but never
// register a blocking interaction. Returns immediately.
const body = (await readJson(req).catch(() => null)) as {
session?: string;
summary?: string;
payload?: { toolName?: string; input?: Record<string, unknown>; cwd?: string };
} | null;
const sessionId = body?.session?.trim();
if (!body || !sessionId) {
send(res, 400, { ok: false, error: "session is required" });
return;
}
const toolName = body.payload?.toolName;
const rawInput = body.payload?.input;
const input = rawInput && typeof rawInput === "object" && !Array.isArray(rawInput) ? rawInput : {};
const cwd = typeof body.payload?.cwd === "string" ? body.payload.cwd : undefined;
const summary = body.summary?.trim() || undefined;
this.tracker.setAttention(sessionId, "needs_input");
this.emitAlert({
kind: "interaction_request",
sessionId,
title: `${sessionId} needs a response`,
message: summary ?? "Agent is waiting on a permission response.",
worktreePath: cwd,
interaction: {
id: randomUUID(),
type: "permission",
summary,
telemetry: true,
toolName,
toolInputJSON: JSON.stringify(input),
},
});
this.options.onChange?.();
send(res, 200, { ok: true, telemetry: true });
return;
}

if (req.method === "POST" && url.pathname === "/agents/interaction/request") {
const body = (await readJson(req)) as {
session?: string;
Expand Down
13 changes: 11 additions & 2 deletions src/project-events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,17 @@ export interface AlertEvent {
worktreePath?: string;
dedupeKey?: string;
forceNotify?: boolean;
/** Present on actionable interaction_request alerts so clients can resolve them. */
interaction?: { id: string; type: InteractionType; summary?: string };
/** Present on actionable interaction_request alerts so clients can resolve them.
* `telemetry: true` marks a read-only notice (e.g. Codex, whose native TUI owns
* the decision) — clients render it as a non-actionable Feed row. */
interaction?: {
id: string;
type: InteractionType;
summary?: string;
telemetry?: boolean;
toolName?: string;
toolInputJSON?: string;
};
}

export type ProjectStreamEvent = AlertEvent;
Expand Down