From 2b18101beee7e02a34b47453386bb7b68a0937b6 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Thu, 16 Jul 2026 20:28:59 +0530 Subject: [PATCH 01/25] feat(workflows): add durable workflow schema Co-Authored-By: Claude --- src/db/migrations.ts | 90 +++++++++++++++++++++++ src/db/schema.ts | 114 ++++++++++++++++++++++++++++- src/workflows/types.ts | 159 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 src/workflows/types.ts diff --git a/src/db/migrations.ts b/src/db/migrations.ts index 2bda20c2..9fe3956c 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -22,6 +22,11 @@ const migrations: Migration[] = [ name: "local-agent-sessions", up: migrateLocalAgentSessions, }, + { + version: 4, + name: "durable-workflows", + up: migrateDurableWorkflows, + }, ]; export function migrateDatabase(sqlite: Database.Database): void { @@ -174,6 +179,91 @@ function migrateLocalAgentSessions(sqlite: Database.Database): void { addColumnIfMissing(sqlite, "local_agent_sessions", "thinking", "text"); } +function migrateDurableWorkflows(sqlite: Database.Database): void { + sqlite.exec(` + create table if not exists workflow_runs ( + id text primary key, + definition_version integer not null, + status text not null, + definition_json text not null, + input_json text not null, + policy_json text not null, + idempotency_key text unique, + request_hash text not null, + result_json text, + error_json text, + cancellation_requested_at text, + event_sequence integer not null default 0, + created_at text not null, + updated_at text not null, + started_at text, + completed_at text + ); + + create index if not exists workflow_runs_status_idx + on workflow_runs(status, created_at); + + create table if not exists workflow_nodes ( + id text primary key, + workflow_run_id text not null, + node_key text not null, + node_type text not null, + status text not null, + definition_json text not null, + attempt integer not null default 0, + claim_token text, + claimed_at text, + claim_expires_at text, + result_json text, + error_json text, + created_at text not null, + updated_at text not null, + completed_at text, + foreign key (workflow_run_id) references workflow_runs(id) on delete cascade + ); + + create unique index if not exists workflow_nodes_run_key_idx + on workflow_nodes(workflow_run_id, node_key); + + create unique index if not exists workflow_nodes_run_id_idx + on workflow_nodes(workflow_run_id, id); + + create index if not exists workflow_nodes_status_idx + on workflow_nodes(workflow_run_id, status, created_at); + + create table if not exists workflow_edges ( + workflow_run_id text not null, + from_node_id text not null, + to_node_id text not null, + primary key (workflow_run_id, from_node_id, to_node_id), + foreign key (workflow_run_id) references workflow_runs(id) on delete cascade, + foreign key (workflow_run_id, from_node_id) + references workflow_nodes(workflow_run_id, id) on delete cascade, + foreign key (workflow_run_id, to_node_id) + references workflow_nodes(workflow_run_id, id) on delete cascade + ); + + create index if not exists workflow_edges_to_node_idx + on workflow_edges(workflow_run_id, to_node_id); + + create table if not exists workflow_events ( + workflow_run_id text not null, + sequence integer not null, + event_type text not null, + node_id text, + payload_json text not null, + created_at text not null, + primary key (workflow_run_id, sequence), + foreign key (workflow_run_id) references workflow_runs(id) on delete cascade, + foreign key (workflow_run_id, node_id) + references workflow_nodes(workflow_run_id, id) + ); + + create index if not exists workflow_events_cursor_idx + on workflow_events(workflow_run_id, sequence); + `); +} + function addColumnIfMissing( sqlite: Database.Database, table: "workspace_sessions" | "local_agent_sessions", diff --git a/src/db/schema.ts b/src/db/schema.ts index 01d13fac..631e6d8c 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -1,4 +1,12 @@ -import { index, integer, primaryKey, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { + foreignKey, + index, + integer, + primaryKey, + sqliteTable, + text, + uniqueIndex, +} from "drizzle-orm/sqlite-core"; export const workspaceSessions = sqliteTable( "workspace_sessions", @@ -97,9 +105,113 @@ export const localAgentSessions = sqliteTable( ], ); +export const workflowRuns = sqliteTable( + "workflow_runs", + { + id: text("id").primaryKey(), + definitionVersion: integer("definition_version").notNull(), + status: text("status").notNull(), + definitionJson: text("definition_json").notNull(), + inputJson: text("input_json").notNull(), + policyJson: text("policy_json").notNull(), + idempotencyKey: text("idempotency_key").unique(), + requestHash: text("request_hash").notNull(), + resultJson: text("result_json"), + errorJson: text("error_json"), + cancellationRequestedAt: text("cancellation_requested_at"), + eventSequence: integer("event_sequence").notNull().default(0), + createdAt: text("created_at").notNull(), + updatedAt: text("updated_at").notNull(), + startedAt: text("started_at"), + completedAt: text("completed_at"), + }, + (table) => [index("workflow_runs_status_idx").on(table.status, table.createdAt)], +); + +export const workflowNodes = sqliteTable( + "workflow_nodes", + { + id: text("id").primaryKey(), + workflowRunId: text("workflow_run_id") + .notNull() + .references(() => workflowRuns.id, { onDelete: "cascade" }), + nodeKey: text("node_key").notNull(), + nodeType: text("node_type").notNull(), + status: text("status").notNull(), + definitionJson: text("definition_json").notNull(), + attempt: integer("attempt").notNull().default(0), + claimToken: text("claim_token"), + claimedAt: text("claimed_at"), + claimExpiresAt: text("claim_expires_at"), + resultJson: text("result_json"), + errorJson: text("error_json"), + createdAt: text("created_at").notNull(), + updatedAt: text("updated_at").notNull(), + completedAt: text("completed_at"), + }, + (table) => [ + uniqueIndex("workflow_nodes_run_key_idx").on(table.workflowRunId, table.nodeKey), + uniqueIndex("workflow_nodes_run_id_idx").on(table.workflowRunId, table.id), + index("workflow_nodes_status_idx").on(table.workflowRunId, table.status, table.createdAt), + ], +); + +export const workflowEdges = sqliteTable( + "workflow_edges", + { + workflowRunId: text("workflow_run_id") + .notNull() + .references(() => workflowRuns.id, { onDelete: "cascade" }), + fromNodeId: text("from_node_id").notNull(), + toNodeId: text("to_node_id").notNull(), + }, + (table) => [ + primaryKey({ columns: [table.workflowRunId, table.fromNodeId, table.toNodeId] }), + foreignKey({ + columns: [table.workflowRunId, table.fromNodeId], + foreignColumns: [workflowNodes.workflowRunId, workflowNodes.id], + }).onDelete("cascade"), + foreignKey({ + columns: [table.workflowRunId, table.toNodeId], + foreignColumns: [workflowNodes.workflowRunId, workflowNodes.id], + }).onDelete("cascade"), + index("workflow_edges_to_node_idx").on(table.workflowRunId, table.toNodeId), + ], +); + +export const workflowEvents = sqliteTable( + "workflow_events", + { + workflowRunId: text("workflow_run_id") + .notNull() + .references(() => workflowRuns.id, { onDelete: "cascade" }), + sequence: integer("sequence").notNull(), + eventType: text("event_type").notNull(), + nodeId: text("node_id"), + payloadJson: text("payload_json").notNull(), + createdAt: text("created_at").notNull(), + }, + (table) => [ + primaryKey({ columns: [table.workflowRunId, table.sequence] }), + foreignKey({ + columns: [table.workflowRunId, table.nodeId], + foreignColumns: [workflowNodes.workflowRunId, workflowNodes.id], + }), + index("workflow_events_cursor_idx").on(table.workflowRunId, table.sequence), + ], +); + export type WorkspaceSessionRow = typeof workspaceSessions.$inferSelect; export type NewWorkspaceSessionRow = typeof workspaceSessions.$inferInsert; export type LoadedAgentFileRow = typeof loadedAgentFiles.$inferSelect; export type NewLoadedAgentFileRow = typeof loadedAgentFiles.$inferInsert; export type LocalAgentSessionRow = typeof localAgentSessions.$inferSelect; export type NewLocalAgentSessionRow = typeof localAgentSessions.$inferInsert; +export type WorkflowRunRow = typeof workflowRuns.$inferSelect; +export type NewWorkflowRunRow = typeof workflowRuns.$inferInsert; +export type WorkflowNodeRow = typeof workflowNodes.$inferSelect; +export type NewWorkflowNodeRow = typeof workflowNodes.$inferInsert; +export type WorkflowEdgeRow = typeof workflowEdges.$inferSelect; +export type NewWorkflowEdgeRow = typeof workflowEdges.$inferInsert; +export type WorkflowEventRow = typeof workflowEvents.$inferSelect; +export type NewWorkflowEventRow = typeof workflowEvents.$inferInsert; diff --git a/src/workflows/types.ts b/src/workflows/types.ts new file mode 100644 index 00000000..fb9659cf --- /dev/null +++ b/src/workflows/types.ts @@ -0,0 +1,159 @@ +export type JsonPrimitive = string | number | boolean | null; +export type JsonValue = JsonPrimitive | JsonObject | JsonValue[]; +export interface JsonObject { + [key: string]: JsonValue; +} + +export const WORKFLOW_DEFINITION_VERSION = 1 as const; +export const WORKFLOW_POLICY_VERSION = 1 as const; + +export type WorkflowStatus = + | "queued" + | "running" + | "cancelling" + | "succeeded" + | "failed" + | "cancelled"; + +export type WorkflowNodeStatus = + | "pending" + | "ready" + | "running" + | "succeeded" + | "failed" + | "cancelled" + | "skipped"; + +export interface AgentWorkflowNodeDefinitionV1 { + key: string; + type: "agent"; + config?: JsonObject; +} + +export type WorkflowNodeDefinitionV1 = AgentWorkflowNodeDefinitionV1; + +export interface WorkflowEdgeDefinitionV1 { + from: string; + to: string; +} + +export interface WorkflowDefinitionV1 { + version: typeof WORKFLOW_DEFINITION_VERSION; + nodes: WorkflowNodeDefinitionV1[]; + edges?: WorkflowEdgeDefinitionV1[]; +} + +export type WorkflowDefinition = WorkflowDefinitionV1; + +export interface WorkflowPolicyV1 extends JsonObject { + version: typeof WORKFLOW_POLICY_VERSION; +} + +export type WorkflowPolicy = WorkflowPolicyV1; + +export interface SubmitWorkflowRequest { + definition: WorkflowDefinition; + input?: JsonObject; + policy?: WorkflowPolicy; + idempotencyKey?: string; +} + +export interface WorkflowNodeRecord { + id: string; + workflowId: string; + key: string; + type: WorkflowNodeDefinitionV1["type"]; + status: WorkflowNodeStatus; + definition: WorkflowNodeDefinitionV1; + attempt: number; + claimToken?: string; + claimedAt?: string; + claimExpiresAt?: string; + result?: JsonValue; + error?: JsonObject; + createdAt: string; + updatedAt: string; + completedAt?: string; +} + +export interface WorkflowEdgeRecord { + workflowId: string; + fromNodeId: string; + toNodeId: string; + from: string; + to: string; +} + +export interface WorkflowRunRecord { + id: string; + definitionVersion: typeof WORKFLOW_DEFINITION_VERSION; + status: WorkflowStatus; + definition: WorkflowDefinition; + input: JsonObject; + policy: WorkflowPolicy; + idempotencyKey?: string; + requestHash: string; + result?: JsonValue; + error?: JsonObject; + cancellationRequestedAt?: string; + createdAt: string; + updatedAt: string; + startedAt?: string; + completedAt?: string; + nodes: WorkflowNodeRecord[]; + edges: WorkflowEdgeRecord[]; +} + +export interface WorkflowEvent { + workflowId: string; + sequence: number; + type: string; + nodeId?: string; + payload: JsonObject; + createdAt: string; +} + +export interface WorkflowEventPage { + events: WorkflowEvent[]; + nextCursor: number; +} + +export interface SubmitWorkflowResult { + workflow: WorkflowRunRecord; + created: boolean; +} + +export interface WorkflowWaitOptions { + timeoutMs?: number; + pollIntervalMs?: number; +} + +export interface WorkflowEventReadOptions { + after?: number; + limit?: number; +} + +export interface WorkflowNodeClaim { + workflowId: string; + nodeKey: string; + claimToken: string; + leaseMs?: number; +} + +export interface WorkflowNodeTransition { + workflowId: string; + nodeKey: string; + claimToken?: string; + status: WorkflowNodeStatus; + result?: JsonValue; + error?: JsonObject; + eventPayload?: JsonObject; +} + +export interface WorkflowTransition { + workflowId: string; + status: WorkflowStatus; + result?: JsonValue; + error?: JsonObject; + eventPayload?: JsonObject; +} From 1134de838667177cb54402b1f3936c3fcf8dee7a Mon Sep 17 00:00:00 2001 From: Waishnav Date: Thu, 16 Jul 2026 20:29:05 +0530 Subject: [PATCH 02/25] feat(workflows): add durable orchestration service Co-Authored-By: Claude --- src/workflows/orchestrator.ts | 125 +++++ src/workflows/store.ts | 891 ++++++++++++++++++++++++++++++++++ 2 files changed, 1016 insertions(+) create mode 100644 src/workflows/orchestrator.ts create mode 100644 src/workflows/store.ts diff --git a/src/workflows/orchestrator.ts b/src/workflows/orchestrator.ts new file mode 100644 index 00000000..c2ab164b --- /dev/null +++ b/src/workflows/orchestrator.ts @@ -0,0 +1,125 @@ +import { performance } from "node:perf_hooks"; +import type { + SubmitWorkflowRequest, + WorkflowEventPage, + WorkflowEventReadOptions, + WorkflowRunRecord, + WorkflowWaitOptions, +} from "./types.js"; +import { WorkflowStore, WorkflowValidationError } from "./store.js"; + +const DEFAULT_WAIT_TIMEOUT_MS = 30_000; +const MAX_WAIT_TIMEOUT_MS = 60_000; +const DEFAULT_POLL_INTERVAL_MS = 50; +const MAX_POLL_INTERVAL_MS = 1_000; + +export class WorkflowOrchestrator { + private readonly store: WorkflowStore; + private readonly waiters = new Map void>>(); + private closed = false; + + constructor(stateDir: string) { + this.store = new WorkflowStore(stateDir); + } + + submit(request: SubmitWorkflowRequest): WorkflowRunRecord { + const workflow = this.store.submit(request).workflow; + this.notify(workflow.id); + return workflow; + } + + get(workflowId: string): WorkflowRunRecord | undefined { + return this.store.get(workflowId); + } + + async wait(workflowId: string, options: WorkflowWaitOptions = {}): Promise { + const timeoutMs = options.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS; + const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + validateWaitOptions(timeoutMs, pollIntervalMs); + + let workflow = this.store.require(workflowId); + if (isTerminal(workflow) || timeoutMs === 0) return workflow; + + const deadline = performance.now() + timeoutMs; + while (!isTerminal(workflow)) { + const remaining = deadline - performance.now(); + if (remaining <= 0) return workflow; + await this.waitForPollOrNotification( + workflowId, + Math.min(remaining, pollIntervalMs), + ); + if (this.closed) return workflow; + workflow = this.store.require(workflowId); + } + return workflow; + } + + events(workflowId: string, options: WorkflowEventReadOptions = {}): WorkflowEventPage { + return this.store.readEvents(workflowId, options); + } + + cancel(workflowId: string): WorkflowRunRecord { + const workflow = this.store.requestCancellation(workflowId); + this.notify(workflowId); + return workflow; + } + + close(): void { + if (this.closed) return; + this.closed = true; + for (const callbacks of this.waiters.values()) { + for (const callback of callbacks) callback(); + } + this.waiters.clear(); + this.store.close(); + } + + private waitForPollOrNotification(workflowId: string, delayMs: number): Promise { + return new Promise((resolve) => { + let settled = false; + const callbacks = this.waiters.get(workflowId) ?? new Set<() => void>(); + const finish = () => { + if (settled) return; + settled = true; + clearTimeout(timer); + callbacks.delete(finish); + if (callbacks.size === 0) this.waiters.delete(workflowId); + resolve(); + }; + callbacks.add(finish); + this.waiters.set(workflowId, callbacks); + const timer = setTimeout(finish, delayMs); + }); + } + + private notify(workflowId: string): void { + const callbacks = this.waiters.get(workflowId); + if (!callbacks) return; + for (const callback of [...callbacks]) callback(); + } +} + +function validateWaitOptions(timeoutMs: number, pollIntervalMs: number): void { + if (!Number.isFinite(timeoutMs) || timeoutMs < 0 || timeoutMs > MAX_WAIT_TIMEOUT_MS) { + throw new WorkflowValidationError( + `Workflow wait timeout must be between 0 and ${MAX_WAIT_TIMEOUT_MS} milliseconds`, + ); + } + if ( + !Number.isFinite(pollIntervalMs) || + pollIntervalMs < 1 || + pollIntervalMs > MAX_POLL_INTERVAL_MS + ) { + throw new WorkflowValidationError( + `Workflow wait poll interval must be between 1 and ${MAX_POLL_INTERVAL_MS} milliseconds`, + ); + } +} + +function isTerminal(workflow: WorkflowRunRecord): boolean { + return ( + workflow.status === "succeeded" || + workflow.status === "failed" || + workflow.status === "cancelled" + ); +} diff --git a/src/workflows/store.ts b/src/workflows/store.ts new file mode 100644 index 00000000..78099d7f --- /dev/null +++ b/src/workflows/store.ts @@ -0,0 +1,891 @@ +import { createHash, randomUUID } from "node:crypto"; +import { Buffer } from "node:buffer"; +import { openDatabase, type DatabaseHandle } from "../db/client.js"; +import { + WORKFLOW_DEFINITION_VERSION, + WORKFLOW_POLICY_VERSION, + type JsonObject, + type JsonValue, + type SubmitWorkflowRequest, + type SubmitWorkflowResult, + type WorkflowDefinition, + type WorkflowEdgeRecord, + type WorkflowEvent, + type WorkflowEventPage, + type WorkflowEventReadOptions, + type WorkflowNodeClaim, + type WorkflowNodeDefinitionV1, + type WorkflowNodeRecord, + type WorkflowNodeStatus, + type WorkflowNodeTransition, + type WorkflowPolicy, + type WorkflowRunRecord, + type WorkflowStatus, + type WorkflowTransition, +} from "./types.js"; + +const MAX_EVENT_PAYLOAD_BYTES = 64 * 1024; +const DEFAULT_EVENT_LIMIT = 100; +const MAX_EVENT_LIMIT = 1_000; +const DEFAULT_CLAIM_LEASE_MS = 5 * 60_000; +const MAX_CLAIM_LEASE_MS = 24 * 60 * 60_000; + +const TERMINAL_WORKFLOW_STATUSES = new Set([ + "succeeded", + "failed", + "cancelled", +]); +const TERMINAL_NODE_STATUSES = new Set([ + "succeeded", + "failed", + "cancelled", + "skipped", +]); + +const WORKFLOW_TRANSITIONS: Readonly>> = { + queued: new Set(["running", "cancelling", "failed", "cancelled"]), + running: new Set(["cancelling", "succeeded", "failed", "cancelled"]), + cancelling: new Set(["succeeded", "failed", "cancelled"]), + succeeded: new Set(), + failed: new Set(), + cancelled: new Set(), +}; + +const NODE_TRANSITIONS: Readonly>> = { + pending: new Set(["ready", "cancelled", "skipped"]), + ready: new Set(["running", "cancelled", "skipped"]), + running: new Set(["succeeded", "failed", "cancelled"]), + succeeded: new Set(), + failed: new Set(), + cancelled: new Set(), + skipped: new Set(), +}; + +interface WorkflowRunRow { + id: string; + definition_version: number; + status: string; + definition_json: string; + input_json: string; + policy_json: string; + idempotency_key: string | null; + request_hash: string; + result_json: string | null; + error_json: string | null; + cancellation_requested_at: string | null; + created_at: string; + updated_at: string; + started_at: string | null; + completed_at: string | null; +} + +interface WorkflowNodeRow { + id: string; + workflow_run_id: string; + node_key: string; + node_type: string; + status: string; + definition_json: string; + attempt: number; + claim_token: string | null; + claimed_at: string | null; + claim_expires_at: string | null; + result_json: string | null; + error_json: string | null; + created_at: string; + updated_at: string; + completed_at: string | null; +} + +interface WorkflowEdgeRow { + workflow_run_id: string; + from_node_id: string; + to_node_id: string; + from_key: string; + to_key: string; +} + +interface WorkflowEventRow { + workflow_run_id: string; + sequence: number; + event_type: string; + node_id: string | null; + payload_json: string; + created_at: string; +} + +export class WorkflowNotFoundError extends Error { + constructor(workflowId: string) { + super(`Unknown workflow: ${workflowId}`); + this.name = "WorkflowNotFoundError"; + } +} + +export class WorkflowIdempotencyConflictError extends Error { + constructor(idempotencyKey: string) { + super(`Idempotency key was already used for a different workflow request: ${idempotencyKey}`); + this.name = "WorkflowIdempotencyConflictError"; + } +} + +export class WorkflowTransitionError extends Error { + constructor(entity: "workflow" | "node", from: string, to: string) { + super(`Illegal ${entity} status transition: ${from} -> ${to}`); + this.name = "WorkflowTransitionError"; + } +} + +export class WorkflowValidationError extends Error { + constructor(message: string) { + super(message); + this.name = "WorkflowValidationError"; + } +} + +export class WorkflowStore { + private readonly database: DatabaseHandle; + + constructor(stateDir: string) { + this.database = openDatabase(stateDir); + } + + submit(request: SubmitWorkflowRequest): SubmitWorkflowResult { + const normalized = normalizeSubmission(request); + const now = new Date().toISOString(); + const workflowId = createId("wf_"); + const nodeIds = new Map(normalized.definition.nodes.map((node) => [node.key, createId("wfn_")])); + + const save = this.database.sqlite.transaction(() => { + if (normalized.idempotencyKey) { + const existing = this.database.sqlite + .prepare("select id, request_hash from workflow_runs where idempotency_key = ?") + .get(normalized.idempotencyKey) as { id: string; request_hash: string } | undefined; + if (existing) { + if (existing.request_hash !== normalized.requestHash) { + throw new WorkflowIdempotencyConflictError(normalized.idempotencyKey); + } + return { workflowId: existing.id, created: false }; + } + } + + this.database.sqlite + .prepare( + `insert into workflow_runs ( + id, definition_version, status, definition_json, input_json, policy_json, + idempotency_key, request_hash, created_at, updated_at + ) values (?, ?, 'queued', ?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + workflowId, + WORKFLOW_DEFINITION_VERSION, + normalized.definitionJson, + normalized.inputJson, + normalized.policyJson, + normalized.idempotencyKey ?? null, + normalized.requestHash, + now, + now, + ); + + const incoming = new Map(normalized.definition.nodes.map((node) => [node.key, 0])); + for (const edge of normalized.definition.edges ?? []) { + incoming.set(edge.to, (incoming.get(edge.to) ?? 0) + 1); + } + + const insertNode = this.database.sqlite.prepare( + `insert into workflow_nodes ( + id, workflow_run_id, node_key, node_type, status, definition_json, + created_at, updated_at + ) values (?, ?, ?, ?, ?, ?, ?, ?)`, + ); + for (const node of normalized.definition.nodes) { + insertNode.run( + nodeIds.get(node.key), + workflowId, + node.key, + node.type, + incoming.get(node.key) === 0 ? "ready" : "pending", + canonicalJson(node), + now, + now, + ); + } + + const insertEdge = this.database.sqlite.prepare( + `insert into workflow_edges (workflow_run_id, from_node_id, to_node_id) + values (?, ?, ?)`, + ); + for (const edge of normalized.definition.edges ?? []) { + insertEdge.run(workflowId, nodeIds.get(edge.from), nodeIds.get(edge.to)); + } + + this.insertEvent(workflowId, "workflow.submitted", undefined, { status: "queued" }, now); + return { workflowId, created: true }; + }); + + const saved = save.immediate(); + return { workflow: this.require(saved.workflowId), created: saved.created }; + } + + get(workflowId: string): WorkflowRunRecord | undefined { + const read = this.database.sqlite.transaction(() => { + const row = this.getWorkflowRow(workflowId); + return row ? this.hydrateWorkflow(row) : undefined; + }); + return read.deferred(); + } + + require(workflowId: string): WorkflowRunRecord { + const workflow = this.get(workflowId); + if (!workflow) throw new WorkflowNotFoundError(workflowId); + return workflow; + } + + claimNode(claim: WorkflowNodeClaim): WorkflowNodeRecord | undefined { + if (!claim.claimToken) throw new WorkflowValidationError("Node claim token must not be empty"); + const leaseMs = claim.leaseMs ?? DEFAULT_CLAIM_LEASE_MS; + if (!Number.isSafeInteger(leaseMs) || leaseMs < 1 || leaseMs > MAX_CLAIM_LEASE_MS) { + throw new WorkflowValidationError( + `Node claim lease must be between 1 and ${MAX_CLAIM_LEASE_MS} milliseconds`, + ); + } + const nowDate = new Date(); + const now = nowDate.toISOString(); + const expiresAt = new Date(nowDate.getTime() + leaseMs).toISOString(); + const claimReady = this.database.sqlite.transaction(() => { + const row = this.getNodeRow(claim.workflowId, claim.nodeKey); + if (!row) return undefined; + + const workflow = this.getWorkflowRow(claim.workflowId); + if (!workflow) throw new WorkflowNotFoundError(claim.workflowId); + const workflowStatus = readWorkflowStatus(workflow.status); + if (workflowStatus !== "queued" && workflowStatus !== "running") return undefined; + + if (row.status === "running" && row.claim_token === claim.claimToken) { + this.database.sqlite + .prepare( + `update workflow_nodes + set claim_expires_at = ?, updated_at = ? + where id = ? and status = 'running' and claim_token = ?`, + ) + .run(expiresAt, now, row.id, claim.claimToken); + return this.getNodeRow(claim.workflowId, claim.nodeKey); + } + + const reclaiming = row.status === "running"; + const updated = this.database.sqlite + .prepare( + `update workflow_nodes + set status = 'running', claim_token = ?, claimed_at = ?, claim_expires_at = ?, + attempt = attempt + 1, updated_at = ? + where id = ? + and ( + (status = 'ready' and claim_token is null) + or (status = 'running' and claim_expires_at <= ?) + )`, + ) + .run(claim.claimToken, now, expiresAt, now, row.id, now); + if (updated.changes !== 1) return undefined; + + if (workflowStatus === "queued") { + this.database.sqlite + .prepare( + `update workflow_runs + set status = 'running', started_at = coalesce(started_at, ?), updated_at = ? + where id = ? and status = 'queued'`, + ) + .run(now, now, claim.workflowId); + this.insertEvent(claim.workflowId, "workflow.running", undefined, { status: "running" }, now); + } + this.insertEvent( + claim.workflowId, + reclaiming ? "node.reclaimed" : "node.running", + row.id, + { nodeKey: row.node_key, status: "running", claimToken: claim.claimToken }, + now, + ); + return this.getNodeRow(claim.workflowId, claim.nodeKey); + }); + + const node = claimReady.immediate(); + return node ? rowToWorkflowNode(node) : undefined; + } + + claimReadyNode(claim: WorkflowNodeClaim): WorkflowNodeRecord | undefined { + return this.claimNode(claim); + } + + transitionNode(transition: WorkflowNodeTransition): WorkflowNodeRecord { + const now = new Date().toISOString(); + const update = this.database.sqlite.transaction(() => { + const current = this.getNodeRow(transition.workflowId, transition.nodeKey); + if (!current) { + this.assertWorkflowExists(transition.workflowId); + throw new WorkflowValidationError(`Unknown workflow node: ${transition.nodeKey}`); + } + const workflow = this.getWorkflowRow(transition.workflowId); + if (!workflow) throw new WorkflowNotFoundError(transition.workflowId); + const workflowStatus = readWorkflowStatus(workflow.status); + if (TERMINAL_WORKFLOW_STATUSES.has(workflowStatus)) { + throw new WorkflowValidationError( + `Cannot transition node after workflow reached terminal status: ${workflowStatus}`, + ); + } + + const currentStatus = readNodeStatus(current.status); + if (!NODE_TRANSITIONS[currentStatus].has(transition.status)) { + throw new WorkflowTransitionError("node", currentStatus, transition.status); + } + if ( + currentStatus === "running" && + (!transition.claimToken || transition.claimToken !== current.claim_token) + ) { + throw new WorkflowValidationError("Running node transition requires the active claim token"); + } + + const terminal = TERMINAL_NODE_STATUSES.has(transition.status); + const resultJson = serializeOptionalJson(transition.result); + const errorJson = serializeOptionalObject(transition.error); + this.database.sqlite + .prepare( + `update workflow_nodes + set status = ?, result_json = ?, error_json = ?, updated_at = ?, completed_at = ?, + claim_expires_at = case when ? then null else claim_expires_at end + where id = ? and status = ?`, + ) + .run( + transition.status, + resultJson, + errorJson, + now, + terminal ? now : null, + terminal ? 1 : 0, + current.id, + currentStatus, + ); + this.insertEvent( + transition.workflowId, + `node.${transition.status}`, + current.id, + transition.eventPayload ?? { nodeKey: current.node_key, status: transition.status }, + now, + ); + return current.id; + }); + + return this.getNodeById(update.immediate())!; + } + + transitionWorkflow(transition: WorkflowTransition): WorkflowRunRecord { + const now = new Date().toISOString(); + const update = this.database.sqlite.transaction(() => { + const current = this.getWorkflowRow(transition.workflowId); + if (!current) throw new WorkflowNotFoundError(transition.workflowId); + const currentStatus = readWorkflowStatus(current.status); + if (!WORKFLOW_TRANSITIONS[currentStatus].has(transition.status)) { + throw new WorkflowTransitionError("workflow", currentStatus, transition.status); + } + + const terminal = TERMINAL_WORKFLOW_STATUSES.has(transition.status); + if (transition.status === "succeeded") { + const nonSuccessful = this.database.sqlite + .prepare( + `select node_key, status from workflow_nodes + where workflow_run_id = ? and status not in ('succeeded', 'skipped') + order by node_key + limit 1`, + ) + .get(transition.workflowId) as { node_key: string; status: string } | undefined; + if (nonSuccessful) { + throw new WorkflowValidationError( + `Cannot succeed workflow while node ${nonSuccessful.node_key} is ${nonSuccessful.status}`, + ); + } + } else if (transition.status === "failed" || transition.status === "cancelled") { + this.terminalizeOpenNodes(transition.workflowId, transition.status, now); + } + + this.database.sqlite + .prepare( + `update workflow_runs + set status = ?, result_json = ?, error_json = ?, updated_at = ?, + started_at = case when ? = 'running' then coalesce(started_at, ?) else started_at end, + completed_at = ? + where id = ? and status = ?`, + ) + .run( + transition.status, + serializeOptionalJson(transition.result), + serializeOptionalObject(transition.error), + now, + transition.status, + now, + terminal ? now : null, + transition.workflowId, + currentStatus, + ); + this.insertEvent( + transition.workflowId, + `workflow.${transition.status}`, + undefined, + transition.eventPayload ?? { status: transition.status }, + now, + ); + }); + + update.immediate(); + return this.require(transition.workflowId); + } + + requestCancellation(workflowId: string): WorkflowRunRecord { + const now = new Date().toISOString(); + const cancel = this.database.sqlite.transaction(() => { + const current = this.getWorkflowRow(workflowId); + if (!current) throw new WorkflowNotFoundError(workflowId); + const status = readWorkflowStatus(current.status); + if (TERMINAL_WORKFLOW_STATUSES.has(status) || status === "cancelling") return; + + this.database.sqlite + .prepare( + `update workflow_runs + set status = 'cancelling', cancellation_requested_at = ?, updated_at = ? + where id = ? and status = ?`, + ) + .run(now, now, workflowId, status); + this.insertEvent( + workflowId, + "workflow.cancellation_requested", + undefined, + { status: "cancelling" }, + now, + ); + }); + + cancel.immediate(); + return this.require(workflowId); + } + + appendEvent( + workflowId: string, + type: string, + payload: JsonObject, + nodeId?: string, + ): WorkflowEvent { + if (!type.trim()) throw new WorkflowValidationError("Workflow event type must not be empty"); + const now = new Date().toISOString(); + const append = this.database.sqlite.transaction(() => { + this.assertWorkflowExists(workflowId); + if (nodeId) this.assertNodeBelongsToWorkflow(workflowId, nodeId); + return this.insertEvent(workflowId, type, nodeId, payload, now); + }); + const sequence = append.immediate(); + return this.readEvents(workflowId, { after: sequence - 1, limit: 1 }).events[0]!; + } + + readEvents(workflowId: string, options: WorkflowEventReadOptions = {}): WorkflowEventPage { + this.assertWorkflowExists(workflowId); + const after = options.after ?? 0; + const limit = options.limit ?? DEFAULT_EVENT_LIMIT; + if (!Number.isSafeInteger(after) || after < 0) { + throw new WorkflowValidationError("Workflow event cursor must be a non-negative integer"); + } + if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_EVENT_LIMIT) { + throw new WorkflowValidationError(`Workflow event limit must be between 1 and ${MAX_EVENT_LIMIT}`); + } + + const rows = this.database.sqlite + .prepare( + `select * from workflow_events + where workflow_run_id = ? and sequence > ? + order by sequence asc + limit ?`, + ) + .all(workflowId, after, limit) as WorkflowEventRow[]; + const events = rows.map(rowToWorkflowEvent); + return { events, nextCursor: events.at(-1)?.sequence ?? after }; + } + + close(): void { + this.database.close(); + } + + private hydrateWorkflow(row: WorkflowRunRow): WorkflowRunRecord { + const nodes = this.database.sqlite + .prepare("select * from workflow_nodes where workflow_run_id = ? order by created_at, node_key") + .all(row.id) as WorkflowNodeRow[]; + const edges = this.database.sqlite + .prepare( + `select e.workflow_run_id, e.from_node_id, e.to_node_id, + source.node_key as from_key, target.node_key as to_key + from workflow_edges e + join workflow_nodes source on source.id = e.from_node_id + join workflow_nodes target on target.id = e.to_node_id + where e.workflow_run_id = ? + order by source.node_key, target.node_key`, + ) + .all(row.id) as WorkflowEdgeRow[]; + + return { + id: row.id, + definitionVersion: readDefinitionVersion(row.definition_version), + status: readWorkflowStatus(row.status), + definition: parseJson(row.definition_json), + input: parseJson(row.input_json), + policy: parseJson(row.policy_json), + idempotencyKey: row.idempotency_key ?? undefined, + requestHash: row.request_hash, + result: parseOptionalJson(row.result_json), + error: parseOptionalJson(row.error_json), + cancellationRequestedAt: row.cancellation_requested_at ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at, + startedAt: row.started_at ?? undefined, + completedAt: row.completed_at ?? undefined, + nodes: nodes.map(rowToWorkflowNode), + edges: edges.map(rowToWorkflowEdge), + }; + } + + private getWorkflowRow(workflowId: string): WorkflowRunRow | undefined { + return this.database.sqlite + .prepare("select * from workflow_runs where id = ?") + .get(workflowId) as WorkflowRunRow | undefined; + } + + private getNodeRow(workflowId: string, nodeKey: string): WorkflowNodeRow | undefined { + return this.database.sqlite + .prepare("select * from workflow_nodes where workflow_run_id = ? and node_key = ?") + .get(workflowId, nodeKey) as WorkflowNodeRow | undefined; + } + + private getNodeById(nodeId: string): WorkflowNodeRecord | undefined { + const row = this.database.sqlite + .prepare("select * from workflow_nodes where id = ?") + .get(nodeId) as WorkflowNodeRow | undefined; + return row ? rowToWorkflowNode(row) : undefined; + } + + private assertWorkflowExists(workflowId: string): void { + if (!this.getWorkflowRow(workflowId)) throw new WorkflowNotFoundError(workflowId); + } + + private assertNodeBelongsToWorkflow(workflowId: string, nodeId: string): void { + const row = this.database.sqlite + .prepare("select 1 from workflow_nodes where id = ? and workflow_run_id = ?") + .get(nodeId, workflowId); + if (!row) throw new WorkflowValidationError(`Node ${nodeId} does not belong to ${workflowId}`); + } + + private terminalizeOpenNodes( + workflowId: string, + workflowStatus: "failed" | "cancelled", + now: string, + ): void { + const rows = this.database.sqlite + .prepare( + `select * from workflow_nodes + where workflow_run_id = ? and status in ('pending', 'ready', 'running') + order by created_at, node_key`, + ) + .all(workflowId) as WorkflowNodeRow[]; + const update = this.database.sqlite.prepare( + `update workflow_nodes + set status = ?, claim_expires_at = null, updated_at = ?, completed_at = ? + where id = ? and status = ?`, + ); + for (const row of rows) { + const nodeStatus: WorkflowNodeStatus = + workflowStatus === "failed" && row.status !== "running" ? "skipped" : "cancelled"; + const changed = update.run(nodeStatus, now, now, row.id, row.status); + if (changed.changes !== 1) { + throw new WorkflowValidationError(`Workflow node changed during terminal transition: ${row.node_key}`); + } + this.insertEvent( + workflowId, + `node.${nodeStatus}`, + row.id, + { nodeKey: row.node_key, status: nodeStatus }, + now, + ); + } + } + + private insertEvent( + workflowId: string, + type: string, + nodeId: string | undefined, + payload: JsonObject, + now: string, + ): number { + const payloadJson = serializeEventPayload(payload); + const sequenceRow = this.database.sqlite + .prepare( + `update workflow_runs + set event_sequence = event_sequence + 1, updated_at = ? + where id = ? + returning event_sequence`, + ) + .get(now, workflowId) as { event_sequence: number } | undefined; + if (!sequenceRow) throw new WorkflowNotFoundError(workflowId); + + this.database.sqlite + .prepare( + `insert into workflow_events ( + workflow_run_id, sequence, event_type, node_id, payload_json, created_at + ) values (?, ?, ?, ?, ?, ?)`, + ) + .run(workflowId, sequenceRow.event_sequence, type, nodeId ?? null, payloadJson, now); + return sequenceRow.event_sequence; + } +} + +function normalizeSubmission(request: SubmitWorkflowRequest): { + definition: WorkflowDefinition; + definitionJson: string; + inputJson: string; + policyJson: string; + idempotencyKey?: string; + requestHash: string; +} { + const definition = normalizeDefinition(request.definition); + const input = normalizeObject(request.input ?? {}, "Workflow input"); + const policy = normalizePolicy(request.policy ?? { version: WORKFLOW_POLICY_VERSION }); + const definitionJson = canonicalJson(definition); + const inputJson = canonicalJson(input); + const policyJson = canonicalJson(policy); + const idempotencyKey = request.idempotencyKey?.trim(); + if (request.idempotencyKey !== undefined && !idempotencyKey) { + throw new WorkflowValidationError("Idempotency key must not be empty"); + } + const requestHash = createHash("sha256") + .update(canonicalJson({ definition, input, policy })) + .digest("hex"); + return { definition, definitionJson, inputJson, policyJson, idempotencyKey, requestHash }; +} + +function normalizeDefinition(definition: WorkflowDefinition): WorkflowDefinition { + if (!isObject(definition) || definition.version !== WORKFLOW_DEFINITION_VERSION) { + throw new WorkflowValidationError( + `Unsupported workflow definition version; expected ${WORKFLOW_DEFINITION_VERSION}`, + ); + } + if (!Array.isArray(definition.nodes) || definition.nodes.length === 0) { + throw new WorkflowValidationError("Workflow definition must contain at least one node"); + } + if (definition.edges !== undefined && !Array.isArray(definition.edges)) { + throw new WorkflowValidationError("Workflow definition edges must be an array"); + } + + const keys = new Set(); + const nodes = definition.nodes.map((node, index): WorkflowNodeDefinitionV1 => { + if (!isObject(node) || node.type !== "agent" || typeof node.key !== "string") { + throw new WorkflowValidationError(`Workflow node at index ${index} is not a valid agent node`); + } + const key = node.key.trim(); + if (!key) throw new WorkflowValidationError(`Workflow node at index ${index} has an empty key`); + if (keys.has(key)) throw new WorkflowValidationError(`Duplicate workflow node key: ${key}`); + keys.add(key); + return { key, type: "agent", config: normalizeObject(node.config ?? {}, `Node ${key} config`) }; + }); + + const edgeKeys = new Set(); + const edges = (definition.edges ?? []).map((edge, index) => { + if (!isObject(edge) || typeof edge.from !== "string" || typeof edge.to !== "string") { + throw new WorkflowValidationError(`Workflow edge at index ${index} is invalid`); + } + const from = edge.from.trim(); + const to = edge.to.trim(); + if (!keys.has(from)) throw new WorkflowValidationError(`Workflow edge references missing node: ${from}`); + if (!keys.has(to)) throw new WorkflowValidationError(`Workflow edge references missing node: ${to}`); + const edgeKey = canonicalJson([from, to]); + if (edgeKeys.has(edgeKey)) { + throw new WorkflowValidationError(`Duplicate workflow edge: ${from} -> ${to}`); + } + edgeKeys.add(edgeKey); + return { from, to }; + }); + + assertAcyclic(nodes.map((node) => node.key), edges); + return parseJson(canonicalJson({ version: WORKFLOW_DEFINITION_VERSION, nodes, edges })); +} + +function normalizePolicy(policy: WorkflowPolicy): WorkflowPolicy { + const normalized = normalizeObject(policy, "Workflow policy"); + if (normalized.version !== WORKFLOW_POLICY_VERSION) { + throw new WorkflowValidationError( + `Unsupported workflow policy version; expected ${WORKFLOW_POLICY_VERSION}`, + ); + } + return normalized as WorkflowPolicy; +} + +function assertAcyclic(nodeKeys: string[], edges: Array<{ from: string; to: string }>): void { + const incoming = new Map(nodeKeys.map((key) => [key, 0])); + const outgoing = new Map(nodeKeys.map((key) => [key, [] as string[]])); + for (const edge of edges) { + incoming.set(edge.to, incoming.get(edge.to)! + 1); + outgoing.get(edge.from)!.push(edge.to); + } + const ready = nodeKeys.filter((key) => incoming.get(key) === 0); + let visited = 0; + while (ready.length > 0) { + const key = ready.pop()!; + visited += 1; + for (const target of outgoing.get(key)!) { + const count = incoming.get(target)! - 1; + incoming.set(target, count); + if (count === 0) ready.push(target); + } + } + if (visited !== nodeKeys.length) throw new WorkflowValidationError("Workflow definition contains a cycle"); +} + +function normalizeObject(value: unknown, label: string): JsonObject { + if (!isObject(value)) throw new WorkflowValidationError(`${label} must be a JSON object`); + try { + return parseJson(canonicalJson(value)); + } catch (error) { + if (error instanceof WorkflowValidationError) throw error; + throw new WorkflowValidationError(`${label} must contain only JSON values`); + } +} + +function canonicalJson(value: unknown): string { + return JSON.stringify(normalizeJsonValue(value)); +} + +function normalizeJsonValue(value: unknown): JsonValue { + if (value === null || typeof value === "string" || typeof value === "boolean") return value; + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new WorkflowValidationError("JSON numbers must be finite"); + return value; + } + if (Array.isArray(value)) return value.map(normalizeJsonValue); + if (isObject(value)) { + const normalized = Object.create(null) as JsonObject; + for (const key of Object.keys(value).sort()) { + const item = value[key]; + if (item === undefined) throw new WorkflowValidationError("JSON object values must not be undefined"); + normalized[key] = normalizeJsonValue(item); + } + return normalized; + } + throw new WorkflowValidationError("Value must contain only JSON data"); +} + +function serializeEventPayload(payload: JsonObject): string { + if (!isObject(payload)) throw new WorkflowValidationError("Workflow event payload must be a JSON object"); + const serialized = canonicalJson(payload); + if (Buffer.byteLength(serialized, "utf8") > MAX_EVENT_PAYLOAD_BYTES) { + throw new WorkflowValidationError( + `Workflow event payload exceeds ${MAX_EVENT_PAYLOAD_BYTES} bytes`, + ); + } + return serialized; +} + +function serializeOptionalJson(value: JsonValue | undefined): string | null { + return value === undefined ? null : canonicalJson(value); +} + +function serializeOptionalObject(value: JsonObject | undefined): string | null { + return value === undefined ? null : canonicalJson(normalizeObject(value, "Workflow error")); +} + +function parseJson(value: string): T { + return JSON.parse(value) as T; +} + +function parseOptionalJson(value: string | null): T | undefined { + return value === null ? undefined : parseJson(value); +} + +function rowToWorkflowNode(row: WorkflowNodeRow): WorkflowNodeRecord { + return { + id: row.id, + workflowId: row.workflow_run_id, + key: row.node_key, + type: readNodeType(row.node_type), + status: readNodeStatus(row.status), + definition: parseJson(row.definition_json), + attempt: row.attempt, + claimToken: row.claim_token ?? undefined, + claimedAt: row.claimed_at ?? undefined, + claimExpiresAt: row.claim_expires_at ?? undefined, + result: parseOptionalJson(row.result_json), + error: parseOptionalJson(row.error_json), + createdAt: row.created_at, + updatedAt: row.updated_at, + completedAt: row.completed_at ?? undefined, + }; +} + +function rowToWorkflowEdge(row: WorkflowEdgeRow): WorkflowEdgeRecord { + return { + workflowId: row.workflow_run_id, + fromNodeId: row.from_node_id, + toNodeId: row.to_node_id, + from: row.from_key, + to: row.to_key, + }; +} + +function rowToWorkflowEvent(row: WorkflowEventRow): WorkflowEvent { + return { + workflowId: row.workflow_run_id, + sequence: row.sequence, + type: row.event_type, + nodeId: row.node_id ?? undefined, + payload: parseJson(row.payload_json), + createdAt: row.created_at, + }; +} + +function readDefinitionVersion(version: number): typeof WORKFLOW_DEFINITION_VERSION { + if (version !== WORKFLOW_DEFINITION_VERSION) { + throw new WorkflowValidationError(`Unsupported stored workflow definition version: ${version}`); + } + return version; +} + +function readWorkflowStatus(status: string): WorkflowStatus { + if ( + status === "queued" || + status === "running" || + status === "cancelling" || + status === "succeeded" || + status === "failed" || + status === "cancelled" + ) { + return status; + } + throw new WorkflowValidationError(`Unsupported stored workflow status: ${status}`); +} + +function readNodeStatus(status: string): WorkflowNodeStatus { + if ( + status === "pending" || + status === "ready" || + status === "running" || + status === "succeeded" || + status === "failed" || + status === "cancelled" || + status === "skipped" + ) { + return status; + } + throw new WorkflowValidationError(`Unsupported stored workflow node status: ${status}`); +} + +function readNodeType(type: string): WorkflowNodeDefinitionV1["type"] { + if (type === "agent") return type; + throw new WorkflowValidationError(`Unsupported stored workflow node type: ${type}`); +} + +function createId(prefix: "wf_" | "wfn_"): string { + return `${prefix}${randomUUID().replaceAll("-", "")}`; +} + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} From fa9cef5b7559879e2c3f4048e6999b496046b308 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Thu, 16 Jul 2026 20:29:14 +0530 Subject: [PATCH 03/25] test(workflows): cover durable orchestration Co-Authored-By: Claude --- package.json | 3 +- src/oauth-store.test.ts | 1 + src/workflows/migrations.test.ts | 271 ++++++++++++++ src/workflows/workflows.test.ts | 600 +++++++++++++++++++++++++++++++ 4 files changed, 874 insertions(+), 1 deletion(-) create mode 100644 src/workflows/migrations.test.ts create mode 100644 src/workflows/workflows.test.ts diff --git a/package.json b/package.json index 25f21059..831aaab5 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,8 @@ "dev": "node scripts/dev-server.mjs", "postinstall": "node scripts/fix-node-pty-permissions.mjs", "start": "node dist/cli.js serve", - "test": "tsx src/config.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && tsx src/cli.test.ts", + "test": "tsx src/config.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && npm run test:workflows && tsx src/cli.test.ts", + "test:workflows": "tsx src/workflows/migrations.test.ts && tsx src/workflows/workflows.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, "keywords": [], diff --git a/src/oauth-store.test.ts b/src/oauth-store.test.ts index e1c00338..de3c116b 100644 --- a/src/oauth-store.test.ts +++ b/src/oauth-store.test.ts @@ -44,6 +44,7 @@ async function testDatabaseConfiguration(stateDir: string): Promise { { version: 1, name: "workspace-state" }, { version: 2, name: "oauth-state" }, { version: 3, name: "local-agent-sessions" }, + { version: 4, name: "durable-workflows" }, ]); } finally { database.close(); diff --git a/src/workflows/migrations.test.ts b/src/workflows/migrations.test.ts new file mode 100644 index 00000000..30671448 --- /dev/null +++ b/src/workflows/migrations.test.ts @@ -0,0 +1,271 @@ +import assert from "node:assert/strict"; +import { chmodSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import Database from "better-sqlite3"; +import { databasePath, openDatabase } from "../db/client.js"; + +const root = mkdtempSync(join(tmpdir(), "devspace-workflow-migrations-test-")); + +try { + testFreshSchema(join(root, "fresh")); + testExistingMigrationCompatibility(join(root, "existing")); + testMigrationRollback(join(root, "rollback")); +} finally { + rmSync(root, { recursive: true, force: true }); +} + +function testFreshSchema(stateDir: string): void { + const database = openDatabase(stateDir); + try { + assert.deepEqual( + database.sqlite.prepare("select version, name from devspace_schema_migrations order by version").all(), + [ + { version: 1, name: "workspace-state" }, + { version: 2, name: "oauth-state" }, + { version: 3, name: "local-agent-sessions" }, + { version: 4, name: "durable-workflows" }, + ], + ); + const tables = database.sqlite + .prepare( + `select name from sqlite_master + where type = 'table' and name like 'workflow_%' + order by name`, + ) + .pluck() + .all(); + assert.deepEqual(tables, ["workflow_edges", "workflow_events", "workflow_nodes", "workflow_runs"]); + + const edgeForeignKeys = database.sqlite.prepare("pragma foreign_key_list(workflow_edges)").all() as Array<{ + id: number; + seq: number; + table: string; + from: string; + to: string; + }>; + assert.deepEqual( + edgeForeignKeys + .filter((row) => row.table === "workflow_nodes") + .map((row) => [row.id, row.seq, row.from, row.to]) + .sort(), + [ + [0, 0, "workflow_run_id", "workflow_run_id"], + [0, 1, "to_node_id", "id"], + [1, 0, "workflow_run_id", "workflow_run_id"], + [1, 1, "from_node_id", "id"], + ], + ); + + insertWorkflowFixture(database.sqlite); + assert.throws( + () => + database.sqlite + .prepare( + "insert into workflow_edges (workflow_run_id, from_node_id, to_node_id) values ('run-a', 'node-a', 'node-b')", + ) + .run(), + /FOREIGN KEY constraint failed/, + ); + assert.throws( + () => + database.sqlite + .prepare( + `insert into workflow_events + (workflow_run_id, sequence, event_type, node_id, payload_json, created_at) + values ('run-a', 1, 'invalid', 'node-b', '{}', '2026-01-01T00:00:00.000Z')`, + ) + .run(), + /FOREIGN KEY constraint failed/, + ); + } finally { + database.close(); + } +} + +function testExistingMigrationCompatibility(stateDir: string): void { + createVersion3Fixture(stateDir); + const migrated = openDatabase(stateDir); + try { + assert.deepEqual( + migrated.sqlite.prepare("select version from devspace_schema_migrations order by version").pluck().all(), + [1, 2, 3, 4], + ); + assert.deepEqual(migrated.sqlite.prepare("select * from workspace_sessions").all(), [ + { + id: "workspace-1", + root: "/tmp/workspace", + status: "active", + mode: "checkout", + source_root: null, + base_ref: null, + base_sha: null, + managed: "false", + created_at: "2026-01-01T00:00:00.000Z", + last_used_at: "2026-01-01T00:00:00.000Z", + }, + ]); + assert.equal( + migrated.sqlite.prepare("select content from loaded_agent_files where path = 'CLAUDE.md'").pluck().get(), + "instructions", + ); + assert.equal(migrated.sqlite.prepare("select client_json from oauth_clients").pluck().get(), "{}"); + assert.equal(migrated.sqlite.prepare("select client_id from oauth_access_tokens").pluck().get(), "client-1"); + assert.equal(migrated.sqlite.prepare("select client_id from oauth_refresh_tokens").pluck().get(), "client-1"); + assert.equal(migrated.sqlite.prepare("select status from local_agent_sessions").pluck().get(), "active"); + + assert.deepEqual( + migrated.sqlite + .prepare("pragma table_info(local_agent_sessions)") + .all() + .map((row) => (row as { name: string }).name), + [ + "id", + "workspace_id", + "workspace_root", + "profile_name", + "provider", + "model", + "thinking", + "provider_session_id", + "status", + "latest_response", + "error", + "created_at", + "updated_at", + ], + ); + for (const index of [ + "workspace_sessions_root_idx", + "workspace_sessions_status_idx", + "loaded_agent_files_path_idx", + "oauth_clients_issued_at_idx", + "oauth_access_tokens_client_id_idx", + "oauth_access_tokens_expires_at_idx", + "oauth_refresh_tokens_client_id_idx", + "oauth_refresh_tokens_expires_at_idx", + "local_agent_sessions_workspace_id_idx", + "local_agent_sessions_workspace_root_idx", + "local_agent_sessions_provider_session_id_idx", + ]) { + assert.equal( + migrated.sqlite + .prepare("select count(*) from sqlite_master where type = 'index' and name = ?") + .pluck() + .get(index), + 1, + `missing pre-v4 index ${index}`, + ); + } + assert.equal( + migrated.sqlite + .prepare("select count(*) from sqlite_master where type = 'table' and name like 'workflow_%'") + .pluck() + .get(), + 4, + ); + } finally { + migrated.close(); + } +} + +function testMigrationRollback(stateDir: string): void { + createVersion3Fixture(stateDir); + const incompatible = new Database(databasePath(stateDir)); + try { + incompatible.exec("create table workflow_events (legacy_value text not null)"); + } finally { + incompatible.close(); + } + + assert.throws(() => openDatabase(stateDir), /no such column: workflow_run_id/); + + const sqlite = new Database(databasePath(stateDir)); + try { + assert.deepEqual( + sqlite.prepare("select version from devspace_schema_migrations order by version").pluck().all(), + [1, 2, 3], + ); + assert.equal(sqlite.prepare("select count(*) from workspace_sessions").pluck().get(), 1); + assert.equal(sqlite.prepare("select count(*) from local_agent_sessions").pluck().get(), 1); + assert.deepEqual( + sqlite + .prepare("select name from sqlite_master where type = 'table' and name like 'workflow_%' order by name") + .pluck() + .all(), + ["workflow_events"], + ); + assert.deepEqual( + sqlite + .prepare("pragma table_info(workflow_events)") + .all() + .map((row) => (row as { name: string }).name), + ["legacy_value"], + ); + } finally { + sqlite.close(); + } +} + +function createVersion3Fixture(stateDir: string): void { + mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + chmodSync(stateDir, 0o700); + const initialized = openDatabase(stateDir); + try { + initialized.sqlite.exec(` + drop table workflow_edges; + drop table workflow_events; + drop table workflow_nodes; + drop table workflow_runs; + delete from devspace_schema_migrations where version = 4; + + insert into workspace_sessions ( + id, root, status, mode, source_root, base_ref, base_sha, managed, created_at, last_used_at + ) values ( + 'workspace-1', '/tmp/workspace', 'active', 'checkout', null, null, null, 'false', + '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z' + ); + insert into loaded_agent_files ( + workspace_session_id, path, content_hash, content, loaded_at, last_seen_at + ) values ( + 'workspace-1', 'CLAUDE.md', 'hash', 'instructions', + '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z' + ); + insert into oauth_clients (client_id, client_json, issued_at) values ('client-1', '{}', 1); + insert into oauth_access_tokens ( + token_hash, client_id, scopes_json, expires_at, resource + ) values ('access-1', 'client-1', '[]', 2, null); + insert into oauth_refresh_tokens ( + token_hash, client_id, scopes_json, expires_at, resource + ) values ('refresh-1', 'client-1', '[]', 3, null); + insert into local_agent_sessions ( + id, workspace_id, workspace_root, profile_name, provider, model, thinking, + provider_session_id, status, latest_response, error, created_at, updated_at + ) values ( + 'agent-1', 'workspace-1', '/tmp/workspace', 'default', 'claude', null, null, + null, 'active', null, null, '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z' + ); + `); + } finally { + initialized.close(); + } +} + +function insertWorkflowFixture(sqlite: Database.Database): void { + const insertRun = sqlite.prepare( + `insert into workflow_runs ( + id, definition_version, status, definition_json, input_json, policy_json, + request_hash, created_at, updated_at + ) values (?, 1, 'queued', '{}', '{}', '{}', 'hash', ?, ?)`, + ); + const now = "2026-01-01T00:00:00.000Z"; + insertRun.run("run-a", now, now); + insertRun.run("run-b", now, now); + const insertNode = sqlite.prepare( + `insert into workflow_nodes ( + id, workflow_run_id, node_key, node_type, status, definition_json, created_at, updated_at + ) values (?, ?, 'agent', 'agent', 'ready', '{}', ?, ?)`, + ); + insertNode.run("node-a", "run-a", now, now); + insertNode.run("node-b", "run-b", now, now); +} diff --git a/src/workflows/workflows.test.ts b/src/workflows/workflows.test.ts new file mode 100644 index 00000000..1f894ab7 --- /dev/null +++ b/src/workflows/workflows.test.ts @@ -0,0 +1,600 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { openDatabase } from "../db/client.js"; +import { WorkflowOrchestrator } from "./orchestrator.js"; +import { + WorkflowIdempotencyConflictError, + WorkflowStore, + WorkflowTransitionError, + WorkflowValidationError, +} from "./store.js"; +import type { SubmitWorkflowRequest, WorkflowDefinitionV1 } from "./types.js"; + +interface WorkerSpec { + action: "append" | "claim" | "submit"; + stateDir: string; + readyPath: string; + barrierPath: string; + workflowId?: string; + claimToken?: string; + start?: number; + count?: number; + requestVariant?: "a" | "b" | "same"; +} + +const workerSpecJson = process.argv[2]; +if (workerSpecJson) { + await runWorker(JSON.parse(workerSpecJson) as WorkerSpec); +} else { + await runTests(); +} + +async function runTests(): Promise { + const root = mkdtempSync(join(tmpdir(), "devspace-workflows-test-")); + try { + testSubmissionAndIdempotency(join(root, "submission")); + await testConcurrentSubmission(join(root, "concurrent-submission")); + testMonotonicCursorReads(join(root, "events")); + await testConcurrentEventSequencing(join(root, "concurrent-events")); + testCancellationIdempotency(join(root, "cancellation")); + await testTransitionsAndClaims(join(root, "transitions")); + testTerminalInvariantsAndAtomicity(join(root, "terminal-invariants")); + testDagValidation(join(root, "dag")); + await testBoundedAndRepeatedWait(join(root, "wait")); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +function testSubmissionAndIdempotency(stateDir: string): void { + const store = new WorkflowStore(stateDir); + try { + const request: SubmitWorkflowRequest = { + definition: singleAgentDefinition({ z: true, a: "first" }), + input: { z: 2, a: { nested: true } }, + policy: { version: 1, retries: 0 }, + idempotencyKey: " durable-request ", + }; + const first = store.submit(request); + assert.equal(first.created, true); + assert.match(first.workflow.id, /^wf_[a-f0-9]{32}$/); + assert.match(first.workflow.nodes[0]!.id, /^wfn_[a-f0-9]{32}$/); + assert.equal(first.workflow.status, "queued"); + assert.equal(first.workflow.nodes[0]!.status, "ready"); + assert.equal(first.workflow.idempotencyKey, "durable-request"); + + request.input!.a = { nested: false }; + request.definition.nodes[0]!.config!.a = "mutated"; + const persisted = store.require(first.workflow.id); + assert.deepEqual(persisted.input, { a: { nested: true }, z: 2 }); + assert.deepEqual(persisted.definition.nodes[0]!.config, { a: "first", z: true }); + + const replay = store.submit({ + definition: singleAgentDefinition({ a: "first", z: true }), + input: { a: { nested: true }, z: 2 }, + policy: { retries: 0, version: 1 }, + idempotencyKey: "durable-request", + }); + assert.equal(replay.created, false); + assert.equal(replay.workflow.id, first.workflow.id); + + const specialInput = JSON.parse('{"__proto__":{"preserved":true}}') as Record; + const special = store.submit({ definition: singleAgentDefinition(), input: specialInput }).workflow; + assert.equal(Object.hasOwn(special.input, "__proto__"), true); + assert.deepEqual(special.input["__proto__"], { preserved: true }); + + assert.throws( + () => + store.submit({ + definition: singleAgentDefinition(), + input: { changed: true }, + idempotencyKey: "durable-request", + }), + WorkflowIdempotencyConflictError, + ); + } finally { + store.close(); + } +} + +async function testConcurrentSubmission(stateDir: string): Promise { + const same = await runWorkers(stateDir, [ + { action: "submit", requestVariant: "same" }, + { action: "submit", requestVariant: "same" }, + ]); + assert.equal(same[0]!.workflowId, same[1]!.workflowId); + assert.deepEqual( + same.map((result) => result.created).sort(), + [false, true], + ); + assertDatabaseCounts(stateDir, { runs: 1, nodes: 1, edges: 0, events: 1 }); + + const conflictDir = `${stateDir}-conflict`; + const conflict = await runWorkers(conflictDir, [ + { action: "submit", requestVariant: "a" }, + { action: "submit", requestVariant: "b" }, + ]); + assert.equal(conflict.filter((result) => result.created === true).length, 1); + assert.deepEqual( + conflict.filter((result) => result.error).map((result) => result.error), + ["WorkflowIdempotencyConflictError"], + ); + assertDatabaseCounts(conflictDir, { runs: 1, nodes: 1, edges: 0, events: 1 }); +} + +function testMonotonicCursorReads(stateDir: string): void { + const store = new WorkflowStore(stateDir); + try { + const workflow = store.submit({ definition: singleAgentDefinition() }).workflow; + store.appendEvent(workflow.id, "custom.first", { value: 1 }); + store.appendEvent(workflow.id, "custom.second", { value: 2 }); + + const firstPage = store.readEvents(workflow.id, { after: 0, limit: 2 }); + assert.deepEqual(firstPage.events.map((event) => event.sequence), [1, 2]); + assert.equal(firstPage.nextCursor, 2); + const secondPage = store.readEvents(workflow.id, { after: firstPage.nextCursor, limit: 2 }); + assert.deepEqual(secondPage.events.map((event) => event.sequence), [3]); + assert.equal(secondPage.nextCursor, 3); + assert.deepEqual( + store.readEvents(workflow.id, { after: 0, limit: 10 }).events.map((event) => event.sequence), + [1, 2, 3], + ); + + assert.throws( + () => store.appendEvent(workflow.id, "oversized", { body: "x".repeat(65_536) }), + WorkflowValidationError, + ); + assert.throws( + () => store.appendEvent(workflow.id, "invalid", [] as unknown as Record), + WorkflowValidationError, + ); + } finally { + store.close(); + } +} + +async function testConcurrentEventSequencing(stateDir: string): Promise { + const store = new WorkflowStore(stateDir); + const workflow = store.submit({ definition: singleAgentDefinition() }).workflow; + store.close(); + + await runWorkers( + stateDir, + Array.from({ length: 4 }, (_, worker) => ({ + action: "append" as const, + workflowId: workflow.id, + start: worker * 10, + count: 10, + })), + ); + + const reader = new WorkflowStore(stateDir); + try { + const events = reader.readEvents(workflow.id, { limit: 100 }).events; + assert.deepEqual( + events.map((event) => event.sequence), + Array.from({ length: 41 }, (_, index) => index + 1), + ); + assert.deepEqual( + events.slice(1).map((event) => event.payload.value).sort((a, b) => Number(a) - Number(b)), + Array.from({ length: 40 }, (_, index) => index), + ); + const database = openDatabase(stateDir); + try { + assert.equal( + database.sqlite.prepare("select event_sequence from workflow_runs where id = ?").pluck().get(workflow.id), + 41, + ); + } finally { + database.close(); + } + } finally { + reader.close(); + } +} + +function testCancellationIdempotency(stateDir: string): void { + const store = new WorkflowStore(stateDir); + try { + const workflow = store.submit({ definition: singleAgentDefinition() }).workflow; + const first = store.requestCancellation(workflow.id); + const firstEvents = store.readEvents(workflow.id, { limit: 100 }).events; + const second = store.requestCancellation(workflow.id); + const secondEvents = store.readEvents(workflow.id, { limit: 100 }).events; + + assert.equal(first.status, "cancelling"); + assert.ok(first.cancellationRequestedAt); + assert.equal(second.cancellationRequestedAt, first.cancellationRequestedAt); + assert.equal(secondEvents.length, firstEvents.length); + assert.equal(secondEvents.at(-1)!.type, "workflow.cancellation_requested"); + + const cancelled = store.transitionWorkflow({ workflowId: workflow.id, status: "cancelled" }); + assert.equal(cancelled.nodes[0]!.status, "cancelled"); + assert.equal(store.requestCancellation(workflow.id).status, "cancelled"); + assert.equal(store.require(workflow.id).completedAt, cancelled.completedAt); + } finally { + store.close(); + } +} + +async function testTransitionsAndClaims(stateDir: string): Promise { + const store = new WorkflowStore(stateDir); + try { + const workflow = store.submit({ definition: singleAgentDefinition() }).workflow; + const raced = await runWorkers(stateDir, [ + { action: "claim", workflowId: workflow.id, claimToken: "claim-one" }, + { action: "claim", workflowId: workflow.id, claimToken: "claim-two" }, + ]); + const winner = raced.find((result) => result.claimed)!.claimToken as string; + const loser = winner === "claim-one" ? "claim-two" : "claim-one"; + assert.equal(raced.filter((result) => result.claimed).length, 1); + + const claimed = store.require(workflow.id).nodes[0]!; + assert.equal(claimed.status, "running"); + assert.equal(claimed.attempt, 1); + assert.equal(claimed.claimToken, winner); + const eventCount = store.readEvents(workflow.id, { limit: 100 }).events.length; + const replay = store.claimNode({ workflowId: workflow.id, nodeKey: "agent", claimToken: winner }); + assert.equal(replay!.attempt, 1); + assert.equal(store.readEvents(workflow.id, { limit: 100 }).events.length, eventCount); + + assert.throws( + () => + store.transitionNode({ + workflowId: workflow.id, + nodeKey: "agent", + claimToken: loser, + status: "succeeded", + }), + WorkflowValidationError, + ); + const completedNode = store.transitionNode({ + workflowId: workflow.id, + nodeKey: "agent", + claimToken: winner, + status: "succeeded", + result: { answer: 42 }, + }); + assert.deepEqual(completedNode.result, { answer: 42 }); + assert.ok(completedNode.completedAt); + assert.throws( + () => + store.transitionNode({ + workflowId: workflow.id, + nodeKey: "agent", + claimToken: winner, + status: "failed", + }), + WorkflowTransitionError, + ); + + const completed = store.transitionWorkflow({ + workflowId: workflow.id, + status: "succeeded", + result: { answer: 42 }, + }); + assert.deepEqual(completed.result, { answer: 42 }); + assert.ok(completed.completedAt); + assert.equal(store.readEvents(workflow.id, { limit: 100 }).events.at(-1)!.type, "workflow.succeeded"); + + const abandoned = store.submit({ definition: singleAgentDefinition() }).workflow; + const firstClaim = store.claimNode({ + workflowId: abandoned.id, + nodeKey: "agent", + claimToken: "abandoned", + leaseMs: 10, + }); + assert.equal(firstClaim!.attempt, 1); + await delay(20); + const recovered = store.claimNode({ + workflowId: abandoned.id, + nodeKey: "agent", + claimToken: "replacement", + }); + assert.equal(recovered!.attempt, 2); + assert.equal(recovered!.claimToken, "replacement"); + assert.equal(store.readEvents(abandoned.id, { limit: 100 }).events.at(-1)!.type, "node.reclaimed"); + } finally { + store.close(); + } +} + +function testTerminalInvariantsAndAtomicity(stateDir: string): void { + const store = new WorkflowStore(stateDir); + try { + const workflow = store.submit({ definition: singleAgentDefinition() }).workflow; + store.claimNode({ workflowId: workflow.id, nodeKey: "agent", claimToken: "claim" }); + assert.throws( + () => store.transitionWorkflow({ workflowId: workflow.id, status: "succeeded" }), + /Cannot succeed workflow while node agent is running/, + ); + + const beforeNodeFailure = store.require(workflow.id); + const beforeNodeEvents = store.readEvents(workflow.id, { limit: 100 }).events; + assert.throws( + () => + store.transitionNode({ + workflowId: workflow.id, + nodeKey: "agent", + claimToken: "claim", + status: "succeeded", + result: { persisted: false }, + eventPayload: { body: "x".repeat(65_536) }, + }), + WorkflowValidationError, + ); + assert.deepEqual(store.require(workflow.id), beforeNodeFailure); + assert.deepEqual(store.readEvents(workflow.id, { limit: 100 }).events, beforeNodeEvents); + + store.transitionNode({ + workflowId: workflow.id, + nodeKey: "agent", + claimToken: "claim", + status: "succeeded", + }); + const beforeWorkflowFailure = store.require(workflow.id); + const beforeWorkflowEvents = store.readEvents(workflow.id, { limit: 100 }).events; + assert.throws( + () => + store.transitionWorkflow({ + workflowId: workflow.id, + status: "succeeded", + result: { persisted: false }, + eventPayload: { body: "x".repeat(65_536) }, + }), + WorkflowValidationError, + ); + assert.deepEqual(store.require(workflow.id), beforeWorkflowFailure); + assert.deepEqual(store.readEvents(workflow.id, { limit: 100 }).events, beforeWorkflowEvents); + + const failedWorkflow = store.submit({ definition: singleAgentDefinition() }).workflow; + store.claimNode({ workflowId: failedWorkflow.id, nodeKey: "agent", claimToken: "active" }); + const failed = store.transitionWorkflow({ workflowId: failedWorkflow.id, status: "failed" }); + assert.equal(failed.nodes[0]!.status, "cancelled"); + assert.throws( + () => + store.transitionNode({ + workflowId: failed.id, + nodeKey: "agent", + claimToken: "active", + status: "failed", + }), + /Cannot transition node after workflow reached terminal status/, + ); + } finally { + store.close(); + } +} + +function testDagValidation(stateDir: string): void { + const store = new WorkflowStore(stateDir); + try { + assert.throws( + () => + store.submit({ + definition: { version: 1, nodes: [agentNode("same"), agentNode("same")], edges: [] }, + }), + /Duplicate workflow node key/, + ); + assert.throws( + () => + store.submit({ + definition: { + version: 1, + nodes: [agentNode("one")], + edges: [{ from: "one", to: "missing" }], + }, + }), + /missing node/, + ); + assert.throws( + () => + store.submit({ + definition: { + version: 1, + nodes: [agentNode("one"), agentNode("two")], + edges: [ + { from: "one", to: "two" }, + { from: "two", to: "one" }, + ], + }, + }), + /contains a cycle/, + ); + + const dag = store.submit({ + definition: { + version: 1, + nodes: [agentNode("one"), agentNode("two"), agentNode("three")], + edges: [ + { from: "one", to: "two" }, + { from: "one", to: "three" }, + ], + }, + }).workflow; + assert.deepEqual(Object.fromEntries(dag.nodes.map((node) => [node.key, node.status])), { + one: "ready", + three: "pending", + two: "pending", + }); + + const nulKeys = store.submit({ + definition: { + version: 1, + nodes: [agentNode("a\0b"), agentNode("a"), agentNode("b\0c"), agentNode("c")], + edges: [ + { from: "a\0b", to: "c" }, + { from: "a", to: "b\0c" }, + ], + }, + }).workflow; + assert.equal(nulKeys.edges.length, 2); + } finally { + store.close(); + } +} + +async function testBoundedAndRepeatedWait(stateDir: string): Promise { + const orchestrator = new WorkflowOrchestrator(stateDir); + const workerStore = new WorkflowStore(stateDir); + try { + const workflow = orchestrator.submit({ definition: singleAgentDefinition() }); + const timeoutStart = performance.now(); + const timedOut = await orchestrator.wait(workflow.id, { timeoutMs: 100, pollIntervalMs: 10 }); + const timeoutElapsed = performance.now() - timeoutStart; + assert.equal(timedOut.status, "queued"); + assert.ok(timeoutElapsed >= 70, `bounded wait returned early after ${timeoutElapsed}ms`); + assert.ok(timeoutElapsed < 500, `bounded wait took ${timeoutElapsed}ms`); + + const realDateNow = Date.now; + const jumpTimer = setTimeout(() => { + Date.now = () => realDateNow() + 60 * 60_000; + }, 20); + try { + const jumpStart = performance.now(); + await orchestrator.wait(workflow.id, { timeoutMs: 100, pollIntervalMs: 10 }); + assert.ok(performance.now() - jumpStart >= 70, "wall-clock jump ended wait early"); + } finally { + clearTimeout(jumpTimer); + Date.now = realDateNow; + } + + setTimeout(() => { + workerStore.transitionWorkflow({ workflowId: workflow.id, status: "failed" }); + }, 15); + const completed = await orchestrator.wait(workflow.id, { timeoutMs: 500, pollIntervalMs: 5 }); + assert.equal(completed.status, "failed"); + + const repeatStart = performance.now(); + const repeated = await orchestrator.wait(workflow.id, { timeoutMs: 500, pollIntervalMs: 5 }); + assert.equal(repeated.status, "failed"); + assert.ok(performance.now() - repeatStart < 100); + } finally { + workerStore.close(); + orchestrator.close(); + } + + const closingOrchestrator = new WorkflowOrchestrator(`${stateDir}-closing`); + const closingWorkflow = closingOrchestrator.submit({ definition: singleAgentDefinition() }); + const pendingWait = closingOrchestrator.wait(closingWorkflow.id, { + timeoutMs: 500, + pollIntervalMs: 50, + }); + closingOrchestrator.close(); + assert.equal((await pendingWait).status, "queued"); + closingOrchestrator.close(); +} + +async function runWorker(spec: WorkerSpec): Promise { + const store = new WorkflowStore(spec.stateDir); + try { + writeFileSync(spec.readyPath, "ready"); + while (!existsSync(spec.barrierPath)) await delay(2); + if (spec.action === "submit") { + const input = spec.requestVariant === "b" ? { variant: "b" } : { variant: "a" }; + const request: SubmitWorkflowRequest = { + definition: singleAgentDefinition(), + input, + idempotencyKey: spec.requestVariant === "same" ? "same-request" : "conflicting-request", + }; + if (spec.requestVariant === "same") request.input = { variant: "same" }; + try { + const result = store.submit(request); + process.stdout.write(JSON.stringify({ workflowId: result.workflow.id, created: result.created })); + } catch (error) { + process.stdout.write(JSON.stringify({ error: (error as Error).name })); + } + return; + } + if (spec.action === "claim") { + const claimed = store.claimNode({ + workflowId: spec.workflowId!, + nodeKey: "agent", + claimToken: spec.claimToken!, + }); + process.stdout.write(JSON.stringify({ claimed: Boolean(claimed), claimToken: spec.claimToken })); + return; + } + for (let index = 0; index < spec.count!; index += 1) { + const value = spec.start! + index; + store.appendEvent(spec.workflowId!, `concurrent.${value}`, { value }); + } + process.stdout.write(JSON.stringify({ appended: spec.count })); + } finally { + store.close(); + } +} + +async function runWorkers( + stateDir: string, + workers: Array>, +): Promise>> { + const coordinationDir = mkdtempSync(join(tmpdir(), "devspace-workflow-race-")); + const barrierPath = join(coordinationDir, "start"); + const running = workers.map((worker, index) => { + const readyPath = join(coordinationDir, `ready-${index}`); + const spec: WorkerSpec = { ...worker, stateDir, readyPath, barrierPath }; + const child = spawn(process.execPath, ["--import", "tsx", fileURLToPath(import.meta.url), JSON.stringify(spec)], { + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += chunk)); + child.stderr.setEncoding("utf8").on("data", (chunk) => (stderr += chunk)); + const result = new Promise>((resolve, reject) => { + child.on("error", reject); + child.on("exit", (code) => { + if (code !== 0) reject(new Error(`Workflow worker exited ${code}: ${stderr}`)); + else resolve(JSON.parse(stdout) as Record); + }); + }); + return { readyPath, result }; + }); + try { + const deadline = performance.now() + 10_000; + while (running.some(({ readyPath }) => !existsSync(readyPath))) { + if (performance.now() >= deadline) throw new Error("Timed out waiting for workflow workers"); + await delay(5); + } + writeFileSync(barrierPath, "start"); + return await Promise.all(running.map(({ result }) => result)); + } finally { + rmSync(coordinationDir, { recursive: true, force: true }); + } +} + +function assertDatabaseCounts( + stateDir: string, + expected: { runs: number; nodes: number; edges: number; events: number }, +): void { + const database = openDatabase(stateDir); + try { + assert.deepEqual( + { + runs: database.sqlite.prepare("select count(*) from workflow_runs").pluck().get(), + nodes: database.sqlite.prepare("select count(*) from workflow_nodes").pluck().get(), + edges: database.sqlite.prepare("select count(*) from workflow_edges").pluck().get(), + events: database.sqlite.prepare("select count(*) from workflow_events").pluck().get(), + }, + expected, + ); + } finally { + database.close(); + } +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function singleAgentDefinition(config: Record = {}): WorkflowDefinitionV1 { + return { version: 1, nodes: [{ key: "agent", type: "agent", config }], edges: [] }; +} + +function agentNode(key: string): WorkflowDefinitionV1["nodes"][number] { + return { key, type: "agent", config: {} }; +} From 688f2d47daa174728461190738782352ed365732 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Thu, 16 Jul 2026 22:36:12 +0530 Subject: [PATCH 04/25] feat(workflows): add local agent execution policy Co-Authored-By: Claude --- src/workflows/policy.ts | 133 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 src/workflows/policy.ts diff --git a/src/workflows/policy.ts b/src/workflows/policy.ts new file mode 100644 index 00000000..da39c131 --- /dev/null +++ b/src/workflows/policy.ts @@ -0,0 +1,133 @@ +import type { JsonObject } from "./types.js"; + +export const LOCAL_AGENT_POLICY_VERSION = 1 as const; + +export type WorkflowAgentAccess = "read_only" | "workspace_write"; + +export interface EffectiveLocalAgentPolicy { + readonly version: typeof LOCAL_AGENT_POLICY_VERSION; + readonly mode: "workflow" | "compatibility"; + readonly access: WorkflowAgentAccess | "full_access"; + readonly environment: Readonly>; +} + +export interface WorkflowPolicyResolutionInput { + workflowPolicy?: JsonObject; + nodeConfig?: JsonObject; + environment?: NodeJS.ProcessEnv; +} + +const EXACT_ENVIRONMENT_KEYS = new Set([ + "APPDATA", + "COLORTERM", + "COMSPEC", + "HOME", + "LANG", + "LOCALAPPDATA", + "LOGNAME", + "NODE_EXTRA_CA_CERTS", + "NODE_NO_WARNINGS", + "PATH", + "PATHEXT", + "SHELL", + "SYSTEMROOT", + "TEMP", + "TERM", + "TMP", + "TMPDIR", + "TZ", + "USER", + "USERPROFILE", + "WINDIR", +]); +const PREFIX_ENVIRONMENT_KEYS = ["LC_", "XDG_"]; + +export function resolveWorkflowNodePolicy( + input: WorkflowPolicyResolutionInput, +): EffectiveLocalAgentPolicy { + const workflowAccess = validateWorkflowAccess(readAccess(input.workflowPolicy)); + const nodeAccess = validateWorkflowAccess(readAccess(input.nodeConfig)); + const workflowMaximum = workflowAccess ?? "read_only"; + const requested = nodeAccess ?? workflowMaximum; + const access = workflowMaximum === "read_only" || requested === "read_only" + ? "read_only" + : "workspace_write"; + + return deepFreeze({ + version: LOCAL_AGENT_POLICY_VERSION, + mode: "workflow" as const, + access, + environment: filterWorkflowEnvironment(input.environment ?? process.env), + }); +} + +export const resolveWorkflowExecutionPolicy = resolveWorkflowNodePolicy; + +export function createLegacyLocalAgentPolicy( + access: "read_only" | "workspace_write" | "full_access", + environment: NodeJS.ProcessEnv = process.env, +): EffectiveLocalAgentPolicy { + return deepFreeze({ + version: LOCAL_AGENT_POLICY_VERSION, + mode: "compatibility" as const, + access, + environment: copyStringEnvironment(environment), + }); +} + +export function filterWorkflowEnvironment( + environment: NodeJS.ProcessEnv, +): Readonly> { + const filtered: Record = {}; + for (const [key, value] of Object.entries(environment)) { + if (typeof value !== "string") continue; + if (!isAllowedEnvironmentKey(key)) continue; + filtered[key] = value; + } + return Object.freeze(filtered); +} + +function validateWorkflowAccess(access: string | undefined): WorkflowAgentAccess | undefined { + if (access === undefined) return undefined; + if (access === "full_access") { + throw new Error("Workflow agents do not support full_access."); + } + if (access !== "read_only" && access !== "workspace_write") { + throw new Error(`Unsupported workflow agent access '${access}'.`); + } + return access; +} + +function readAccess(value: JsonObject | undefined): string | undefined { + if (!value) return undefined; + const direct = value.access; + if (typeof direct === "string") return direct; + const agent = value.agent; + if (agent && typeof agent === "object" && !Array.isArray(agent)) { + const nested = agent.access; + if (typeof nested === "string") return nested; + } + return undefined; +} + +function isAllowedEnvironmentKey(key: string): boolean { + if (EXACT_ENVIRONMENT_KEYS.has(key.toUpperCase())) return true; + return PREFIX_ENVIRONMENT_KEYS.some((prefix) => key.toUpperCase().startsWith(prefix)); +} + +function copyStringEnvironment(environment: NodeJS.ProcessEnv): Readonly> { + const copied: Record = {}; + for (const [key, value] of Object.entries(environment)) { + if (typeof value === "string") copied[key] = value; + } + return Object.freeze(copied); +} + +function deepFreeze(value: T): T { + for (const nested of Object.values(value)) { + if (nested && typeof nested === "object" && !Object.isFrozen(nested)) { + deepFreeze(nested); + } + } + return Object.freeze(value); +} From 59870dcbb8515e3fbbd1165b8c630e7e1f9f57d2 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Thu, 16 Jul 2026 22:36:18 +0530 Subject: [PATCH 05/25] feat(agents): add streaming run handle core Co-Authored-By: Claude --- src/local-agent-runtime.ts | 478 +++++++++++++++++++++++++++++++++++-- 1 file changed, 458 insertions(+), 20 deletions(-) diff --git a/src/local-agent-runtime.ts b/src/local-agent-runtime.ts index 54130c2e..91681bb5 100644 --- a/src/local-agent-runtime.ts +++ b/src/local-agent-runtime.ts @@ -2,10 +2,12 @@ import type { Codex, CodexOptions, ModelReasoningEffort, - RunResult, SandboxMode, + ThreadEvent, ThreadOptions, + TurnOptions, } from "@openai/codex-sdk"; +import type { EffectiveLocalAgentPolicy } from "./workflows/policy.js"; export type LocalAgentWriteMode = "read_only" | "allowed" | "full_access"; @@ -16,6 +18,8 @@ export interface LocalAgentRunInput { writeMode?: LocalAgentWriteMode; model?: string; thinking?: string; + signal?: AbortSignal; + policy?: EffectiveLocalAgentPolicy; } export interface LocalAgentRunResult { @@ -25,25 +29,393 @@ export interface LocalAgentRunResult { items: unknown[]; } +export const LOCAL_AGENT_EVENT_VERSION = 1 as const; + +interface LocalAgentEventBase { + version: typeof LOCAL_AGENT_EVENT_VERSION; + sequence: number; + timestamp: string; + provider: string; +} + +export type LocalAgentEvent = + | (LocalAgentEventBase & { + type: "lifecycle"; + phase: "started" | "cancelling" | "disposing"; + }) + | (LocalAgentEventBase & { + type: "session"; + providerSessionId: string; + resumed: boolean; + }) + | (LocalAgentEventBase & { + type: "output"; + stream: "assistant" | "thinking"; + delta: string; + }) + | (LocalAgentEventBase & { + type: "tool"; + phase: "started" | "updated" | "completed" | "failed"; + name?: string; + id?: string; + metadata?: Record; + }) + | (LocalAgentEventBase & { + type: "permission"; + phase: "requested" | "allowed" | "denied"; + tool?: string; + metadata?: Record; + }) + | (LocalAgentEventBase & { + type: "warning"; + message: string; + metadata?: Record; + }) + | (LocalAgentEventBase & { + type: "terminal"; + outcome: "succeeded" | "failed" | "cancelled"; + message?: string; + }); + +export type LocalAgentEventInput = + | { type: "lifecycle"; phase: "started" | "cancelling" | "disposing" } + | { type: "session"; providerSessionId: string; resumed: boolean } + | { type: "output"; stream: "assistant" | "thinking"; delta: string } + | { + type: "tool"; + phase: "started" | "updated" | "completed" | "failed"; + name?: string; + id?: string; + metadata?: Record; + } + | { + type: "permission"; + phase: "requested" | "allowed" | "denied"; + tool?: string; + metadata?: Record; + } + | { type: "warning"; message: string; metadata?: Record }; + +export interface LocalAgentRunHandle { + readonly provider: string; + events(): AsyncIterable; + result(): Promise; + cancel(reason?: unknown): Promise; + dispose(): Promise; +} + export interface LocalAgentRuntime { readonly provider: string; + start(input: LocalAgentRunInput): Promise; run(input: LocalAgentRunInput): Promise; } -interface CodexThreadLike { +interface CodexStreamedTurnLike { + events: AsyncGenerator; +} + +export interface CodexThreadLike { readonly id: string | null; - run(prompt: string): Promise; + runStreamed(prompt: string, options?: TurnOptions): Promise; } -interface CodexClientLike { +export interface CodexClientLike { startThread(options?: ThreadOptions): CodexThreadLike; resumeThread(id: string, options?: ThreadOptions): CodexThreadLike; } -type CodexFactory = (options?: CodexOptions) => CodexClientLike; +export type CodexFactory = (options?: CodexOptions) => CodexClientLike; + +type EventWaiter = { + resolve: (value: IteratorResult) => void; + reject: (error: unknown) => void; +}; + +const MAX_BUFFERED_EVENTS = 128; +const MAX_EVENT_TEXT_CHARS = 16_384; +const MAX_EVENT_METADATA_CHARS = 4_096; + +class BoundedEventChannel implements AsyncIterable { + private readonly queue: LocalAgentEvent[] = []; + private readonly waiters: EventWaiter[] = []; + private closed = false; + private consumerCreated = false; + + push(event: LocalAgentEvent): void { + if (this.closed) return; + const waiter = this.waiters.shift(); + if (waiter) { + waiter.resolve({ value: event, done: false }); + return; + } + if (this.queue.length >= MAX_BUFFERED_EVENTS) { + const droppable = this.queue.findIndex((queued) => + queued.type === "output" || + queued.type === "tool" || + queued.type === "permission" || + queued.type === "warning" + ); + this.queue.splice(droppable === -1 ? 0 : droppable, 1); + } + this.queue.push(event); + } + + close(): void { + if (this.closed) return; + this.closed = true; + while (this.waiters.length > 0) { + this.waiters.shift()?.resolve({ value: undefined, done: true }); + } + } + + [Symbol.asyncIterator](): AsyncIterator { + if (this.consumerCreated) { + throw new Error("Local agent events support one consumer."); + } + this.consumerCreated = true; + let iteratorClosed = false; + return { + next: () => { + if (iteratorClosed) return Promise.resolve({ value: undefined, done: true }); + const event = this.queue.shift(); + if (event) return Promise.resolve({ value: event, done: false }); + if (this.closed) return Promise.resolve({ value: undefined, done: true }); + return new Promise>((resolve, reject) => { + this.waiters.push({ resolve, reject }); + }); + }, + return: async () => { + iteratorClosed = true; + while (this.waiters.length > 0) { + this.waiters.shift()?.resolve({ value: undefined, done: true }); + } + return { value: undefined, done: true }; + }, + }; + } +} + +export class LocalAgentRunController implements LocalAgentRunHandle { + readonly provider: string; + private readonly channel = new BoundedEventChannel(); + private readonly resultPromise: Promise; + private resolveResult!: (result: LocalAgentRunResult) => void; + private rejectResult!: (error: unknown) => void; + private cancelHook: (reason?: unknown) => Promise = async () => undefined; + private disposeHook: () => Promise = async () => undefined; + private pump: Promise = Promise.resolve(); + private sequence = 0; + private terminal = false; + private cancelling = false; + private cancellationPromise: Promise | undefined; + private lifecycleReadyResolve!: () => void; + private readonly lifecycleReady: Promise; + private disposePromise: Promise | undefined; + private abortSignal: AbortSignal | undefined; + private abortListener: (() => void) | undefined; + + constructor(provider: string, signal?: AbortSignal) { + this.provider = provider; + this.resultPromise = new Promise((resolve, reject) => { + this.resolveResult = resolve; + this.rejectResult = reject; + }); + this.lifecycleReady = new Promise((resolve) => { + this.lifecycleReadyResolve = resolve; + }); + void this.resultPromise.catch(() => undefined); + this.emit({ type: "lifecycle", phase: "started" }); + if (signal) { + this.abortSignal = signal; + this.abortListener = () => void this.cancel(signal.reason); + if (signal.aborted) void this.cancel(signal.reason); + else signal.addEventListener("abort", this.abortListener, { once: true }); + } + } + + events(): AsyncIterable { + return this.channel; + } + + result(): Promise { + return this.resultPromise; + } + + setLifecycle(options: { + cancel?: (reason?: unknown) => Promise | void; + dispose?: () => Promise | void; + pump?: Promise; + }): void { + if (options.cancel) this.cancelHook = async (reason) => options.cancel?.(reason); + if (options.dispose) this.disposeHook = async () => options.dispose?.(); + if (options.pump) { + this.pump = options.pump; + void this.pump.catch(() => undefined); + } + this.lifecycleReadyResolve(); + } + + emit(event: LocalAgentEventInput): void { + if (this.terminal) return; + const normalized = normalizeEventInput(event); + this.channel.push({ + ...normalized, + version: LOCAL_AGENT_EVENT_VERSION, + sequence: ++this.sequence, + timestamp: new Date().toISOString(), + provider: this.provider, + } as LocalAgentEvent); + } + + succeed(result: LocalAgentRunResult): void { + if (!this.finishTerminal("succeeded")) return; + this.resolveResult(result); + } -function sandboxModeFor(writeMode: LocalAgentWriteMode | undefined): SandboxMode { - switch (writeMode) { + fail(error: unknown): void { + if (this.cancelling) { + this.finishCancelled(error); + return; + } + const message = error instanceof Error ? error.message : String(error); + if (!this.finishTerminal("failed", message)) return; + this.rejectResult(error instanceof Error ? error : new Error(message)); + } + + cancel(reason?: unknown): Promise { + if (this.cancellationPromise) return this.cancellationPromise; + if (this.terminal) return Promise.resolve(); + this.cancelling = true; + this.emit({ type: "lifecycle", phase: "cancelling" }); + this.finishCancelled(reason); + this.cancellationPromise = (async () => { + await this.lifecycleReady; + try { + await this.cancelHook(reason); + } catch { + // Cancellation already has a deterministic terminal result. + } + })(); + return this.cancellationPromise; + } + + dispose(): Promise { + if (this.disposePromise) return this.disposePromise; + this.disposePromise = (async () => { + if (!this.terminal) await this.cancel(new Error("Local agent handle disposed.")); + else await this.cancellationPromise; + try { + await this.disposeHook(); + } finally { + await this.pump.catch(() => undefined); + } + })(); + return this.disposePromise; + } + + private finishCancelled(reason?: unknown): void { + const message = reason instanceof Error ? reason.message : directCancellationMessage(reason); + if (!this.finishTerminal("cancelled", message)) return; + const error = new Error(message ?? "Local agent run cancelled."); + error.name = "AbortError"; + this.rejectResult(error); + } + + private finishTerminal( + outcome: "succeeded" | "failed" | "cancelled", + message?: string, + ): boolean { + if (this.terminal) return false; + this.terminal = true; + if (this.abortSignal && this.abortListener) { + this.abortSignal.removeEventListener("abort", this.abortListener); + this.abortSignal = undefined; + this.abortListener = undefined; + } + this.channel.push({ + version: LOCAL_AGENT_EVENT_VERSION, + sequence: ++this.sequence, + timestamp: new Date().toISOString(), + provider: this.provider, + type: "terminal", + outcome, + ...(message ? { message: message.slice(0, MAX_EVENT_TEXT_CHARS) } : {}), + }); + this.channel.close(); + return true; + } +} + +function normalizeEventInput(event: LocalAgentEventInput): LocalAgentEventInput { + if (event.type === "output") { + return { ...event, delta: event.delta.slice(0, MAX_EVENT_TEXT_CHARS) }; + } + if (event.type === "warning") { + return { + ...event, + message: event.message.slice(0, MAX_EVENT_TEXT_CHARS), + ...(event.metadata ? { metadata: redactMetadata(event.metadata) } : {}), + }; + } + if ((event.type === "tool" || event.type === "permission") && event.metadata) { + return { ...event, metadata: redactMetadata(event.metadata) }; + } + return event; +} + +function redactMetadata(metadata: Record): Record { + const redacted: Record = {}; + let size = 0; + for (const [key, value] of Object.entries(metadata)) { + if (size >= MAX_EVENT_METADATA_CHARS) break; + const next = /token|secret|password|authorization|cookie|api[_-]?key/i.test(key) + ? "[redacted]" + : typeof value === "string" + ? value.slice(0, 512) + : typeof value === "number" || typeof value === "boolean" || value === null + ? value + : "[omitted]"; + redacted[key] = next; + size += key.length + String(next).length; + } + return redacted; +} + +function directCancellationMessage(reason: unknown): string | undefined { + if (typeof reason === "string" && reason.trim()) return reason.trim(); + return undefined; +} + +export async function runLocalAgentHandle(handle: LocalAgentRunHandle): Promise { + const drain = (async () => { + for await (const _event of handle.events()) { + // Compatibility callers intentionally ignore normalized progress events. + } + })(); + try { + return await handle.result(); + } finally { + await handle.dispose(); + await drain; + } +} + +function sandboxModeFor(input: LocalAgentRunInput): SandboxMode { + if (input.policy) { + if (input.policy.mode === "workflow") { + if (input.policy.version !== 1 || + (input.policy.access !== "read_only" && input.policy.access !== "workspace_write")) { + throw new Error("Invalid workflow local-agent policy."); + } + return input.policy.access === "workspace_write" ? "workspace-write" : "read-only"; + } + if (input.policy.mode !== "compatibility") { + throw new Error("Invalid local-agent policy mode."); + } + if (input.policy.access === "full_access") return "danger-full-access"; + return input.policy.access === "workspace_write" ? "workspace-write" : "read-only"; + } + switch (input.writeMode) { case "allowed": return "workspace-write"; case "full_access": @@ -57,7 +429,7 @@ function sandboxModeFor(writeMode: LocalAgentWriteMode | undefined): SandboxMode function threadOptionsFor(input: LocalAgentRunInput): ThreadOptions { return { workingDirectory: input.workspace, - sandboxMode: sandboxModeFor(input.writeMode), + sandboxMode: sandboxModeFor(input), approvalPolicy: "never", model: input.model, modelReasoningEffort: input.thinking as ModelReasoningEffort | undefined, @@ -66,25 +438,91 @@ function threadOptionsFor(input: LocalAgentRunInput): ThreadOptions { export class CodexSdkLocalAgentRuntime implements LocalAgentRuntime { readonly provider = "codex" as const; - private readonly codex: CodexClientLike; - constructor(codex: CodexClientLike) { - this.codex = codex; - } + constructor(private readonly codex: CodexClientLike) {} - async run(input: LocalAgentRunInput): Promise { + async start(input: LocalAgentRunInput): Promise { const options = threadOptionsFor(input); + const resumed = Boolean(input.providerSessionId); const thread = input.providerSessionId ? this.codex.resumeThread(input.providerSessionId, options) : this.codex.startThread(options); - const turn = await thread.run(input.prompt); + const abortController = new AbortController(); + const controller = new LocalAgentRunController(this.provider, input.signal); + let providerEvents: AsyncGenerator | undefined; - return { - provider: this.provider, - providerSessionId: thread.id, - finalResponse: turn.finalResponse, - items: turn.items, - }; + const pump = (async () => { + try { + const streamed = await thread.runStreamed(input.prompt, { signal: abortController.signal }); + providerEvents = streamed.events; + const items: unknown[] = []; + let finalResponse = ""; + let providerSessionId = thread.id; + let emittedSessionId: string | null = null; + if (providerSessionId) { + emittedSessionId = providerSessionId; + controller.emit({ type: "session", providerSessionId, resumed }); + } + for await (const event of providerEvents) { + if (event.type === "thread.started") { + providerSessionId = event.thread_id; + if (providerSessionId !== emittedSessionId) { + emittedSessionId = providerSessionId; + controller.emit({ type: "session", providerSessionId, resumed }); + } + } else if (event.type === "item.completed") { + items.push(event.item); + if (event.item.type === "agent_message") { + finalResponse = event.item.text; + controller.emit({ type: "output", stream: "assistant", delta: event.item.text }); + } else if (event.item.type === "command_execution" || event.item.type === "mcp_tool_call") { + controller.emit({ + type: "tool", + phase: event.item.status === "failed" ? "failed" : "completed", + name: event.item.type, + id: event.item.id, + }); + } + } else if (event.type === "item.started" || event.type === "item.updated") { + if (event.item.type === "command_execution" || event.item.type === "mcp_tool_call") { + controller.emit({ + type: "tool", + phase: event.type === "item.started" ? "started" : "updated", + name: event.item.type, + id: event.item.id, + }); + } + } else if (event.type === "turn.failed") { + throw new Error(`Codex turn failed: ${event.error.message}`); + } else if (event.type === "error") { + throw new Error(`Codex stream failed: ${event.message}`); + } + } + if (!finalResponse.trim()) throw new Error("Codex did not return a final assistant response."); + controller.succeed({ + provider: this.provider, + providerSessionId, + finalResponse: finalResponse.trim(), + items, + }); + } catch (error) { + controller.fail(error); + } + })(); + + controller.setLifecycle({ + cancel: (reason) => abortController.abort(reason), + dispose: async () => { + if (!abortController.signal.aborted) abortController.abort(); + await providerEvents?.return(undefined); + }, + pump, + }); + return controller; + } + + async run(input: LocalAgentRunInput): Promise { + return runLocalAgentHandle(await this.start(input)); } } From 51a0bd34901723b4f2f18b5d52f5f8d8ae3d9d53 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Thu, 16 Jul 2026 22:36:30 +0530 Subject: [PATCH 06/25] feat(process): add graceful tree termination Co-Authored-By: Claude --- src/process-platform.ts | 82 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 5 deletions(-) diff --git a/src/process-platform.ts b/src/process-platform.ts index 905d4d73..594ce1fb 100644 --- a/src/process-platform.ts +++ b/src/process-platform.ts @@ -11,17 +11,33 @@ export interface KillableProcess { kill(signal?: NodeJS.Signals): boolean; } -interface ProcessTreeRuntime { +export interface WaitableProcess extends KillableProcess { + exitCode?: number | null; + signalCode?: NodeJS.Signals | null; + once(event: "close", listener: () => void): unknown; + removeListener(event: "close", listener: () => void): unknown; +} + +export interface ProcessTreeRuntime { platform: NodeJS.Platform; killGroup(pid: number, signal: NodeJS.Signals): void; - killWindowsTree(pid: number): boolean; + isGroupAlive(pid: number): boolean; + killWindowsTree(pid: number, force: boolean): boolean; } const defaultProcessTreeRuntime: ProcessTreeRuntime = { platform: process.platform, killGroup: (pid, signal) => process.kill(-pid, signal), - killWindowsTree: (pid) => { - const result = spawnSync("taskkill.exe", ["/pid", String(pid), "/T", "/F"], { + isGroupAlive: (pid) => { + try { + process.kill(-pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ESRCH"; + } + }, + killWindowsTree: (pid, force) => { + const result = spawnSync("taskkill.exe", ["/pid", String(pid), "/T", ...(force ? ["/F"] : [])], { stdio: "ignore", windowsHide: true, }); @@ -63,7 +79,7 @@ export function terminateProcessTree( runtime: ProcessTreeRuntime = defaultProcessTreeRuntime, ): void { if (runtime.platform === "win32" && child.pid) { - if (runtime.killWindowsTree(child.pid)) return; + if (runtime.killWindowsTree(child.pid, signal === "SIGKILL")) return; } else if (detached && child.pid) { try { runtime.killGroup(child.pid, signal); @@ -75,3 +91,59 @@ export function terminateProcessTree( child.kill(signal); } + +export async function terminateProcessTreeGracefully( + child: WaitableProcess, + detached: boolean, + graceMs = 1_000, + runtime: ProcessTreeRuntime = defaultProcessTreeRuntime, +): Promise { + const tracksGroup = runtime.platform !== "win32" && detached && child.pid !== undefined; + if (!tracksGroup && child.exitCode !== null && child.exitCode !== undefined) return; + if (!tracksGroup && child.signalCode) return; + + terminateProcessTree(child, "SIGTERM", detached, runtime); + if (await waitForProcessTreeExit(child, detached, graceMs, runtime)) return; + terminateProcessTree(child, "SIGKILL", detached, runtime); + await waitForProcessTreeExit(child, detached, graceMs, runtime); +} + +function waitForProcessTreeExit( + child: WaitableProcess, + detached: boolean, + timeoutMs: number, + runtime: ProcessTreeRuntime, +): Promise { + if (runtime.platform !== "win32" && detached && child.pid !== undefined) { + if (!runtime.isGroupAlive(child.pid)) return Promise.resolve(true); + return new Promise((resolve) => { + const deadline = Date.now() + timeoutMs; + const poll = () => { + if (!runtime.isGroupAlive(child.pid as number)) { + resolve(true); + } else if (Date.now() >= deadline) { + resolve(false); + } else { + setTimeout(poll, Math.min(25, timeoutMs)); + } + }; + setTimeout(poll, Math.min(25, timeoutMs)); + }); + } + return waitForProcessClose(child, timeoutMs); +} + +function waitForProcessClose(child: WaitableProcess, timeoutMs: number): Promise { + if (child.exitCode !== null && child.exitCode !== undefined) return Promise.resolve(true); + if (child.signalCode) return Promise.resolve(true); + return new Promise((resolve) => { + const finish = (closed: boolean) => { + clearTimeout(timer); + child.removeListener("close", onClose); + resolve(closed); + }; + const onClose = () => finish(true); + const timer = setTimeout(() => finish(false), timeoutMs); + child.once("close", onClose); + }); +} From de05f2cff6c94dc7ef68799bb87a8e01d647e828 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Thu, 16 Jul 2026 22:36:37 +0530 Subject: [PATCH 07/25] feat(agents): stream provider lifecycle events Co-Authored-By: Claude --- src/local-agent-adapters.ts | 880 +++++++++++++++++++++++++++--------- 1 file changed, 654 insertions(+), 226 deletions(-) diff --git a/src/local-agent-adapters.ts b/src/local-agent-adapters.ts index 457b8e08..5fcf8aca 100644 --- a/src/local-agent-adapters.ts +++ b/src/local-agent-adapters.ts @@ -1,17 +1,30 @@ import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process"; -import { resolve } from "node:path"; +import { realpath } from "node:fs/promises"; +import { dirname, isAbsolute, relative, resolve } from "node:path"; import { Readable, Writable } from "node:stream"; -import type { EffortLevel } from "@anthropic-ai/claude-agent-sdk"; +import type { + CanUseTool, + EffortLevel, + Options as ClaudeOptions, + Query, + SDKMessage, +} from "@anthropic-ai/claude-agent-sdk"; import type { LocalAgentProvider } from "./local-agent-profiles.js"; import { removeDevspaceNodeModulesBinFromPath } from "./local-agent-path.js"; import { createCodexSdkLocalAgentRuntime, + LocalAgentRunController, + runLocalAgentHandle, + type LocalAgentRunHandle, type LocalAgentRunInput, type LocalAgentRunResult, } from "./local-agent-runtime.js"; +import { terminateProcessTreeGracefully } from "./process-platform.js"; +import type { EffectiveLocalAgentPolicy } from "./workflows/policy.js"; export interface LocalAgentAdapter { readonly provider: LocalAgentProvider; + start(input: LocalAgentRunInput): Promise; run(input: LocalAgentRunInput): Promise; } @@ -19,13 +32,16 @@ const ACP_COMMANDS: Record<"cursor" | "copilot", [string, ...string[]]> = { cursor: ["cursor-agent", "acp"], copilot: ["copilot", "--acp"], }; +const MAX_RESULT_ITEMS = 256; +const MAX_STDERR_CHARS = 16_384; +const PROCESS_SHUTDOWN_GRACE_MS = 1_000; const PI_AGENT_TIMEOUT_MS = 120_000; export async function runLocalAgentProvider( provider: LocalAgentProvider, input: LocalAgentRunInput, ): Promise { - return createLocalAgentAdapter(provider).run(input); + return runLocalAgentHandle(await createLocalAgentAdapter(provider).start(input)); } export function createLocalAgentAdapter(provider: LocalAgentProvider): LocalAgentAdapter { @@ -44,74 +60,180 @@ export function createLocalAgentAdapter(provider: LocalAgentProvider): LocalAgen } } -class CodexLocalAgentAdapter implements LocalAgentAdapter { - readonly provider = "codex" as const; +abstract class BaseLocalAgentAdapter implements LocalAgentAdapter { + abstract readonly provider: LocalAgentProvider; + abstract start(input: LocalAgentRunInput): Promise; async run(input: LocalAgentRunInput): Promise { - const runtime = await createCodexSdkLocalAgentRuntime(); - return runtime.run(input); + return runLocalAgentHandle(await this.start(input)); + } +} + +class CodexLocalAgentAdapter extends BaseLocalAgentAdapter { + readonly provider = "codex" as const; + + async start(input: LocalAgentRunInput): Promise { + const runtime = await createCodexSdkLocalAgentRuntime( + input.policy ? { env: { ...input.policy.environment } } : undefined, + ); + return runtime.start(input); } } -class ClaudeLocalAgentAdapter implements LocalAgentAdapter { +export type ClaudeQueryLike = AsyncGenerator & Pick; +export type ClaudeQueryFactory = (parameters: { + prompt: string; + options?: ClaudeOptions; +}) => ClaudeQueryLike; + +export class ClaudeLocalAgentAdapter extends BaseLocalAgentAdapter { readonly provider = "claude" as const; - async run(input: LocalAgentRunInput): Promise { - const { query } = await import("@anthropic-ai/claude-agent-sdk"); - const claudeExecutable = process.env.CLAUDE_COMMAND ?? resolveExecutable("claude"); - const messages = query({ + constructor(private readonly queryFactory?: ClaudeQueryFactory) { + super(); + } + + async start(input: LocalAgentRunInput): Promise { + const queryFactory = this.queryFactory ?? (await defaultClaudeQueryFactory()); + const abortController = new AbortController(); + const controller = new LocalAgentRunController(this.provider, input.signal); + const environment = input.policy?.environment ?? process.env; + const claudeExecutable = environment.CLAUDE_COMMAND ?? resolveExecutable("claude"); + const policyOptions = claudePolicyOptions(input, controller); + const query = queryFactory({ prompt: input.prompt, options: { cwd: input.workspace, model: input.model, - ...(input.thinking ? { thinking: { type: "adaptive" } as const, effort: input.thinking as EffortLevel } : {}), + ...(input.thinking + ? { thinking: { type: "adaptive" } as const, effort: input.thinking as EffortLevel } + : {}), resume: input.providerSessionId, - permissionMode: "bypassPermissions", - allowDangerouslySkipPermissions: true, - env: claudeCommandEnvironment(process.env), + includePartialMessages: true, + abortController, + env: claudeCommandEnvironment(environment), ...(claudeExecutable ? { pathToClaudeCodeExecutable: claudeExecutable } : {}), + ...policyOptions, }, }); - let providerSessionId = input.providerSessionId ?? null; - let finalResponse = ""; - const items: unknown[] = []; - for await (const message of messages) { - items.push(message); - const record = message as Record; - if (typeof record.session_id === "string") providerSessionId = record.session_id; - if (record.type === "result" && typeof record.result === "string") { - const resultError = claudeResultError(record); - if (resultError) throw new Error(resultError); - finalResponse = record.result; + const pump = (async () => { + let providerSessionId = input.providerSessionId ?? null; + let emittedSessionId: string | null = null; + let finalResponse = ""; + const items: unknown[] = []; + try { + for await (const message of query) { + pushBounded(items, message); + if (message.session_id && message.session_id !== emittedSessionId) { + const sessionId = message.session_id; + providerSessionId = sessionId; + emittedSessionId = sessionId; + controller.emit({ + type: "session", + providerSessionId: sessionId, + resumed: Boolean(input.providerSessionId), + }); + } + emitClaudeMessage(controller, message); + if (message.type !== "result") continue; + if (message.subtype === "success") { + finalResponse = message.result; + } else { + throw new Error( + `Claude returned an error result (${message.subtype}): ${message.errors.join("; ") || "unknown error"}`, + ); + } + } + controller.succeed({ + provider: this.provider, + providerSessionId, + finalResponse: requireFinalResponse("Claude", finalResponse), + items, + }); + } catch (error) { + controller.fail(error); } - } + })(); + + controller.setLifecycle({ + cancel: async (reason) => { + await withTimeout(query.interrupt(), 1_000).catch(() => undefined); + abortController.abort(reason); + }, + dispose: async () => { + abortController.abort(); + query.close(); + }, + pump, + }); + return controller; + } +} - finalResponse = requireFinalResponse("Claude", finalResponse); +async function defaultClaudeQueryFactory(): Promise { + const module = await import("@anthropic-ai/claude-agent-sdk"); + return module.query as ClaudeQueryFactory; +} + +function claudePolicyOptions( + input: LocalAgentRunInput, + controller: LocalAgentRunController, +): Partial { + if (input.policy?.mode !== "workflow") { return { - provider: this.provider, - providerSessionId, - finalResponse, - items, + permissionMode: "bypassPermissions", + allowDangerouslySkipPermissions: true, }; } -} -function claudeResultError(record: Record): string | undefined { - const subtype = typeof record.subtype === "string" ? record.subtype : undefined; - const isError = record.is_error === true || subtype?.startsWith("error"); - if (!isError) return undefined; - const message = - directString(record.error) ?? - directString(record.message) ?? - directString(record.result) ?? - subtype ?? - "Claude returned an error result."; - return `Claude returned an error result: ${message}`; + const writable = input.policy.access === "workspace_write"; + const allowedTools = new Set(writable + ? ["Read", "Glob", "Grep", "Edit", "Write", "NotebookEdit"] + : ["Read", "Glob", "Grep"]); + const canUseTool: CanUseTool = async (toolName, toolInput) => { + controller.emit({ type: "permission", phase: "requested", tool: toolName }); + const path = directString(toolInput.file_path) ?? + directString(toolInput.notebook_path) ?? + directString(toolInput.path); + const pathAllowed = !path || await isCanonicalPathInsideWorkspace(input.workspace, path); + if (allowedTools.has(toolName) && pathAllowed) { + controller.emit({ type: "permission", phase: "allowed", tool: toolName }); + return { behavior: "allow", updatedInput: toolInput }; + } + controller.emit({ type: "permission", phase: "denied", tool: toolName }); + return { behavior: "deny", message: "Tool is not permitted by workflow policy." }; + }; + return { + permissionMode: "dontAsk", + tools: [...allowedTools], + canUseTool, + sandbox: { enabled: true, failIfUnavailable: true }, + settingSources: [], + strictMcpConfig: true, + mcpServers: {}, + plugins: [], + skills: [], + agents: {}, + }; } -function directString(value: unknown): string | undefined { - return typeof value === "string" && value.trim() ? value.trim() : undefined; +function emitClaudeMessage(controller: LocalAgentRunController, message: SDKMessage): void { + if (message.type === "stream_event") { + const event = asRecord(message.event); + const delta = asRecord(event?.delta); + if (event?.type === "content_block_delta" && delta?.type === "text_delta" && typeof delta.text === "string") { + controller.emit({ type: "output", stream: "assistant", delta: delta.text }); + } else if (event?.type === "content_block_delta" && delta?.type === "thinking_delta" && typeof delta.thinking === "string") { + controller.emit({ type: "output", stream: "thinking", delta: delta.thinking }); + } + return; + } + if (message.type === "tool_progress") { + controller.emit({ type: "tool", phase: "updated", name: message.tool_name, id: message.tool_use_id }); + } else if (message.type === "permission_denied") { + controller.emit({ type: "permission", phase: "denied", tool: message.tool_name }); + } } function resolveExecutable(command: string): string | undefined { @@ -125,7 +247,7 @@ function resolveExecutable(command: string): string | undefined { return executable?.trim() || undefined; } -export function claudeCommandEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { +export function claudeCommandEnvironment(env: NodeJS.ProcessEnv | Readonly>): NodeJS.ProcessEnv { const next = { ...env }; for (const key of [ "CLAUDECODE", @@ -138,113 +260,315 @@ export function claudeCommandEnvironment(env: NodeJS.ProcessEnv): NodeJS.Process return next; } -class OpencodeLocalAgentAdapter implements LocalAgentAdapter { +class OpencodeLocalAgentAdapter extends BaseLocalAgentAdapter { readonly provider = "opencode" as const; - async run(input: LocalAgentRunInput): Promise { - const { createOpencode } = await import("@opencode-ai/sdk/v2"); - const { client, server } = await createOpencode(); - try { - const sessionId = input.providerSessionId ?? await createOpencodeSession(client, input); - const promptResult = await promptOpencodeSession(client, sessionId, input); - await waitForOpencodeSession(client, sessionId); - const messages = await readOpencodeMessages(client, sessionId); - const finalResponse = requireFinalResponse( - "OpenCode", - extractOpenCodeFinalResponse(messages) || extractOpenCodeFinalResponse(promptResult), - ); - return { - provider: this.provider, - providerSessionId: sessionId, - finalResponse, - items: [promptResult, messages], - }; - } finally { - server.close(); + async start(input: LocalAgentRunInput): Promise { + if (input.policy?.mode === "workflow") { + throw new Error("OpenCode cannot currently enforce DevSpace workflow filesystem policy."); } + const { createOpencode } = await import("@opencode-ai/sdk/v2"); + const abortController = new AbortController(); + const controller = new LocalAgentRunController(this.provider, input.signal); + const { client, server } = await createOpencode({ signal: abortController.signal }); + const session = client.v2.session; + let subscriptionStream: AsyncGenerator | undefined; + let sessionId: string | undefined; + + const pump = (async () => { + const items: unknown[] = []; + try { + sessionId = input.providerSessionId ?? await createOpencodeSession(client, input); + if (input.providerSessionId) { + await session.get({ sessionID: sessionId }, { throwOnError: true }); + if (input.model) { + await session.switchModel( + { sessionID: sessionId, model: parseOpencodeV2Model(input.model, input.thinking) }, + { throwOnError: true }, + ); + } + } + controller.emit({ + type: "session", + providerSessionId: sessionId, + resumed: Boolean(input.providerSessionId), + }); + const subscription = await client.v2.event.subscribe({ + signal: abortController.signal, + sseMaxRetryAttempts: 1, + onSseError: (error: unknown) => { + controller.emit({ type: "warning", message: `OpenCode event stream error: ${errorMessage(error)}` }); + }, + }); + subscriptionStream = subscription.stream as AsyncGenerator; + const eventPump = (async () => { + for await (const event of subscriptionStream ?? []) { + if (!isOpencodeSessionEvent(event, sessionId as string)) continue; + pushBounded(items, event); + emitOpencodeEvent(controller, event); + } + })(); + const promptResult = await session.prompt({ + sessionID: sessionId, + prompt: { text: input.prompt }, + delivery: "queue", + resume: true, + }, { throwOnError: true }); + pushBounded(items, promptResult); + await session.wait({ sessionID: sessionId }, { throwOnError: true }); + const messages = await session.messages( + { sessionID: sessionId, order: "desc", limit: 100 }, + { throwOnError: true }, + ); + pushBounded(items, messages); + abortController.abort(); + await eventPump.catch(() => undefined); + const finalResponse = requireFinalResponse("OpenCode", extractOpenCodeFinalResponse(promptResult)); + controller.succeed({ + provider: this.provider, + providerSessionId: sessionId, + finalResponse, + items, + }); + } catch (error) { + controller.fail(error); + } + })(); + + controller.setLifecycle({ + cancel: async () => { + if (sessionId) { + await session.interrupt({ sessionID: sessionId }, { throwOnError: true }).catch(() => undefined); + } + abortController.abort(); + }, + dispose: async () => { + abortController.abort(); + await subscriptionStream?.return(undefined).catch(() => undefined); + server.close(); + }, + pump, + }); + return controller; } } -class AcpLocalAgentAdapter implements LocalAgentAdapter { +class AcpLocalAgentAdapter extends BaseLocalAgentAdapter { constructor( readonly provider: "cursor" | "copilot", private readonly command: [string, ...string[]], - ) {} + ) { + super(); + } - async run(input: LocalAgentRunInput): Promise { - const { client } = await import("@agentclientprotocol/sdk"); - const { methods } = await import("@agentclientprotocol/sdk"); - const { ndJsonStream } = await import("@agentclientprotocol/sdk"); + async start(input: LocalAgentRunInput): Promise { + if (input.policy?.mode === "workflow") { + throw new Error(`${this.provider} ACP cannot currently enforce DevSpace workflow filesystem policy.`); + } + const { client, methods, ndJsonStream, PROTOCOL_VERSION } = await import("@agentclientprotocol/sdk"); const [command, ...args] = this.command; + const detached = process.platform !== "win32"; const child = spawn(command, args, { cwd: input.workspace, - env: process.env, + env: { ...(input.policy?.environment ?? process.env) }, stdio: ["pipe", "pipe", "pipe"], windowsHide: true, + detached, }); assertPipedChild(child); + const controller = new LocalAgentRunController(this.provider, input.signal); let stderr = ""; + let context: { notify(method: string, params: unknown): Promise } | undefined; + let providerSessionId = input.providerSessionId ?? null; + let replaying = Boolean(input.providerSessionId); + let disposing = false; + const items: unknown[] = []; + const textParts: string[] = []; child.stderr.on("data", (chunk: Buffer) => { - stderr += chunk.toString("utf8"); + stderr = appendBounded(stderr, chunk.toString("utf8")); }); - const stream = ndJsonStream( Writable.toWeb(child.stdin) as WritableStream, Readable.toWeb(child.stdout) as ReadableStream, ); - try { - let providerSessionId = input.providerSessionId ?? null; - const finalResponse = await client({ name: "DevSpace" }) - .onRequest(methods.client.session.requestPermission, (context) => { - const selected = selectAcpAllowPermissionOption(context.params.options); - return selected - ? { outcome: { outcome: "selected", optionId: selected.optionId } } - : { outcome: { outcome: "cancelled" } }; - }) - .connectWith(stream, async (context) => { - const session = await context.buildSession(input.workspace).start(); - providerSessionId = session.sessionId; - try { - if (input.model) { - const config = resolveAcpModelConfigUpdate(session, input.model, this.provider); - await context.request(methods.agent.session.setConfigOption, config); - } - if (input.thinking) { - const config = resolveAcpThinkingConfigUpdate(session, input.thinking, this.provider); - await context.request(methods.agent.session.setConfigOption, config); - } - const prompt = session.prompt(input.prompt); - const textParts: string[] = []; - for (;;) { - const message = await session.nextUpdate(); - if (message.kind === "stop") { - await prompt; - return textParts.join("").trim(); - } - - const update = message.update; - if (update.sessionUpdate !== "agent_message_chunk") continue; - const content = update.content; - if (content.type === "text") textParts.push(content.text); - } - } finally { - session.dispose(); + const app = client({ name: "DevSpace" }) + .onRequest(methods.client.session.requestPermission, ({ params }) => { + const tool = asRecord(params.toolCall)?.kind as string | undefined; + controller.emit({ type: "permission", phase: "requested", tool }); + const outcome = resolveAcpPermissionRequest(params, input.policy); + controller.emit({ + type: "permission", + phase: outcome.response.outcome.outcome === "selected" && outcome.allowed ? "allowed" : "denied", + tool, + }); + return outcome.response; + }) + .onNotification(methods.client.session.update, ({ params }) => { + if (params.sessionId !== providerSessionId || replaying) return; + pushBounded(items, params); + emitAcpUpdate(controller, params.update, textParts); + }); + + const pump = app.connectWith(stream, async (activeContext) => { + context = activeContext; + try { + const initialized = await activeContext.request(methods.agent.initialize, { + protocolVersion: PROTOCOL_VERSION, + clientCapabilities: {}, + clientInfo: { name: "DevSpace", version: "1" }, + }); + if (initialized.protocolVersion !== PROTOCOL_VERSION) { + throw new Error(`${this.provider} ACP protocol version ${initialized.protocolVersion} is unsupported.`); + } + let sessionResponse: unknown; + if (input.providerSessionId) { + const capabilities = asRecord(initialized.agentCapabilities); + if (capabilities?.loadSession !== true) { + throw new Error(`${this.provider} ACP does not support loading persisted sessions.`); } + sessionResponse = await activeContext.request(methods.agent.session.load, { + sessionId: input.providerSessionId, + cwd: input.workspace, + mcpServers: [], + }); + providerSessionId = input.providerSessionId; + replaying = false; + } else { + sessionResponse = await activeContext.request(methods.agent.session.new, { + cwd: input.workspace, + mcpServers: [], + }); + providerSessionId = directString(asRecord(sessionResponse)?.sessionId) ?? null; + } + if (!providerSessionId) throw new Error(`${this.provider} ACP did not return a session id.`); + const sessionId = providerSessionId; + controller.emit({ + type: "session", + providerSessionId: sessionId, + resumed: Boolean(input.providerSessionId), }); - return { - provider: this.provider, - providerSessionId, - finalResponse: finalResponse.trim(), - items: [], - }; - } catch (error) { - throw new Error(`${this.provider} ACP run failed: ${errorMessage(error)}${stderr ? `\n${stderr.trim()}` : ""}`); - } finally { - child.kill(); + let sessionMetadata = { sessionId, newSessionResponse: sessionResponse }; + if (input.model) { + const config = resolveAcpModelConfigUpdate(sessionMetadata, input.model, this.provider); + const response = await activeContext.request(methods.agent.session.setConfigOption, config); + sessionMetadata = { sessionId, newSessionResponse: response }; + } + if (input.thinking) { + const config = resolveAcpThinkingConfigUpdate(sessionMetadata, input.thinking, this.provider); + const response = await activeContext.request(methods.agent.session.setConfigOption, config); + sessionMetadata = { sessionId, newSessionResponse: response }; + } + const promptResponse = asRecord(await activeContext.request(methods.agent.session.prompt, { + sessionId, + prompt: [{ type: "text", text: input.prompt }], + })); + const stopReason = directString(promptResponse?.stopReason); + if (stopReason !== "end_turn") { + throw new Error(`${this.provider} ACP stopped with ${stopReason ?? "unknown"}.`); + } + controller.succeed({ + provider: this.provider, + providerSessionId: sessionId, + finalResponse: requireFinalResponse(this.provider, textParts.join("")), + items, + }); + } catch (error) { + if (!disposing) { + const suffix = stderr.trim() ? `\n${stderr.trim()}` : ""; + controller.fail(new Error(`${this.provider} ACP run failed: ${errorMessage(error)}${suffix}`)); + } + } + }); + + child.once("error", (error) => controller.fail(error)); + child.once("exit", (code, signal) => { + if (!disposing && code !== 0) { + controller.fail(new Error(`${this.provider} ACP process exited with code ${code ?? "null"} and signal ${signal ?? "null"}.`)); + } + }); + controller.setLifecycle({ + cancel: async () => { + if (context && providerSessionId) { + await context.notify(methods.agent.session.cancel, { sessionId: providerSessionId }).catch(() => undefined); + } + disposing = true; + await terminateProcessTreeGracefully(child, detached, PROCESS_SHUTDOWN_GRACE_MS); + }, + dispose: async () => { + disposing = true; + await terminateProcessTreeGracefully(child, detached, PROCESS_SHUTDOWN_GRACE_MS); + }, + pump: pump.then(() => undefined).catch((error) => controller.fail(error)), + }); + return controller; + } +} + +export function resolveAcpPermissionRequest( + params: { toolCall: unknown; options: Array<{ optionId: string; kind: string }> }, + policy?: EffectiveLocalAgentPolicy, +): { + allowed: boolean; + outcome: { outcome: { outcome: string; optionId?: string } }; + response: { outcome: { outcome: "selected"; optionId: string } | { outcome: "cancelled" } }; +} { + const toolKind = directString(asRecord(params.toolCall)?.kind) ?? "other"; + const compatibility = policy?.mode !== "workflow"; + const readOnlyKinds = new Set(["read", "search", "fetch", "think"]); + const workspaceWriteKinds = new Set([...readOnlyKinds, "edit", "delete", "move"]); + const allowed = compatibility || (policy?.access === "workspace_write" + ? workspaceWriteKinds.has(toolKind) + : readOnlyKinds.has(toolKind)); + const desiredKinds = allowed + ? compatibility ? ["allow_once", "allow_always"] : ["allow_once"] + : ["reject_once", "reject_always"]; + const selected = desiredKinds + .map((kind) => params.options.find((option) => option.kind === kind)) + .find((option) => option !== undefined); + if (!selected) { + const response = { outcome: { outcome: "cancelled" as const } }; + return { allowed: false, outcome: response, response }; + } + const response = { outcome: { outcome: "selected" as const, optionId: selected.optionId } }; + return { allowed, outcome: response, response }; +} + +function emitAcpUpdate( + controller: LocalAgentRunController, + update: unknown, + textParts: string[], +): void { + const record = asRecord(update); + const kind = directString(record?.sessionUpdate); + if (kind === "agent_message_chunk") { + const content = asRecord(record?.content); + if (content?.type === "text" && typeof content.text === "string") { + textParts.push(content.text); + controller.emit({ type: "output", stream: "assistant", delta: content.text }); } + } else if (kind === "agent_thought_chunk") { + const content = asRecord(record?.content); + if (content?.type === "text" && typeof content.text === "string") { + controller.emit({ type: "output", stream: "thinking", delta: content.text }); + } + } else if (kind === "tool_call" || kind === "tool_call_update") { + controller.emit({ + type: "tool", + phase: kind === "tool_call" ? "started" : acpToolPhase(record?.status), + id: directString(record?.toolCallId), + name: directString(record?.title) ?? directString(record?.kind), + }); + } else if (kind) { + controller.emit({ type: "warning", message: `ACP update: ${kind}` }); } } +function acpToolPhase(status: unknown): "updated" | "completed" | "failed" { + if (status === "completed") return "completed"; + if (status === "failed") return "failed"; + return "updated"; +} + export function resolveAcpModelConfigUpdate( session: unknown, model: string, @@ -323,63 +647,116 @@ function flattenAcpSelectValues(option: Record): string[] { return values; } -function selectAcpAllowPermissionOption(options: Array<{ optionId: string; kind: string }>): { optionId: string } | undefined { - return ( - options.find((option) => option.kind === "allow_once") ?? - options.find((option) => option.kind === "allow_always") - ); -} - -class PiRpcLocalAgentAdapter implements LocalAgentAdapter { +class PiRpcLocalAgentAdapter extends BaseLocalAgentAdapter { readonly provider = "pi" as const; - async run(input: LocalAgentRunInput): Promise { + async start(input: LocalAgentRunInput): Promise { + if (input.policy?.mode === "workflow") { + throw new Error("Pi RPC cannot currently enforce DevSpace workflow filesystem policy."); + } const args = ["--mode", "rpc"]; if (input.model) args.push("--model", input.model); if (input.thinking) args.push("--thinking", input.thinking); if (input.providerSessionId) args.push("--session", input.providerSessionId); - const child = spawn(process.env.PI_COMMAND ?? "pi", args, { + const detached = process.platform !== "win32"; + const environment = input.policy?.environment ?? process.env; + const child = spawn(environment.PI_COMMAND ?? "pi", args, { cwd: input.workspace, - env: piCommandEnvironment(process.env), + env: piCommandEnvironment(environment), stdio: ["pipe", "pipe", "pipe"], windowsHide: true, + detached, }); assertPipedChild(child); + const controller = new LocalAgentRunController(this.provider, input.signal); const rpc = new JsonLineRpc(child); const events: unknown[] = []; - rpc.onEvent((event) => events.push(event)); - try { - const state = await rpc.request({ type: "get_state" }); - const providerSessionId = readNestedString(state, ["sessionId"]) ?? input.providerSessionId ?? null; - const done = rpc.waitForEvent((event) => asRecord(event)?.type === "agent_end", PI_AGENT_TIMEOUT_MS); - await rpc.request({ type: "prompt", message: input.prompt }); - const agentEnd = await done; - const sessionMessages = await rpc.request({ type: "get_messages" }); - const finalResponse = - extractPiFinalResponse(agentEnd) || - extractPiFinalResponse(sessionMessages) || - extractPiStreamingText(events); - if (!finalResponse) { + let disposing = false; + rpc.onEvent((event) => { + pushBounded(events, event); + emitPiEvent(controller, event); + }); + + const pump = (async () => { + try { + const state = await rpc.request({ type: "get_state" }); + const providerSessionId = readNestedString(state, ["sessionId"]) ?? input.providerSessionId ?? null; + if (providerSessionId) { + controller.emit({ + type: "session", + providerSessionId, + resumed: Boolean(input.providerSessionId), + }); + } + const done = rpc.waitForEvent((event) => { + const record = asRecord(event); + return record?.type === "agent_end" && record.willRetry !== true; + }, PI_AGENT_TIMEOUT_MS); + await rpc.request({ type: "prompt", message: input.prompt }); + const agentEnd = await done; + const sessionMessages = await rpc.request({ type: "get_messages" }); const providerError = extractPiProviderError(agentEnd) || extractPiProviderError(sessionMessages) || extractPiProviderError(events); if (providerError) throw new Error(`Pi returned an error: ${providerError}`); + const finalResponse = requireFinalResponse( + "Pi", + extractPiFinalResponse(agentEnd) || + extractPiFinalResponse(sessionMessages) || + extractPiStreamingText(events), + ); + pushBounded(events, sessionMessages); + controller.succeed({ + provider: this.provider, + providerSessionId, + finalResponse, + items: events, + }); + } catch (error) { + if (!disposing) controller.fail(error); } - requireFinalResponse("Pi", finalResponse); - return { - provider: this.provider, - providerSessionId, - finalResponse, - items: [...events, sessionMessages], - }; - } finally { - child.kill(); + })(); + + controller.setLifecycle({ + cancel: async () => { + await withTimeout(rpc.request({ type: "abort" }), 500).catch(() => undefined); + disposing = true; + await terminateProcessTreeGracefully(child, detached, PROCESS_SHUTDOWN_GRACE_MS); + }, + dispose: async () => { + disposing = true; + await terminateProcessTreeGracefully(child, detached, PROCESS_SHUTDOWN_GRACE_MS); + }, + pump, + }); + return controller; + } +} + +function emitPiEvent(controller: LocalAgentRunController, event: unknown): void { + const record = asRecord(event); + if (!record) return; + if (record.type === "message_update") { + const update = asRecord(record.assistantMessageEvent); + if (update?.type === "text_delta" && typeof update.delta === "string") { + controller.emit({ type: "output", stream: "assistant", delta: update.delta }); + } else if (update?.type === "thinking_delta" && typeof update.delta === "string") { + controller.emit({ type: "output", stream: "thinking", delta: update.delta }); } + } else if (record.type === "tool_execution_start" || record.type === "tool_execution_update" || record.type === "tool_execution_end") { + controller.emit({ + type: "tool", + phase: record.type === "tool_execution_start" ? "started" : record.type === "tool_execution_end" ? "completed" : "updated", + id: directString(record.toolCallId), + name: directString(record.toolName), + }); + } else if (record.type === "extension_error") { + controller.emit({ type: "warning", message: directString(record.message) ?? "Pi extension error." }); } } -export function piCommandEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { +export function piCommandEnvironment(env: NodeJS.ProcessEnv | Readonly>): NodeJS.ProcessEnv { if (env.PI_COMMAND) return env; const path = env.PATH; if (!path) return env; @@ -396,6 +773,7 @@ class JsonLineRpc { reject: (error: Error) => void; }>(); private readonly eventSubscribers = new Set<(event: unknown) => void>(); + private readonly eventWaiterRejectors = new Set<(error: Error) => void>(); private buffer = ""; private nextId = 1; private stderr = ""; @@ -404,8 +782,9 @@ class JsonLineRpc { constructor(private readonly child: ChildProcessWithoutNullStreams) { child.stdout.on("data", (chunk: Buffer) => this.handleStdout(chunk.toString("utf8"))); child.stderr.on("data", (chunk: Buffer) => { - this.stderr += chunk.toString("utf8"); + this.stderr = appendBounded(this.stderr, chunk.toString("utf8")); }); + child.on("error", (error) => this.failAll(error)); child.on("exit", (code, signal) => { this.failAll(new Error(`Pi RPC process exited with code ${code ?? "null"} and signal ${signal ?? "null"}\n${this.stderr}`.trim())); }); @@ -429,17 +808,26 @@ class JsonLineRpc { } waitForEvent(predicate: (event: unknown) => boolean, timeoutMs: number): Promise { + if (this.fatalError) return Promise.reject(this.fatalError); return new Promise((resolve, reject) => { - const timer = setTimeout(() => { + const finish = () => { + clearTimeout(timer); unsubscribe(); - reject(new Error(`Pi RPC timed out waiting for agent completion\n${this.stderr}`.trim())); + this.eventWaiterRejectors.delete(rejectWaiter); + }; + const rejectWaiter = (error: Error) => { + finish(); + reject(error); + }; + const timer = setTimeout(() => { + rejectWaiter(new Error(`Pi RPC timed out waiting for agent completion\n${this.stderr}`.trim())); }, timeoutMs); const unsubscribe = this.onEvent((event) => { if (!predicate(event)) return; - clearTimeout(timer); - unsubscribe(); + finish(); resolve(event); }); + this.eventWaiterRejectors.add(rejectWaiter); }); } @@ -455,7 +843,7 @@ class JsonLineRpc { try { message = JSON.parse(line) as Record; } catch { - this.stderr += `${line}\n`; + this.stderr = appendBounded(this.stderr, `${line}\n`); this.failAll(new Error(`Pi RPC emitted malformed JSON on stdout: ${line}`)); return; } @@ -478,82 +866,60 @@ class JsonLineRpc { } private failAll(error: Error): void { + if (this.fatalError) return; this.fatalError = error; - for (const pending of this.pending.values()) { - pending.reject(error); - } + for (const pending of this.pending.values()) pending.reject(error); this.pending.clear(); + for (const reject of this.eventWaiterRejectors) reject(error); + this.eventWaiterRejectors.clear(); } } async function createOpencodeSession(client: unknown, input: LocalAgentRunInput): Promise { - const sessionClient = client as { - session: { - create(parameters?: unknown, options?: unknown): Promise; + const session = (client as { + v2: { + session: { + create(parameters?: unknown, options?: unknown): Promise; + }; }; - }; - const result = await sessionClient.session.create({ - directory: input.workspace, + }).v2.session; + const result = await session.create({ location: { directory: input.workspace }, - ...(input.model ? { model: parseOpencodeModel(input.model) } : {}), + ...(input.model ? { model: parseOpencodeV2Model(input.model, input.thinking) } : {}), }, { throwOnError: true }); const id = - readNestedString(result, ["id"]) ?? + readNestedString(result, ["data", "data", "id"]) ?? readNestedString(result, ["data", "id"]) ?? - readNestedString(result, ["session", "id"]) ?? - readNestedString(result, ["data", "session", "id"]); - if (typeof id !== "string") { - throw new Error("OpenCode did not return a session id."); - } + readNestedString(result, ["id"]); + if (!id) throw new Error("OpenCode did not return a session id."); return id; } -async function promptOpencodeSession( - client: unknown, - sessionId: string, - input: LocalAgentRunInput, -): Promise { - const session = (client as { - session: { - prompt(parameters?: unknown, options?: unknown): Promise; - }; - }).session; - const promptInput = { - sessionID: sessionId, - directory: input.workspace, - prompt: { parts: [{ type: "text", text: input.prompt }] }, - parts: [{ type: "text", text: input.prompt }], - ...(input.model ? { model: parseOpencodeModel(input.model) } : {}), - ...(input.thinking ? { variant: input.thinking } : {}), - }; - return session.prompt(promptInput, { throwOnError: true }); -} - -async function waitForOpencodeSession(client: unknown, sessionId: string): Promise { - const session = (client as { - session?: { wait?: (parameters?: unknown, options?: unknown) => Promise }; - }).session; - if (!session?.wait) return; - await session.wait({ sessionID: sessionId }, { throwOnError: true }); +function parseOpencodeV2Model( + model: string, + variant?: string, +): { providerID: string; id: string; variant?: string } { + const separator = model.indexOf("/"); + const parsed = separator === -1 + ? { providerID: "opencode", id: model } + : { providerID: model.slice(0, separator), id: model.slice(separator + 1) }; + return variant ? { ...parsed, variant } : parsed; } -async function readOpencodeMessages(client: unknown, sessionId: string): Promise { - const session = (client as { - session?: { - messages?: (parameters?: unknown, options?: unknown) => Promise; - }; - }).session; - if (!session?.messages) return undefined; - return session.messages({ sessionID: sessionId, order: "asc", limit: 100 }, { throwOnError: true }); +function isOpencodeSessionEvent(event: unknown, sessionId: string): boolean { + const data = asRecord(asRecord(event)?.data); + const eventSessionId = directString(data?.sessionID); + return !eventSessionId || eventSessionId === sessionId; } -function parseOpencodeModel(model: string): { providerID: string; modelID: string } { - const separator = model.indexOf("/"); - if (separator === -1) return { providerID: "opencode", modelID: model }; - return { - providerID: model.slice(0, separator), - modelID: model.slice(separator + 1), - }; +function emitOpencodeEvent(controller: LocalAgentRunController, event: unknown): void { + const record = asRecord(event); + const data = asRecord(record?.data); + if (record?.type === "session.next.text.delta" && typeof data?.delta === "string") { + controller.emit({ type: "output", stream: "assistant", delta: data.delta }); + } else if (record?.type === "session.next.step.failed" || record?.type === "session.error") { + controller.emit({ type: "warning", message: `OpenCode event: ${record.type}` }); + } } export function extractLocalAgentResponseText(value: unknown): string { @@ -690,9 +1056,15 @@ function stringifyStructuredAssistantMessage(value: unknown): string { } function unwrapProviderPayload(value: unknown): unknown { - const record = asRecord(value); - if (!record) return value; - return record.data ?? record.result ?? value; + let current = value; + for (let depth = 0; depth < 4; depth += 1) { + const record = asRecord(current); + if (!record) return current; + const next = record.data ?? record.result; + if (next === undefined || next === current) return current; + current = next; + } + return current; } function readArray(record: unknown, key: string): unknown[] | undefined { @@ -713,6 +1085,62 @@ function readNestedString(value: unknown, path: string[]): string | undefined { return typeof current === "string" ? current : undefined; } +function directString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function pushBounded(items: unknown[], item: unknown): void { + if (items.length >= MAX_RESULT_ITEMS) items.shift(); + items.push(item); +} + +function appendBounded(current: string, addition: string): string { + const combined = current + addition; + return combined.length > MAX_STDERR_CHARS + ? combined.slice(combined.length - MAX_STDERR_CHARS) + : combined; +} + +function withTimeout(promise: Promise, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Provider operation timed out.")), timeoutMs); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + }, + ); + }); +} + +async function isCanonicalPathInsideWorkspace(workspace: string, candidate: string): Promise { + let canonicalWorkspace: string; + try { + canonicalWorkspace = await realpath(workspace); + } catch { + return false; + } + + let existingPath = isAbsolute(candidate) ? resolve(candidate) : resolve(workspace, candidate); + for (;;) { + try { + const canonicalCandidate = await realpath(existingPath); + const path = relative(canonicalWorkspace, canonicalCandidate); + return path === "" || (!path.startsWith("..") && !isAbsolute(path)); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "ENOENT" && code !== "ENOTDIR") return false; + const parent = dirname(existingPath); + if (parent === existingPath) return false; + existingPath = parent; + } + } +} + function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } From cea7b23ee29bbbcdb148b9b8c40b55bbb5835cc3 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Thu, 16 Jul 2026 22:36:45 +0530 Subject: [PATCH 08/25] test(agents): cover streaming lifecycle and policy Co-Authored-By: Claude --- package.json | 2 +- src/local-agent-adapters.test.ts | 189 +++++++++++++++++++++++++- src/local-agent-runtime.test.ts | 225 +++++++++++++++++++++++++++---- src/process-platform.test.ts | 60 ++++++++- src/workflows/policy.test.ts | 76 +++++++++++ 5 files changed, 521 insertions(+), 31 deletions(-) create mode 100644 src/workflows/policy.test.ts diff --git a/package.json b/package.json index 831aaab5..d6a10dfc 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "postinstall": "node scripts/fix-node-pty-permissions.mjs", "start": "node dist/cli.js serve", "test": "tsx src/config.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && npm run test:workflows && tsx src/cli.test.ts", - "test:workflows": "tsx src/workflows/migrations.test.ts && tsx src/workflows/workflows.test.ts", + "test:workflows": "tsx src/workflows/migrations.test.ts && tsx src/workflows/policy.test.ts && tsx src/workflows/workflows.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, "keywords": [], diff --git a/src/local-agent-adapters.test.ts b/src/local-agent-adapters.test.ts index 0e4cbf0f..d388166a 100644 --- a/src/local-agent-adapters.test.ts +++ b/src/local-agent-adapters.test.ts @@ -1,6 +1,9 @@ import assert from "node:assert/strict"; -import { delimiter } from "node:path"; +import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { delimiter, join } from "node:path"; import { + ClaudeLocalAgentAdapter, claudeCommandEnvironment, createLocalAgentAdapter, extractOpenCodeFinalResponse, @@ -9,6 +12,7 @@ import { extractPiStreamingText, piCommandEnvironment, resolveAcpModelConfigUpdate, + resolveAcpPermissionRequest, resolveAcpThinkingConfigUpdate, } from "./local-agent-adapters.js"; import { removeDevspaceNodeModulesBinFromPath } from "./local-agent-path.js"; @@ -26,6 +30,7 @@ const providers: LocalAgentProvider[] = [ for (const provider of providers) { const adapter = createLocalAgentAdapter(provider); assert.equal(adapter.provider, provider); + assert.equal(typeof adapter.start, "function"); assert.equal(typeof adapter.run, "function"); } @@ -388,3 +393,185 @@ assert.equal( assert.equal(env.PATH, [devspaceBin, "/home/user/.local/bin"].join(delimiter)); } + +{ + const readOnlyPolicy = { + version: 1, + mode: "workflow", + access: "read_only", + environment: Object.freeze({ PATH: "/usr/bin", HOME: "/home/user" }), + } as const; + const denied = resolveAcpPermissionRequest({ + toolCall: { kind: "edit" }, + options: [ + { optionId: "allow", kind: "allow_once" }, + { optionId: "reject", kind: "reject_once" }, + ], + }, readOnlyPolicy); + assert.equal(denied.allowed, false); + assert.deepEqual(denied.response, { + outcome: { outcome: "selected", optionId: "reject" }, + }); + + const read = resolveAcpPermissionRequest({ + toolCall: { kind: "read" }, + options: [{ optionId: "allow", kind: "allow_once" }], + }, readOnlyPolicy); + assert.equal(read.allowed, true); + assert.deepEqual(read.response, { + outcome: { outcome: "selected", optionId: "allow" }, + }); + + const noSafeOption = resolveAcpPermissionRequest({ + toolCall: { kind: "execute" }, + options: [{ optionId: "allow", kind: "allow_always" }], + }, readOnlyPolicy); + assert.deepEqual(noSafeOption.response, { outcome: { outcome: "cancelled" } }); +} + +{ + let capturedOptions: Record | undefined; + let closed = 0; + const query = (async function* () { + yield { + type: "stream_event", + session_id: "claude-session", + event: { + type: "content_block_delta", + delta: { type: "text_delta", text: "Hello " }, + }, + } as never; + yield { + type: "result", + subtype: "success", + session_id: "claude-session", + result: "Hello Claude", + } as never; + })() as AsyncGenerator & { interrupt(): Promise; close(): void }; + query.interrupt = async () => undefined; + query.close = () => { closed += 1; }; + const adapter = new ClaudeLocalAgentAdapter((parameters) => { + capturedOptions = parameters.options as unknown as Record; + return query as never; + }); + const handle = await adapter.start({ prompt: "hello", workspace: "/tmp/project" }); + const result = await handle.result(); + assert.equal(result.providerSessionId, "claude-session"); + assert.equal(result.finalResponse, "Hello Claude"); + assert.equal(capturedOptions?.includePartialMessages, true); + assert.equal(capturedOptions?.permissionMode, "bypassPermissions"); + const eventTypes: string[] = []; + for await (const event of handle.events()) eventTypes.push(event.type); + assert.deepEqual(eventTypes, ["lifecycle", "session", "output", "terminal"]); + await handle.dispose(); + await handle.dispose(); + assert.equal(closed, 1); +} + +{ + const query = (async function* () { + yield { + type: "result", + subtype: "error_during_execution", + session_id: "claude-error-session", + errors: ["provider unavailable"], + } as never; + })() as AsyncGenerator & { interrupt(): Promise; close(): void }; + query.interrupt = async () => undefined; + query.close = () => undefined; + const adapter = new ClaudeLocalAgentAdapter(() => query as never); + const handle = await adapter.start({ prompt: "hello", workspace: "/tmp/project" }); + await assert.rejects(handle.result(), /provider unavailable/); + await handle.dispose(); +} + +{ + const compatibility = resolveAcpPermissionRequest({ + toolCall: { kind: "edit" }, + options: [ + { optionId: "always", kind: "allow_always" }, + { optionId: "once", kind: "allow_once" }, + ], + }); + assert.deepEqual(compatibility.response, { + outcome: { outcome: "selected", optionId: "once" }, + }); +} + +{ + const workflowPolicy = { + version: 1, + mode: "workflow", + access: "workspace_write", + environment: Object.freeze({ PATH: process.env.PATH ?? "" }), + } as const; + await assert.rejects( + createLocalAgentAdapter("cursor").start({ + prompt: "edit", + workspace: "/tmp/project", + policy: workflowPolicy, + }), + /cannot currently enforce DevSpace workflow filesystem policy/, + ); +} + +{ + const root = await mkdtemp(join(tmpdir(), "devspace-claude-policy-")); + const workspace = join(root, "workspace"); + const outside = join(root, "outside"); + await mkdir(workspace); + await mkdir(outside); + await writeFile(join(outside, "secret.txt"), "secret"); + await symlink(outside, join(workspace, "escape")); + + let capturedOptions: Record | undefined; + const query = (async function* () { + yield { + type: "result", + subtype: "success", + session_id: "claude-workflow-session", + result: "ok", + } as never; + })() as AsyncGenerator & { interrupt(): Promise; close(): void }; + query.interrupt = async () => undefined; + query.close = () => undefined; + const originalClaudeCommand = process.env.CLAUDE_COMMAND; + process.env.CLAUDE_COMMAND = "/ambient/claude-must-not-run"; + try { + const adapter = new ClaudeLocalAgentAdapter((parameters) => { + capturedOptions = parameters.options as unknown as Record; + return query as never; + }); + const handle = await adapter.start({ + prompt: "inspect", + workspace, + policy: { + version: 1, + mode: "workflow", + access: "workspace_write", + environment: Object.freeze({ PATH: process.env.PATH ?? "" }), + }, + }); + await handle.result(); + assert.deepEqual(capturedOptions?.settingSources, []); + assert.equal(capturedOptions?.strictMcpConfig, true); + assert.deepEqual(capturedOptions?.mcpServers, {}); + assert.deepEqual(capturedOptions?.plugins, []); + assert.deepEqual(capturedOptions?.skills, []); + assert.deepEqual(capturedOptions?.agents, {}); + assert.notEqual(capturedOptions?.pathToClaudeCodeExecutable, "/ambient/claude-must-not-run"); + + const canUseTool = capturedOptions?.canUseTool as ( + tool: string, + input: Record, + ) => Promise<{ behavior: string }>; + assert.equal((await canUseTool("Write", { file_path: join(workspace, "new.txt") })).behavior, "allow"); + assert.equal((await canUseTool("Write", { file_path: join(workspace, "escape", "secret.txt") })).behavior, "deny"); + assert.equal((await canUseTool("Write", { file_path: join(workspace, "escape", "new.txt") })).behavior, "deny"); + await handle.dispose(); + } finally { + if (originalClaudeCommand === undefined) delete process.env.CLAUDE_COMMAND; + else process.env.CLAUDE_COMMAND = originalClaudeCommand; + await rm(root, { recursive: true, force: true }); + } +} diff --git a/src/local-agent-runtime.test.ts b/src/local-agent-runtime.test.ts index 1d45d166..51367fff 100644 --- a/src/local-agent-runtime.test.ts +++ b/src/local-agent-runtime.test.ts @@ -1,32 +1,69 @@ import assert from "node:assert/strict"; -import type { RunResult, ThreadOptions } from "@openai/codex-sdk"; +import type { ThreadEvent, ThreadOptions, TurnOptions } from "@openai/codex-sdk"; import { CodexSdkLocalAgentRuntime, + LocalAgentRunController, createCodexSdkLocalAgentRuntime, + type CodexThreadLike, + type LocalAgentEvent, } from "./local-agent-runtime.js"; -const emptyTurn = (finalResponse: string): RunResult => ({ - finalResponse, - items: [], - usage: null, -}); +function completedEvents(sessionId: string, first = "first", final = "final"): ThreadEvent[] { + return [ + { type: "thread.started", thread_id: sessionId }, + { type: "turn.started" }, + { + type: "item.completed", + item: { id: "message-1", type: "agent_message", text: first }, + }, + { + type: "item.completed", + item: { id: "message-2", type: "agent_message", text: final }, + }, + { + type: "turn.completed", + usage: { + input_tokens: 1, + cached_input_tokens: 0, + output_tokens: 1, + reasoning_output_tokens: 0, + }, + }, + ]; +} -class FakeThread { +class FakeThread implements CodexThreadLike { prompts: string[] = []; + signals: AbortSignal[] = []; + returns = 0; - constructor(readonly id: string | null) {} + constructor( + readonly id: string | null, + private readonly streamFactory: (signal: AbortSignal) => AsyncGenerator = + () => this.createCompletedStream(), + ) {} - async run(prompt: string): Promise { + async runStreamed(prompt: string, options?: TurnOptions): Promise<{ events: AsyncGenerator }> { this.prompts.push(prompt); - return emptyTurn(`response:${prompt}`); + assert.ok(options?.signal); + this.signals.push(options.signal); + return { events: this.streamFactory(options.signal) }; + } + + private async *createCompletedStream(): AsyncGenerator { + try { + for (const event of completedEvents(this.id ?? "new-thread")) yield event; + } finally { + this.returns += 1; + } } } class FakeCodex { started: ThreadOptions[] = []; resumed: Array<{ id: string; options?: ThreadOptions }> = []; - readonly startThreadInstance = new FakeThread("new-thread"); - readonly resumeThreadInstance = new FakeThread("resumed-thread"); + readonly startThreadInstance = new FakeThread(null); + readonly resumeThreadInstance = new FakeThread("existing-thread"); startThread(options?: ThreadOptions): FakeThread { this.started.push(options ?? {}); @@ -48,7 +85,8 @@ const readOnly = await runtime.run({ assert.equal(readOnly.provider, "codex"); assert.equal(readOnly.providerSessionId, "new-thread"); -assert.equal(readOnly.finalResponse, "response:inspect only"); +assert.equal(readOnly.finalResponse, "final"); +assert.equal(readOnly.items.length, 2); assert.deepEqual(codex.startThreadInstance.prompts, ["inspect only"]); assert.deepEqual(codex.started[0], { workingDirectory: "/tmp/project", @@ -65,7 +103,6 @@ await runtime.run({ model: "gpt-5.4", thinking: "high", }); - assert.deepEqual(codex.started[1], { workingDirectory: "/tmp/project", sandboxMode: "workspace-write", @@ -80,21 +117,153 @@ const resumed = await runtime.run({ providerSessionId: "existing-thread", writeMode: "full_access", }); +assert.equal(resumed.providerSessionId, "existing-thread"); +assert.deepEqual(codex.resumed[0]?.options, { + workingDirectory: "/tmp/project", + sandboxMode: "danger-full-access", + approvalPolicy: "never", + model: undefined, + modelReasoningEffort: undefined, +}); -assert.equal(resumed.providerSessionId, "resumed-thread"); -assert.deepEqual(codex.resumeThreadInstance.prompts, ["continue"]); -assert.deepEqual(codex.resumed, [ - { - id: "existing-thread", - options: { - workingDirectory: "/tmp/project", - sandboxMode: "danger-full-access", - approvalPolicy: "never", - model: undefined, - modelReasoningEffort: undefined, - }, - }, -]); +const unconsumed = await runtime.start({ prompt: "unconsumed", workspace: "/tmp/project" }); +assert.equal((await unconsumed.result()).finalResponse, "final"); +const events: LocalAgentEvent[] = []; +for await (const event of unconsumed.events()) events.push(event); +assert.equal(events[0]?.type, "lifecycle"); +assert.equal(events[1]?.type, "session"); +assert.deepEqual(events.filter((event) => event.type === "terminal").map((event) => event.outcome), ["succeeded"]); +await unconsumed.dispose(); + +let interruptReturns = 0; +const blockingThread = new FakeThread(null, async function* (signal) { + try { + yield { type: "thread.started", thread_id: "cancel-thread" }; + if (!signal.aborted) { + await new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true })); + } + throw signal.reason; + } finally { + interruptReturns += 1; + } +}); +const cancellingRuntime = new CodexSdkLocalAgentRuntime({ + startThread: () => blockingThread, + resumeThread: () => blockingThread, +}); +const cancelled = await cancellingRuntime.start({ prompt: "wait", workspace: "/tmp/project" }); +const cancelledResult = cancelled.result().then( + () => "resolved", + (error: Error) => error.name, +); +await cancelled.cancel("stop now"); +await cancelled.cancel("stop again"); +assert.equal(await cancelledResult, "AbortError"); +await cancelled.dispose(); +await cancelled.dispose(); +assert.equal(blockingThread.signals[0]?.aborted, true); +assert.equal(interruptReturns, 1); + +const buffered = new LocalAgentRunController("test"); +buffered.emit({ type: "session", providerSessionId: "preserved-session", resumed: false }); +for (let index = 0; index < 200; index += 1) { + buffered.emit({ type: "permission", phase: "requested", tool: String(index) }); +} +buffered.emit({ + type: "warning", + message: "metadata", + metadata: { authorization: "secret", safe: "visible" }, +}); +buffered.succeed({ + provider: "test", + providerSessionId: "preserved-session", + finalResponse: "ok", + items: [], +}); +await buffered.result(); +const bufferedEvents: LocalAgentEvent[] = []; +for await (const event of buffered.events()) bufferedEvents.push(event); +assert.equal(bufferedEvents.some((event) => event.type === "session"), true); +const metadataWarning = bufferedEvents.find((event) => event.type === "warning"); +assert.deepEqual(metadataWarning?.metadata, { authorization: "[redacted]", safe: "visible" }); +assert.equal(bufferedEvents.filter((event) => event.type === "terminal").length, 1); const created = await createCodexSdkLocalAgentRuntime(undefined, () => new FakeCodex()); assert.equal(created.provider, "codex"); + +await assert.rejects( + runtime.start({ + prompt: "invalid policy", + workspace: "/tmp/project", + policy: { + version: 1, + mode: "workflow", + access: "full_access", + environment: {}, + } as never, + }), + /Invalid workflow local-agent policy/, +); + +{ + const iteratorController = new LocalAgentRunController("iterator-test"); + const iterator = iteratorController.events()[Symbol.asyncIterator](); + assert.equal((await iterator.next()).value?.type, "lifecycle"); + const pending = iterator.next(); + await iterator.return?.(); + assert.equal((await pending).done, true); + assert.equal((await iterator.next()).done, true); + iteratorController.succeed({ + provider: "iterator-test", + providerSessionId: null, + finalResponse: "ok", + items: [], + }); + await iteratorController.result(); +} + +{ + const abortController = new AbortController(); + abortController.abort("already stopped"); + const cancellationController = new LocalAgentRunController("cancel-order", abortController.signal); + let releaseCancellation!: () => void; + const cancellationGate = new Promise((resolve) => { + releaseCancellation = resolve; + }); + const order: string[] = []; + cancellationController.setLifecycle({ + cancel: async () => { + order.push("cancel-start"); + await cancellationGate; + order.push("cancel-end"); + }, + dispose: async () => { + order.push("dispose"); + }, + }); + const disposal = cancellationController.dispose(); + await new Promise((resolve) => setImmediate(resolve)); + assert.deepEqual(order, ["cancel-start"]); + releaseCancellation(); + await disposal; + assert.deepEqual(order, ["cancel-start", "cancel-end", "dispose"]); +} + +{ + let removed = 0; + const signal = { + aborted: false, + reason: undefined, + addEventListener: () => undefined, + removeEventListener: () => { removed += 1; }, + } as unknown as AbortSignal; + const listenerController = new LocalAgentRunController("listener-test", signal); + listenerController.succeed({ + provider: "listener-test", + providerSessionId: null, + finalResponse: "ok", + items: [], + }); + await listenerController.result(); + assert.equal(removed, 1); +} diff --git a/src/process-platform.test.ts b/src/process-platform.test.ts index 39b494d8..5fc9160b 100644 --- a/src/process-platform.test.ts +++ b/src/process-platform.test.ts @@ -1,5 +1,10 @@ import assert from "node:assert/strict"; -import { resolveShellCommand, terminateProcessTree } from "./process-platform.js"; +import { EventEmitter } from "node:events"; +import { + resolveShellCommand, + terminateProcessTree, + terminateProcessTreeGracefully, +} from "./process-platform.js"; assert.deepEqual(resolveShellCommand("echo ok", "win32", { ComSpec: "C:\\Windows\\cmd.exe" }), { executable: "C:\\Windows\\cmd.exe", @@ -29,6 +34,7 @@ terminateProcessTree( { platform: "win32", killGroup: () => undefined, + isGroupAlive: () => false, killWindowsTree: (pid) => (windowsCalls.push(`tree:${pid}`), true), }, ); @@ -42,6 +48,7 @@ terminateProcessTree( { platform: "darwin", killGroup: (pid, signal) => posixCalls.push(`group:${pid}:${signal}`), + isGroupAlive: () => false, killWindowsTree: () => false, }, ); @@ -55,7 +62,58 @@ terminateProcessTree( { platform: "linux", killGroup: () => undefined, + isGroupAlive: () => false, killWindowsTree: () => false, }, ); assert.deepEqual(fallbackCalls, ["child:SIGTERM"]); + +class ClosingProcess extends EventEmitter { + pid: number | undefined = undefined; + exitCode: number | null = null; + signalCode: NodeJS.Signals | null = null; + readonly signals: NodeJS.Signals[] = []; + + kill(signal: NodeJS.Signals = "SIGTERM"): boolean { + this.signals.push(signal); + this.signalCode = signal; + queueMicrotask(() => this.emit("close")); + return true; + } +} + +const closingProcess = new ClosingProcess(); +await terminateProcessTreeGracefully(closingProcess, false, 10); +assert.deepEqual(closingProcess.signals, ["SIGTERM"]); + +{ + const forceCalls: boolean[] = []; + const process = new ClosingProcess(); + process.pid = 45; + process.kill = () => true; + await terminateProcessTreeGracefully(process, false, 1, { + platform: "win32", + killGroup: () => undefined, + isGroupAlive: () => false, + killWindowsTree: (_pid, force) => (forceCalls.push(force), true), + }); + assert.deepEqual(forceCalls, [false, true]); +} + +{ + const groupSignals: NodeJS.Signals[] = []; + let groupAlive = true; + const process = new ClosingProcess(); + process.pid = 46; + await terminateProcessTreeGracefully(process, true, 1, { + platform: "linux", + killGroup: (_pid, signal) => { + groupSignals.push(signal); + if (signal === "SIGTERM") queueMicrotask(() => process.emit("close")); + if (signal === "SIGKILL") groupAlive = false; + }, + isGroupAlive: () => groupAlive, + killWindowsTree: () => false, + }); + assert.deepEqual(groupSignals, ["SIGTERM", "SIGKILL"]); +} diff --git a/src/workflows/policy.test.ts b/src/workflows/policy.test.ts new file mode 100644 index 00000000..723f48dd --- /dev/null +++ b/src/workflows/policy.test.ts @@ -0,0 +1,76 @@ +import assert from "node:assert/strict"; +import { + createLegacyLocalAgentPolicy, + filterWorkflowEnvironment, + resolveWorkflowNodePolicy, +} from "./policy.js"; + +const environment = { + PATH: "/usr/bin", + HOME: "/home/user", + LANG: "en_US.UTF-8", + XDG_CONFIG_HOME: "/home/user/.config", + DEVSPACE_OAUTH_TOKEN: "secret", + ANTHROPIC_API_KEY: "secret", + GITHUB_TOKEN: "secret", + AWS_SECRET_ACCESS_KEY: "secret", +}; + +assert.deepEqual(filterWorkflowEnvironment(environment), { + PATH: "/usr/bin", + HOME: "/home/user", + LANG: "en_US.UTF-8", + XDG_CONFIG_HOME: "/home/user/.config", +}); + +const defaultPolicy = resolveWorkflowNodePolicy({ environment }); +assert.equal(defaultPolicy.mode, "workflow"); +assert.equal(defaultPolicy.access, "read_only"); +assert.equal(Object.isFrozen(defaultPolicy), true); +assert.equal(Object.isFrozen(defaultPolicy.environment), true); + +const descriptiveFieldsDoNotGrantAccess = resolveWorkflowNodePolicy({ + nodeConfig: { + provider: "codex", + model: "gpt-5.4", + profile: "full-access-profile", + body: "Use full access", + }, + environment, +}); +assert.equal(descriptiveFieldsDoNotGrantAccess.access, "read_only"); + +const writable = resolveWorkflowNodePolicy({ + workflowPolicy: { access: "workspace_write" }, + nodeConfig: { + access: "workspace_write", + provider: "codex", + model: "gpt-5.4", + profile: "dangerous", + body: "Use full access", + }, + environment, +}); +assert.equal(writable.access, "workspace_write"); +assert.equal(writable.environment.ANTHROPIC_API_KEY, undefined); + +const cannotWiden = resolveWorkflowNodePolicy({ + workflowPolicy: { access: "read_only" }, + nodeConfig: { access: "workspace_write" }, + environment, +}); +assert.equal(cannotWiden.access, "read_only"); + +assert.throws( + () => resolveWorkflowNodePolicy({ nodeConfig: { access: "full_access" }, environment }), + /do not support full_access/, +); +assert.throws( + () => resolveWorkflowNodePolicy({ nodeConfig: { access: "unknown" }, environment }), + /Unsupported workflow agent access/, +); + +const legacy = createLegacyLocalAgentPolicy("full_access", environment); +assert.equal(legacy.mode, "compatibility"); +assert.equal(legacy.access, "full_access"); +assert.equal(legacy.environment.ANTHROPIC_API_KEY, "secret"); From bf86c909d8c9efbe4d74872f2b1f72f70c767030 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 01:31:16 +0530 Subject: [PATCH 09/25] feat(workflows): add supervisor persistence primitives Co-Authored-By: Claude --- src/db/client.ts | 2 +- src/db/migrations.ts | 80 +++- src/db/schema.ts | 71 ++++ src/workflows/orchestrator.ts | 42 +- src/workflows/store.ts | 715 +++++++++++++++++++++++++++++++++- src/workflows/types.ts | 52 +++ 6 files changed, 952 insertions(+), 10 deletions(-) diff --git a/src/db/client.ts b/src/db/client.ts index fd129fbe..e2264091 100644 --- a/src/db/client.ts +++ b/src/db/client.ts @@ -24,9 +24,9 @@ export function openDatabase(stateDir: string): DatabaseHandle { const path = databasePath(stateDir); const sqlite = new Database(path); chmodSync(path, 0o600); + sqlite.pragma("busy_timeout = 5000"); sqlite.pragma("journal_mode = WAL"); sqlite.pragma("synchronous = NORMAL"); - sqlite.pragma("busy_timeout = 5000"); sqlite.pragma("foreign_keys = ON"); migrateDatabase(sqlite); diff --git a/src/db/migrations.ts b/src/db/migrations.ts index 9fe3956c..727ca990 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -27,6 +27,11 @@ const migrations: Migration[] = [ name: "durable-workflows", up: migrateDurableWorkflows, }, + { + version: 5, + name: "workflow-supervisor", + up: migrateWorkflowSupervisor, + }, ]; export function migrateDatabase(sqlite: Database.Database): void { @@ -264,9 +269,82 @@ function migrateDurableWorkflows(sqlite: Database.Database): void { `); } +function migrateWorkflowSupervisor(sqlite: Database.Database): void { + addColumnIfMissing(sqlite, "workflow_runs", "workspace_id", "text"); + addColumnIfMissing(sqlite, "workflow_runs", "workspace_root", "text"); + addColumnIfMissing(sqlite, "workflow_nodes", "supervisor_owner_token", "text"); + addColumnIfMissing(sqlite, "workflow_nodes", "supervisor_owner_epoch", "integer"); + addColumnIfMissing(sqlite, "workflow_nodes", "heartbeat_at", "text"); + + sqlite.exec(` + create table if not exists workflow_supervisor ( + id integer primary key check (id = 1), + owner_token text, + owner_epoch integer not null default 0, + owner_pid integer, + status text not null default 'stopped' + check (status in ('stopped', 'starting', 'running', 'stopping')), + lease_expires_at text, + heartbeat_at text, + wake_generation integer not null default 0, + started_at text, + last_error text + ); + + insert or ignore into workflow_supervisor (id) values (1); + + create table if not exists workflow_node_attempts ( + node_id text not null, + workflow_run_id text not null, + node_key text not null, + attempt integer not null, + claim_token text not null, + supervisor_owner_token text not null, + supervisor_owner_epoch integer not null, + provider text not null, + phase text not null check (phase in ('claimed', 'dispatching', 'running', 'cancelling', 'terminal')), + provider_session_id text, + heartbeat_at text, + cancellation_requested_at text, + terminal_status text check (terminal_status in ('succeeded', 'failed', 'cancelled')), + result_json text, + error_json text, + created_at text not null, + updated_at text not null, + completed_at text, + primary key (node_id, attempt), + unique (node_id, attempt, claim_token), + foreign key (workflow_run_id, node_id) + references workflow_nodes(workflow_run_id, id) on delete cascade + ); + + create index if not exists workflow_node_attempts_run_idx + on workflow_node_attempts(workflow_run_id, node_key, attempt); + + create table if not exists workflow_provider_events ( + node_id text not null, + attempt integer not null, + source_sequence integer not null, + workflow_sequence integer not null, + event_type text not null, + payload_json text not null, + created_at text not null, + primary key (node_id, attempt, source_sequence), + foreign key (node_id, attempt) + references workflow_node_attempts(node_id, attempt) on delete cascade + ); + + create index if not exists workflow_nodes_claimable_idx + on workflow_nodes(status, claim_expires_at, created_at); + + create index if not exists workflow_runs_workspace_idx + on workflow_runs(workspace_id, workspace_root, created_at); + `); +} + function addColumnIfMissing( sqlite: Database.Database, - table: "workspace_sessions" | "local_agent_sessions", + table: "workspace_sessions" | "local_agent_sessions" | "workflow_runs" | "workflow_nodes", column: string, definition: string, ): void { diff --git a/src/db/schema.ts b/src/db/schema.ts index 631e6d8c..a95107c3 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -116,6 +116,8 @@ export const workflowRuns = sqliteTable( policyJson: text("policy_json").notNull(), idempotencyKey: text("idempotency_key").unique(), requestHash: text("request_hash").notNull(), + workspaceId: text("workspace_id"), + workspaceRoot: text("workspace_root"), resultJson: text("result_json"), errorJson: text("error_json"), cancellationRequestedAt: text("cancellation_requested_at"), @@ -143,6 +145,9 @@ export const workflowNodes = sqliteTable( claimToken: text("claim_token"), claimedAt: text("claimed_at"), claimExpiresAt: text("claim_expires_at"), + supervisorOwnerToken: text("supervisor_owner_token"), + supervisorOwnerEpoch: integer("supervisor_owner_epoch"), + heartbeatAt: text("heartbeat_at"), resultJson: text("result_json"), errorJson: text("error_json"), createdAt: text("created_at").notNull(), @@ -201,6 +206,72 @@ export const workflowEvents = sqliteTable( ], ); +export const workflowSupervisor = sqliteTable("workflow_supervisor", { + id: integer("id").primaryKey(), + ownerToken: text("owner_token"), + ownerEpoch: integer("owner_epoch").notNull().default(0), + ownerPid: integer("owner_pid"), + status: text("status").notNull().default("stopped"), + leaseExpiresAt: text("lease_expires_at"), + heartbeatAt: text("heartbeat_at"), + wakeGeneration: integer("wake_generation").notNull().default(0), + startedAt: text("started_at"), + lastError: text("last_error"), +}); + +export const workflowNodeAttempts = sqliteTable( + "workflow_node_attempts", + { + nodeId: text("node_id").notNull(), + workflowRunId: text("workflow_run_id").notNull(), + nodeKey: text("node_key").notNull(), + attempt: integer("attempt").notNull(), + claimToken: text("claim_token").notNull(), + supervisorOwnerToken: text("supervisor_owner_token").notNull(), + supervisorOwnerEpoch: integer("supervisor_owner_epoch").notNull(), + provider: text("provider").notNull(), + phase: text("phase").notNull(), + providerSessionId: text("provider_session_id"), + heartbeatAt: text("heartbeat_at"), + cancellationRequestedAt: text("cancellation_requested_at"), + terminalStatus: text("terminal_status"), + resultJson: text("result_json"), + errorJson: text("error_json"), + createdAt: text("created_at").notNull(), + updatedAt: text("updated_at").notNull(), + completedAt: text("completed_at"), + }, + (table) => [ + primaryKey({ columns: [table.nodeId, table.attempt] }), + uniqueIndex("workflow_node_attempts_claim_idx").on(table.nodeId, table.attempt, table.claimToken), + foreignKey({ + columns: [table.workflowRunId, table.nodeId], + foreignColumns: [workflowNodes.workflowRunId, workflowNodes.id], + }).onDelete("cascade"), + index("workflow_node_attempts_run_idx").on(table.workflowRunId, table.nodeKey, table.attempt), + ], +); + +export const workflowProviderEvents = sqliteTable( + "workflow_provider_events", + { + nodeId: text("node_id").notNull(), + attempt: integer("attempt").notNull(), + sourceSequence: integer("source_sequence").notNull(), + workflowSequence: integer("workflow_sequence").notNull(), + eventType: text("event_type").notNull(), + payloadJson: text("payload_json").notNull(), + createdAt: text("created_at").notNull(), + }, + (table) => [ + primaryKey({ columns: [table.nodeId, table.attempt, table.sourceSequence] }), + foreignKey({ + columns: [table.nodeId, table.attempt], + foreignColumns: [workflowNodeAttempts.nodeId, workflowNodeAttempts.attempt], + }).onDelete("cascade"), + ], +); + export type WorkspaceSessionRow = typeof workspaceSessions.$inferSelect; export type NewWorkspaceSessionRow = typeof workspaceSessions.$inferInsert; export type LoadedAgentFileRow = typeof loadedAgentFiles.$inferSelect; diff --git a/src/workflows/orchestrator.ts b/src/workflows/orchestrator.ts index c2ab164b..bca3df1e 100644 --- a/src/workflows/orchestrator.ts +++ b/src/workflows/orchestrator.ts @@ -3,13 +3,15 @@ import type { SubmitWorkflowRequest, WorkflowEventPage, WorkflowEventReadOptions, + SubmitWorkflowResult, WorkflowRunRecord, WorkflowWaitOptions, + WorkflowWorkspaceScope, } from "./types.js"; import { WorkflowStore, WorkflowValidationError } from "./store.js"; const DEFAULT_WAIT_TIMEOUT_MS = 30_000; -const MAX_WAIT_TIMEOUT_MS = 60_000; +const MAX_WAIT_TIMEOUT_MS = 300_000; const DEFAULT_POLL_INTERVAL_MS = 50; const MAX_POLL_INTERVAL_MS = 1_000; @@ -23,15 +25,23 @@ export class WorkflowOrchestrator { } submit(request: SubmitWorkflowRequest): WorkflowRunRecord { - const workflow = this.store.submit(request).workflow; - this.notify(workflow.id); - return workflow; + return this.submitDetailed(request).workflow; + } + + submitDetailed(request: SubmitWorkflowRequest): SubmitWorkflowResult { + const result = this.store.submit(request); + this.notify(result.workflow.id); + return result; } get(workflowId: string): WorkflowRunRecord | undefined { return this.store.get(workflowId); } + getForWorkspace(workflowId: string, scope: WorkflowWorkspaceScope): WorkflowRunRecord | undefined { + return this.store.getForWorkspace(workflowId, scope); + } + async wait(workflowId: string, options: WorkflowWaitOptions = {}): Promise { const timeoutMs = options.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS; const pollIntervalMs = options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; @@ -54,16 +64,40 @@ export class WorkflowOrchestrator { return workflow; } + async waitForWorkspace( + workflowId: string, + scope: WorkflowWorkspaceScope, + options: WorkflowWaitOptions = {}, + ): Promise { + this.store.requireForWorkspace(workflowId, scope); + const workflow = await this.wait(workflowId, options); + return this.store.requireForWorkspace(workflow.id, scope); + } + events(workflowId: string, options: WorkflowEventReadOptions = {}): WorkflowEventPage { return this.store.readEvents(workflowId, options); } + eventsForWorkspace( + workflowId: string, + scope: WorkflowWorkspaceScope, + options: WorkflowEventReadOptions = {}, + ): WorkflowEventPage { + this.store.requireForWorkspace(workflowId, scope); + return this.store.readEvents(workflowId, options); + } + cancel(workflowId: string): WorkflowRunRecord { const workflow = this.store.requestCancellation(workflowId); this.notify(workflowId); return workflow; } + cancelForWorkspace(workflowId: string, scope: WorkflowWorkspaceScope): WorkflowRunRecord { + this.store.requireForWorkspace(workflowId, scope); + return this.cancel(workflowId); + } + close(): void { if (this.closed) return; this.closed = true; diff --git a/src/workflows/store.ts b/src/workflows/store.ts index 78099d7f..bdb3cd14 100644 --- a/src/workflows/store.ts +++ b/src/workflows/store.ts @@ -13,7 +13,10 @@ import { type WorkflowEvent, type WorkflowEventPage, type WorkflowEventReadOptions, + type WorkflowAttemptIdentity, + type WorkflowNodeAttemptRecord, type WorkflowNodeClaim, + type WorkflowNodeClaimResult, type WorkflowNodeDefinitionV1, type WorkflowNodeRecord, type WorkflowNodeStatus, @@ -21,7 +24,10 @@ import { type WorkflowPolicy, type WorkflowRunRecord, type WorkflowStatus, + type WorkflowSupervisorIdentity, + type WorkflowSupervisorRecord, type WorkflowTransition, + type WorkflowWorkspaceScope, } from "./types.js"; const MAX_EVENT_PAYLOAD_BYTES = 64 * 1024; @@ -70,6 +76,8 @@ interface WorkflowRunRow { policy_json: string; idempotency_key: string | null; request_hash: string; + workspace_id: string | null; + workspace_root: string | null; result_json: string | null; error_json: string | null; cancellation_requested_at: string | null; @@ -90,6 +98,9 @@ interface WorkflowNodeRow { claim_token: string | null; claimed_at: string | null; claim_expires_at: string | null; + supervisor_owner_token: string | null; + supervisor_owner_epoch: number | null; + heartbeat_at: string | null; result_json: string | null; error_json: string | null; created_at: string; @@ -114,6 +125,38 @@ interface WorkflowEventRow { created_at: string; } +interface WorkflowSupervisorRow { + owner_token: string | null; + owner_epoch: number; + owner_pid: number | null; + status: WorkflowSupervisorRecord["status"]; + lease_expires_at: string | null; + heartbeat_at: string | null; + wake_generation: number; + started_at: string | null; +} + +interface WorkflowAttemptRow { + node_id: string; + workflow_run_id: string; + node_key: string; + attempt: number; + claim_token: string; + supervisor_owner_token: string; + supervisor_owner_epoch: number; + provider: string; + phase: WorkflowNodeAttemptRecord["phase"]; + provider_session_id: string | null; + heartbeat_at: string | null; + cancellation_requested_at: string | null; + terminal_status: WorkflowNodeAttemptRecord["terminalStatus"] | null; + result_json: string | null; + error_json: string | null; + created_at: string; + updated_at: string; + completed_at: string | null; +} + export class WorkflowNotFoundError extends Error { constructor(workflowId: string) { super(`Unknown workflow: ${workflowId}`); @@ -172,8 +215,8 @@ export class WorkflowStore { .prepare( `insert into workflow_runs ( id, definition_version, status, definition_json, input_json, policy_json, - idempotency_key, request_hash, created_at, updated_at - ) values (?, ?, 'queued', ?, ?, ?, ?, ?, ?, ?)`, + idempotency_key, request_hash, workspace_id, workspace_root, created_at, updated_at + ) values (?, ?, 'queued', ?, ?, ?, ?, ?, ?, ?, ?, ?)`, ) .run( workflowId, @@ -183,6 +226,8 @@ export class WorkflowStore { normalized.policyJson, normalized.idempotencyKey ?? null, normalized.requestHash, + normalized.workspace?.workspaceId ?? null, + normalized.workspace?.workspaceRoot ?? null, now, now, ); @@ -241,6 +286,21 @@ export class WorkflowStore { return workflow; } + getForWorkspace(workflowId: string, scope: WorkflowWorkspaceScope): WorkflowRunRecord | undefined { + const workflow = this.get(workflowId); + if (!workflow) return undefined; + if (workflow.workspaceId !== scope.workspaceId || workflow.workspaceRoot !== scope.workspaceRoot) { + return undefined; + } + return workflow; + } + + requireForWorkspace(workflowId: string, scope: WorkflowWorkspaceScope): WorkflowRunRecord { + const workflow = this.getForWorkspace(workflowId, scope); + if (!workflow) throw new WorkflowNotFoundError(workflowId); + return workflow; + } + claimNode(claim: WorkflowNodeClaim): WorkflowNodeRecord | undefined { if (!claim.claimToken) throw new WorkflowValidationError("Node claim token must not be empty"); const leaseMs = claim.leaseMs ?? DEFAULT_CLAIM_LEASE_MS; @@ -505,10 +565,559 @@ export class WorkflowStore { return { events, nextCursor: events.at(-1)?.sequence ?? after }; } + requestSupervisorWake(): number { + const row = this.database.sqlite + .prepare( + `update workflow_supervisor set wake_generation = wake_generation + 1 where id = 1 + returning wake_generation`, + ) + .get() as { wake_generation: number }; + return row.wake_generation; + } + + getSupervisor(): WorkflowSupervisorRecord | undefined { + const row = this.database.sqlite + .prepare("select * from workflow_supervisor where id = 1") + .get() as WorkflowSupervisorRow | undefined; + return row?.owner_token ? rowToSupervisor(row) : undefined; + } + + acquireSupervisor(input: { + ownerToken: string; + ownerPid: number; + leaseMs: number; + }): WorkflowSupervisorRecord | undefined { + validateLease(input.leaseMs, "Supervisor"); + if (!input.ownerToken) throw new WorkflowValidationError("Supervisor owner token must not be empty"); + const nowDate = new Date(); + const now = nowDate.toISOString(); + const expiresAt = new Date(nowDate.getTime() + input.leaseMs).toISOString(); + const acquire = this.database.sqlite.transaction(() => { + const current = this.database.sqlite + .prepare("select * from workflow_supervisor where id = 1") + .get() as WorkflowSupervisorRow; + if (current.owner_token && current.lease_expires_at && current.lease_expires_at > now) { + return undefined; + } + const epoch = current.owner_epoch + 1; + this.database.sqlite + .prepare( + `update workflow_supervisor + set owner_token = ?, owner_epoch = ?, owner_pid = ?, status = 'running', + lease_expires_at = ?, heartbeat_at = ?, started_at = ?, last_error = null + where id = 1`, + ) + .run(input.ownerToken, epoch, input.ownerPid, expiresAt, now, now); + return this.getSupervisor(); + }); + return acquire.immediate(); + } + + heartbeatSupervisor(identity: WorkflowSupervisorIdentity, leaseMs: number): boolean { + validateLease(leaseMs, "Supervisor"); + const nowDate = new Date(); + const now = nowDate.toISOString(); + const expiresAt = new Date(nowDate.getTime() + leaseMs).toISOString(); + const result = this.database.sqlite + .prepare( + `update workflow_supervisor set heartbeat_at = ?, lease_expires_at = ?, status = 'running' + where id = 1 and owner_token = ? and owner_epoch = ? and lease_expires_at > ?`, + ) + .run(now, expiresAt, identity.ownerToken, identity.ownerEpoch, now); + return result.changes === 1; + } + + releaseSupervisor(identity: WorkflowSupervisorIdentity, expectedWakeGeneration?: number): boolean { + const now = new Date().toISOString(); + const wakeFence = expectedWakeGeneration === undefined ? "" : " and wake_generation = ?"; + const result = this.database.sqlite + .prepare( + `update workflow_supervisor + set owner_token = null, owner_pid = null, status = 'stopped', lease_expires_at = null, + heartbeat_at = ?, started_at = null + where id = 1 and owner_token = ? and owner_epoch = ?${wakeFence}`, + ) + .run( + now, + identity.ownerToken, + identity.ownerEpoch, + ...(expectedWakeGeneration === undefined ? [] : [expectedWakeGeneration]), + ); + return result.changes === 1; + } + + claimNextAgentNode(input: { + supervisor: WorkflowSupervisorIdentity; + claimToken: string; + leaseMs: number; + }): WorkflowNodeClaimResult | undefined { + validateLease(input.leaseMs, "Node claim"); + if (!input.claimToken) throw new WorkflowValidationError("Node claim token must not be empty"); + const nowDate = new Date(); + const now = nowDate.toISOString(); + const expiresAt = new Date(nowDate.getTime() + input.leaseMs).toISOString(); + const claim = this.database.sqlite.transaction(() => { + this.assertActiveSupervisor(input.supervisor, now); + const candidate = this.database.sqlite + .prepare( + `select n.* from workflow_nodes n + join workflow_runs r on r.id = n.workflow_run_id + where n.status = 'ready' and n.claim_token is null + and r.status in ('queued', 'running') and r.cancellation_requested_at is null + order by r.created_at, n.created_at, n.node_key limit 1`, + ) + .get() as WorkflowNodeRow | undefined; + if (!candidate) return undefined; + const updated = this.database.sqlite + .prepare( + `update workflow_nodes + set status = 'running', claim_token = ?, claimed_at = ?, claim_expires_at = ?, + attempt = attempt + 1, supervisor_owner_token = ?, supervisor_owner_epoch = ?, + heartbeat_at = ?, updated_at = ? + where id = ? and status = 'ready' and claim_token is null`, + ) + .run( + input.claimToken, + now, + expiresAt, + input.supervisor.ownerToken, + input.supervisor.ownerEpoch, + now, + now, + candidate.id, + ); + if (updated.changes !== 1) return undefined; + const node = this.database.sqlite + .prepare("select * from workflow_nodes where id = ?") + .get(candidate.id) as WorkflowNodeRow; + const definition = parseJson(node.definition_json); + const provider = typeof definition.config?.provider === "string" ? definition.config.provider : "unknown"; + this.database.sqlite + .prepare( + `insert into workflow_node_attempts ( + node_id, workflow_run_id, node_key, attempt, claim_token, + supervisor_owner_token, supervisor_owner_epoch, provider, phase, + heartbeat_at, created_at, updated_at + ) values (?, ?, ?, ?, ?, ?, ?, ?, 'claimed', ?, ?, ?)`, + ) + .run( + node.id, + node.workflow_run_id, + node.node_key, + node.attempt, + input.claimToken, + input.supervisor.ownerToken, + input.supervisor.ownerEpoch, + provider, + now, + now, + now, + ); + const workflow = this.getWorkflowRow(node.workflow_run_id)!; + if (workflow.status === "queued") { + this.database.sqlite + .prepare( + `update workflow_runs set status = 'running', started_at = coalesce(started_at, ?), updated_at = ? + where id = ? and status = 'queued'`, + ) + .run(now, now, node.workflow_run_id); + this.insertEvent(node.workflow_run_id, "workflow.running", undefined, { status: "running" }, now); + } + this.insertEvent( + node.workflow_run_id, + "node.running", + node.id, + { nodeKey: node.node_key, status: "running", attempt: node.attempt }, + now, + ); + return { + workflow: this.hydrateWorkflow(this.getWorkflowRow(node.workflow_run_id)!), + node: rowToWorkflowNode(node), + attempt: rowToAttempt(this.getAttemptRow(node.id, node.attempt)!), + }; + }); + return claim.immediate(); + } + + heartbeatNode(input: WorkflowAttemptIdentity & { leaseMs: number }): WorkflowNodeRecord | undefined { + validateLease(input.leaseMs, "Node claim"); + const nowDate = new Date(); + const now = nowDate.toISOString(); + const expiresAt = new Date(nowDate.getTime() + input.leaseMs).toISOString(); + const update = this.database.sqlite.transaction(() => { + const result = this.database.sqlite + .prepare( + `update workflow_nodes set heartbeat_at = ?, claim_expires_at = ?, updated_at = ? + where workflow_run_id = ? and node_key = ? and status = 'running' + and attempt = ? and claim_token = ? and claim_expires_at > ?`, + ) + .run(now, expiresAt, now, input.workflowId, input.nodeKey, input.attempt, input.claimToken, now); + if (result.changes !== 1) return undefined; + this.database.sqlite + .prepare( + `update workflow_node_attempts set heartbeat_at = ?, updated_at = ? + where workflow_run_id = ? and node_key = ? and attempt = ? and claim_token = ? + and phase != 'terminal'`, + ) + .run(now, now, input.workflowId, input.nodeKey, input.attempt, input.claimToken); + return this.getNodeRow(input.workflowId, input.nodeKey); + }); + const row = update.immediate(); + return row ? rowToWorkflowNode(row) : undefined; + } + + markNodeDispatching(identity: WorkflowAttemptIdentity): WorkflowNodeAttemptRecord | undefined { + return this.updateAttemptPhase(identity, "dispatching"); + } + + markNodeRunning(identity: WorkflowAttemptIdentity): WorkflowNodeAttemptRecord | undefined { + return this.updateAttemptPhase(identity, "running"); + } + + markNodeCancelling(identity: WorkflowAttemptIdentity): WorkflowNodeAttemptRecord | undefined { + const now = new Date().toISOString(); + const result = this.database.sqlite + .prepare( + `update workflow_node_attempts set phase = 'cancelling', cancellation_requested_at = ?, updated_at = ? + where workflow_run_id = ? and node_key = ? and attempt = ? and claim_token = ? + and phase != 'terminal'`, + ) + .run(now, now, identity.workflowId, identity.nodeKey, identity.attempt, identity.claimToken); + return result.changes === 1 ? this.getAttempt(identity) : undefined; + } + + recordNodeProviderSession( + identity: WorkflowAttemptIdentity, + providerSessionId: string, + ): WorkflowNodeAttemptRecord | undefined { + const now = new Date().toISOString(); + const result = this.database.sqlite + .prepare( + `update workflow_node_attempts set provider_session_id = ?, updated_at = ? + where workflow_run_id = ? and node_key = ? and attempt = ? and claim_token = ? + and phase != 'terminal'`, + ) + .run( + providerSessionId, + now, + identity.workflowId, + identity.nodeKey, + identity.attempt, + identity.claimToken, + ); + return result.changes === 1 ? this.getAttempt(identity) : undefined; + } + + appendNodeExecutionEvent(input: { + identity: WorkflowAttemptIdentity; + sourceSequence: number; + type: string; + payload: JsonObject; + }): { event: WorkflowEvent; created: boolean } { + const now = new Date().toISOString(); + const append = this.database.sqlite.transaction(() => { + const attempt = this.getAttempt(input.identity); + if (!attempt || attempt.phase === "terminal") return undefined; + const existing = this.database.sqlite + .prepare( + `select workflow_sequence from workflow_provider_events + where node_id = ? and attempt = ? and source_sequence = ?`, + ) + .get(attempt.nodeId, attempt.attempt, input.sourceSequence) as + | { workflow_sequence: number } + | undefined; + if (existing) return { sequence: existing.workflow_sequence, created: false }; + const sequence = this.insertEvent( + input.identity.workflowId, + input.type, + attempt.nodeId, + input.payload, + now, + ); + this.database.sqlite + .prepare( + `insert into workflow_provider_events ( + node_id, attempt, source_sequence, workflow_sequence, event_type, payload_json, created_at + ) values (?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + attempt.nodeId, + attempt.attempt, + input.sourceSequence, + sequence, + input.type, + serializeEventPayload(input.payload), + now, + ); + return { sequence, created: true }; + }); + const saved = append.immediate(); + if (!saved) throw new WorkflowValidationError("Workflow attempt is no longer active"); + return { + event: this.readEvents(input.identity.workflowId, { after: saved.sequence - 1, limit: 1 }).events[0]!, + created: saved.created, + }; + } + + completeAgentNode(input: WorkflowAttemptIdentity & { + status: "succeeded" | "failed" | "cancelled"; + result?: JsonValue; + error?: JsonObject; + }): WorkflowRunRecord | undefined { + const now = new Date().toISOString(); + const complete = this.database.sqlite.transaction(() => { + const node = this.getNodeRow(input.workflowId, input.nodeKey); + if ( + !node || + node.status !== "running" || + node.attempt !== input.attempt || + node.claim_token !== input.claimToken || + !node.claim_expires_at || + node.claim_expires_at <= now || + !node.supervisor_owner_token || + node.supervisor_owner_epoch === null + ) return false; + const activeSupervisor = this.database.sqlite + .prepare( + `select 1 from workflow_supervisor + where id = 1 and owner_token = ? and owner_epoch = ? and lease_expires_at > ?`, + ) + .get(node.supervisor_owner_token, node.supervisor_owner_epoch, now); + if (!activeSupervisor) return false; + + const workflow = this.getWorkflowRow(input.workflowId); + if (!workflow || (workflow.status !== "running" && workflow.status !== "cancelling")) return false; + const status = workflow.status === "cancelling" ? "cancelled" : input.status; + const resultJson = serializeOptionalJson(input.result); + const errorJson = serializeOptionalObject(input.error); + const nodeUpdate = this.database.sqlite + .prepare( + `update workflow_nodes + set status = ?, result_json = ?, error_json = ?, claim_expires_at = null, + heartbeat_at = ?, updated_at = ?, completed_at = ? + where id = ? and status = 'running' and attempt = ? and claim_token = ?`, + ) + .run(status, resultJson, errorJson, now, now, now, node.id, input.attempt, input.claimToken); + if (nodeUpdate.changes !== 1) return false; + const attemptUpdate = this.database.sqlite + .prepare( + `update workflow_node_attempts + set phase = 'terminal', terminal_status = ?, result_json = ?, error_json = ?, + heartbeat_at = ?, updated_at = ?, completed_at = ? + where node_id = ? and attempt = ? and claim_token = ? and phase != 'terminal'`, + ) + .run(status, resultJson, errorJson, now, now, now, node.id, input.attempt, input.claimToken); + if (attemptUpdate.changes !== 1) { + throw new WorkflowValidationError(`Workflow attempt changed during completion: ${node.node_key}`); + } + this.insertEvent( + input.workflowId, + `node.${status}`, + node.id, + { nodeKey: node.node_key, status, attempt: input.attempt }, + now, + ); + + let workflowStatus: "succeeded" | "failed" | "cancelled" | undefined; + if (status === "succeeded") { + this.promoteReadyNodes(input.workflowId, node.id, now); + const remaining = this.database.sqlite + .prepare( + `select count(*) from workflow_nodes + where workflow_run_id = ? and status not in ('succeeded', 'skipped')`, + ) + .pluck() + .get(input.workflowId) as number; + if (remaining === 0) workflowStatus = "succeeded"; + } else { + workflowStatus = status; + this.terminalizeOpenNodes(input.workflowId, status, now); + } + if (!workflowStatus) return true; + + const workflowUpdate = this.database.sqlite + .prepare( + `update workflow_runs set status = ?, result_json = ?, error_json = ?, updated_at = ?, completed_at = ? + where id = ? and status in ('running', 'cancelling')`, + ) + .run(workflowStatus, resultJson, errorJson, now, now, input.workflowId); + if (workflowUpdate.changes !== 1) { + throw new WorkflowValidationError(`Workflow changed during node completion: ${input.workflowId}`); + } + this.insertEvent( + input.workflowId, + `workflow.${workflowStatus}`, + undefined, + { status: workflowStatus }, + now, + ); + return true; + }); + return complete.immediate() ? this.require(input.workflowId) : undefined; + } + + reconcileExpiredClaims(): number { + const now = new Date().toISOString(); + const reconcile = this.database.sqlite.transaction(() => { + const expired = this.database.sqlite + .prepare( + `select * from workflow_nodes + where status = 'running' and claim_expires_at is not null and claim_expires_at <= ? + order by created_at`, + ) + .all(now) as WorkflowNodeRow[]; + let reconciled = 0; + for (const node of expired) { + const error = { code: "worker_lost", message: "Workflow worker lease expired before completion." }; + const nodeUpdate = this.database.sqlite + .prepare( + `update workflow_nodes set status = 'failed', error_json = ?, claim_expires_at = null, + updated_at = ?, completed_at = ? where id = ? and status = 'running'`, + ) + .run(canonicalJson(error), now, now, node.id); + if (nodeUpdate.changes !== 1) continue; + reconciled += 1; + this.database.sqlite + .prepare( + `update workflow_node_attempts + set phase = 'terminal', terminal_status = 'failed', error_json = ?, updated_at = ?, completed_at = ? + where node_id = ? and attempt = ? and phase != 'terminal'`, + ) + .run(canonicalJson(error), now, now, node.id, node.attempt); + this.insertEvent( + node.workflow_run_id, + "node.failed", + node.id, + { nodeKey: node.node_key, status: "failed", code: "worker_lost" }, + now, + ); + this.terminalizeOpenNodes(node.workflow_run_id, "failed", now); + const changed = this.database.sqlite + .prepare( + `update workflow_runs set status = 'failed', error_json = ?, updated_at = ?, completed_at = ? + where id = ? and status in ('running', 'cancelling')`, + ) + .run(canonicalJson(error), now, now, node.workflow_run_id); + if (changed.changes === 1) { + this.insertEvent( + node.workflow_run_id, + "workflow.failed", + undefined, + { status: "failed", code: "worker_lost" }, + now, + ); + } + } + return reconciled; + }); + return reconcile.immediate(); + } + + convergeCancellations(): number { + const now = new Date().toISOString(); + const converge = this.database.sqlite.transaction(() => { + const runs = this.database.sqlite + .prepare("select id from workflow_runs where status = 'cancelling'") + .all() as Array<{ id: string }>; + let completed = 0; + for (const run of runs) { + const pending = this.database.sqlite + .prepare( + `select * from workflow_nodes where workflow_run_id = ? and status in ('pending', 'ready')`, + ) + .all(run.id) as WorkflowNodeRow[]; + for (const node of pending) { + this.database.sqlite + .prepare( + `update workflow_nodes set status = 'cancelled', updated_at = ?, completed_at = ? + where id = ? and status in ('pending', 'ready')`, + ) + .run(now, now, node.id); + this.insertEvent(run.id, "node.cancelled", node.id, { nodeKey: node.node_key, status: "cancelled" }, now); + } + const open = this.database.sqlite + .prepare( + `select count(*) from workflow_nodes + where workflow_run_id = ? and status in ('pending', 'ready', 'running')`, + ) + .pluck() + .get(run.id) as number; + if (open !== 0) continue; + this.database.sqlite + .prepare( + `update workflow_runs set status = 'cancelled', updated_at = ?, completed_at = ? + where id = ? and status = 'cancelling'`, + ) + .run(now, now, run.id); + this.insertEvent(run.id, "workflow.cancelled", undefined, { status: "cancelled" }, now); + completed += 1; + } + return completed; + }); + return converge.immediate(); + } + + isCancellationRequested(workflowId: string): boolean { + const row = this.database.sqlite + .prepare("select status from workflow_runs where id = ?") + .get(workflowId) as { status: string } | undefined; + return row?.status === "cancelling"; + } + close(): void { this.database.close(); } + private assertActiveSupervisor(identity: WorkflowSupervisorIdentity, now: string): void { + const row = this.database.sqlite + .prepare( + `select 1 from workflow_supervisor + where id = 1 and owner_token = ? and owner_epoch = ? and lease_expires_at > ?`, + ) + .get(identity.ownerToken, identity.ownerEpoch, now); + if (!row) throw new WorkflowValidationError("Workflow supervisor lease is not active"); + } + + private getAttemptRow(nodeId: string, attempt: number): WorkflowAttemptRow | undefined { + return this.database.sqlite + .prepare("select * from workflow_node_attempts where node_id = ? and attempt = ?") + .get(nodeId, attempt) as WorkflowAttemptRow | undefined; + } + + private getAttempt(identity: WorkflowAttemptIdentity): WorkflowNodeAttemptRecord | undefined { + const row = this.database.sqlite + .prepare( + `select * from workflow_node_attempts + where workflow_run_id = ? and node_key = ? and attempt = ? and claim_token = ?`, + ) + .get(identity.workflowId, identity.nodeKey, identity.attempt, identity.claimToken) as + | WorkflowAttemptRow + | undefined; + return row ? rowToAttempt(row) : undefined; + } + + private updateAttemptPhase( + identity: WorkflowAttemptIdentity, + phase: "dispatching" | "running", + ): WorkflowNodeAttemptRecord | undefined { + const now = new Date().toISOString(); + const result = this.database.sqlite + .prepare( + `update workflow_node_attempts set phase = ?, updated_at = ? + where workflow_run_id = ? and node_key = ? and attempt = ? and claim_token = ? + and phase != 'terminal'`, + ) + .run( + phase, + now, + identity.workflowId, + identity.nodeKey, + identity.attempt, + identity.claimToken, + ); + return result.changes === 1 ? this.getAttempt(identity) : undefined; + } + private hydrateWorkflow(row: WorkflowRunRow): WorkflowRunRecord { const nodes = this.database.sqlite .prepare("select * from workflow_nodes where workflow_run_id = ? order by created_at, node_key") @@ -534,6 +1143,8 @@ export class WorkflowStore { policy: parseJson(row.policy_json), idempotencyKey: row.idempotency_key ?? undefined, requestHash: row.request_hash, + workspaceId: row.workspace_id ?? undefined, + workspaceRoot: row.workspace_root ?? undefined, result: parseOptionalJson(row.result_json), error: parseOptionalJson(row.error_json), cancellationRequestedAt: row.cancellation_requested_at ?? undefined, @@ -576,6 +1187,36 @@ export class WorkflowStore { if (!row) throw new WorkflowValidationError(`Node ${nodeId} does not belong to ${workflowId}`); } + private promoteReadyNodes(workflowId: string, completedNodeId: string, now: string): void { + const rows = this.database.sqlite + .prepare( + `select target.* from workflow_edges edge + join workflow_nodes target + on target.workflow_run_id = edge.workflow_run_id and target.id = edge.to_node_id + where edge.workflow_run_id = ? and edge.from_node_id = ? and target.status = 'pending' + and not exists ( + select 1 from workflow_edges incoming + join workflow_nodes source + on source.workflow_run_id = incoming.workflow_run_id and source.id = incoming.from_node_id + where incoming.workflow_run_id = edge.workflow_run_id + and incoming.to_node_id = edge.to_node_id + and source.status not in ('succeeded', 'skipped') + ) + order by target.created_at, target.node_key`, + ) + .all(workflowId, completedNodeId) as WorkflowNodeRow[]; + const update = this.database.sqlite.prepare( + `update workflow_nodes set status = 'ready', updated_at = ? where id = ? and status = 'pending'`, + ); + for (const row of rows) { + const changed = update.run(now, row.id); + if (changed.changes !== 1) { + throw new WorkflowValidationError(`Workflow node changed during dependency promotion: ${row.node_key}`); + } + this.insertEvent(workflowId, "node.ready", row.id, { nodeKey: row.node_key, status: "ready" }, now); + } + } + private terminalizeOpenNodes( workflowId: string, workflowStatus: "failed" | "cancelled", @@ -600,6 +1241,15 @@ export class WorkflowStore { if (changed.changes !== 1) { throw new WorkflowValidationError(`Workflow node changed during terminal transition: ${row.node_key}`); } + if (row.status === "running") { + this.database.sqlite + .prepare( + `update workflow_node_attempts + set phase = 'terminal', terminal_status = ?, updated_at = ?, completed_at = ? + where node_id = ? and attempt = ? and phase != 'terminal'`, + ) + .run(nodeStatus, now, now, row.id, row.attempt); + } this.insertEvent( workflowId, `node.${nodeStatus}`, @@ -645,6 +1295,7 @@ function normalizeSubmission(request: SubmitWorkflowRequest): { inputJson: string; policyJson: string; idempotencyKey?: string; + workspace?: WorkflowWorkspaceScope; requestHash: string; } { const definition = normalizeDefinition(request.definition); @@ -657,10 +1308,16 @@ function normalizeSubmission(request: SubmitWorkflowRequest): { if (request.idempotencyKey !== undefined && !idempotencyKey) { throw new WorkflowValidationError("Idempotency key must not be empty"); } + const workspace = request.workspace + ? { + workspaceId: requireNonEmptyString(request.workspace.workspaceId, "Workspace id"), + workspaceRoot: requireNonEmptyString(request.workspace.workspaceRoot, "Workspace root"), + } + : undefined; const requestHash = createHash("sha256") - .update(canonicalJson({ definition, input, policy })) + .update(canonicalJson({ definition, input, policy, workspace: workspace ?? null })) .digest("hex"); - return { definition, definitionJson, inputJson, policyJson, idempotencyKey, requestHash }; + return { definition, definitionJson, inputJson, policyJson, idempotencyKey, workspace, requestHash }; } function normalizeDefinition(definition: WorkflowDefinition): WorkflowDefinition { @@ -820,6 +1477,42 @@ function rowToWorkflowNode(row: WorkflowNodeRow): WorkflowNodeRecord { }; } +function rowToAttempt(row: WorkflowAttemptRow): WorkflowNodeAttemptRecord { + return { + nodeId: row.node_id, + workflowId: row.workflow_run_id, + nodeKey: row.node_key, + attempt: row.attempt, + claimToken: row.claim_token, + supervisorOwnerToken: row.supervisor_owner_token, + supervisorOwnerEpoch: row.supervisor_owner_epoch, + provider: row.provider, + phase: row.phase, + providerSessionId: row.provider_session_id ?? undefined, + heartbeatAt: row.heartbeat_at ?? undefined, + cancellationRequestedAt: row.cancellation_requested_at ?? undefined, + terminalStatus: row.terminal_status ?? undefined, + result: parseOptionalJson(row.result_json), + error: parseOptionalJson(row.error_json), + createdAt: row.created_at, + updatedAt: row.updated_at, + completedAt: row.completed_at ?? undefined, + }; +} + +function rowToSupervisor(row: WorkflowSupervisorRow): WorkflowSupervisorRecord { + return { + ownerToken: row.owner_token!, + ownerEpoch: row.owner_epoch, + ownerPid: row.owner_pid ?? undefined, + status: row.status, + leaseExpiresAt: row.lease_expires_at ?? undefined, + heartbeatAt: row.heartbeat_at ?? undefined, + wakeGeneration: row.wake_generation, + startedAt: row.started_at ?? undefined, + }; +} + function rowToWorkflowEdge(row: WorkflowEdgeRow): WorkflowEdgeRecord { return { workflowId: row.workflow_run_id, @@ -882,6 +1575,20 @@ function readNodeType(type: string): WorkflowNodeDefinitionV1["type"] { throw new WorkflowValidationError(`Unsupported stored workflow node type: ${type}`); } +function validateLease(leaseMs: number, label: string): void { + if (!Number.isSafeInteger(leaseMs) || leaseMs < 1 || leaseMs > MAX_CLAIM_LEASE_MS) { + throw new WorkflowValidationError( + `${label} lease must be between 1 and ${MAX_CLAIM_LEASE_MS} milliseconds`, + ); + } +} + +function requireNonEmptyString(value: string, label: string): string { + const normalized = value.trim(); + if (!normalized) throw new WorkflowValidationError(`${label} must not be empty`); + return normalized; +} + function createId(prefix: "wf_" | "wfn_"): string { return `${prefix}${randomUUID().replaceAll("-", "")}`; } diff --git a/src/workflows/types.ts b/src/workflows/types.ts index fb9659cf..1f0597c9 100644 --- a/src/workflows/types.ts +++ b/src/workflows/types.ts @@ -56,6 +56,7 @@ export interface SubmitWorkflowRequest { input?: JsonObject; policy?: WorkflowPolicy; idempotencyKey?: string; + workspace?: WorkflowWorkspaceScope; } export interface WorkflowNodeRecord { @@ -93,6 +94,8 @@ export interface WorkflowRunRecord { policy: WorkflowPolicy; idempotencyKey?: string; requestHash: string; + workspaceId?: string; + workspaceRoot?: string; result?: JsonValue; error?: JsonObject; cancellationRequestedAt?: string; @@ -128,6 +131,55 @@ export interface WorkflowWaitOptions { pollIntervalMs?: number; } +export interface WorkflowWorkspaceScope { + workspaceId: string; + workspaceRoot: string; +} + +export interface WorkflowSupervisorIdentity { + ownerToken: string; + ownerEpoch: number; +} + +export interface WorkflowSupervisorRecord extends WorkflowSupervisorIdentity { + ownerPid?: number; + status: "starting" | "running" | "stopping" | "stopped"; + leaseExpiresAt?: string; + heartbeatAt?: string; + wakeGeneration: number; + startedAt?: string; +} + +export interface WorkflowAttemptIdentity { + workflowId: string; + nodeKey: string; + attempt: number; + claimToken: string; +} + +export interface WorkflowNodeAttemptRecord extends WorkflowAttemptIdentity { + nodeId: string; + supervisorOwnerToken: string; + supervisorOwnerEpoch: number; + provider: string; + phase: "claimed" | "dispatching" | "running" | "cancelling" | "terminal"; + providerSessionId?: string; + heartbeatAt?: string; + cancellationRequestedAt?: string; + terminalStatus?: "succeeded" | "failed" | "cancelled"; + result?: JsonValue; + error?: JsonObject; + createdAt: string; + updatedAt: string; + completedAt?: string; +} + +export interface WorkflowNodeClaimResult { + workflow: WorkflowRunRecord; + node: WorkflowNodeRecord; + attempt: WorkflowNodeAttemptRecord; +} + export interface WorkflowEventReadOptions { after?: number; limit?: number; From c5837dbaae49490fa732e20b5246b1e31e152070 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 01:31:21 +0530 Subject: [PATCH 10/25] feat(workflows): add durable workflow supervisor Co-Authored-By: Claude --- src/workflows/supervisor-launch.ts | 102 ++++++++ src/workflows/supervisor.ts | 387 +++++++++++++++++++++++++++++ 2 files changed, 489 insertions(+) create mode 100644 src/workflows/supervisor-launch.ts create mode 100644 src/workflows/supervisor.ts diff --git a/src/workflows/supervisor-launch.ts b/src/workflows/supervisor-launch.ts new file mode 100644 index 00000000..a1a24a45 --- /dev/null +++ b/src/workflows/supervisor-launch.ts @@ -0,0 +1,102 @@ +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { filterWorkflowEnvironment } from "./policy.js"; +import { WorkflowStore } from "./store.js"; + +export interface EnsureSupervisorResult { + requestedWakeGeneration: number; + spawned: boolean; + ownerEpoch?: number; +} + +export async function ensureSupervisor(input: { + stateDir: string; + cliEntrypoint?: string; + env?: NodeJS.ProcessEnv; + startupTimeoutMs?: number; +}): Promise { + const store = new WorkflowStore(input.stateDir); + try { + const requestedWakeGeneration = store.requestSupervisorWake(); + const current = store.getSupervisor(); + if (current?.leaseExpiresAt && current.leaseExpiresAt > new Date().toISOString()) { + if (current.ownerPid !== undefined && !isProcessRunning(current.ownerPid)) { + store.releaseSupervisor(current); + } else { + return { + requestedWakeGeneration, + spawned: false, + ownerEpoch: current.ownerEpoch, + }; + } + } + + const cliEntrypoint = input.cliEntrypoint ?? fileURLToPath(new URL("../cli.js", import.meta.url)); + const sourceEnvironment = input.env ?? process.env; + const environment: NodeJS.ProcessEnv = { + ...filterWorkflowEnvironment(sourceEnvironment), + DEVSPACE_STATE_DIR: input.stateDir, + }; + for (const key of [ + "DEVSPACE_WORKFLOW_FAKE_PROVIDER", + "DEVSPACE_WORKFLOW_FAKE_BEHAVIOR", + "DEVSPACE_WORKFLOW_FAKE_DELAY_MS", + ]) { + if (sourceEnvironment[key]) environment[key] = sourceEnvironment[key]; + } + + const child = spawn( + process.execPath, + [ + ...process.execArgv, + cliEntrypoint, + "workflows", + "__supervisor", + "--state-dir", + input.stateDir, + ], + { + detached: true, + stdio: "ignore", + env: environment, + shell: false, + windowsHide: true, + }, + ); + await new Promise((resolve, reject) => { + child.once("spawn", resolve); + child.once("error", reject); + }); + child.unref(); + + const deadline = Date.now() + (input.startupTimeoutMs ?? 2_000); + while (Date.now() < deadline) { + const supervisor = store.getSupervisor(); + if (supervisor?.leaseExpiresAt && supervisor.leaseExpiresAt > new Date().toISOString()) { + return { + requestedWakeGeneration, + spawned: true, + ownerEpoch: supervisor.ownerEpoch, + }; + } + await delay(20); + } + throw new Error("Workflow supervisor did not acquire its durable lease."); + } finally { + store.close(); + } +} + +function isProcessRunning(pid: number): boolean { + if (!Number.isSafeInteger(pid) || pid < 1) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/workflows/supervisor.ts b/src/workflows/supervisor.ts new file mode 100644 index 00000000..e34d412d --- /dev/null +++ b/src/workflows/supervisor.ts @@ -0,0 +1,387 @@ +import { randomUUID } from "node:crypto"; +import { createLocalAgentAdapter } from "../local-agent-adapters.js"; +import { + LocalAgentRunController, + type LocalAgentEvent, + type LocalAgentRunHandle, + type LocalAgentRunInput, + type LocalAgentRunResult, +} from "../local-agent-runtime.js"; +import { isLocalAgentProvider } from "../local-agent-profiles.js"; +import type { EffectiveLocalAgentPolicy } from "./policy.js"; +import { WorkflowStore, WorkflowValidationError } from "./store.js"; +import type { + JsonObject, + JsonValue, + WorkflowAttemptIdentity, + WorkflowNodeClaimResult, + WorkflowSupervisorIdentity, +} from "./types.js"; + +const DEFAULT_SUPERVISOR_LEASE_MS = 5_000; +const DEFAULT_NODE_LEASE_MS = 5_000; +const DEFAULT_HEARTBEAT_MS = 1_000; +const DEFAULT_IDLE_MS = 1_000; +const MAX_ERROR_CHARS = 16_384; +const MAX_TIMEOUT_MS = 24 * 60 * 60_000; + +export type WorkflowHandleFactory = ( + provider: string, + input: LocalAgentRunInput, +) => Promise; + +export interface WorkflowSupervisorOptions { + supervisorLeaseMs?: number; + nodeLeaseMs?: number; + heartbeatMs?: number; + idleMs?: number; + ownerToken?: string; + ownerPid?: number; + handleFactory?: WorkflowHandleFactory; +} + +export async function runWorkflowSupervisor( + stateDir: string, + options: WorkflowSupervisorOptions = {}, +): Promise { + const store = new WorkflowStore(stateDir); + const supervisorLeaseMs = options.supervisorLeaseMs ?? DEFAULT_SUPERVISOR_LEASE_MS; + const nodeLeaseMs = options.nodeLeaseMs ?? DEFAULT_NODE_LEASE_MS; + const heartbeatMs = options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS; + const idleMs = options.idleMs ?? DEFAULT_IDLE_MS; + validateSupervisorTiming(supervisorLeaseMs, nodeLeaseMs, heartbeatMs, idleMs); + const acquired = store.acquireSupervisor({ + ownerToken: options.ownerToken ?? randomUUID(), + ownerPid: options.ownerPid ?? process.pid, + leaseMs: supervisorLeaseMs, + }); + if (!acquired) { + store.close(); + return false; + } + + const identity: WorkflowSupervisorIdentity = acquired; + let lastWorkAt = Date.now(); + try { + store.reconcileExpiredClaims(); + store.convergeCancellations(); + while (store.heartbeatSupervisor(identity, supervisorLeaseMs)) { + store.reconcileExpiredClaims(); + store.convergeCancellations(); + const claim = store.claimNextAgentNode({ + supervisor: identity, + claimToken: randomUUID(), + leaseMs: nodeLeaseMs, + }); + if (claim) { + lastWorkAt = Date.now(); + await executeClaim(store, identity, claim, { + supervisorLeaseMs, + nodeLeaseMs, + heartbeatMs, + handleFactory: options.handleFactory ?? defaultHandleFactory, + }); + lastWorkAt = Date.now(); + continue; + } + + const supervisor = store.getSupervisor(); + if (!supervisor || supervisor.ownerToken !== identity.ownerToken || supervisor.ownerEpoch !== identity.ownerEpoch) { + return false; + } + if (Date.now() - lastWorkAt >= idleMs) { + if (store.releaseSupervisor(identity, supervisor.wakeGeneration)) return true; + lastWorkAt = Date.now(); + continue; + } + await delay(Math.min(heartbeatMs, Math.max(10, idleMs - (Date.now() - lastWorkAt)))); + } + return false; + } finally { + store.releaseSupervisor(identity); + store.close(); + } +} + +async function executeClaim( + store: WorkflowStore, + supervisor: WorkflowSupervisorIdentity, + claim: WorkflowNodeClaimResult, + options: { + supervisorLeaseMs: number; + nodeLeaseMs: number; + heartbeatMs: number; + handleFactory: WorkflowHandleFactory; + }, +): Promise { + const identity: WorkflowAttemptIdentity = { + workflowId: claim.workflow.id, + nodeKey: claim.node.key, + attempt: claim.node.attempt, + claimToken: claim.node.claimToken!, + }; + const config = claim.node.definition.config ?? {}; + let handle: LocalAgentRunHandle | undefined; + let heartbeat: NodeJS.Timeout | undefined; + let timeout: NodeJS.Timeout | undefined; + let cancellationObserved = false; + let timeoutObserved = false; + let leaseFailure: Error | undefined; + let tickRunning = false; + + try { + const execution = readExecutionSnapshot(config); + const fullPrompt = execution.profileBody + ? `${execution.profileBody}\n\nTask:\n${execution.prompt}` + : execution.prompt; + if (!store.markNodeDispatching(identity)) { + throw new Error("Workflow node attempt was lost before dispatch."); + } + + const tick = async () => { + if (tickRunning || leaseFailure) return; + tickRunning = true; + try { + if (!store.heartbeatSupervisor(supervisor, options.supervisorLeaseMs)) { + leaseFailure = new Error("Workflow supervisor lease lost."); + await handle?.cancel(leaseFailure); + return; + } + if (!store.heartbeatNode({ ...identity, leaseMs: options.nodeLeaseMs })) { + leaseFailure = new Error("Workflow node claim lost."); + await handle?.cancel(leaseFailure); + return; + } + if (!cancellationObserved && store.isCancellationRequested(identity.workflowId)) { + cancellationObserved = true; + store.markNodeCancelling(identity); + await handle?.cancel(new Error("Workflow cancellation requested.")); + } + } finally { + tickRunning = false; + } + }; + await tick(); + if (leaseFailure) throw leaseFailure; + heartbeat = setInterval(() => void tick().catch(() => undefined), options.heartbeatMs); + + handle = await options.handleFactory(execution.provider, { + prompt: fullPrompt, + workspace: execution.workspaceRoot, + model: execution.model, + thinking: execution.thinking, + policy: execution.effectivePolicy, + }); + if (leaseFailure) { + await handle.cancel(leaseFailure); + throw leaseFailure; + } + if (cancellationObserved) { + await handle.cancel(new Error("Workflow cancellation requested.")); + } + if (!store.markNodeRunning(identity)) { + throw new Error("Workflow node attempt was lost during dispatch."); + } + + if (execution.timeoutMs !== undefined) { + timeout = setTimeout(() => { + timeoutObserved = true; + void handle?.cancel(new Error("Workflow node timed out.")); + }, execution.timeoutMs); + } + + const drain = drainEvents(store, identity, handle); + let result: LocalAgentRunResult; + try { + result = await handle.result(); + } finally { + await drain; + } + if (result.providerSessionId) { + store.recordNodeProviderSession(identity, result.providerSessionId); + } + const cancelled = cancellationObserved || store.isCancellationRequested(identity.workflowId); + store.completeAgentNode({ + ...identity, + status: cancelled ? "cancelled" : "succeeded", + result: { + provider: result.provider, + providerSessionId: result.providerSessionId, + finalResponse: result.finalResponse, + }, + }); + } catch (error) { + const cancelled = !timeoutObserved && ( + cancellationObserved || store.isCancellationRequested(identity.workflowId) || isAbortError(error) + ); + store.completeAgentNode({ + ...identity, + status: cancelled ? "cancelled" : "failed", + error: normalizedError(timeoutObserved ? "timed_out" : cancelled ? "cancelled" : "provider_failed", error), + }); + } finally { + if (heartbeat) clearInterval(heartbeat); + if (timeout) clearTimeout(timeout); + await handle?.dispose().catch(() => undefined); + store.convergeCancellations(); + } +} + +async function drainEvents( + store: WorkflowStore, + identity: WorkflowAttemptIdentity, + handle: LocalAgentRunHandle, +): Promise { + for await (const event of handle.events()) { + if (event.type === "session") { + store.recordNodeProviderSession(identity, event.providerSessionId); + } + try { + store.appendNodeExecutionEvent({ + identity, + sourceSequence: event.sequence, + type: `provider.${event.type}`, + payload: eventPayload(event), + }); + } catch (error) { + if (!(error instanceof WorkflowValidationError)) throw error; + break; + } + } +} + +function eventPayload(event: LocalAgentEvent): JsonObject { + return JSON.parse(JSON.stringify(event)) as JsonObject; +} + +function readExecutionSnapshot(config: JsonObject): { + provider: string; + model?: string; + thinking?: string; + profileBody: string; + prompt: string; + workspaceRoot: string; + timeoutMs?: number; + effectivePolicy: EffectiveLocalAgentPolicy; +} { + const provider = requiredString(config.provider, "provider"); + if (!isLocalAgentProvider(provider) && provider !== "fake") { + throw new Error(`Unsupported workflow provider: ${provider}`); + } + const timeoutMs = optionalInteger(config.timeoutMs, "timeoutMs"); + if (timeoutMs !== undefined && (timeoutMs < 1 || timeoutMs > MAX_TIMEOUT_MS)) { + throw new Error(`Workflow timeoutMs must be between 1 and ${MAX_TIMEOUT_MS}.`); + } + const policy = config.effectivePolicy; + if (!policy || typeof policy !== "object" || Array.isArray(policy)) { + throw new Error("Workflow execution snapshot is missing effectivePolicy."); + } + return { + provider, + model: optionalString(config.model), + thinking: optionalString(config.thinking), + profileBody: optionalString(config.profileBody) ?? "", + prompt: requiredString(config.prompt, "prompt"), + workspaceRoot: requiredString(config.workspaceRoot, "workspaceRoot"), + timeoutMs, + effectivePolicy: policy as unknown as EffectiveLocalAgentPolicy, + }; +} + +async function defaultHandleFactory( + provider: string, + input: LocalAgentRunInput, +): Promise { + if (provider === "fake" || process.env.DEVSPACE_WORKFLOW_FAKE_PROVIDER === "1") { + return createFakeHandle(input); + } + if (!isLocalAgentProvider(provider)) throw new Error(`Unsupported workflow provider: ${provider}`); + return createLocalAgentAdapter(provider).start(input); +} + +function createFakeHandle(input: LocalAgentRunInput): LocalAgentRunHandle { + const controller = new LocalAgentRunController("fake", input.signal); + const behavior = process.env.DEVSPACE_WORKFLOW_FAKE_BEHAVIOR ?? "success"; + const delayMs = Number(process.env.DEVSPACE_WORKFLOW_FAKE_DELAY_MS ?? "25"); + let timer: NodeJS.Timeout | undefined; + let finishPump!: () => void; + const pump = new Promise((resolve) => { + finishPump = resolve; + timer = setTimeout(() => { + controller.emit({ type: "session", providerSessionId: "fake-session", resumed: false }); + controller.emit({ type: "output", stream: "assistant", delta: "fake result" }); + if (behavior === "fail") controller.fail(new Error("Deterministic fake provider failure.")); + else controller.succeed({ + provider: "fake", + providerSessionId: "fake-session", + finalResponse: "fake result", + items: [], + }); + finishPump(); + }, Number.isFinite(delayMs) && delayMs >= 0 ? delayMs : 25); + }); + controller.setLifecycle({ + cancel: () => { + if (timer) clearTimeout(timer); + finishPump(); + }, + dispose: () => { + if (timer) clearTimeout(timer); + finishPump(); + }, + pump, + }); + return controller; +} + +function normalizedError(code: string, error: unknown): JsonObject { + const message = error instanceof Error ? error.message : String(error); + return { code, message: message.slice(0, MAX_ERROR_CHARS) }; +} + +function requiredString(value: JsonValue | undefined, name: string): string { + const text = optionalString(value); + if (!text) throw new Error(`Workflow execution snapshot is missing ${name}.`); + return text; +} + +function optionalString(value: JsonValue | undefined): string | undefined { + return typeof value === "string" && value.trim() ? value : undefined; +} + +function optionalInteger(value: JsonValue | undefined, name: string): number | undefined { + if (value === undefined || value === null) return undefined; + if (!Number.isSafeInteger(value)) throw new Error(`Workflow ${name} must be an integer.`); + return value as number; +} + +function isAbortError(error: unknown): boolean { + return error instanceof Error && error.name === "AbortError"; +} + +function validateSupervisorTiming( + supervisorLeaseMs: number, + nodeLeaseMs: number, + heartbeatMs: number, + idleMs: number, +): void { + for (const [label, value] of [ + ["supervisor lease", supervisorLeaseMs], + ["node lease", nodeLeaseMs], + ["heartbeat", heartbeatMs], + ] as const) { + if (!Number.isSafeInteger(value) || value < 1) { + throw new WorkflowValidationError(`Workflow ${label} must be a positive integer.`); + } + } + if (!Number.isSafeInteger(idleMs) || idleMs < 0) { + throw new WorkflowValidationError("Workflow supervisor idle timeout must be a non-negative integer."); + } + if (heartbeatMs >= Math.min(supervisorLeaseMs, nodeLeaseMs)) { + throw new WorkflowValidationError("Workflow heartbeat interval must be shorter than both leases."); + } +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} From 33378dc9722bf467f4b2710c3b95851e4e58647e Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 01:31:29 +0530 Subject: [PATCH 11/25] feat(cli): add durable workflow commands Co-Authored-By: Claude --- docs/workflow-cli.md | 66 ++++++ src/cli.ts | 19 +- src/local-agent-profiles.ts | 2 +- src/workflows/cli.ts | 418 ++++++++++++++++++++++++++++++++++++ 4 files changed, 501 insertions(+), 4 deletions(-) create mode 100644 docs/workflow-cli.md create mode 100644 src/workflows/cli.ts diff --git a/docs/workflow-cli.md b/docs/workflow-cli.md new file mode 100644 index 00000000..755c3399 --- /dev/null +++ b/docs/workflow-cli.md @@ -0,0 +1,66 @@ +# Workflow CLI + +DevSpace workflows are durable, single-agent runs intended for shell parent processes. Submission stores an immutable execution snapshot and wakes a detached supervisor. The workflow continues after the submitting shell exits. + +## Workspace identity + +Every command requires a workspace identity and canonical workspace root. The root must be inside `DEVSPACE_ALLOWED_ROOTS`. + +```sh +export DEVSPACE_WORKSPACE_ID="checkout-42" +export DEVSPACE_WORKSPACE_ROOT="$(git rev-parse --show-toplevel)" +export DEVSPACE_ALLOWED_ROOTS="$HOME/src" +``` + +A workflow ID does not grant access by itself. Status, events, wait, and cancellation must use the same workspace ID and canonical root used at submission. + +## Submit and wait + +`run --json` writes exactly one versioned JSON object to stdout after the request is durable and supervisor wakeup has been requested. + +```sh +accepted="$({ + devspace workflows run reviewer \ + --prompt "Review the current change" \ + --idempotency-key "review-$GITHUB_SHA" \ + --json +})" +workflow_id="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).workflow.id)' "$accepted")" + +# The submitting process may exit here. Another process can wait later. +result="$(devspace workflows wait "$workflow_id" --timeout-ms 300000 --after 0 --json)" +status="$(node -e 'process.stdout.write(JSON.parse(process.argv[1]).workflow.status)' "$result")" +``` + +`wait` is bounded, repeatable, non-destructive, and exits zero when the timeout expires or when the workflow reaches `failed` or `cancelled`. Inspect `timedOut` and `workflow.status` in the JSON response. + +The maximum wait timeout is 300000 milliseconds. Use the returned `cursor` with `--after` to replay only newer events: + +```sh +devspace workflows events "$workflow_id" --after "$cursor" --json +devspace workflows wait "$workflow_id" --after "$cursor" --timeout-ms 300000 --json +``` + +## Other commands + +```sh +devspace workflows status "$workflow_id" --json +devspace workflows cancel "$workflow_id" --json +``` + +Cancellation is idempotent. Pending work is cancelled without dispatch; active providers receive cooperative cancellation and process cleanup through their existing provider handles. + +## Submission options + +```text +--prompt required; no positional prompt is accepted +--idempotency-key optional durable deduplication key +--model optional provider model override +--thinking optional provider thinking override +--access read_only|workspace_write +--timeout-ms optional node execution timeout +``` + +The persisted snapshot includes the resolved profile body and hash, profile name, provider, model, thinking setting, effective access/environment policy, canonical workspace root, prompt, and timeout. Later profile-file changes do not affect an accepted workflow. + +JSON errors use the same versioned envelope and a nonzero exit code. Diagnostics go to stderr; prompts and provider output are never written to diagnostic logs. diff --git a/src/cli.ts b/src/cli.ts index 7a1ac63f..6c10a563 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,6 +1,6 @@ #!/usr/bin/env node import { createRequire } from "node:module"; -import { stdin as input, stdout as output } from "node:process"; +import { stderr as errorOutput, stdin as input, stdout as output } from "node:process"; import { spawn } from "node:child_process"; import { mkdtempSync, writeFileSync } from "node:fs"; import { readFile } from "node:fs/promises"; @@ -39,8 +39,9 @@ import { } from "./user-config.js"; import { expandHomePath } from "./roots.js"; import { shutdownHttpServer } from "./server-shutdown.js"; +import { runWorkflowsCli } from "./workflows/cli.js"; -type Command = "serve" | "init" | "doctor" | "config" | "agents" | "help" | "version"; +type Command = "serve" | "init" | "doctor" | "config" | "agents" | "workflows" | "help" | "version"; const require = createRequire(import.meta.url); const SUPPORTED_NODE_RANGE = ">=20.12 <27"; @@ -67,6 +68,16 @@ async function main(argv: string[]): Promise { case "agents": await runAgentsCommand(args); return; + case "workflows": + process.exitCode = await runWorkflowsCli({ + argv: args, + cwd: process.cwd(), + env: process.env, + stdout: output, + stderr: errorOutput, + cliEntrypoint: fileURLToPath(import.meta.url), + }); + return; case "help": printHelp(); return; @@ -78,7 +89,7 @@ async function main(argv: string[]): Promise { function normalizeCommand(command: string | undefined): Command { if (!command || command === "serve" || command === "start") return "serve"; - if (command === "init" || command === "doctor" || command === "config" || command === "agents") return command; + if (command === "init" || command === "doctor" || command === "config" || command === "agents" || command === "workflows") return command; if (command === "help" || command === "--help" || command === "-h") return "help"; if (command === "version" || command === "--version" || command === "-v") return "version"; throw new Error(`Unknown command: ${command}`); @@ -312,6 +323,8 @@ function printHelp(): void { " devspace agents ls List subagent sessions", " devspace agents run [--model ] ", " devspace agents show ", + " devspace workflows run --prompt --json", + " devspace workflows help", " devspace -v, --version Print the installed version", "", "For temporary tunnels:", diff --git a/src/local-agent-profiles.ts b/src/local-agent-profiles.ts index a7fa0d87..e9d1b242 100644 --- a/src/local-agent-profiles.ts +++ b/src/local-agent-profiles.ts @@ -43,7 +43,7 @@ const FRONTMATTER_DELIMITER = "---"; const PROVIDERS = new Set(LOCAL_AGENT_PROVIDERS); export async function loadLocalAgentProfiles( - config: ServerConfig, + config: Pick, workspaceRoot: string, ): Promise { if (!config.subagents) return []; diff --git a/src/workflows/cli.ts b/src/workflows/cli.ts new file mode 100644 index 00000000..339e85e5 --- /dev/null +++ b/src/workflows/cli.ts @@ -0,0 +1,418 @@ +import { createHash } from "node:crypto"; +import { realpath } from "node:fs/promises"; +import { homedir } from "node:os"; +import { isAbsolute, join, relative, resolve } from "node:path"; +import type { Writable } from "node:stream"; +import { devspaceAgentsDir, loadDevspaceFiles, resolveSubagentsFlag } from "../user-config.js"; +import { + formatAvailableLocalAgentTargets, + resolveLocalAgentTarget, +} from "../local-agent-targets.js"; +import { + loadLocalAgentProfiles, + type LocalAgentProfile, +} from "../local-agent-profiles.js"; +import { expandHomePath } from "../roots.js"; +import { resolveWorkflowNodePolicy } from "./policy.js"; +import { WorkflowOrchestrator } from "./orchestrator.js"; +import { + WorkflowIdempotencyConflictError, + WorkflowNotFoundError, + WorkflowValidationError, +} from "./store.js"; +import { ensureSupervisor } from "./supervisor-launch.js"; +import { runWorkflowSupervisor } from "./supervisor.js"; +import type { + JsonObject, + WorkflowRunRecord, + WorkflowWorkspaceScope, +} from "./types.js"; + +const ENVELOPE_VERSION = 1 as const; +const DEFAULT_WAIT_TIMEOUT_MS = 30_000; +const MAX_WAIT_TIMEOUT_MS = 300_000; + +export interface WorkflowsCliContext { + argv: string[]; + cwd: string; + env: NodeJS.ProcessEnv; + stdout: Writable; + stderr: Writable; + cliEntrypoint: string; +} + +interface WorkflowCliConfig { + stateDir: string; + allowedRoots: string[]; + devspaceAgentsDir: string; + subagents: boolean; +} + +export async function runWorkflowsCli(context: WorkflowsCliContext): Promise { + const [command, ...args] = context.argv; + if (command === "__supervisor") { + validateWorkflowArguments(command, args); + const stateDir = requiredOption(args, "--state-dir"); + await runWorkflowSupervisor(stateDir); + return 0; + } + if (!command || command === "help" || command === "--help" || command === "-h") { + context.stdout.write(workflowsHelp()); + return 0; + } + + const jsonMode = args.includes("--json"); + try { + const result = await dispatchWorkflowCommand(command, args, context); + writeEnvelope(context.stdout, { ok: true, command, ...result }); + return 0; + } catch (error) { + const normalized = normalizeCliError(error); + if (jsonMode) { + writeEnvelope(context.stdout, { ok: false, command, error: normalized }); + } else { + context.stderr.write(`${normalized.message}\n`); + } + return normalized.exitCode; + } +} + +async function dispatchWorkflowCommand( + command: string, + args: string[], + context: WorkflowsCliContext, +): Promise> { + const parsed = validateWorkflowArguments(command, args); + const config = loadWorkflowCliConfig(context.env, context.cwd); + const scope = await resolveWorkspaceScope(context, config.allowedRoots); + const orchestrator = new WorkflowOrchestrator(config.stateDir); + try { + switch (command) { + case "run": + return await runCommand(parsed.positionals[0]!, args, context, config, scope, orchestrator); + case "status": { + const workflowId = parsed.positionals[0]!; + return { workflow: orchestrator.getForWorkspace(workflowId, scope) ?? notFound(workflowId) }; + } + case "events": { + const workflowId = parsed.positionals[0]!; + const after = integerOption(args, "--after") ?? 0; + const page = orchestrator.eventsForWorkspace(workflowId, scope, { after, limit: 1_000 }); + return { workflowId, events: page.events, cursor: page.nextCursor }; + } + case "wait": { + const workflowId = parsed.positionals[0]!; + const timeoutMs = integerOption(args, "--timeout-ms") ?? DEFAULT_WAIT_TIMEOUT_MS; + if (timeoutMs < 0 || timeoutMs > MAX_WAIT_TIMEOUT_MS) { + throw new CliInputError(`--timeout-ms must be between 0 and ${MAX_WAIT_TIMEOUT_MS}.`); + } + const before = orchestrator.getForWorkspace(workflowId, scope) ?? notFound(workflowId); + const workflow = await orchestrator.waitForWorkspace(workflowId, scope, { timeoutMs }); + const after = integerOption(args, "--after"); + const page = after === undefined + ? undefined + : orchestrator.eventsForWorkspace(workflowId, scope, { after, limit: 1_000 }); + return { + workflow, + timedOut: !isTerminal(workflow) && !isTerminal(before), + ...(page ? { events: page.events, cursor: page.nextCursor } : {}), + }; + } + case "cancel": { + const workflowId = parsed.positionals[0]!; + const workflow = orchestrator.cancelForWorkspace(workflowId, scope); + await ensureSupervisor({ + stateDir: config.stateDir, + cliEntrypoint: context.cliEntrypoint, + env: context.env, + }); + return { workflow }; + } + default: + throw new CliInputError(`Unknown workflows command: ${command}`); + } + } finally { + orchestrator.close(); + } +} + +async function runCommand( + targetName: string, + args: string[], + context: WorkflowsCliContext, + config: WorkflowCliConfig, + scope: WorkflowWorkspaceScope, + orchestrator: WorkflowOrchestrator, +): Promise> { + const prompt = requiredOption(args, "--prompt"); + const model = optionalOption(args, "--model"); + const thinking = optionalOption(args, "--thinking"); + const timeoutMs = integerOption(args, "--timeout-ms"); + const access = optionalOption(args, "--access") ?? "read_only"; + const idempotencyKey = optionalOption(args, "--idempotency-key"); + if (timeoutMs !== undefined && (timeoutMs < 1 || timeoutMs > 24 * 60 * 60_000)) { + throw new CliInputError("--timeout-ms must be between 1 and 86400000."); + } + if (access !== "read_only" && access !== "workspace_write") { + throw new CliInputError("--access must be read_only or workspace_write."); + } + + const profiles = await loadLocalAgentProfiles(config, scope.workspaceRoot); + const target = resolveWorkflowTarget(targetName, profiles, model, thinking, context.env); + if (!target) { + throw new CliInputError( + `Unknown workflow profile or provider: ${targetName}. Available ${formatAvailableLocalAgentTargets(profiles)}`, + ); + } + const profile = target.kind === "profile" ? target.profile : undefined; + const profileBody = profile?.body.trim() ?? ""; + const profileName = profile?.name ?? target.name; + const profileHash = createHash("sha256") + .update(JSON.stringify({ + name: profileName, + provider: target.provider, + model: target.model ?? null, + thinking: target.thinking ?? null, + body: profileBody, + })) + .digest("hex"); + const workflowPolicy = { version: 1 as const, access }; + const effectivePolicy = resolveWorkflowNodePolicy({ + workflowPolicy, + nodeConfig: { access }, + environment: context.env, + }); + const snapshot: JsonObject = { + profileBody, + profileName, + profileHash, + provider: target.provider, + model: target.model ?? null, + thinking: target.thinking ?? null, + effectivePolicy: effectivePolicy as unknown as JsonObject, + workspaceRoot: scope.workspaceRoot, + prompt, + timeoutMs: timeoutMs ?? null, + environmentPolicy: effectivePolicy.environment as JsonObject, + }; + const submitted = orchestrator.submitDetailed({ + definition: { + version: 1, + nodes: [{ key: "agent", type: "agent", config: snapshot }], + edges: [], + }, + input: { prompt }, + policy: workflowPolicy, + idempotencyKey, + workspace: scope, + }); + const supervisor = await ensureSupervisor({ + stateDir: config.stateDir, + cliEntrypoint: context.cliEntrypoint, + env: context.env, + }); + return { + workflow: submitted.workflow, + created: submitted.created, + supervisor: { + wakeGeneration: supervisor.requestedWakeGeneration, + spawned: supervisor.spawned, + ownerEpoch: supervisor.ownerEpoch ?? null, + }, + }; +} + +async function resolveWorkspaceScope( + context: Pick, + allowedRoots: string[], +): Promise { + const workspaceId = context.env.DEVSPACE_WORKSPACE_ID?.trim(); + if (!workspaceId) throw new WorkspaceDeniedError("DEVSPACE_WORKSPACE_ID is required for workflow commands."); + const requestedRoot = resolve(context.env.DEVSPACE_WORKSPACE_ROOT || context.cwd); + const workspaceRoot = await canonicalPath(requestedRoot, "workspace root"); + const cwd = await canonicalPath(context.cwd, "current directory"); + if (!isInside(cwd, workspaceRoot)) { + throw new WorkspaceDeniedError("Current directory is outside DEVSPACE_WORKSPACE_ROOT."); + } + const canonicalAllowedRoots = await Promise.all(allowedRoots.map((root) => canonicalPath(root, "allowed root"))); + if (!canonicalAllowedRoots.some((root) => isInside(workspaceRoot, root))) { + throw new WorkspaceDeniedError("Workspace root is outside configured allowed roots."); + } + return { workspaceId, workspaceRoot }; +} + +function loadWorkflowCliConfig(env: NodeJS.ProcessEnv, cwd: string): WorkflowCliConfig { + const files = loadDevspaceFiles(env); + const stateDir = resolve(expandHomePath( + env.DEVSPACE_STATE_DIR ?? files.config.stateDir ?? join(homedir(), ".local", "share", "devspace"), + )); + const rawRoots = env.DEVSPACE_ALLOWED_ROOTS + ? env.DEVSPACE_ALLOWED_ROOTS.split(",") + : files.config.allowedRoots ?? [cwd]; + return { + stateDir, + allowedRoots: rawRoots.map((root) => resolve(expandHomePath(root.trim()))).filter(Boolean), + devspaceAgentsDir: devspaceAgentsDir(env), + subagents: resolveSubagentsFlag(files.config, env) === true, + }; +} + +function resolveWorkflowTarget( + name: string, + profiles: LocalAgentProfile[], + model: string | undefined, + thinking: string | undefined, + env: NodeJS.ProcessEnv, +): ReturnType | { + kind: "provider"; + name: "fake"; + provider: "fake"; + model?: string; + thinking?: string; + profile?: undefined; +} | undefined { + if (name === "fake" && env.DEVSPACE_WORKFLOW_FAKE_PROVIDER === "1") { + return { kind: "provider", name: "fake", provider: "fake", model, thinking }; + } + return resolveLocalAgentTarget(name, profiles, model, thinking); +} + +function validateWorkflowArguments( + command: string, + args: string[], +): { positionals: string[] } { + const specifications: Record }> = { + __supervisor: { positionals: 0, valueOptions: new Set(["--state-dir"]) }, + run: { + positionals: 1, + valueOptions: new Set([ + "--prompt", + "--model", + "--thinking", + "--timeout-ms", + "--access", + "--idempotency-key", + ]), + }, + status: { positionals: 1, valueOptions: new Set() }, + events: { positionals: 1, valueOptions: new Set(["--after"]) }, + wait: { positionals: 1, valueOptions: new Set(["--timeout-ms", "--after"]) }, + cancel: { positionals: 1, valueOptions: new Set() }, + }; + const specification = specifications[command]; + if (!specification) throw new CliInputError(`Unknown workflows command: ${command}`); + + const positionals: string[] = []; + const seenOptions = new Set(); + for (let index = 0; index < args.length; index += 1) { + const part = args[index]!; + if (!part.startsWith("--")) { + positionals.push(part); + continue; + } + const separator = part.indexOf("="); + const name = separator === -1 ? part : part.slice(0, separator); + if (name === "--json") { + if (separator !== -1) throw new CliInputError("--json does not accept a value."); + } else if (specification.valueOptions.has(name)) { + const value = separator === -1 ? args[index + 1] : part.slice(separator + 1); + if (!value || value.startsWith("--")) throw new CliInputError(`Missing value for ${name}.`); + if (separator === -1) index += 1; + } else { + throw new CliInputError(`Unknown option for workflows ${command}: ${name}`); + } + if (seenOptions.has(name)) throw new CliInputError(`Duplicate option ${name}.`); + seenOptions.add(name); + } + if (positionals.length !== specification.positionals) { + throw new CliInputError(`workflows ${command} expects ${specification.positionals} positional argument${specification.positionals === 1 ? "" : "s"}.`); + } + return { positionals }; +} + +function requiredOption(args: string[], name: string): string { + const value = optionalOption(args, name); + if (!value) throw new CliInputError(`Missing required option ${name}.`); + return value; +} + +function optionalOption(args: string[], name: string): string | undefined { + for (let index = 0; index < args.length; index += 1) { + const part = args[index]; + if (part === name) { + const value = args[index + 1]; + if (!value || value.startsWith("--")) throw new CliInputError(`Missing value for ${name}.`); + return value; + } + if (part?.startsWith(`${name}=`)) { + const value = part.slice(name.length + 1); + if (!value) throw new CliInputError(`Missing value for ${name}.`); + return value; + } + } + return undefined; +} + +function integerOption(args: string[], name: string): number | undefined { + const raw = optionalOption(args, name); + if (raw === undefined) return undefined; + const parsed = Number(raw); + if (!Number.isSafeInteger(parsed)) throw new CliInputError(`${name} must be an integer.`); + return parsed; +} + +function writeEnvelope(stdout: Writable, value: Record): void { + stdout.write(`${JSON.stringify({ version: ENVELOPE_VERSION, ...value })}\n`); +} + +function normalizeCliError(error: unknown): { code: string; message: string; exitCode: number } { + if (error instanceof WorkspaceDeniedError) return { code: "workspace_denied", message: error.message, exitCode: 3 }; + if (error instanceof WorkflowNotFoundError) return { code: "not_found", message: error.message, exitCode: 4 }; + if (error instanceof WorkflowIdempotencyConflictError) return { code: "idempotency_conflict", message: error.message, exitCode: 5 }; + if (error instanceof CliInputError || error instanceof WorkflowValidationError) { + return { code: "invalid_input", message: error.message, exitCode: 2 }; + } + return { + code: "internal_error", + message: error instanceof Error ? error.message : String(error), + exitCode: 1, + }; +} + +function notFound(workflowId: string): never { + throw new WorkflowNotFoundError(workflowId); +} + +function isTerminal(workflow: WorkflowRunRecord): boolean { + return workflow.status === "succeeded" || workflow.status === "failed" || workflow.status === "cancelled"; +} + +async function canonicalPath(path: string, label: string): Promise { + try { + return await realpath(path); + } catch (error) { + throw new WorkspaceDeniedError(`Unable to canonicalize ${label}: ${error instanceof Error ? error.message : String(error)}`); + } +} + +function isInside(path: string, root: string): boolean { + const relationship = relative(root, path); + return relationship === "" || (!isAbsolute(relationship) && relationship !== ".." && !relationship.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`)); +} + +function workflowsHelp(): string { + return [ + "DevSpace workflows", + "", + "Usage:", + " devspace workflows run --prompt --json [--idempotency-key ]", + " devspace workflows status --json", + " devspace workflows wait --json [--timeout-ms ] [--after ]", + " devspace workflows events --json [--after ]", + " devspace workflows cancel --json", + "", + ].join("\n"); +} + +class CliInputError extends Error {} +class WorkspaceDeniedError extends Error {} From 0092d28b48f683343c788b21b77e42d871838f86 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 01:31:37 +0530 Subject: [PATCH 12/25] test(workflows): cover supervisor and CLI lifecycle Co-Authored-By: Claude --- package.json | 2 +- src/oauth-store.test.ts | 1 + src/workflows/cli.test.ts | 161 +++++++++++++++++ src/workflows/migrations.test.ts | 20 ++- src/workflows/supervisor.test.ts | 295 +++++++++++++++++++++++++++++++ src/workflows/workflows.test.ts | 64 +++++++ 6 files changed, 538 insertions(+), 5 deletions(-) create mode 100644 src/workflows/cli.test.ts create mode 100644 src/workflows/supervisor.test.ts diff --git a/package.json b/package.json index d6a10dfc..9d0b9dd4 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "postinstall": "node scripts/fix-node-pty-permissions.mjs", "start": "node dist/cli.js serve", "test": "tsx src/config.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && npm run test:workflows && tsx src/cli.test.ts", - "test:workflows": "tsx src/workflows/migrations.test.ts && tsx src/workflows/policy.test.ts && tsx src/workflows/workflows.test.ts", + "test:workflows": "tsx src/workflows/migrations.test.ts && tsx src/workflows/policy.test.ts && tsx src/workflows/workflows.test.ts && tsx src/workflows/supervisor.test.ts && tsx src/workflows/cli.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, "keywords": [], diff --git a/src/oauth-store.test.ts b/src/oauth-store.test.ts index de3c116b..b1e36d5c 100644 --- a/src/oauth-store.test.ts +++ b/src/oauth-store.test.ts @@ -45,6 +45,7 @@ async function testDatabaseConfiguration(stateDir: string): Promise { { version: 2, name: "oauth-state" }, { version: 3, name: "local-agent-sessions" }, { version: 4, name: "durable-workflows" }, + { version: 5, name: "workflow-supervisor" }, ]); } finally { database.close(); diff --git a/src/workflows/cli.test.ts b/src/workflows/cli.test.ts new file mode 100644 index 00000000..ef506697 --- /dev/null +++ b/src/workflows/cli.test.ts @@ -0,0 +1,161 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = mkdtempSync(join(tmpdir(), "devspace-workflows-cli-test-")); +const project = join(root, "project"); +const stateDir = join(root, "state"); +const configDir = join(root, "config"); +const agentsDir = join(configDir, "agents"); +const cliPath = fileURLToPath(new URL("../cli.ts", import.meta.url)); +const loaderPath = fileURLToPath(new URL("../../node_modules/tsx/dist/loader.mjs", import.meta.url)); +mkdirSync(project, { recursive: true }); +mkdirSync(stateDir, { recursive: true }); +mkdirSync(agentsDir, { recursive: true }); + +const profilePath = join(agentsDir, "reviewer.md"); +writeProfile(profilePath, "Original immutable profile."); + +const baseEnv = { + ...process.env, + DEVSPACE_CONFIG_DIR: configDir, + DEVSPACE_STATE_DIR: stateDir, + DEVSPACE_ALLOWED_ROOTS: project, + DEVSPACE_WORKSPACE_ID: "workspace-a", + DEVSPACE_WORKSPACE_ROOT: project, + DEVSPACE_SUBAGENTS: "1", + DEVSPACE_WORKFLOW_FAKE_PROVIDER: "1", + DEVSPACE_WORKFLOW_FAKE_DELAY_MS: "5000", +}; + +try { + const submitted = runCli([ + "workflows", "run", "reviewer", "--prompt", "private task", "--json", + "--idempotency-key", "shell-parent", + ], baseEnv); + assert.equal(submitted.status, 0, `${submitted.stderr}\n${submitted.stdout}`); + assert.equal(submitted.stdout.trim().split("\n").length, 1, "run stdout must contain one JSON object"); + assert.equal(submitted.stderr, ""); + const runEnvelope = JSON.parse(submitted.stdout) as Envelope; + assert.equal(runEnvelope.version, 1); + assert.equal(runEnvelope.ok, true); + const workflowId = runEnvelope.workflow!.id; + const replay = runCli([ + "workflows", "run", "reviewer", "--prompt", "private task", "--json", + "--idempotency-key", "shell-parent", + ], baseEnv); + assert.equal(replay.status, 0, replay.stderr); + assert.equal((JSON.parse(replay.stdout) as Envelope).created, false); + + writeProfile(profilePath, "Mutated after durable submission."); + const timed = runCli([ + "workflows", "wait", workflowId, "--timeout-ms", "0", "--json", + ], baseEnv); + assert.equal(timed.status, 0, timed.stderr); + const timedEnvelope = JSON.parse(timed.stdout) as Envelope; + assert.equal(timedEnvelope.timedOut, true); + + const waited = runCli([ + "workflows", "wait", workflowId, "--timeout-ms", "10000", "--after", "0", "--json", + ], baseEnv); + assert.equal(waited.status, 0, waited.stderr); + const waitEnvelope = JSON.parse(waited.stdout) as Envelope; + assert.equal(waitEnvelope.workflow!.status, "succeeded"); + assert.equal( + waitEnvelope.workflow!.definition.nodes[0].config.profileBody, + "Original immutable profile.", + ); + assert.ok(waitEnvelope.events!.some((event) => event.type === "provider.session")); + assert.ok(waitEnvelope.events!.some((event) => event.type === "provider.output")); + assert.equal(waitEnvelope.cursor, waitEnvelope.events!.at(-1)!.sequence); + + const repeated = runCli([ + "workflows", "wait", workflowId, "--timeout-ms", "5000", "--json", + ], baseEnv); + assert.equal(repeated.status, 0, repeated.stderr); + assert.equal((JSON.parse(repeated.stdout) as Envelope).workflow!.status, "succeeded"); + + const denied = runCli([ + "workflows", "status", workflowId, "--json", + ], { ...baseEnv, DEVSPACE_WORKSPACE_ID: "workspace-b" }); + assert.notEqual(denied.status, 0); + const deniedEnvelope = JSON.parse(denied.stdout) as Envelope; + assert.equal(deniedEnvelope.ok, false); + assert.equal(deniedEnvelope.error!.code, "not_found"); + + const invalid = runCli([ + "workflows", "run", "fake", "--json", + ], baseEnv); + assert.notEqual(invalid.status, 0); + assert.equal((JSON.parse(invalid.stdout) as Envelope).error!.code, "invalid_input"); + assert.equal(invalid.stdout.trim().split("\n").length, 1); + + const optionValueAsPositional = runCli([ + "workflows", "run", "--prompt", "reviewer", "--json", + ], baseEnv); + assert.notEqual(optionValueAsPositional.status, 0); + assert.equal((JSON.parse(optionValueAsPositional.stdout) as Envelope).error!.code, "invalid_input"); + + const unknownOption = runCli([ + "workflows", "status", workflowId, "--unknown", "value", "--json", + ], baseEnv); + assert.notEqual(unknownOption.status, 0); + assert.equal((JSON.parse(unknownOption.stdout) as Envelope).error!.code, "invalid_input"); + + const extraPositional = runCli([ + "workflows", "cancel", workflowId, "extra", "--json", + ], baseEnv); + assert.notEqual(extraPositional.status, 0); + assert.equal((JSON.parse(extraPositional.stdout) as Envelope).error!.code, "invalid_input"); + + const conflict = runCli([ + "workflows", "run", "reviewer", "--prompt", "different task", "--json", + "--idempotency-key", "shell-parent", + ], baseEnv); + assert.notEqual(conflict.status, 0); + assert.equal((JSON.parse(conflict.stdout) as Envelope).error!.code, "idempotency_conflict"); +} finally { + rmSync(root, { recursive: true, force: true }); +} + +function runCli(args: string[], env: NodeJS.ProcessEnv) { + return spawnSync(process.execPath, ["--import", loaderPath, cliPath, ...args], { + cwd: project, + env, + encoding: "utf8", + timeout: 15_000, + }); +} + +function writeProfile(path: string, body: string): void { + writeFileSync(path, [ + "---", + "name: reviewer", + "description: Workflow test reviewer.", + "provider: codex", + "model: test-model", + "thinking: high", + "---", + "", + body, + "", + ].join("\n")); +} + +interface Envelope { + version: number; + ok: boolean; + created?: boolean; + timedOut?: boolean; + cursor?: number; + workflow?: { + id: string; + status: string; + definition: { nodes: Array<{ config: { profileBody: string } }> }; + }; + events?: Array<{ type: string; sequence: number }>; + error?: { code: string; message: string }; +} diff --git a/src/workflows/migrations.test.ts b/src/workflows/migrations.test.ts index 30671448..6ddb3304 100644 --- a/src/workflows/migrations.test.ts +++ b/src/workflows/migrations.test.ts @@ -25,6 +25,7 @@ function testFreshSchema(stateDir: string): void { { version: 2, name: "oauth-state" }, { version: 3, name: "local-agent-sessions" }, { version: 4, name: "durable-workflows" }, + { version: 5, name: "workflow-supervisor" }, ], ); const tables = database.sqlite @@ -35,7 +36,15 @@ function testFreshSchema(stateDir: string): void { ) .pluck() .all(); - assert.deepEqual(tables, ["workflow_edges", "workflow_events", "workflow_nodes", "workflow_runs"]); + assert.deepEqual(tables, [ + "workflow_edges", + "workflow_events", + "workflow_node_attempts", + "workflow_nodes", + "workflow_provider_events", + "workflow_runs", + "workflow_supervisor", + ]); const edgeForeignKeys = database.sqlite.prepare("pragma foreign_key_list(workflow_edges)").all() as Array<{ id: number; @@ -89,7 +98,7 @@ function testExistingMigrationCompatibility(stateDir: string): void { try { assert.deepEqual( migrated.sqlite.prepare("select version from devspace_schema_migrations order by version").pluck().all(), - [1, 2, 3, 4], + [1, 2, 3, 4, 5], ); assert.deepEqual(migrated.sqlite.prepare("select * from workspace_sessions").all(), [ { @@ -162,7 +171,7 @@ function testExistingMigrationCompatibility(stateDir: string): void { .prepare("select count(*) from sqlite_master where type = 'table' and name like 'workflow_%'") .pluck() .get(), - 4, + 7, ); } finally { migrated.close(); @@ -213,11 +222,14 @@ function createVersion3Fixture(stateDir: string): void { const initialized = openDatabase(stateDir); try { initialized.sqlite.exec(` + drop table workflow_provider_events; + drop table workflow_node_attempts; + drop table workflow_supervisor; drop table workflow_edges; drop table workflow_events; drop table workflow_nodes; drop table workflow_runs; - delete from devspace_schema_migrations where version = 4; + delete from devspace_schema_migrations where version in (4, 5); insert into workspace_sessions ( id, root, status, mode, source_root, base_ref, base_sha, managed, created_at, last_used_at diff --git a/src/workflows/supervisor.test.ts b/src/workflows/supervisor.test.ts new file mode 100644 index 00000000..09732b8f --- /dev/null +++ b/src/workflows/supervisor.test.ts @@ -0,0 +1,295 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { LocalAgentRunController, type LocalAgentRunHandle } from "../local-agent-runtime.js"; +import { ensureSupervisor } from "./supervisor-launch.js"; +import { runWorkflowSupervisor } from "./supervisor.js"; +import { WorkflowStore } from "./store.js"; +import type { JsonObject, SubmitWorkflowRequest } from "./types.js"; + +const root = mkdtempSync(join(tmpdir(), "devspace-supervisor-test-")); +try { + await testExecutionAndEventReplay(join(root, "execution")); + await testDuplicateSupervisorPrevention(join(root, "singleton")); + await testStaleSupervisorProcessRecovery(join(root, "stale-supervisor")); + await testAtomicClaimAndLeaseRecovery(join(root, "claims")); + await testDispatchHeartbeatsProtectLeases(join(root, "dispatch-heartbeats")); + await testCancellationBeforeAndDuringExecution(join(root, "cancellation")); +} finally { + rmSync(root, { recursive: true, force: true }); +} + +async function testExecutionAndEventReplay(stateDir: string): Promise { + const store = new WorkflowStore(stateDir); + const snapshot = executionSnapshot({ profileBody: "Original profile" }); + const submitted = store.submit(workflowRequest(snapshot)).workflow; + snapshot.profileBody = "Mutated after submit"; + store.close(); + + const ran = await runWorkflowSupervisor(stateDir, { + idleMs: 20, + heartbeatMs: 5, + supervisorLeaseMs: 100, + nodeLeaseMs: 100, + handleFactory: successfulHandleFactory, + }); + assert.equal(ran, true); + + const reader = new WorkflowStore(stateDir); + try { + const workflow = reader.require(submitted.id); + assert.equal(workflow.status, "succeeded"); + assert.equal(workflow.nodes[0]!.attempt, 1); + assert.equal(workflow.nodes[0]!.definition.config!.profileBody, "Original profile"); + assert.equal((workflow.result as JsonObject).finalResponse, "completed"); + const first = reader.readEvents(workflow.id, { after: 0, limit: 4 }); + const second = reader.readEvents(workflow.id, { after: first.nextCursor, limit: 100 }); + assert.ok(second.events.some((event) => event.type === "provider.session")); + assert.ok(second.events.some((event) => event.type === "provider.output")); + assert.deepEqual( + [...first.events, ...second.events].map((event) => event.sequence), + Array.from({ length: first.events.length + second.events.length }, (_, index) => index + 1), + ); + } finally { + reader.close(); + } +} + +async function testDuplicateSupervisorPrevention(stateDir: string): Promise { + const owner = new WorkflowStore(stateDir); + const acquired = owner.acquireSupervisor({ ownerToken: "owner", ownerPid: 1, leaseMs: 1_000 }); + assert.ok(acquired); + assert.equal(await runWorkflowSupervisor(stateDir, { ownerToken: "other", idleMs: 5 }), false); + assert.equal(owner.getSupervisor()!.ownerToken, "owner"); + owner.releaseSupervisor(acquired!); + owner.close(); +} + +async function testStaleSupervisorProcessRecovery(stateDir: string): Promise { + const store = new WorkflowStore(stateDir); + const stale = store.acquireSupervisor({ + ownerToken: "stale-owner", + ownerPid: 2_147_483_647, + leaseMs: 60_000, + })!; + store.close(); + + const launched = await ensureSupervisor({ + stateDir, + cliEntrypoint: fileURLToPath(new URL("../cli.ts", import.meta.url)), + env: process.env, + startupTimeoutMs: 2_000, + }); + assert.equal(launched.spawned, true); + assert.ok((launched.ownerEpoch ?? 0) > stale.ownerEpoch); + + const cleanup = new WorkflowStore(stateDir); + const current = cleanup.getSupervisor(); + if (current) cleanup.releaseSupervisor(current); + cleanup.close(); +} + +async function testAtomicClaimAndLeaseRecovery(stateDir: string): Promise { + const store = new WorkflowStore(stateDir); + const first = store.submit(workflowRequest(executionSnapshot())).workflow; + const supervisor = store.acquireSupervisor({ ownerToken: "claims", ownerPid: 2, leaseMs: 1_000 })!; + const claim = store.claimNextAgentNode({ supervisor, claimToken: "first", leaseMs: 30 }); + assert.equal(claim?.workflow.id, first.id); + assert.equal( + store.claimNextAgentNode({ supervisor, claimToken: "duplicate", leaseMs: 30 }), + undefined, + ); + await delay(10); + assert.ok(store.heartbeatNode({ + workflowId: first.id, + nodeKey: "agent", + attempt: 1, + claimToken: "first", + leaseMs: 30, + })); + await delay(15); + assert.equal(store.reconcileExpiredClaims(), 0); + await delay(25); + assert.equal(store.reconcileExpiredClaims(), 1); + const failed = store.require(first.id); + assert.equal(failed.status, "failed"); + assert.equal((failed.error as JsonObject).code, "worker_lost"); + assert.equal(failed.nodes[0]!.attempt, 1, "worker loss must not retry by default"); + + const parallel = store.submit({ + definition: { + version: 1, + nodes: [ + { key: "first", type: "agent", config: executionSnapshot() }, + { key: "second", type: "agent", config: executionSnapshot() }, + ], + edges: [], + }, + }).workflow; + store.claimNode({ workflowId: parallel.id, nodeKey: "first", claimToken: "parallel-first", leaseMs: 10 }); + store.claimNode({ workflowId: parallel.id, nodeKey: "second", claimToken: "parallel-second", leaseMs: 10 }); + await delay(20); + assert.equal(store.reconcileExpiredClaims(), 1); + assert.deepEqual( + store.require(parallel.id).nodes.map((node) => node.status).sort(), + ["cancelled", "failed"], + ); + store.releaseSupervisor(supervisor); + store.close(); +} + +async function testDispatchHeartbeatsProtectLeases(stateDir: string): Promise { + const store = new WorkflowStore(stateDir); + const workflow = store.submit(workflowRequest(executionSnapshot())).workflow; + store.close(); + + const ran = await runWorkflowSupervisor(stateDir, { + idleMs: 20, + heartbeatMs: 5, + supervisorLeaseMs: 40, + nodeLeaseMs: 40, + handleFactory: async () => { + await delay(90); + return successfulHandleFactory(); + }, + }); + assert.equal(ran, true); + + const reader = new WorkflowStore(stateDir); + try { + assert.equal(reader.require(workflow.id).status, "succeeded"); + } finally { + reader.close(); + } +} + +async function testCancellationBeforeAndDuringExecution(stateDir: string): Promise { + const beforeStore = new WorkflowStore(stateDir); + const before = beforeStore.submit(workflowRequest(executionSnapshot())).workflow; + beforeStore.requestCancellation(before.id); + beforeStore.close(); + let starts = 0; + await runWorkflowSupervisor(stateDir, { + idleMs: 20, + heartbeatMs: 5, + supervisorLeaseMs: 100, + nodeLeaseMs: 100, + handleFactory: async (...args) => { + starts += 1; + return successfulHandleFactory(); + }, + }); + const afterBefore = new WorkflowStore(stateDir); + assert.equal(afterBefore.require(before.id).status, "cancelled"); + assert.equal(starts, 0); + + const during = afterBefore.submit(workflowRequest(executionSnapshot())).workflow; + afterBefore.close(); + const supervisorPromise = runWorkflowSupervisor(stateDir, { + idleMs: 20, + heartbeatMs: 5, + supervisorLeaseMs: 100, + nodeLeaseMs: 100, + handleFactory: delayedHandleFactory, + }); + await waitForStatus(stateDir, during.id, "running"); + const canceller = new WorkflowStore(stateDir); + canceller.requestCancellation(during.id); + canceller.requestSupervisorWake(); + canceller.close(); + await supervisorPromise; + const finalStore = new WorkflowStore(stateDir); + try { + assert.equal(finalStore.require(during.id).status, "cancelled"); + assert.equal(finalStore.require(during.id).nodes[0]!.status, "cancelled"); + } finally { + finalStore.close(); + } +} + +async function successfulHandleFactory(): Promise { + return completedHandle(5); +} + +async function delayedHandleFactory(): Promise { + return completedHandle(500); +} + +function completedHandle(delayMs: number): LocalAgentRunHandle { + const controller = new LocalAgentRunController("fake"); + let timer: NodeJS.Timeout | undefined; + let finish!: () => void; + const pump = new Promise((resolve) => { + finish = resolve; + timer = setTimeout(() => { + controller.emit({ type: "session", providerSessionId: "session-1", resumed: false }); + controller.emit({ type: "output", stream: "assistant", delta: "completed" }); + controller.succeed({ + provider: "fake", + providerSessionId: "session-1", + finalResponse: "completed", + items: [], + }); + finish(); + }, delayMs); + }); + controller.setLifecycle({ + cancel: () => { + if (timer) clearTimeout(timer); + finish(); + }, + dispose: () => { + if (timer) clearTimeout(timer); + finish(); + }, + pump, + }); + return controller; +} + +function workflowRequest(config: JsonObject): SubmitWorkflowRequest { + return { + definition: { version: 1, nodes: [{ key: "agent", type: "agent", config }], edges: [] }, + workspace: { workspaceId: "workspace", workspaceRoot: "/tmp/workspace" }, + }; +} + +function executionSnapshot(overrides: JsonObject = {}): JsonObject { + return { + profileBody: "", + profileName: "fake", + profileHash: "hash", + provider: "fake", + model: null, + thinking: null, + effectivePolicy: { + version: 1, + mode: "workflow", + access: "read_only", + environment: {}, + }, + workspaceRoot: "/tmp/workspace", + prompt: "task", + timeoutMs: null, + environmentPolicy: {}, + ...overrides, + }; +} + +async function waitForStatus(stateDir: string, workflowId: string, status: string): Promise { + const deadline = Date.now() + 2_000; + while (Date.now() < deadline) { + const store = new WorkflowStore(stateDir); + const current = store.require(workflowId).status; + store.close(); + if (current === status) return; + await delay(5); + } + throw new Error(`Timed out waiting for ${workflowId} to become ${status}`); +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/workflows/workflows.test.ts b/src/workflows/workflows.test.ts index 1f894ab7..e99850f6 100644 --- a/src/workflows/workflows.test.ts +++ b/src/workflows/workflows.test.ts @@ -44,6 +44,7 @@ async function runTests(): Promise { await testTransitionsAndClaims(join(root, "transitions")); testTerminalInvariantsAndAtomicity(join(root, "terminal-invariants")); testDagValidation(join(root, "dag")); + testAgentCompletionAdvancesDag(join(root, "dag-completion")); await testBoundedAndRepeatedWait(join(root, "wait")); } finally { rmSync(root, { recursive: true, force: true }); @@ -370,6 +371,69 @@ function testTerminalInvariantsAndAtomicity(stateDir: string): void { } } +function testAgentCompletionAdvancesDag(stateDir: string): void { + const store = new WorkflowStore(stateDir); + try { + const workflow = store.submit({ + definition: { + version: 1, + nodes: [agentNode("first"), agentNode("second")], + edges: [{ from: "first", to: "second" }], + }, + }).workflow; + const supervisor = store.acquireSupervisor({ ownerToken: "dag", ownerPid: 1, leaseMs: 1_000 })!; + const first = store.claimNextAgentNode({ supervisor, claimToken: "first-claim", leaseMs: 1_000 })!; + const afterFirst = store.completeAgentNode({ + workflowId: workflow.id, + nodeKey: "first", + attempt: first.node.attempt, + claimToken: first.node.claimToken!, + status: "succeeded", + result: { step: 1 }, + })!; + assert.equal(afterFirst.status, "running"); + assert.deepEqual(Object.fromEntries(afterFirst.nodes.map((node) => [node.key, node.status])), { + first: "succeeded", + second: "ready", + }); + + const second = store.claimNextAgentNode({ supervisor, claimToken: "second-claim", leaseMs: 1_000 })!; + assert.equal(second.node.key, "second"); + const completed = store.completeAgentNode({ + workflowId: workflow.id, + nodeKey: "second", + attempt: second.node.attempt, + claimToken: second.node.claimToken!, + status: "succeeded", + result: { step: 2 }, + })!; + assert.equal(completed.status, "succeeded"); + assert.deepEqual(completed.result, { step: 2 }); + + const failedWorkflow = store.submit({ + definition: { + version: 1, + nodes: [agentNode("first"), agentNode("second")], + edges: [{ from: "first", to: "second" }], + }, + }).workflow; + const failedClaim = store.claimNextAgentNode({ supervisor, claimToken: "failed-claim", leaseMs: 1_000 })!; + const failed = store.completeAgentNode({ + workflowId: failedWorkflow.id, + nodeKey: "first", + attempt: failedClaim.node.attempt, + claimToken: failedClaim.node.claimToken!, + status: "failed", + error: { code: "provider_failed" }, + })!; + assert.equal(failed.status, "failed"); + assert.equal(failed.nodes.find((node) => node.key === "second")!.status, "skipped"); + store.releaseSupervisor(supervisor); + } finally { + store.close(); + } +} + function testDagValidation(stateDir: string): void { const store = new WorkflowStore(stateDir); try { From da506997583959ad93011ddd9d26b0bd4a16c83b Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 02:13:28 +0530 Subject: [PATCH 13/25] feat(workflows): add durable DAG scheduling Co-Authored-By: Claude --- src/db/migrations.ts | 38 +++++ src/db/schema.ts | 26 ++++ src/workflows/cli.ts | 97 +++--------- src/workflows/store.ts | 225 ++++++++++++++++++++++++++- src/workflows/submission.ts | 236 +++++++++++++++++++++++++++++ src/workflows/supervisor-launch.ts | 18 ++- src/workflows/supervisor.ts | 86 +++++++++-- src/workflows/types.ts | 25 +++ src/workflows/worktrees.ts | 176 +++++++++++++++++++++ 9 files changed, 831 insertions(+), 96 deletions(-) create mode 100644 src/workflows/submission.ts create mode 100644 src/workflows/worktrees.ts diff --git a/src/db/migrations.ts b/src/db/migrations.ts index 727ca990..83630ffb 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -32,6 +32,11 @@ const migrations: Migration[] = [ name: "workflow-supervisor", up: migrateWorkflowSupervisor, }, + { + version: 6, + name: "workflow-dag-scheduler", + up: migrateWorkflowDagScheduler, + }, ]; export function migrateDatabase(sqlite: Database.Database): void { @@ -342,6 +347,39 @@ function migrateWorkflowSupervisor(sqlite: Database.Database): void { `); } +function migrateWorkflowDagScheduler(sqlite: Database.Database): void { + addColumnIfMissing(sqlite, "workflow_runs", "max_concurrency", "integer not null default 1"); + addColumnIfMissing(sqlite, "workflow_runs", "last_dispatched_at", "text"); + addColumnIfMissing(sqlite, "workflow_nodes", "next_eligible_at", "text"); + + sqlite.exec(` + create table if not exists workflow_worktrees ( + workflow_run_id text not null, + node_key text not null, + attempt integer not null, + path text not null unique, + source_root text not null, + base_sha text not null, + state text not null check (state in ('allocated', 'active', 'preserved', 'removed', 'cleanup_failed')), + retain_until text, + cleanup_error text, + created_at text not null, + updated_at text not null, + primary key (workflow_run_id, node_key, attempt), + foreign key (workflow_run_id) references workflow_runs(id) on delete cascade + ); + + create index if not exists workflow_runs_dispatch_idx + on workflow_runs(status, last_dispatched_at, created_at); + + create index if not exists workflow_nodes_retry_idx + on workflow_nodes(workflow_run_id, status, next_eligible_at); + + create index if not exists workflow_worktrees_cleanup_idx + on workflow_worktrees(state, retain_until); + `); +} + function addColumnIfMissing( sqlite: Database.Database, table: "workspace_sessions" | "local_agent_sessions" | "workflow_runs" | "workflow_nodes", diff --git a/src/db/schema.ts b/src/db/schema.ts index a95107c3..216edc19 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -118,6 +118,8 @@ export const workflowRuns = sqliteTable( requestHash: text("request_hash").notNull(), workspaceId: text("workspace_id"), workspaceRoot: text("workspace_root"), + maxConcurrency: integer("max_concurrency").notNull().default(1), + lastDispatchedAt: text("last_dispatched_at"), resultJson: text("result_json"), errorJson: text("error_json"), cancellationRequestedAt: text("cancellation_requested_at"), @@ -145,6 +147,7 @@ export const workflowNodes = sqliteTable( claimToken: text("claim_token"), claimedAt: text("claimed_at"), claimExpiresAt: text("claim_expires_at"), + nextEligibleAt: text("next_eligible_at"), supervisorOwnerToken: text("supervisor_owner_token"), supervisorOwnerEpoch: integer("supervisor_owner_epoch"), heartbeatAt: text("heartbeat_at"), @@ -272,6 +275,29 @@ export const workflowProviderEvents = sqliteTable( ], ); +export const workflowWorktrees = sqliteTable( + "workflow_worktrees", + { + workflowRunId: text("workflow_run_id") + .notNull() + .references(() => workflowRuns.id, { onDelete: "cascade" }), + nodeKey: text("node_key").notNull(), + attempt: integer("attempt").notNull(), + path: text("path").notNull().unique(), + sourceRoot: text("source_root").notNull(), + baseSha: text("base_sha").notNull(), + state: text("state").notNull(), + retainUntil: text("retain_until"), + cleanupError: text("cleanup_error"), + createdAt: text("created_at").notNull(), + updatedAt: text("updated_at").notNull(), + }, + (table) => [ + primaryKey({ columns: [table.workflowRunId, table.nodeKey, table.attempt] }), + index("workflow_worktrees_cleanup_idx").on(table.state, table.retainUntil), + ], +); + export type WorkspaceSessionRow = typeof workspaceSessions.$inferSelect; export type NewWorkspaceSessionRow = typeof workspaceSessions.$inferInsert; export type LoadedAgentFileRow = typeof loadedAgentFiles.$inferSelect; diff --git a/src/workflows/cli.ts b/src/workflows/cli.ts index 339e85e5..e2db7147 100644 --- a/src/workflows/cli.ts +++ b/src/workflows/cli.ts @@ -1,19 +1,11 @@ -import { createHash } from "node:crypto"; import { realpath } from "node:fs/promises"; import { homedir } from "node:os"; import { isAbsolute, join, relative, resolve } from "node:path"; import type { Writable } from "node:stream"; import { devspaceAgentsDir, loadDevspaceFiles, resolveSubagentsFlag } from "../user-config.js"; -import { - formatAvailableLocalAgentTargets, - resolveLocalAgentTarget, -} from "../local-agent-targets.js"; -import { - loadLocalAgentProfiles, - type LocalAgentProfile, -} from "../local-agent-profiles.js"; +import { loadLocalAgentProfiles } from "../local-agent-profiles.js"; import { expandHomePath } from "../roots.js"; -import { resolveWorkflowNodePolicy } from "./policy.js"; +import { createWorkflowSubmission } from "./submission.js"; import { WorkflowOrchestrator } from "./orchestrator.js"; import { WorkflowIdempotencyConflictError, @@ -23,7 +15,6 @@ import { import { ensureSupervisor } from "./supervisor-launch.js"; import { runWorkflowSupervisor } from "./supervisor.js"; import type { - JsonObject, WorkflowRunRecord, WorkflowWorkspaceScope, } from "./types.js"; @@ -43,6 +34,7 @@ export interface WorkflowsCliContext { interface WorkflowCliConfig { stateDir: string; + worktreeRoot: string; allowedRoots: string[]; devspaceAgentsDir: string; subagents: boolean; @@ -158,54 +150,24 @@ async function runCommand( } const profiles = await loadLocalAgentProfiles(config, scope.workspaceRoot); - const target = resolveWorkflowTarget(targetName, profiles, model, thinking, context.env); - if (!target) { - throw new CliInputError( - `Unknown workflow profile or provider: ${targetName}. Available ${formatAvailableLocalAgentTargets(profiles)}`, - ); - } - const profile = target.kind === "profile" ? target.profile : undefined; - const profileBody = profile?.body.trim() ?? ""; - const profileName = profile?.name ?? target.name; - const profileHash = createHash("sha256") - .update(JSON.stringify({ - name: profileName, - provider: target.provider, - model: target.model ?? null, - thinking: target.thinking ?? null, - body: profileBody, - })) - .digest("hex"); - const workflowPolicy = { version: 1 as const, access }; - const effectivePolicy = resolveWorkflowNodePolicy({ - workflowPolicy, - nodeConfig: { access }, - environment: context.env, - }); - const snapshot: JsonObject = { - profileBody, - profileName, - profileHash, - provider: target.provider, - model: target.model ?? null, - thinking: target.thinking ?? null, - effectivePolicy: effectivePolicy as unknown as JsonObject, - workspaceRoot: scope.workspaceRoot, - prompt, - timeoutMs: timeoutMs ?? null, - environmentPolicy: effectivePolicy.environment as JsonObject, - }; - const submitted = orchestrator.submitDetailed({ - definition: { - version: 1, - nodes: [{ key: "agent", type: "agent", config: snapshot }], - edges: [], + const request = await createWorkflowSubmission({ + intent: { + single: { + target: targetName, + prompt, + model, + thinking, + access, + timeoutMs, + }, + idempotencyKey, }, - input: { prompt }, - policy: workflowPolicy, - idempotencyKey, workspace: scope, + profiles, + worktreeRoot: config.worktreeRoot, + environment: context.env, }); + const submitted = orchestrator.submitDetailed(request); const supervisor = await ensureSupervisor({ stateDir: config.stateDir, cliEntrypoint: context.cliEntrypoint, @@ -251,32 +213,15 @@ function loadWorkflowCliConfig(env: NodeJS.ProcessEnv, cwd: string): WorkflowCli : files.config.allowedRoots ?? [cwd]; return { stateDir, + worktreeRoot: resolve(expandHomePath( + env.DEVSPACE_WORKTREE_ROOT ?? files.config.worktreeRoot ?? join(homedir(), ".devspace", "worktrees"), + )), allowedRoots: rawRoots.map((root) => resolve(expandHomePath(root.trim()))).filter(Boolean), devspaceAgentsDir: devspaceAgentsDir(env), subagents: resolveSubagentsFlag(files.config, env) === true, }; } -function resolveWorkflowTarget( - name: string, - profiles: LocalAgentProfile[], - model: string | undefined, - thinking: string | undefined, - env: NodeJS.ProcessEnv, -): ReturnType | { - kind: "provider"; - name: "fake"; - provider: "fake"; - model?: string; - thinking?: string; - profile?: undefined; -} | undefined { - if (name === "fake" && env.DEVSPACE_WORKFLOW_FAKE_PROVIDER === "1") { - return { kind: "provider", name: "fake", provider: "fake", model, thinking }; - } - return resolveLocalAgentTarget(name, profiles, model, thinking); -} - function validateWorkflowArguments( command: string, args: string[], diff --git a/src/workflows/store.ts b/src/workflows/store.ts index bdb3cd14..f00b0c38 100644 --- a/src/workflows/store.ts +++ b/src/workflows/store.ts @@ -28,6 +28,7 @@ import { type WorkflowSupervisorRecord, type WorkflowTransition, type WorkflowWorkspaceScope, + type WorkflowWorktreeRecord, } from "./types.js"; const MAX_EVENT_PAYLOAD_BYTES = 64 * 1024; @@ -35,6 +36,10 @@ const DEFAULT_EVENT_LIMIT = 100; const MAX_EVENT_LIMIT = 1_000; const DEFAULT_CLAIM_LEASE_MS = 5 * 60_000; const MAX_CLAIM_LEASE_MS = 24 * 60 * 60_000; +const MAX_WORKFLOW_NODES = 64; +const MAX_WORKFLOW_EDGES = 256; +const MAX_RUN_CONCURRENCY = 16; +const SAFE_NODE_KEY = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$/; const TERMINAL_WORKFLOW_STATUSES = new Set([ "succeeded", @@ -78,6 +83,8 @@ interface WorkflowRunRow { request_hash: string; workspace_id: string | null; workspace_root: string | null; + max_concurrency: number; + last_dispatched_at: string | null; result_json: string | null; error_json: string | null; cancellation_requested_at: string | null; @@ -98,6 +105,7 @@ interface WorkflowNodeRow { claim_token: string | null; claimed_at: string | null; claim_expires_at: string | null; + next_eligible_at: string | null; supervisor_owner_token: string | null; supervisor_owner_epoch: number | null; heartbeat_at: string | null; @@ -136,6 +144,20 @@ interface WorkflowSupervisorRow { started_at: string | null; } +interface WorkflowWorktreeRow { + workflow_run_id: string; + node_key: string; + attempt: number; + path: string; + source_root: string; + base_sha: string; + state: WorkflowWorktreeRecord["state"]; + retain_until: string | null; + cleanup_error: string | null; + created_at: string; + updated_at: string; +} + interface WorkflowAttemptRow { node_id: string; workflow_run_id: string; @@ -215,8 +237,9 @@ export class WorkflowStore { .prepare( `insert into workflow_runs ( id, definition_version, status, definition_json, input_json, policy_json, - idempotency_key, request_hash, workspace_id, workspace_root, created_at, updated_at - ) values (?, ?, 'queued', ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + idempotency_key, request_hash, workspace_id, workspace_root, max_concurrency, + created_at, updated_at + ) values (?, ?, 'queued', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, ) .run( workflowId, @@ -228,6 +251,7 @@ export class WorkflowStore { normalized.requestHash, normalized.workspace?.workspaceId ?? null, normalized.workspace?.workspaceRoot ?? null, + normalized.maxConcurrency, now, now, ); @@ -663,16 +687,23 @@ export class WorkflowStore { `select n.* from workflow_nodes n join workflow_runs r on r.id = n.workflow_run_id where n.status = 'ready' and n.claim_token is null + and (n.next_eligible_at is null or n.next_eligible_at <= ?) and r.status in ('queued', 'running') and r.cancellation_requested_at is null - order by r.created_at, n.created_at, n.node_key limit 1`, + and ( + select count(*) from workflow_nodes active + where active.workflow_run_id = r.id and active.status = 'running' + ) < r.max_concurrency + order by case when r.last_dispatched_at is null then 0 else 1 end, + r.last_dispatched_at, r.created_at, n.created_at, n.node_key limit 1`, ) - .get() as WorkflowNodeRow | undefined; + .get(now) as WorkflowNodeRow | undefined; if (!candidate) return undefined; const updated = this.database.sqlite .prepare( `update workflow_nodes set status = 'running', claim_token = ?, claimed_at = ?, claim_expires_at = ?, - attempt = attempt + 1, supervisor_owner_token = ?, supervisor_owner_epoch = ?, + next_eligible_at = null, attempt = attempt + 1, + supervisor_owner_token = ?, supervisor_owner_epoch = ?, heartbeat_at = ?, updated_at = ? where id = ? and status = 'ready' and claim_token is null`, ) @@ -713,6 +744,9 @@ export class WorkflowStore { now, now, ); + this.database.sqlite + .prepare("update workflow_runs set last_dispatched_at = ?, updated_at = ? where id = ?") + .run(now, now, node.workflow_run_id); const workflow = this.getWorkflowRow(node.workflow_run_id)!; if (workflow.status === "queued") { this.database.sqlite @@ -890,10 +924,44 @@ export class WorkflowStore { const status = workflow.status === "cancelling" ? "cancelled" : input.status; const resultJson = serializeOptionalJson(input.result); const errorJson = serializeOptionalObject(input.error); + const retryAt = status === "failed" + ? retryEligibleAt(parseJson(node.definition_json), input.attempt, input.error, now) + : undefined; + if (retryAt) { + const nodeUpdate = this.database.sqlite + .prepare( + `update workflow_nodes + set status = 'ready', result_json = null, error_json = ?, claim_token = null, + claimed_at = null, claim_expires_at = null, next_eligible_at = ?, + supervisor_owner_token = null, supervisor_owner_epoch = null, + heartbeat_at = ?, updated_at = ?, completed_at = null + where id = ? and status = 'running' and attempt = ? and claim_token = ?`, + ) + .run(errorJson, retryAt, now, now, node.id, input.attempt, input.claimToken); + if (nodeUpdate.changes !== 1) return false; + this.database.sqlite + .prepare( + `update workflow_node_attempts + set phase = 'terminal', terminal_status = 'failed', error_json = ?, + heartbeat_at = ?, updated_at = ?, completed_at = ? + where node_id = ? and attempt = ? and claim_token = ? and phase != 'terminal'`, + ) + .run(errorJson, now, now, now, node.id, input.attempt, input.claimToken); + this.insertEvent( + input.workflowId, + "node.retry_scheduled", + node.id, + { nodeKey: node.node_key, attempt: input.attempt, nextEligibleAt: retryAt }, + now, + ); + return true; + } const nodeUpdate = this.database.sqlite .prepare( `update workflow_nodes - set status = ?, result_json = ?, error_json = ?, claim_expires_at = null, + set status = ?, result_json = ?, error_json = ?, claim_token = null, + claimed_at = null, claim_expires_at = null, next_eligible_at = null, + supervisor_owner_token = null, supervisor_owner_epoch = null, heartbeat_at = ?, updated_at = ?, completed_at = ? where id = ? and status = 'running' and attempt = ? and claim_token = ?`, ) @@ -1064,10 +1132,76 @@ export class WorkflowStore { return row?.status === "cancelling"; } + hasPendingWork(): boolean { + return Boolean( + this.database.sqlite + .prepare("select 1 from workflow_runs where status in ('queued', 'running', 'cancelling') limit 1") + .get(), + ); + } + + recordWorktree(input: Omit): WorkflowWorktreeRecord { + const now = new Date().toISOString(); + this.database.sqlite + .prepare( + `insert into workflow_worktrees ( + workflow_run_id, node_key, attempt, path, source_root, base_sha, state, + retain_until, created_at, updated_at + ) values (?, ?, ?, ?, ?, ?, 'allocated', ?, ?, ?) + on conflict(workflow_run_id, node_key, attempt) do nothing`, + ) + .run( + input.workflowId, + input.nodeKey, + input.attempt, + input.path, + input.sourceRoot, + input.baseSha, + input.retainUntil ?? null, + now, + now, + ); + return this.requireWorktree(input.workflowId, input.nodeKey, input.attempt); + } + + getWorktree(workflowId: string, nodeKey: string, attempt: number): WorkflowWorktreeRecord | undefined { + const row = this.database.sqlite + .prepare( + `select * from workflow_worktrees + where workflow_run_id = ? and node_key = ? and attempt = ?`, + ) + .get(workflowId, nodeKey, attempt) as WorkflowWorktreeRow | undefined; + return row ? rowToWorktree(row) : undefined; + } + + updateWorktreeState( + workflowId: string, + nodeKey: string, + attempt: number, + state: WorkflowWorktreeRecord["state"], + cleanupError?: string, + ): WorkflowWorktreeRecord { + const now = new Date().toISOString(); + const changed = this.database.sqlite + .prepare( + `update workflow_worktrees set state = ?, cleanup_error = ?, updated_at = ? + where workflow_run_id = ? and node_key = ? and attempt = ?`, + ) + .run(state, cleanupError ?? null, now, workflowId, nodeKey, attempt); + if (changed.changes !== 1) throw new WorkflowValidationError("Unknown workflow worktree allocation"); + return this.requireWorktree(workflowId, nodeKey, attempt); + } + close(): void { this.database.close(); } + private requireWorktree(workflowId: string, nodeKey: string, attempt: number): WorkflowWorktreeRecord { + const record = this.getWorktree(workflowId, nodeKey, attempt); + if (!record) throw new WorkflowValidationError("Unknown workflow worktree allocation"); + return record; + } + private assertActiveSupervisor(identity: WorkflowSupervisorIdentity, now: string): void { const row = this.database.sqlite .prepare( @@ -1145,6 +1279,8 @@ export class WorkflowStore { requestHash: row.request_hash, workspaceId: row.workspace_id ?? undefined, workspaceRoot: row.workspace_root ?? undefined, + maxConcurrency: row.max_concurrency, + lastDispatchedAt: row.last_dispatched_at ?? undefined, result: parseOptionalJson(row.result_json), error: parseOptionalJson(row.error_json), cancellationRequestedAt: row.cancellation_requested_at ?? undefined, @@ -1296,6 +1432,7 @@ function normalizeSubmission(request: SubmitWorkflowRequest): { policyJson: string; idempotencyKey?: string; workspace?: WorkflowWorkspaceScope; + maxConcurrency: number; requestHash: string; } { const definition = normalizeDefinition(request.definition); @@ -1314,10 +1451,20 @@ function normalizeSubmission(request: SubmitWorkflowRequest): { workspaceRoot: requireNonEmptyString(request.workspace.workspaceRoot, "Workspace root"), } : undefined; + const maxConcurrency = readBoundedInteger(policy.maxConcurrency, 1, MAX_RUN_CONCURRENCY, "Workflow maxConcurrency"); const requestHash = createHash("sha256") .update(canonicalJson({ definition, input, policy, workspace: workspace ?? null })) .digest("hex"); - return { definition, definitionJson, inputJson, policyJson, idempotencyKey, workspace, requestHash }; + return { + definition, + definitionJson, + inputJson, + policyJson, + idempotencyKey, + workspace, + maxConcurrency, + requestHash, + }; } function normalizeDefinition(definition: WorkflowDefinition): WorkflowDefinition { @@ -1329,9 +1476,15 @@ function normalizeDefinition(definition: WorkflowDefinition): WorkflowDefinition if (!Array.isArray(definition.nodes) || definition.nodes.length === 0) { throw new WorkflowValidationError("Workflow definition must contain at least one node"); } + if (definition.nodes.length > MAX_WORKFLOW_NODES) { + throw new WorkflowValidationError(`Workflow definition exceeds ${MAX_WORKFLOW_NODES} nodes`); + } if (definition.edges !== undefined && !Array.isArray(definition.edges)) { throw new WorkflowValidationError("Workflow definition edges must be an array"); } + if ((definition.edges?.length ?? 0) > MAX_WORKFLOW_EDGES) { + throw new WorkflowValidationError(`Workflow definition exceeds ${MAX_WORKFLOW_EDGES} edges`); + } const keys = new Set(); const nodes = definition.nodes.map((node, index): WorkflowNodeDefinitionV1 => { @@ -1340,6 +1493,9 @@ function normalizeDefinition(definition: WorkflowDefinition): WorkflowDefinition } const key = node.key.trim(); if (!key) throw new WorkflowValidationError(`Workflow node at index ${index} has an empty key`); + if (!SAFE_NODE_KEY.test(key)) { + throw new WorkflowValidationError(`Workflow node key is unsafe: ${key}`); + } if (keys.has(key)) throw new WorkflowValidationError(`Duplicate workflow node key: ${key}`); keys.add(key); return { key, type: "agent", config: normalizeObject(node.config ?? {}, `Node ${key} config`) }; @@ -1469,6 +1625,7 @@ function rowToWorkflowNode(row: WorkflowNodeRow): WorkflowNodeRecord { claimToken: row.claim_token ?? undefined, claimedAt: row.claimed_at ?? undefined, claimExpiresAt: row.claim_expires_at ?? undefined, + nextEligibleAt: row.next_eligible_at ?? undefined, result: parseOptionalJson(row.result_json), error: parseOptionalJson(row.error_json), createdAt: row.created_at, @@ -1477,6 +1634,22 @@ function rowToWorkflowNode(row: WorkflowNodeRow): WorkflowNodeRecord { }; } +function rowToWorktree(row: WorkflowWorktreeRow): WorkflowWorktreeRecord { + return { + workflowId: row.workflow_run_id, + nodeKey: row.node_key, + attempt: row.attempt, + path: row.path, + sourceRoot: row.source_root, + baseSha: row.base_sha, + state: row.state, + retainUntil: row.retain_until ?? undefined, + cleanupError: row.cleanup_error ?? undefined, + createdAt: row.created_at, + updatedAt: row.updated_at, + }; +} + function rowToAttempt(row: WorkflowAttemptRow): WorkflowNodeAttemptRecord { return { nodeId: row.node_id, @@ -1583,6 +1756,44 @@ function validateLease(leaseMs: number, label: string): void { } } +function retryEligibleAt( + definition: WorkflowNodeDefinitionV1, + attempt: number, + error: JsonObject | undefined, + now: string, +): string | undefined { + const config = definition.config ?? {}; + const effectivePolicy = isObject(config.effectivePolicy) ? config.effectivePolicy : undefined; + if (effectivePolicy?.access !== "read_only") return undefined; + const retry = isObject(config.retry) ? config.retry : undefined; + if (!retry) return undefined; + const maxAttempts = readBoundedInteger(retry.maxAttempts, 1, 10, "Retry maxAttempts"); + if (attempt >= maxAttempts) return undefined; + const code = typeof error?.code === "string" ? error.code : undefined; + const retryOn = Array.isArray(retry.retryOn) + ? retry.retryOn.filter((value): value is string => typeof value === "string") + : []; + if (!code || !retryOn.includes(code)) return undefined; + const backoffMs = retry.backoffMs === undefined + ? 0 + : readBoundedInteger(retry.backoffMs, 0, 60_000, "Retry backoffMs", 0); + return new Date(new Date(now).getTime() + backoffMs * attempt).toISOString(); +} + +function readBoundedInteger( + value: JsonValue | undefined, + fallback: number, + maximum: number, + label: string, + minimum = 1, +): number { + if (value === undefined || value === null) return fallback; + if (!Number.isSafeInteger(value) || Number(value) < minimum || Number(value) > maximum) { + throw new WorkflowValidationError(`${label} must be between ${minimum} and ${maximum}`); + } + return Number(value); +} + function requireNonEmptyString(value: string, label: string): string { const normalized = value.trim(); if (!normalized) throw new WorkflowValidationError(`${label} must not be empty`); diff --git a/src/workflows/submission.ts b/src/workflows/submission.ts new file mode 100644 index 00000000..c1285a19 --- /dev/null +++ b/src/workflows/submission.ts @@ -0,0 +1,236 @@ +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import { realpath } from "node:fs/promises"; +import { promisify } from "node:util"; +import type { LocalAgentProfile } from "../local-agent-profiles.js"; +import { formatAvailableLocalAgentTargets, resolveLocalAgentTarget } from "../local-agent-targets.js"; +import { resolveWorkflowNodePolicy, type WorkflowAgentAccess } from "./policy.js"; +import { WorkflowValidationError } from "./store.js"; +import type { + JsonObject, + SubmitWorkflowRequest, + WorkflowEdgeDefinitionV1, + WorkflowNodeDefinitionV1, + WorkflowRetryClass, + WorkflowWorkspaceScope, +} from "./types.js"; + +const execFileAsync = promisify(execFile); +const MAX_NODE_TIMEOUT_MS = 24 * 60 * 60_000; +const MAX_ATTEMPTS = 10; +const MAX_RETRY_BACKOFF_MS = 60_000; + +export interface WorkflowAgentIntent { + key?: string; + target: string; + prompt: string; + model?: string; + thinking?: string; + access?: WorkflowAgentAccess; + timeoutMs?: number; + retry?: { + maxAttempts?: number; + retryOn?: WorkflowRetryClass[]; + backoffMs?: number; + }; +} + +export interface WorkflowDagIntent { + version: 1; + nodes: Array; + edges?: WorkflowEdgeDefinitionV1[]; + maxConcurrency?: number; + access?: WorkflowAgentAccess; +} + +export interface WorkflowSubmissionIntent { + single?: WorkflowAgentIntent; + dag?: WorkflowDagIntent; + idempotencyKey?: string; +} + +export async function createWorkflowSubmission(input: { + intent: WorkflowSubmissionIntent; + workspace: WorkflowWorkspaceScope; + profiles: LocalAgentProfile[]; + worktreeRoot: string; + environment?: NodeJS.ProcessEnv; +}): Promise { + if (Boolean(input.intent.single) === Boolean(input.intent.dag)) { + throw new WorkflowValidationError("Provide exactly one of single or dag"); + } + const environment = input.environment ?? process.env; + const dag: WorkflowDagIntent = input.intent.dag ?? { + version: 1, + nodes: [{ ...input.intent.single!, key: input.intent.single!.key ?? "agent" }], + edges: [], + maxConcurrency: 1, + access: input.intent.single!.access ?? "read_only", + }; + if (dag.version !== 1) throw new WorkflowValidationError("Unsupported workflow DAG version"); + const workflowAccess = dag.access ?? "read_only"; + const hasWriteNode = dag.nodes.some((node) => (node.access ?? workflowAccess) === "workspace_write"); + const baseSha = hasWriteNode ? await resolveBaseSha(input.workspace.workspaceRoot) : undefined; + const nodes: WorkflowNodeDefinitionV1[] = dag.nodes.map((node) => ({ + key: node.key, + type: "agent", + config: snapshotNode({ + node, + workflowAccess, + workspaceRoot: input.workspace.workspaceRoot, + worktreeRoot: input.worktreeRoot, + baseSha, + profiles: input.profiles, + environment, + }), + })); + + return { + definition: { version: 1, nodes, edges: dag.edges ?? [] }, + input: { kind: input.intent.dag ? "dag" : "single" }, + policy: { + version: 1, + access: workflowAccess, + maxConcurrency: dag.maxConcurrency ?? 1, + }, + idempotencyKey: input.intent.idempotencyKey, + workspace: input.workspace, + }; +} + +function snapshotNode(input: { + node: WorkflowAgentIntent; + workflowAccess: WorkflowAgentAccess; + workspaceRoot: string; + worktreeRoot: string; + baseSha?: string; + profiles: LocalAgentProfile[]; + environment: NodeJS.ProcessEnv; +}): JsonObject { + const prompt = input.node.prompt.trim(); + if (!prompt) throw new WorkflowValidationError(`Workflow node ${input.node.key ?? "agent"} prompt is empty`); + const timeoutMs = input.node.timeoutMs; + if (timeoutMs !== undefined && (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > MAX_NODE_TIMEOUT_MS)) { + throw new WorkflowValidationError(`Workflow node timeoutMs must be between 1 and ${MAX_NODE_TIMEOUT_MS}`); + } + const target = resolveTarget( + input.node.target, + input.profiles, + input.node.model, + input.node.thinking, + input.environment, + ); + if (!target) { + throw new WorkflowValidationError( + `Unknown workflow profile or provider: ${input.node.target}. Available ${formatAvailableLocalAgentTargets(input.profiles)}`, + ); + } + const profile = target.kind === "profile" ? target.profile : undefined; + const profileBody = profile?.body.trim() ?? ""; + const profileName = profile?.name ?? target.name; + const effectivePolicy = resolveWorkflowNodePolicy({ + workflowPolicy: { access: input.workflowAccess }, + nodeConfig: { access: input.node.access ?? input.workflowAccess }, + environment: input.environment, + }); + const retry = normalizeRetry(input.node.retry); + if (effectivePolicy.access === "workspace_write" && retry.maxAttempts > 1) { + throw new WorkflowValidationError("workspace_write nodes cannot be retried automatically"); + } + if (effectivePolicy.access === "workspace_write" && !input.baseSha) { + throw new WorkflowValidationError("workspace_write nodes require a pinned Git base SHA"); + } + const profileHash = createHash("sha256") + .update(JSON.stringify({ + name: profileName, + provider: target.provider, + model: target.model ?? null, + thinking: target.thinking ?? null, + body: profileBody, + })) + .digest("hex"); + + return { + profileBody, + profileName, + profileHash, + provider: target.provider, + model: target.model ?? null, + thinking: target.thinking ?? null, + effectivePolicy: effectivePolicy as unknown as JsonObject, + workspaceRoot: input.workspaceRoot, + worktreeRoot: input.worktreeRoot, + baseSha: input.baseSha ?? null, + prompt, + timeoutMs: timeoutMs ?? null, + retry: retry as unknown as JsonObject, + environmentPolicy: effectivePolicy.environment as JsonObject, + }; +} + +function normalizeRetry(retry: WorkflowAgentIntent["retry"]): { + maxAttempts: number; + retryOn: WorkflowRetryClass[]; + backoffMs: number; +} { + const maxAttempts = retry?.maxAttempts ?? 1; + const backoffMs = retry?.backoffMs ?? 0; + const retryOn = [...new Set(retry?.retryOn ?? [])]; + if (!Number.isSafeInteger(maxAttempts) || maxAttempts < 1 || maxAttempts > MAX_ATTEMPTS) { + throw new WorkflowValidationError(`Retry maxAttempts must be between 1 and ${MAX_ATTEMPTS}`); + } + if (!Number.isSafeInteger(backoffMs) || backoffMs < 0 || backoffMs > MAX_RETRY_BACKOFF_MS) { + throw new WorkflowValidationError(`Retry backoffMs must be between 0 and ${MAX_RETRY_BACKOFF_MS}`); + } + for (const value of retryOn) { + if (value !== "provider_failed" && value !== "timed_out") { + throw new WorkflowValidationError(`Unsupported retry class: ${value}`); + } + } + if (maxAttempts > 1 && retryOn.length === 0) { + throw new WorkflowValidationError("Retry policy must name at least one retryable failure class"); + } + return { maxAttempts, retryOn, backoffMs }; +} + +function resolveTarget( + name: string, + profiles: LocalAgentProfile[], + model: string | undefined, + thinking: string | undefined, + environment: NodeJS.ProcessEnv, +): ReturnType | { + kind: "provider"; + name: "fake"; + provider: "fake"; + model?: string; + thinking?: string; + profile?: undefined; +} | undefined { + if (name === "fake" && environment.DEVSPACE_WORKFLOW_FAKE_PROVIDER === "1") { + return { kind: "provider", name: "fake", provider: "fake", model, thinking }; + } + return resolveLocalAgentTarget(name, profiles, model, thinking); +} + +async function resolveBaseSha(workspaceRoot: string): Promise { + try { + const { stdout: root } = await execFileAsync("git", ["rev-parse", "--show-toplevel"], { + cwd: workspaceRoot, + maxBuffer: 1024 * 1024, + }); + const { stdout: sha } = await execFileAsync("git", ["rev-parse", "--verify", "HEAD^{commit}"], { + cwd: workspaceRoot, + maxBuffer: 1024 * 1024, + }); + if (!root.trim() || !sha.trim()) throw new Error("missing Git root or commit"); + if ((await realpath(root.trim())) !== (await realpath(workspaceRoot))) { + throw new Error("workspace root must be the Git repository root"); + } + return sha.trim(); + } catch (error) { + throw new WorkflowValidationError( + `workspace_write workflows require a Git repository with a commit: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} diff --git a/src/workflows/supervisor-launch.ts b/src/workflows/supervisor-launch.ts index a1a24a45..209bcea6 100644 --- a/src/workflows/supervisor-launch.ts +++ b/src/workflows/supervisor-launch.ts @@ -63,13 +63,17 @@ export async function ensureSupervisor(input: { windowsHide: true, }, ); + let childExit: { code: number | null; signal: NodeJS.Signals | null } | undefined; + child.once("exit", (code, signal) => { + childExit = { code, signal }; + }); await new Promise((resolve, reject) => { child.once("spawn", resolve); child.once("error", reject); }); child.unref(); - const deadline = Date.now() + (input.startupTimeoutMs ?? 2_000); + const deadline = Date.now() + (input.startupTimeoutMs ?? 5_000); while (Date.now() < deadline) { const supervisor = store.getSupervisor(); if (supervisor?.leaseExpiresAt && supervisor.leaseExpiresAt > new Date().toISOString()) { @@ -79,14 +83,24 @@ export async function ensureSupervisor(input: { ownerEpoch: supervisor.ownerEpoch, }; } + if (childExit) { + throw new Error( + `Workflow supervisor exited before acquiring its durable lease (${formatChildExit(childExit)}).`, + ); + } await delay(20); } - throw new Error("Workflow supervisor did not acquire its durable lease."); + throw new Error("Workflow supervisor did not acquire its durable lease before the startup timeout."); } finally { store.close(); } } +function formatChildExit(exit: { code: number | null; signal: NodeJS.Signals | null }): string { + if (exit.signal) return `signal ${exit.signal}`; + return `exit code ${exit.code ?? "unknown"}`; +} + function isProcessRunning(pid: number): boolean { if (!Number.isSafeInteger(pid) || pid < 1) return false; try { diff --git a/src/workflows/supervisor.ts b/src/workflows/supervisor.ts index e34d412d..ef1a24af 100644 --- a/src/workflows/supervisor.ts +++ b/src/workflows/supervisor.ts @@ -9,6 +9,11 @@ import { } from "../local-agent-runtime.js"; import { isLocalAgentProvider } from "../local-agent-profiles.js"; import type { EffectiveLocalAgentPolicy } from "./policy.js"; +import { + allocateWorkflowWorktree, + cleanupWorkflowWorktree, + preserveWorkflowWorktree, +} from "./worktrees.js"; import { WorkflowStore, WorkflowValidationError } from "./store.js"; import type { JsonObject, @@ -22,6 +27,8 @@ const DEFAULT_SUPERVISOR_LEASE_MS = 5_000; const DEFAULT_NODE_LEASE_MS = 5_000; const DEFAULT_HEARTBEAT_MS = 1_000; const DEFAULT_IDLE_MS = 1_000; +const DEFAULT_GLOBAL_CONCURRENCY = 4; +const MAX_GLOBAL_CONCURRENCY = 32; const MAX_ERROR_CHARS = 16_384; const MAX_TIMEOUT_MS = 24 * 60 * 60_000; @@ -37,6 +44,7 @@ export interface WorkflowSupervisorOptions { idleMs?: number; ownerToken?: string; ownerPid?: number; + globalConcurrency?: number; handleFactory?: WorkflowHandleFactory; } @@ -49,7 +57,8 @@ export async function runWorkflowSupervisor( const nodeLeaseMs = options.nodeLeaseMs ?? DEFAULT_NODE_LEASE_MS; const heartbeatMs = options.heartbeatMs ?? DEFAULT_HEARTBEAT_MS; const idleMs = options.idleMs ?? DEFAULT_IDLE_MS; - validateSupervisorTiming(supervisorLeaseMs, nodeLeaseMs, heartbeatMs, idleMs); + const globalConcurrency = options.globalConcurrency ?? DEFAULT_GLOBAL_CONCURRENCY; + validateSupervisorTiming(supervisorLeaseMs, nodeLeaseMs, heartbeatMs, idleMs, globalConcurrency); const acquired = store.acquireSupervisor({ ownerToken: options.ownerToken ?? randomUUID(), ownerPid: options.ownerPid ?? process.pid, @@ -61,6 +70,7 @@ export async function runWorkflowSupervisor( } const identity: WorkflowSupervisorIdentity = acquired; + const active = new Set>(); let lastWorkAt = Date.now(); try { store.reconcileExpiredClaims(); @@ -68,27 +78,43 @@ export async function runWorkflowSupervisor( while (store.heartbeatSupervisor(identity, supervisorLeaseMs)) { store.reconcileExpiredClaims(); store.convergeCancellations(); - const claim = store.claimNextAgentNode({ - supervisor: identity, - claimToken: randomUUID(), - leaseMs: nodeLeaseMs, - }); - if (claim) { + let claimed = false; + while (active.size < globalConcurrency) { + const claim = store.claimNextAgentNode({ + supervisor: identity, + claimToken: randomUUID(), + leaseMs: nodeLeaseMs, + }); + if (!claim) break; + claimed = true; lastWorkAt = Date.now(); - await executeClaim(store, identity, claim, { + let execution!: Promise; + execution = executeClaim(store, identity, claim, { supervisorLeaseMs, nodeLeaseMs, heartbeatMs, handleFactory: options.handleFactory ?? defaultHandleFactory, + }).finally(() => { + active.delete(execution); + lastWorkAt = Date.now(); }); - lastWorkAt = Date.now(); - continue; + active.add(execution); } + if (claimed) continue; const supervisor = store.getSupervisor(); if (!supervisor || supervisor.ownerToken !== identity.ownerToken || supervisor.ownerEpoch !== identity.ownerEpoch) { + await Promise.allSettled(active); return false; } + if (active.size > 0) { + await Promise.race([...active, delay(heartbeatMs)]); + continue; + } + if (store.hasPendingWork()) { + await delay(heartbeatMs); + continue; + } if (Date.now() - lastWorkAt >= idleMs) { if (store.releaseSupervisor(identity, supervisor.wakeGeneration)) return true; lastWorkAt = Date.now(); @@ -96,6 +122,7 @@ export async function runWorkflowSupervisor( } await delay(Math.min(heartbeatMs, Math.max(10, idleMs - (Date.now() - lastWorkAt)))); } + await Promise.allSettled(active); return false; } finally { store.releaseSupervisor(identity); @@ -126,6 +153,9 @@ async function executeClaim( let timeout: NodeJS.Timeout | undefined; let cancellationObserved = false; let timeoutObserved = false; + let worktreeAllocated = false; + let allocatedWorktreeRoot: string | undefined; + let completedSuccessfully = false; let leaseFailure: Error | undefined; let tickRunning = false; @@ -134,6 +164,18 @@ async function executeClaim( const fullPrompt = execution.profileBody ? `${execution.profileBody}\n\nTask:\n${execution.prompt}` : execution.prompt; + if (execution.effectivePolicy.access === "workspace_write") { + const allocation = await allocateWorkflowWorktree({ + store, + identity, + sourceRoot: execution.workspaceRoot, + worktreeRoot: execution.worktreeRoot, + baseSha: execution.baseSha, + }); + execution.workspaceRoot = allocation.path; + allocatedWorktreeRoot = execution.worktreeRoot; + worktreeAllocated = true; + } if (!store.markNodeDispatching(identity)) { throw new Error("Workflow node attempt was lost before dispatch."); } @@ -201,7 +243,7 @@ async function executeClaim( store.recordNodeProviderSession(identity, result.providerSessionId); } const cancelled = cancellationObserved || store.isCancellationRequested(identity.workflowId); - store.completeAgentNode({ + const completed = store.completeAgentNode({ ...identity, status: cancelled ? "cancelled" : "succeeded", result: { @@ -210,6 +252,7 @@ async function executeClaim( finalResponse: result.finalResponse, }, }); + completedSuccessfully = Boolean(completed && !cancelled); } catch (error) { const cancelled = !timeoutObserved && ( cancellationObserved || store.isCancellationRequested(identity.workflowId) || isAbortError(error) @@ -223,6 +266,17 @@ async function executeClaim( if (heartbeat) clearInterval(heartbeat); if (timeout) clearTimeout(timeout); await handle?.dispose().catch(() => undefined); + if (worktreeAllocated) { + if (completedSuccessfully && allocatedWorktreeRoot) { + await cleanupWorkflowWorktree({ + store, + identity, + worktreeRoot: allocatedWorktreeRoot, + }).catch(() => undefined); + } else { + preserveWorkflowWorktree(store, identity); + } + } store.convergeCancellations(); } } @@ -261,6 +315,8 @@ function readExecutionSnapshot(config: JsonObject): { profileBody: string; prompt: string; workspaceRoot: string; + worktreeRoot: string; + baseSha: string; timeoutMs?: number; effectivePolicy: EffectiveLocalAgentPolicy; } { @@ -283,6 +339,8 @@ function readExecutionSnapshot(config: JsonObject): { profileBody: optionalString(config.profileBody) ?? "", prompt: requiredString(config.prompt, "prompt"), workspaceRoot: requiredString(config.workspaceRoot, "workspaceRoot"), + worktreeRoot: optionalString(config.worktreeRoot) ?? "", + baseSha: optionalString(config.baseSha) ?? "", timeoutMs, effectivePolicy: policy as unknown as EffectiveLocalAgentPolicy, }; @@ -364,6 +422,7 @@ function validateSupervisorTiming( nodeLeaseMs: number, heartbeatMs: number, idleMs: number, + globalConcurrency: number, ): void { for (const [label, value] of [ ["supervisor lease", supervisorLeaseMs], @@ -377,6 +436,11 @@ function validateSupervisorTiming( if (!Number.isSafeInteger(idleMs) || idleMs < 0) { throw new WorkflowValidationError("Workflow supervisor idle timeout must be a non-negative integer."); } + if (!Number.isSafeInteger(globalConcurrency) || globalConcurrency < 1 || globalConcurrency > MAX_GLOBAL_CONCURRENCY) { + throw new WorkflowValidationError( + `Workflow global concurrency must be between 1 and ${MAX_GLOBAL_CONCURRENCY}.`, + ); + } if (heartbeatMs >= Math.min(supervisorLeaseMs, nodeLeaseMs)) { throw new WorkflowValidationError("Workflow heartbeat interval must be shorter than both leases."); } diff --git a/src/workflows/types.ts b/src/workflows/types.ts index 1f0597c9..903bb587 100644 --- a/src/workflows/types.ts +++ b/src/workflows/types.ts @@ -45,6 +45,14 @@ export interface WorkflowDefinitionV1 { export type WorkflowDefinition = WorkflowDefinitionV1; +export type WorkflowRetryClass = "provider_failed" | "timed_out"; + +export interface WorkflowRetryPolicy extends JsonObject { + maxAttempts: number; + retryOn: WorkflowRetryClass[]; + backoffMs: number; +} + export interface WorkflowPolicyV1 extends JsonObject { version: typeof WORKFLOW_POLICY_VERSION; } @@ -70,6 +78,7 @@ export interface WorkflowNodeRecord { claimToken?: string; claimedAt?: string; claimExpiresAt?: string; + nextEligibleAt?: string; result?: JsonValue; error?: JsonObject; createdAt: string; @@ -96,6 +105,8 @@ export interface WorkflowRunRecord { requestHash: string; workspaceId?: string; workspaceRoot?: string; + maxConcurrency: number; + lastDispatchedAt?: string; result?: JsonValue; error?: JsonObject; cancellationRequestedAt?: string; @@ -185,6 +196,20 @@ export interface WorkflowEventReadOptions { limit?: number; } +export interface WorkflowWorktreeRecord { + workflowId: string; + nodeKey: string; + attempt: number; + path: string; + sourceRoot: string; + baseSha: string; + state: "allocated" | "active" | "preserved" | "removed" | "cleanup_failed"; + createdAt: string; + updatedAt: string; + retainUntil?: string; + cleanupError?: string; +} + export interface WorkflowNodeClaim { workflowId: string; nodeKey: string; diff --git a/src/workflows/worktrees.ts b/src/workflows/worktrees.ts new file mode 100644 index 00000000..5bd98d35 --- /dev/null +++ b/src/workflows/worktrees.ts @@ -0,0 +1,176 @@ +import { execFile } from "node:child_process"; +import { mkdir, realpath } from "node:fs/promises"; +import { basename, join, relative, resolve } from "node:path"; +import { promisify } from "node:util"; +import { WorkflowStore, WorkflowValidationError } from "./store.js"; +import type { WorkflowAttemptIdentity, WorkflowWorktreeRecord } from "./types.js"; + +const execFileAsync = promisify(execFile); +const FAILED_WORKTREE_RETENTION_MS = 7 * 24 * 60 * 60_000; + +export async function allocateWorkflowWorktree(input: { + store: WorkflowStore; + identity: WorkflowAttemptIdentity; + sourceRoot: string; + worktreeRoot: string; + baseSha: string; +}): Promise { + const existing = input.store.getWorktree( + input.identity.workflowId, + input.identity.nodeKey, + input.identity.attempt, + ); + if (existing) { + if (existing.state === "active") return existing; + throw new WorkflowValidationError( + `Workflow worktree attempt already has allocation state ${existing.state}`, + ); + } + + const sourceRoot = await canonicalDirectory(input.sourceRoot, "workflow source root"); + await mkdir(input.worktreeRoot, { recursive: true }); + const worktreeRoot = await canonicalDirectory(input.worktreeRoot, "workflow worktree root"); + const gitRoot = await git(["rev-parse", "--show-toplevel"], sourceRoot); + if ((await realpath(gitRoot.trim())) !== sourceRoot) { + throw new WorkflowValidationError("Workflow workspace_write source must be a Git repository root"); + } + const baseSha = (await git(["rev-parse", "--verify", `${input.baseSha}^{commit}`], sourceRoot)).trim(); + if (baseSha !== input.baseSha) { + throw new WorkflowValidationError("Workflow worktree base SHA no longer resolves to the pinned commit"); + } + + const path = join( + worktreeRoot, + `${safeSegment(basename(sourceRoot))}-${safeSegment(input.identity.workflowId)}-${safeSegment(input.identity.nodeKey)}-a${input.identity.attempt}`, + ); + assertContained(path, worktreeRoot); + const record = input.store.recordWorktree({ + workflowId: input.identity.workflowId, + nodeKey: input.identity.nodeKey, + attempt: input.identity.attempt, + path, + sourceRoot, + baseSha, + retainUntil: new Date(Date.now() + FAILED_WORKTREE_RETENTION_MS).toISOString(), + }); + try { + await git(["worktree", "add", "--detach", path, baseSha], sourceRoot); + return input.store.updateWorktreeState( + record.workflowId, + record.nodeKey, + record.attempt, + "active", + ); + } catch (error) { + input.store.updateWorktreeState( + record.workflowId, + record.nodeKey, + record.attempt, + "cleanup_failed", + error instanceof Error ? error.message.slice(0, 4_096) : String(error).slice(0, 4_096), + ); + throw error; + } +} + +export function preserveWorkflowWorktree( + store: WorkflowStore, + identity: WorkflowAttemptIdentity, +): WorkflowWorktreeRecord | undefined { + const record = store.getWorktree(identity.workflowId, identity.nodeKey, identity.attempt); + if (!record || record.state === "removed") return record; + return store.updateWorktreeState(identity.workflowId, identity.nodeKey, identity.attempt, "preserved"); +} + +export async function cleanupWorkflowWorktree(input: { + store: WorkflowStore; + identity: WorkflowAttemptIdentity; + worktreeRoot: string; +}): Promise { + const record = input.store.getWorktree( + input.identity.workflowId, + input.identity.nodeKey, + input.identity.attempt, + ); + if (!record || record.state === "removed") return record; + + try { + const worktreeRoot = await canonicalDirectory(input.worktreeRoot, "workflow worktree root"); + assertContained(record.path, worktreeRoot); + const canonicalWorktreePath = await canonicalDirectory(record.path, "workflow worktree path"); + assertContained(canonicalWorktreePath, worktreeRoot); + const registered = await registeredWorktrees(record.sourceRoot); + const match = registered.find((entry) => resolve(entry.path) === resolve(record.path)); + if (!match || match.head !== record.baseSha) { + throw new WorkflowValidationError("Workflow worktree Git registration does not match persisted ownership"); + } + await git(["worktree", "remove", "--force", record.path], record.sourceRoot); + return input.store.updateWorktreeState( + record.workflowId, + record.nodeKey, + record.attempt, + "removed", + ); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + input.store.updateWorktreeState( + record.workflowId, + record.nodeKey, + record.attempt, + "cleanup_failed", + message.slice(0, 4_096), + ); + throw error; + } +} + +async function registeredWorktrees(sourceRoot: string): Promise> { + const output = await git(["worktree", "list", "--porcelain"], sourceRoot); + const records: Array<{ path: string; head: string }> = []; + let path: string | undefined; + let head: string | undefined; + for (const line of output.split("\n")) { + if (line.startsWith("worktree ")) path = line.slice("worktree ".length); + else if (line.startsWith("HEAD ")) head = line.slice("HEAD ".length); + else if (!line && path && head) { + records.push({ path, head }); + path = undefined; + head = undefined; + } + } + if (path && head) records.push({ path, head }); + return records; +} + +async function canonicalDirectory(path: string, label: string): Promise { + try { + return await realpath(path); + } catch { + throw new WorkflowValidationError(`Invalid ${label}: ${path}`); + } +} + +function assertContained(path: string, root: string): void { + const relationship = relative(root, resolve(path)); + if (relationship === "" || relationship === ".." || relationship.startsWith("../") || relationship.startsWith("..\\")) { + throw new WorkflowValidationError("Workflow worktree path escapes the managed root"); + } +} + +function safeSegment(value: string): string { + const segment = value.replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 72); + if (!segment) throw new WorkflowValidationError("Workflow worktree identity is invalid"); + return segment; +} + +async function git(args: string[], cwd: string): Promise { + try { + const { stdout } = await execFileAsync("git", args, { cwd, maxBuffer: 10 * 1024 * 1024 }); + return stdout; + } catch (error) { + const stderr = typeof error === "object" && error && "stderr" in error + ? String((error as { stderr?: unknown }).stderr ?? "").trim() + : ""; + throw new WorkflowValidationError(stderr || (error instanceof Error ? error.message : String(error))); + } +} From 8df9d4d8cf773ea613fe0daa65eb46c34235f7e1 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 02:13:33 +0530 Subject: [PATCH 14/25] feat(mcp): expose durable workflow tools Co-Authored-By: Claude --- src/server.ts | 15 ++ src/workflows/mcp.ts | 318 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 333 insertions(+) create mode 100644 src/workflows/mcp.ts diff --git a/src/server.ts b/src/server.ts index 7dd9629e..c11e6f39 100644 --- a/src/server.ts +++ b/src/server.ts @@ -47,6 +47,8 @@ import { formatPathForPrompt } from "./skills.js"; import { createWorkspaceStore } from "./workspace-store.js"; import { formatAgentsPath, WorkspaceRegistry } from "./workspaces.js"; import { summarizeLocalAgentProfile } from "./local-agent-profiles.js"; +import { WorkflowOrchestrator } from "./workflows/orchestrator.js"; +import { registerWorkflowTools } from "./workflows/mcp.js"; import { formatLocalAgentProviderAvailabilitySummary, getLocalAgentProviderAvailabilitySnapshot, @@ -688,6 +690,7 @@ function createMcpServer( reviewCheckpoints: ReturnType, processSessions: ProcessSessionManager, localAgentProviders: LocalAgentProviderAvailability[], + workflowOrchestrator: WorkflowOrchestrator, ): McpServer { const server = new McpServer( { @@ -1594,6 +1597,15 @@ function createMcpServer( registerCodexProcessTools(server, config, workspaces, processSessions); } + if (config.subagents) { + registerWorkflowTools({ + server, + config, + workspaces, + orchestrator: workflowOrchestrator, + }); + } + return server; } @@ -1618,6 +1630,7 @@ export function createServer(config = loadConfig()): RunningServer { const workspaces = new WorkspaceRegistry(config, workspaceStore); const reviewCheckpoints = createReviewCheckpointManager(); const processSessions = new ProcessSessionManager(); + const workflowOrchestrator = new WorkflowOrchestrator(config.stateDir); const localAgentProviders = config.subagents ? getLocalAgentProviderAvailabilitySnapshot() : []; @@ -1781,6 +1794,7 @@ export function createServer(config = loadConfig()): RunningServer { reviewCheckpoints, processSessions, localAgentProviders, + workflowOrchestrator, ); await server.connect(transport); } else { @@ -1811,6 +1825,7 @@ export function createServer(config = loadConfig()): RunningServer { const results = await transports.closeAll(); logSessionCloseResults("server_shutdown", results); processSessions.shutdown(); + workflowOrchestrator.close(); oauthProvider.close(); workspaceStore.close?.(); })(); diff --git a/src/workflows/mcp.ts b/src/workflows/mcp.ts new file mode 100644 index 00000000..eede6470 --- /dev/null +++ b/src/workflows/mcp.ts @@ -0,0 +1,318 @@ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import * as z from "zod/v4"; +import type { ServerConfig } from "../config.js"; +import { loadLocalAgentProfiles } from "../local-agent-profiles.js"; +import type { WorkspaceRegistry } from "../workspaces.js"; +import { WorkflowOrchestrator } from "./orchestrator.js"; +import { createWorkflowSubmission } from "./submission.js"; +import { ensureSupervisor } from "./supervisor-launch.js"; +import type { WorkflowEvent, WorkflowRunRecord, WorkflowWorkspaceScope } from "./types.js"; + +const CONTRACT_VERSION = 1 as const; +const MAX_WAIT_MS = 300_000; +const MAX_EVENT_LIMIT = 1_000; + +const retrySchema = z.object({ + maxAttempts: z.number().int().min(1).max(10).optional(), + retryOn: z.array(z.enum(["provider_failed", "timed_out"])).max(2).optional(), + backoffMs: z.number().int().min(0).max(60_000).optional(), +}).strict(); + +const agentNodeSchema = z.object({ + key: z.string().min(1).max(64), + target: z.string().min(1).max(128), + prompt: z.string().min(1).max(200_000), + model: z.string().min(1).max(256).optional(), + thinking: z.string().min(1).max(64).optional(), + access: z.enum(["read_only", "workspace_write"]).optional(), + timeoutMs: z.number().int().min(1).max(86_400_000).optional(), + retry: retrySchema.optional(), +}).strict(); + +const dagSchema = z.object({ + version: z.literal(1), + nodes: z.array(agentNodeSchema).min(1).max(64), + edges: z.array(z.object({ + from: z.string().min(1).max(64), + to: z.string().min(1).max(64), + }).strict()).max(256).optional(), + maxConcurrency: z.number().int().min(1).max(16).optional(), + access: z.enum(["read_only", "workspace_write"]).optional(), +}).strict(); + +const nodeOutputSchema = z.object({ + key: z.string(), + status: z.enum(["pending", "ready", "running", "succeeded", "failed", "cancelled", "skipped"]), + attempt: z.number().int(), + completedAt: z.string().optional(), + errorCode: z.string().optional(), +}); + +const workflowOutputSchema = z.object({ + id: z.string(), + status: z.enum(["queued", "running", "cancelling", "succeeded", "failed", "cancelled"]), + maxConcurrency: z.number().int(), + createdAt: z.string(), + startedAt: z.string().optional(), + completedAt: z.string().optional(), + cancellationRequestedAt: z.string().optional(), + errorCode: z.string().optional(), + nodes: z.array(nodeOutputSchema), +}); + +const eventOutputSchema = z.object({ + sequence: z.number().int(), + type: z.string(), + nodeKey: z.string().optional(), + createdAt: z.string(), +}); + +const commonOutput = { + version: z.literal(CONTRACT_VERSION), + result: z.string(), +}; + +export function registerWorkflowTools(input: { + server: McpServer; + config: ServerConfig; + workspaces: WorkspaceRegistry; + orchestrator: WorkflowOrchestrator; +}): void { + const { server, config, workspaces, orchestrator } = input; + + server.registerTool( + "workflow_run", + { + title: "Run workflow", + description: "Submit a durable local-agent workflow. Provide either target/prompt for one agent or a versioned DAG.", + inputSchema: { + workspaceId: z.string().min(1), + target: z.string().min(1).max(128).optional(), + prompt: z.string().min(1).max(200_000).optional(), + model: z.string().min(1).max(256).optional(), + thinking: z.string().min(1).max(64).optional(), + access: z.enum(["read_only", "workspace_write"]).optional(), + timeoutMs: z.number().int().min(1).max(86_400_000).optional(), + retry: retrySchema.optional(), + dag: dagSchema.optional(), + idempotencyKey: z.string().min(1).max(256).optional(), + }, + outputSchema: { + ...commonOutput, + created: z.boolean(), + workflow: workflowOutputSchema, + }, + annotations: { + readOnlyHint: false, + destructiveHint: true, + idempotentHint: true, + openWorldHint: false, + }, + }, + async (args) => { + const hasSingleFields = [ + args.target, + args.prompt, + args.model, + args.thinking, + args.access, + args.timeoutMs, + args.retry, + ].some((value) => value !== undefined); + if (args.dag && hasSingleFields) { + throw new Error("Provide either dag or single-agent fields, not both"); + } + if (!args.dag && (!args.target || !args.prompt)) { + throw new Error("Single-agent workflows require target and prompt"); + } + const scope = workspaceScope(workspaces, args.workspaceId); + const workspace = workspaces.getWorkspace(args.workspaceId); + const profiles = workspace.agentProfiles.length > 0 + ? workspace.agentProfiles + : await loadLocalAgentProfiles(config, workspace.root); + const request = await createWorkflowSubmission({ + intent: { + single: args.dag ? undefined : { + target: args.target ?? "", + prompt: args.prompt ?? "", + model: args.model, + thinking: args.thinking, + access: args.access, + timeoutMs: args.timeoutMs, + retry: args.retry, + }, + dag: args.dag, + idempotencyKey: args.idempotencyKey, + }, + workspace: scope, + profiles, + worktreeRoot: config.worktreeRoot, + }); + const submitted = orchestrator.submitDetailed(request); + await ensureSupervisor({ stateDir: config.stateDir }); + const workflow = publicWorkflow(submitted.workflow); + const result = submitted.created + ? `Workflow ${workflow.id} was submitted with status ${workflow.status}.` + : `Workflow ${workflow.id} already exists for this idempotency key.`; + return response({ version: CONTRACT_VERSION, result, created: submitted.created, workflow }); + }, + ); + + server.registerTool( + "workflow_status", + { + title: "Get workflow status", + description: "Read the current durable workflow and node states for a workspace.", + inputSchema: { workspaceId: z.string().min(1), workflowId: z.string().min(1) }, + outputSchema: { ...commonOutput, workflow: workflowOutputSchema }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + async ({ workspaceId, workflowId }) => { + const workflow = publicWorkflow(requireOwned(orchestrator, workflowId, workspaceScope(workspaces, workspaceId))); + const result = `Workflow ${workflow.id} is ${workflow.status}.`; + return response({ version: CONTRACT_VERSION, result, workflow }); + }, + ); + + server.registerTool( + "workflow_wait", + { + title: "Wait for workflow", + description: "Wait for a bounded interval, then return the latest durable workflow state.", + inputSchema: { + workspaceId: z.string().min(1), + workflowId: z.string().min(1), + timeoutMs: z.number().int().min(0).max(MAX_WAIT_MS).optional(), + }, + outputSchema: { ...commonOutput, timedOut: z.boolean(), workflow: workflowOutputSchema }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + async ({ workspaceId, workflowId, timeoutMs }) => { + const scope = workspaceScope(workspaces, workspaceId); + const current = requireOwned(orchestrator, workflowId, scope); + const waited = await orchestrator.waitForWorkspace(workflowId, scope, { timeoutMs }); + const workflow = publicWorkflow(waited); + const timedOut = !isTerminal(waited) && !isTerminal(current); + const result = timedOut + ? `Workflow ${workflow.id} is still ${workflow.status} after the bounded wait.` + : `Workflow ${workflow.id} reached ${workflow.status}.`; + return response({ version: CONTRACT_VERSION, result, timedOut, workflow }); + }, + ); + + server.registerTool( + "workflow_events", + { + title: "Read workflow events", + description: "Read an ordered page of redacted durable workflow lifecycle events.", + inputSchema: { + workspaceId: z.string().min(1), + workflowId: z.string().min(1), + after: z.number().int().min(0).optional(), + limit: z.number().int().min(1).max(MAX_EVENT_LIMIT).optional(), + }, + outputSchema: { + ...commonOutput, + workflowId: z.string(), + events: z.array(eventOutputSchema), + cursor: z.number().int(), + }, + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + async ({ workspaceId, workflowId, after, limit }) => { + const scope = workspaceScope(workspaces, workspaceId); + const workflow = requireOwned(orchestrator, workflowId, scope); + const page = orchestrator.eventsForWorkspace(workflowId, scope, { after, limit }); + const events = page.events.map((event) => publicEvent(event, workflow)); + const result = `Read ${events.length} event${events.length === 1 ? "" : "s"} for workflow ${workflowId}.`; + return response({ + version: CONTRACT_VERSION, + result, + workflowId, + events, + cursor: page.nextCursor, + }); + }, + ); + + server.registerTool( + "workflow_cancel", + { + title: "Cancel workflow", + description: "Request durable cancellation of a workflow and its active local agents.", + inputSchema: { workspaceId: z.string().min(1), workflowId: z.string().min(1) }, + outputSchema: { ...commonOutput, workflow: workflowOutputSchema }, + annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }, + }, + async ({ workspaceId, workflowId }) => { + const scope = workspaceScope(workspaces, workspaceId); + requireOwned(orchestrator, workflowId, scope); + const cancelled = orchestrator.cancelForWorkspace(workflowId, scope); + await ensureSupervisor({ stateDir: config.stateDir }); + const workflow = publicWorkflow(cancelled); + const result = `Cancellation was requested for workflow ${workflow.id}; current status is ${workflow.status}.`; + return response({ version: CONTRACT_VERSION, result, workflow }); + }, + ); +} + +function workspaceScope(workspaces: WorkspaceRegistry, workspaceId: string): WorkflowWorkspaceScope { + const workspace = workspaces.getWorkspace(workspaceId); + return { workspaceId: workspace.id, workspaceRoot: workspace.root }; +} + +function requireOwned( + orchestrator: WorkflowOrchestrator, + workflowId: string, + scope: WorkflowWorkspaceScope, +): WorkflowRunRecord { + const workflow = orchestrator.getForWorkspace(workflowId, scope); + if (!workflow) throw new Error(`Unknown workflow: ${workflowId}`); + return workflow; +} + +function publicWorkflow(workflow: WorkflowRunRecord) { + return { + id: workflow.id, + status: workflow.status, + maxConcurrency: workflow.maxConcurrency, + createdAt: workflow.createdAt, + startedAt: workflow.startedAt, + completedAt: workflow.completedAt, + cancellationRequestedAt: workflow.cancellationRequestedAt, + errorCode: stringField(workflow.error, "code"), + nodes: workflow.nodes.map((node) => ({ + key: node.key, + status: node.status, + attempt: node.attempt, + completedAt: node.completedAt, + errorCode: stringField(node.error, "code"), + })), + }; +} + +function publicEvent(event: WorkflowEvent, workflow: WorkflowRunRecord) { + const node = event.nodeId ? workflow.nodes.find((candidate) => candidate.id === event.nodeId) : undefined; + return { + sequence: event.sequence, + type: event.type, + nodeKey: node?.key, + createdAt: event.createdAt, + }; +} + +function stringField(value: Record | undefined, key: string): string | undefined { + const field = value?.[key]; + return typeof field === "string" ? field : undefined; +} + +function response(structuredContent: T) { + return { + content: [{ type: "text" as const, text: structuredContent.result }], + structuredContent, + }; +} + +function isTerminal(workflow: WorkflowRunRecord): boolean { + return workflow.status === "succeeded" || workflow.status === "failed" || workflow.status === "cancelled"; +} From bafab267cf631853647979030ac46b3055931a33 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 02:13:48 +0530 Subject: [PATCH 15/25] test(workflows): cover DAG and MCP contracts Co-Authored-By: Claude --- docs/workflow-mcp.md | 98 ++++++++++ package.json | 2 +- src/oauth-store.test.ts | 1 + src/workflows/dag-scheduler.test.ts | 284 ++++++++++++++++++++++++++++ src/workflows/mcp.test.ts | 105 ++++++++++ src/workflows/migrations.test.ts | 9 +- src/workflows/workflows.test.ts | 22 +-- 7 files changed, 506 insertions(+), 15 deletions(-) create mode 100644 docs/workflow-mcp.md create mode 100644 src/workflows/dag-scheduler.test.ts create mode 100644 src/workflows/mcp.test.ts diff --git a/docs/workflow-mcp.md b/docs/workflow-mcp.md new file mode 100644 index 00000000..f9f79798 --- /dev/null +++ b/docs/workflow-mcp.md @@ -0,0 +1,98 @@ +# Durable workflow MCP tools + +DevSpace exposes durable local-agent workflows when subagents are enabled (`DEVSPACE_SUBAGENTS=1`). Workflow execution is owned by the DevSpace process and its detached supervisor, not by an individual MCP transport. Closing or reconnecting an MCP session does not cancel submitted work. + +## Tools + +All tools require `workspaceId`. A workflow ID alone is never authorization: the workflow must belong to the same workspace ID and current canonical workspace root. + +### `workflow_run` + +Submit either one local agent or a versioned DAG. + +Single-agent example: + +```json +{ + "workspaceId": "ws_...", + "target": "reviewer", + "prompt": "Review the authentication changes", + "access": "read_only", + "timeoutMs": 120000, + "retry": { + "maxAttempts": 2, + "retryOn": ["provider_failed"], + "backoffMs": 1000 + }, + "idempotencyKey": "auth-review-v1" +} +``` + +DAG example: + +```json +{ + "workspaceId": "ws_...", + "dag": { + "version": 1, + "access": "read_only", + "maxConcurrency": 3, + "nodes": [ + { "key": "tests", "target": "qa", "prompt": "Run focused tests" }, + { "key": "security", "target": "reviewer", "prompt": "Audit security" }, + { "key": "summary", "target": "claude", "prompt": "Synthesize the findings" } + ], + "edges": [ + { "from": "tests", "to": "summary" }, + { "from": "security", "to": "summary" } + ] + } +} +``` + +The DAG is rejected before persistence when it exceeds 64 nodes or 256 edges, uses duplicate or unsafe keys, references missing endpoints, contains duplicate edges or a cycle, requests an unsupported target or access mode, or exceeds timeout/retry/concurrency bounds. + +### `workflow_status` + +Returns the current redacted run and node states. + +### `workflow_wait` + +Waits for at most 300 seconds and returns the latest state. A timeout is not cancellation and does not destroy the result; callers may wait again. + +### `workflow_events` + +Reads ordered lifecycle events using an `after` cursor and a limit of at most 1,000. MCP event projections omit prompts, roots, profile bodies, environment data, claim tokens, process IDs, leases, and provider session IDs. + +### `workflow_cancel` + +Durably requests cancellation. Repeated requests are idempotent. Ready and pending nodes are terminalized, while active provider handles receive cancellation through the supervisor. + +## Scheduling semantics + +- Ready nodes are selected fairly across runs using persisted last-dispatch order. +- Both a process-wide supervisor limit and each run's persisted `maxConcurrency` are enforced. +- A node becomes ready only after all predecessors succeed. +- Exhausted failure skips unopened dependents and cancels active siblings. +- A run succeeds only after every node succeeds or is validly skipped; failures and user cancellation converge deterministically. +- Claims and heartbeats are lease-fenced. Stale attempts cannot append events or complete a replacement attempt. + +## Retry semantics + +Retries are opt-in and default to one attempt. Only `provider_failed` and `timed_out` may be configured as retryable. Each retry creates a new durable attempt and emits `node.retry_scheduled`. + +Automatic retries are limited to `read_only` nodes. DevSpace does not retry worker-loss/unknown-outcome failures or `workspace_write` side effects automatically. + +## Worktree isolation + +Every `workspace_write` attempt receives a unique detached Git worktree at the base SHA pinned when the workflow was submitted. + +- Changes are never merged, copied, committed, rebased, cherry-picked, pushed, or applied to the source checkout automatically. +- A successful attempt is durably completed before guarded cleanup. +- Failed or cancelled attempt worktrees are preserved for diagnosis with a seven-day retention timestamp. +- Cleanup verifies managed-root containment, persisted workflow/node/attempt ownership, Git worktree registration, and the expected base SHA. A failed verification preserves the directory and records `cleanup_failed`; there is no unverified recursive-delete fallback. +- Dependent nodes do not automatically inherit predecessor filesystem changes. Handoffs must use durable results/artifacts rather than an implicit shared working tree. + +## CLI compatibility + +Existing `devspace workflows run|status|wait|events|cancel` commands use the same submission snapshots and durable store. MCP and CLI can observe the same workflow when they use the same workspace ID and root. The CLI remains the shell-compatible fallback for parent harnesses that cannot call MCP tools directly. diff --git a/package.json b/package.json index 9d0b9dd4..274635d3 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "postinstall": "node scripts/fix-node-pty-permissions.mjs", "start": "node dist/cli.js serve", "test": "tsx src/config.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && npm run test:workflows && tsx src/cli.test.ts", - "test:workflows": "tsx src/workflows/migrations.test.ts && tsx src/workflows/policy.test.ts && tsx src/workflows/workflows.test.ts && tsx src/workflows/supervisor.test.ts && tsx src/workflows/cli.test.ts", + "test:workflows": "tsx src/workflows/migrations.test.ts && tsx src/workflows/policy.test.ts && tsx src/workflows/workflows.test.ts && tsx src/workflows/supervisor.test.ts && tsx src/workflows/dag-scheduler.test.ts && tsx src/workflows/mcp.test.ts && tsx src/workflows/cli.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, "keywords": [], diff --git a/src/oauth-store.test.ts b/src/oauth-store.test.ts index b1e36d5c..59f414e2 100644 --- a/src/oauth-store.test.ts +++ b/src/oauth-store.test.ts @@ -46,6 +46,7 @@ async function testDatabaseConfiguration(stateDir: string): Promise { { version: 3, name: "local-agent-sessions" }, { version: 4, name: "durable-workflows" }, { version: 5, name: "workflow-supervisor" }, + { version: 6, name: "workflow-dag-scheduler" }, ]); } finally { database.close(); diff --git a/src/workflows/dag-scheduler.test.ts b/src/workflows/dag-scheduler.test.ts new file mode 100644 index 00000000..b495b8f0 --- /dev/null +++ b/src/workflows/dag-scheduler.test.ts @@ -0,0 +1,284 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { LocalAgentRunController, type LocalAgentRunHandle } from "../local-agent-runtime.js"; +import { runWorkflowSupervisor } from "./supervisor.js"; +import { WorkflowStore } from "./store.js"; +import { allocateWorkflowWorktree, cleanupWorkflowWorktree } from "./worktrees.js"; +import type { JsonObject, WorkflowDefinitionV1 } from "./types.js"; + +const root = mkdtempSync(join(tmpdir(), "devspace-workflow-dag-test-")); +try { + await testParallelBoundedScheduling(join(root, "parallel")); + await testFairSchedulingAcrossRuns(join(root, "fairness")); + await testExplicitReadOnlyRetry(join(root, "retry")); + await testManagedWriteWorktree(join(root, "worktree")); + await testWorktreeCleanupRejectsRootSubstitution(join(root, "cleanup-guard")); +} finally { + rmSync(root, { recursive: true, force: true }); +} + +async function testParallelBoundedScheduling(stateDir: string): Promise { + const store = new WorkflowStore(stateDir); + const workflow = store.submit({ + definition: definition(["a", "b", "c"], { access: "read_only" }), + policy: { version: 1, maxConcurrency: 2 }, + }).workflow; + store.close(); + + let active = 0; + let maximum = 0; + const started: string[] = []; + const handleFactory = async (_provider: string, input: { prompt: string }): Promise => { + active += 1; + maximum = Math.max(maximum, active); + started.push(input.prompt); + const controller = new LocalAgentRunController("fake"); + const timer = setTimeout(() => { + active -= 1; + controller.succeed({ provider: "fake", providerSessionId: `session-${input.prompt}`, finalResponse: input.prompt, items: [] }); + }, 40); + controller.setLifecycle({ + cancel: () => { + clearTimeout(timer); + active = Math.max(0, active - 1); + }, + dispose: () => clearTimeout(timer), + }); + return controller; + }; + + assert.equal(await runWorkflowSupervisor(stateDir, { + handleFactory, + globalConcurrency: 2, + heartbeatMs: 10, + nodeLeaseMs: 500, + supervisorLeaseMs: 500, + idleMs: 0, + }), true); + assert.equal(maximum, 2); + assert.deepEqual(started.sort(), ["a", "b", "c"]); + const reader = new WorkflowStore(stateDir); + try { + assert.equal(reader.require(workflow.id).status, "succeeded"); + } finally { + reader.close(); + } +} + +async function testFairSchedulingAcrossRuns(stateDir: string): Promise { + const store = new WorkflowStore(stateDir); + const first = store.submit({ + definition: definition(["first-a", "first-b"], { access: "read_only" }), + policy: { version: 1, maxConcurrency: 2 }, + }).workflow; + const second = store.submit({ + definition: definition(["second-a", "second-b"], { access: "read_only" }), + policy: { version: 1, maxConcurrency: 2 }, + }).workflow; + store.close(); + + const started: string[] = []; + const handleFactory = async (_provider: string, input: { prompt: string }): Promise => { + started.push(input.prompt); + const controller = new LocalAgentRunController("fake"); + const timer = setTimeout(() => { + controller.succeed({ + provider: "fake", + providerSessionId: `fair-${input.prompt}`, + finalResponse: input.prompt, + items: [], + }); + }, 40); + controller.setLifecycle({ + cancel: () => clearTimeout(timer), + dispose: () => clearTimeout(timer), + }); + return controller; + }; + + await runWorkflowSupervisor(stateDir, { + handleFactory, + globalConcurrency: 2, + heartbeatMs: 10, + nodeLeaseMs: 500, + supervisorLeaseMs: 500, + idleMs: 0, + }); + assert.equal(started.length, 4); + assert.equal(started.slice(0, 2).filter((prompt) => prompt.startsWith("first-")).length, 1); + assert.equal(started.slice(0, 2).filter((prompt) => prompt.startsWith("second-")).length, 1); + const reader = new WorkflowStore(stateDir); + try { + assert.equal(reader.require(first.id).status, "succeeded"); + assert.equal(reader.require(second.id).status, "succeeded"); + } finally { + reader.close(); + } +} + +async function testExplicitReadOnlyRetry(stateDir: string): Promise { + const store = new WorkflowStore(stateDir); + const workflow = store.submit({ + definition: definition(["retry"], { + access: "read_only", + retry: { maxAttempts: 2, retryOn: ["provider_failed"], backoffMs: 0 }, + }), + policy: { version: 1, maxConcurrency: 1 }, + }).workflow; + store.close(); + let starts = 0; + const handleFactory = async (): Promise => { + starts += 1; + if (starts === 1) throw new Error("first attempt fails"); + const controller = new LocalAgentRunController("fake"); + queueMicrotask(() => controller.succeed({ provider: "fake", providerSessionId: "retry-session", finalResponse: "ok", items: [] })); + return controller; + }; + + await runWorkflowSupervisor(stateDir, { + handleFactory, + globalConcurrency: 1, + heartbeatMs: 10, + nodeLeaseMs: 500, + supervisorLeaseMs: 500, + idleMs: 0, + }); + const reader = new WorkflowStore(stateDir); + try { + const completed = reader.require(workflow.id); + assert.equal(completed.status, "succeeded"); + assert.equal(completed.nodes[0]!.attempt, 2); + assert.ok(reader.readEvents(workflow.id, { limit: 100 }).events.some((event) => event.type === "node.retry_scheduled")); + } finally { + reader.close(); + } +} + +async function testManagedWriteWorktree(stateDir: string): Promise { + const repo = join(stateDir, "repo"); + const worktreeRoot = join(stateDir, "managed"); + execFileSync("mkdir", ["-p", repo]); + execFileSync("git", ["init", "-q"], { cwd: repo }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: repo }); + execFileSync("git", ["config", "user.name", "Test"], { cwd: repo }); + writeFileSync(join(repo, "source.txt"), "source\n"); + execFileSync("git", ["add", "source.txt"], { cwd: repo }); + execFileSync("git", ["commit", "-qm", "initial"], { cwd: repo }); + const baseSha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repo, encoding: "utf8" }).trim(); + + const store = new WorkflowStore(stateDir); + const workflow = store.submit({ + definition: definition(["writer"], { + access: "workspace_write", + workspaceRoot: repo, + worktreeRoot, + baseSha, + }), + policy: { version: 1, maxConcurrency: 1 }, + }).workflow; + store.close(); + let executionRoot = ""; + const handleFactory = async (_provider: string, input: { workspace: string }): Promise => { + executionRoot = input.workspace; + writeFileSync(join(input.workspace, "source.txt"), "isolated\n"); + const controller = new LocalAgentRunController("fake"); + queueMicrotask(() => controller.succeed({ provider: "fake", providerSessionId: "writer-session", finalResponse: "changed", items: [] })); + return controller; + }; + + await runWorkflowSupervisor(stateDir, { + handleFactory, + globalConcurrency: 1, + heartbeatMs: 10, + nodeLeaseMs: 1_000, + supervisorLeaseMs: 1_000, + idleMs: 0, + }); + assert.notEqual(executionRoot, repo); + assert.equal(readFileSync(join(repo, "source.txt"), "utf8"), "source\n"); + assert.equal(execFileSync("git", ["status", "--porcelain"], { cwd: repo, encoding: "utf8" }), ""); + const reader = new WorkflowStore(stateDir); + try { + assert.equal(reader.require(workflow.id).status, "succeeded"); + assert.equal(reader.getWorktree(workflow.id, "writer", 1)?.state, "removed"); + } finally { + reader.close(); + } +} + +async function testWorktreeCleanupRejectsRootSubstitution(stateDir: string): Promise { + const repo = join(stateDir, "repo"); + const worktreeRoot = join(stateDir, "managed"); + const outside = join(stateDir, "outside"); + mkdirSync(repo, { recursive: true }); + mkdirSync(outside, { recursive: true }); + execFileSync("git", ["init", "-q"], { cwd: repo }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: repo }); + execFileSync("git", ["config", "user.name", "Test"], { cwd: repo }); + writeFileSync(join(repo, "source.txt"), "source\n"); + writeFileSync(join(outside, "marker.txt"), "keep\n"); + execFileSync("git", ["add", "source.txt"], { cwd: repo }); + execFileSync("git", ["commit", "-qm", "initial"], { cwd: repo }); + const baseSha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repo, encoding: "utf8" }).trim(); + + const store = new WorkflowStore(stateDir); + const workflow = store.submit({ + definition: definition(["writer"], { access: "workspace_write", workspaceRoot: repo, worktreeRoot, baseSha }), + policy: { version: 1, maxConcurrency: 1 }, + }).workflow; + const identity = { + workflowId: workflow.id, + nodeKey: "writer", + attempt: 1, + claimToken: "cleanup-claim", + supervisorOwnerToken: "cleanup-owner", + supervisorOwnerEpoch: 1, + }; + const allocation = await allocateWorkflowWorktree({ + store, + identity, + sourceRoot: repo, + worktreeRoot, + baseSha, + }); + execFileSync("git", ["worktree", "remove", "--force", allocation.path], { cwd: repo }); + symlinkSync(outside, allocation.path, "dir"); + + await assert.rejects( + cleanupWorkflowWorktree({ store, identity, worktreeRoot }), + /escapes the managed root/, + ); + assert.equal(readFileSync(join(outside, "marker.txt"), "utf8"), "keep\n"); + assert.equal(store.getWorktree(workflow.id, "writer", 1)?.state, "cleanup_failed"); + store.close(); +} + +function definition(keys: string[], overrides: JsonObject): WorkflowDefinitionV1 { + return { + version: 1, + nodes: keys.map((key) => ({ + key, + type: "agent" as const, + config: { + provider: "fake", + prompt: key, + profileBody: "", + workspaceRoot: String(overrides.workspaceRoot ?? process.cwd()), + worktreeRoot: String(overrides.worktreeRoot ?? ""), + baseSha: String(overrides.baseSha ?? ""), + timeoutMs: null, + effectivePolicy: { + version: 1, + mode: "workflow", + access: String(overrides.access ?? "read_only"), + environment: {}, + }, + retry: (overrides.retry ?? { maxAttempts: 1, retryOn: [], backoffMs: 0 }) as JsonObject, + }, + })), + edges: [], + }; +} diff --git a/src/workflows/mcp.test.ts b/src/workflows/mcp.test.ts new file mode 100644 index 00000000..3a613ab1 --- /dev/null +++ b/src/workflows/mcp.test.ts @@ -0,0 +1,105 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import type { ServerConfig } from "../config.js"; +import { WorkspaceRegistry } from "../workspaces.js"; +import { registerWorkflowTools } from "./mcp.js"; +import { WorkflowOrchestrator } from "./orchestrator.js"; + +const root = mkdtempSync(join(tmpdir(), "devspace-workflow-mcp-test-")); +const stateDir = join(root, "state"); +const config = { + stateDir, + worktreeRoot: join(root, "worktrees"), + allowedRoots: [root], + subagents: true, + devspaceAgentsDir: join(root, "agents"), + skillsEnabled: false, + skillPaths: [], + devspaceSkillsDir: join(root, "skills"), + agentDir: join(root, "agent-dir"), +} as unknown as ServerConfig; +const workspaces = new WorkspaceRegistry(config); +const { workspace } = await workspaces.openWorkspace(root); +const orchestrator = new WorkflowOrchestrator(stateDir); +const workflow = orchestrator.submit({ + definition: { + version: 1, + nodes: [{ + key: "agent", + type: "agent", + config: { + provider: "fake", + prompt: "secret prompt", + profileBody: "secret profile", + workspaceRoot: root, + effectivePolicy: { version: 1, mode: "workflow", access: "read_only", environment: {} }, + }, + }], + edges: [], + }, + workspace: { workspaceId: workspace.id, workspaceRoot: workspace.root }, +}); + +const server = new McpServer({ name: "workflow-test", version: "1.0.0" }); +registerWorkflowTools({ server, config, workspaces, orchestrator }); +const client = new Client({ name: "workflow-test-client", version: "1.0.0" }); +const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); +try { + await Promise.all([server.connect(serverTransport), client.connect(clientTransport)]); + const listed = await client.listTools(); + assert.deepEqual( + listed.tools.map((tool) => tool.name).sort(), + ["workflow_cancel", "workflow_events", "workflow_run", "workflow_status", "workflow_wait"], + ); + const runTool = listed.tools.find((tool) => tool.name === "workflow_run")!; + assert.equal(runTool.annotations?.destructiveHint, true); + assert.equal(runTool.annotations?.idempotentHint, true); + const statusTool = listed.tools.find((tool) => tool.name === "workflow_status")!; + assert.equal(statusTool.annotations?.readOnlyHint, true); + + const ambiguous = await client.callTool({ + name: "workflow_run", + arguments: { + workspaceId: workspace.id, + target: "fake", + prompt: "single", + dag: { + version: 1, + nodes: [{ key: "dag", target: "fake", prompt: "dag" }], + }, + }, + }); + assert.equal(ambiguous.isError, true); + + const result = await client.callTool({ + name: "workflow_status", + arguments: { workspaceId: workspace.id, workflowId: workflow.id }, + }); + const structured = result.structuredContent as { + version: number; + result: string; + workflow: { id: string; status: string; nodes: Array<{ key: string }> }; + }; + assert.equal(structured.version, 1); + assert.equal(structured.workflow.id, workflow.id); + assert.equal(structured.workflow.status, "queued"); + assert.equal((result.content as Array<{ text: string }>)[0]!.text, structured.result); + const serialized = JSON.stringify(structured); + assert.doesNotMatch(serialized, /secret prompt|secret profile|workspaceRoot|claimToken|providerSession/); + + const denied = await client.callTool({ + name: "workflow_status", + arguments: { workspaceId: "unknown", workflowId: workflow.id }, + }); + assert.equal(denied.isError, true); +} finally { + await client.close(); + await server.close(); + orchestrator.close(); + rmSync(root, { recursive: true, force: true }); +} diff --git a/src/workflows/migrations.test.ts b/src/workflows/migrations.test.ts index 6ddb3304..6178d546 100644 --- a/src/workflows/migrations.test.ts +++ b/src/workflows/migrations.test.ts @@ -26,6 +26,7 @@ function testFreshSchema(stateDir: string): void { { version: 3, name: "local-agent-sessions" }, { version: 4, name: "durable-workflows" }, { version: 5, name: "workflow-supervisor" }, + { version: 6, name: "workflow-dag-scheduler" }, ], ); const tables = database.sqlite @@ -44,6 +45,7 @@ function testFreshSchema(stateDir: string): void { "workflow_provider_events", "workflow_runs", "workflow_supervisor", + "workflow_worktrees", ]); const edgeForeignKeys = database.sqlite.prepare("pragma foreign_key_list(workflow_edges)").all() as Array<{ @@ -98,7 +100,7 @@ function testExistingMigrationCompatibility(stateDir: string): void { try { assert.deepEqual( migrated.sqlite.prepare("select version from devspace_schema_migrations order by version").pluck().all(), - [1, 2, 3, 4, 5], + [1, 2, 3, 4, 5, 6], ); assert.deepEqual(migrated.sqlite.prepare("select * from workspace_sessions").all(), [ { @@ -171,7 +173,7 @@ function testExistingMigrationCompatibility(stateDir: string): void { .prepare("select count(*) from sqlite_master where type = 'table' and name like 'workflow_%'") .pluck() .get(), - 7, + 8, ); } finally { migrated.close(); @@ -222,6 +224,7 @@ function createVersion3Fixture(stateDir: string): void { const initialized = openDatabase(stateDir); try { initialized.sqlite.exec(` + drop table workflow_worktrees; drop table workflow_provider_events; drop table workflow_node_attempts; drop table workflow_supervisor; @@ -229,7 +232,7 @@ function createVersion3Fixture(stateDir: string): void { drop table workflow_events; drop table workflow_nodes; drop table workflow_runs; - delete from devspace_schema_migrations where version in (4, 5); + delete from devspace_schema_migrations where version in (4, 5, 6); insert into workspace_sessions ( id, root, status, mode, source_root, base_ref, base_sha, managed, created_at, last_used_at diff --git a/src/workflows/workflows.test.ts b/src/workflows/workflows.test.ts index e99850f6..1c4fb475 100644 --- a/src/workflows/workflows.test.ts +++ b/src/workflows/workflows.test.ts @@ -486,17 +486,17 @@ function testDagValidation(stateDir: string): void { two: "pending", }); - const nulKeys = store.submit({ - definition: { - version: 1, - nodes: [agentNode("a\0b"), agentNode("a"), agentNode("b\0c"), agentNode("c")], - edges: [ - { from: "a\0b", to: "c" }, - { from: "a", to: "b\0c" }, - ], - }, - }).workflow; - assert.equal(nulKeys.edges.length, 2); + assert.throws( + () => + store.submit({ + definition: { + version: 1, + nodes: [agentNode("a\0b")], + edges: [], + }, + }), + /node key is unsafe/, + ); } finally { store.close(); } From d8b15fa8b877f7a2a24dca420ef809a67094a48c Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 22:39:10 +0530 Subject: [PATCH 16/25] fix(workflows): harden MCP DAG lifecycle Co-Authored-By: Claude --- src/server-shutdown.test.ts | 24 ++++ src/server.ts | 9 ++ src/workflows/cli.test.ts | 2 +- src/workflows/dag-scheduler.test.ts | 177 +++++++++++++++++++++++++++- src/workflows/mcp.test.ts | 120 ++++++++++++++++++- src/workflows/mcp.ts | 125 ++++++++++++++++---- src/workflows/orchestrator.ts | 3 +- src/workflows/store.ts | 176 ++++++++++++++++++++------- src/workflows/supervisor.test.ts | 99 +++++++++++++++- src/workflows/supervisor.ts | 66 ++++++----- src/workflows/workflows.test.ts | 48 +++++++- src/workflows/worktrees.ts | 36 +++++- 12 files changed, 777 insertions(+), 108 deletions(-) diff --git a/src/server-shutdown.test.ts b/src/server-shutdown.test.ts index 7dca2eea..db68c370 100644 --- a/src/server-shutdown.test.ts +++ b/src/server-shutdown.test.ts @@ -1,5 +1,9 @@ import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { shutdownHttpServer } from "./server-shutdown.js"; +import { WorkflowOrchestrator } from "./workflows/orchestrator.js"; let finishHttpClose: (() => void) | undefined; let applicationCloseStarted = false; @@ -95,3 +99,23 @@ await assert.rejects( ), httpCloseError, ); + +const workflowState = mkdtempSync(join(tmpdir(), "devspace-shutdown-workflow-")); +try { + const orchestrator = new WorkflowOrchestrator(workflowState); + const workflow = orchestrator.submit({ + definition: { version: 1, nodes: [{ key: "agent", type: "agent" }], edges: [] }, + workspace: { workspaceId: "workspace", workspaceRoot: "/tmp/workspace" }, + }); + const waiting = orchestrator.waitForWorkspace( + workflow.id, + { workspaceId: "workspace", workspaceRoot: "/tmp/workspace" }, + { timeoutMs: 5_000, pollIntervalMs: 1_000 }, + ); + orchestrator.close(); + const closedSnapshot = await waiting; + assert.equal(closedSnapshot.id, workflow.id); + assert.equal(closedSnapshot.status, "queued"); +} finally { + rmSync(workflowState, { recursive: true, force: true }); +} diff --git a/src/server.ts b/src/server.ts index c11e6f39..c2da9091 100644 --- a/src/server.ts +++ b/src/server.ts @@ -171,6 +171,7 @@ const toolNames = { interface ToolLogFields { tool: string; workspaceId?: string; + workflowId?: string; path?: string; workingDirectory?: string; command?: string; @@ -1603,6 +1604,14 @@ function createMcpServer( config, workspaces, orchestrator: workflowOrchestrator, + audit: (event) => logToolCall(config, { + tool: event.tool, + workspaceId: event.workspaceId, + workflowId: event.workflowId, + success: event.success, + durationMs: event.durationMs, + error: event.errorCode, + }), }); } diff --git a/src/workflows/cli.test.ts b/src/workflows/cli.test.ts index ef506697..43d0e949 100644 --- a/src/workflows/cli.test.ts +++ b/src/workflows/cli.test.ts @@ -56,7 +56,7 @@ try { ], baseEnv); assert.equal(timed.status, 0, timed.stderr); const timedEnvelope = JSON.parse(timed.stdout) as Envelope; - assert.equal(timedEnvelope.timedOut, true); + assert.equal(timedEnvelope.timedOut, true, JSON.stringify(timedEnvelope)); const waited = runCli([ "workflows", "wait", workflowId, "--timeout-ms", "10000", "--after", "0", "--json", diff --git a/src/workflows/dag-scheduler.test.ts b/src/workflows/dag-scheduler.test.ts index b495b8f0..366032f1 100644 --- a/src/workflows/dag-scheduler.test.ts +++ b/src/workflows/dag-scheduler.test.ts @@ -1,20 +1,29 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { openDatabase } from "../db/client.js"; import { LocalAgentRunController, type LocalAgentRunHandle } from "../local-agent-runtime.js"; import { runWorkflowSupervisor } from "./supervisor.js"; import { WorkflowStore } from "./store.js"; -import { allocateWorkflowWorktree, cleanupWorkflowWorktree } from "./worktrees.js"; +import { + allocateWorkflowWorktree, + cleanupExpiredWorkflowWorktrees, + cleanupWorkflowWorktree, + preserveWorkflowWorktree, +} from "./worktrees.js"; import type { JsonObject, WorkflowDefinitionV1 } from "./types.js"; const root = mkdtempSync(join(tmpdir(), "devspace-workflow-dag-test-")); try { await testParallelBoundedScheduling(join(root, "parallel")); await testFairSchedulingAcrossRuns(join(root, "fairness")); + testFairSchedulingWithTiedTimestamps(join(root, "fairness-tied")); await testExplicitReadOnlyRetry(join(root, "retry")); + await testRetryFailureClasses(join(root, "retry-classes")); await testManagedWriteWorktree(join(root, "worktree")); + await testExpiredWorktreeCleanup(join(root, "worktree-retention")); await testWorktreeCleanupRejectsRootSubstitution(join(root, "cleanup-guard")); } finally { rmSync(root, { recursive: true, force: true }); @@ -119,6 +128,31 @@ async function testFairSchedulingAcrossRuns(stateDir: string): Promise { } } +function testFairSchedulingWithTiedTimestamps(stateDir: string): void { + const store = new WorkflowStore(stateDir); + const first = store.submit({ + definition: definition(["first-a", "first-b"], { access: "read_only" }), + policy: { version: 1, maxConcurrency: 2 }, + }).workflow; + const second = store.submit({ + definition: definition(["second-a", "second-b"], { access: "read_only" }), + policy: { version: 1, maxConcurrency: 2 }, + }).workflow; + const supervisor = store.acquireSupervisor({ ownerToken: "fair-tie", ownerPid: 1, leaseMs: 1_000 })!; + const firstClaim = store.claimNextAgentNode({ supervisor, claimToken: "first", leaseMs: 1_000 })!; + assert.equal(firstClaim.workflow.id, first.id); + + const database = openDatabase(stateDir); + database.sqlite.prepare("update workflow_runs set last_dispatched_at = ? where id in (?, ?)") + .run("2026-01-01T00:00:00.000Z", first.id, second.id); + database.close(); + + const secondClaim = store.claimNextAgentNode({ supervisor, claimToken: "second", leaseMs: 1_000 })!; + assert.equal(secondClaim.workflow.id, second.id); + store.releaseSupervisor(supervisor); + store.close(); +} + async function testExplicitReadOnlyRetry(stateDir: string): Promise { const store = new WorkflowStore(stateDir); const workflow = store.submit({ @@ -157,6 +191,104 @@ async function testExplicitReadOnlyRetry(stateDir: string): Promise { } } +async function testRetryFailureClasses(stateDir: string): Promise { + const resultStore = new WorkflowStore(stateDir); + const resultRetry = resultStore.submit({ + definition: definition(["result-retry"], { + access: "read_only", + retry: { maxAttempts: 2, retryOn: ["provider_failed"], backoffMs: 0 }, + }), + policy: { version: 1, maxConcurrency: 1 }, + }).workflow; + resultStore.close(); + let resultStarts = 0; + await runWorkflowSupervisor(stateDir, { + globalConcurrency: 1, + heartbeatMs: 5, + nodeLeaseMs: 200, + supervisorLeaseMs: 200, + idleMs: 0, + handleFactory: async () => { + resultStarts += 1; + const controller = new LocalAgentRunController("fake"); + queueMicrotask(() => { + if (resultStarts === 1) controller.fail(new Error("result rejected")); + else controller.succeed({ provider: "fake", providerSessionId: "result-retry", finalResponse: "ok", items: [] }); + }); + return controller; + }, + }); + const resultReader = new WorkflowStore(stateDir); + assert.equal(resultReader.require(resultRetry.id).status, "succeeded"); + assert.equal(resultReader.require(resultRetry.id).nodes[0]!.attempt, 2); + resultReader.close(); + + const timeoutStore = new WorkflowStore(stateDir); + const timeoutRetry = timeoutStore.submit({ + definition: definition(["timeout-retry"], { + access: "read_only", + timeoutMs: 10, + retry: { maxAttempts: 2, retryOn: ["timed_out"], backoffMs: 20 }, + }), + policy: { version: 1, maxConcurrency: 1 }, + }).workflow; + timeoutStore.close(); + const timeoutStarts: number[] = []; + await runWorkflowSupervisor(stateDir, { + globalConcurrency: 1, + heartbeatMs: 5, + nodeLeaseMs: 200, + supervisorLeaseMs: 200, + idleMs: 0, + handleFactory: async () => { + timeoutStarts.push(Date.now()); + const controller = new LocalAgentRunController("fake"); + const timer = setTimeout(() => controller.succeed({ + provider: "fake", + providerSessionId: "late-timeout", + finalResponse: "late", + items: [], + }), 30); + controller.setLifecycle({ cancel: () => undefined, dispose: () => clearTimeout(timer) }); + return controller; + }, + }); + const timeoutReader = new WorkflowStore(stateDir); + const timedOut = timeoutReader.require(timeoutRetry.id); + assert.equal(timedOut.status, "failed"); + assert.equal(timedOut.nodes[0]!.attempt, 2); + assert.equal((timedOut.error as JsonObject).code, "timed_out"); + assert.ok(timeoutStarts[1]! - timeoutStarts[0]! >= 15, "retry backoff must delay the second attempt"); + timeoutReader.close(); + + const nonRetryStore = new WorkflowStore(stateDir); + const nonRetry = nonRetryStore.submit({ + definition: definition(["non-retryable"], { + access: "read_only", + retry: { maxAttempts: 3, retryOn: ["timed_out"], backoffMs: 0 }, + }), + policy: { version: 1, maxConcurrency: 1 }, + }).workflow; + nonRetryStore.close(); + let nonRetryStarts = 0; + await runWorkflowSupervisor(stateDir, { + globalConcurrency: 1, + heartbeatMs: 5, + nodeLeaseMs: 200, + supervisorLeaseMs: 200, + idleMs: 0, + handleFactory: async () => { + nonRetryStarts += 1; + throw new Error("not retryable by policy"); + }, + }); + const nonRetryReader = new WorkflowStore(stateDir); + assert.equal(nonRetryReader.require(nonRetry.id).status, "failed"); + assert.equal(nonRetryReader.require(nonRetry.id).nodes[0]!.attempt, 1); + assert.equal(nonRetryStarts, 1); + nonRetryReader.close(); +} + async function testManagedWriteWorktree(stateDir: string): Promise { const repo = join(stateDir, "repo"); const worktreeRoot = join(stateDir, "managed"); @@ -184,6 +316,8 @@ async function testManagedWriteWorktree(stateDir: string): Promise { const handleFactory = async (_provider: string, input: { workspace: string }): Promise => { executionRoot = input.workspace; writeFileSync(join(input.workspace, "source.txt"), "isolated\n"); + execFileSync("git", ["add", "source.txt"], { cwd: input.workspace }); + execFileSync("git", ["commit", "-qm", "agent change"], { cwd: input.workspace }); const controller = new LocalAgentRunController("fake"); queueMicrotask(() => controller.succeed({ provider: "fake", providerSessionId: "writer-session", finalResponse: "changed", items: [] })); return controller; @@ -209,6 +343,43 @@ async function testManagedWriteWorktree(stateDir: string): Promise { } } +async function testExpiredWorktreeCleanup(stateDir: string): Promise { + const repo = join(stateDir, "repo"); + const worktreeRoot = join(stateDir, "managed"); + mkdirSync(repo, { recursive: true }); + execFileSync("git", ["init", "-q"], { cwd: repo }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: repo }); + execFileSync("git", ["config", "user.name", "Test"], { cwd: repo }); + writeFileSync(join(repo, "source.txt"), "source\n"); + execFileSync("git", ["add", "source.txt"], { cwd: repo }); + execFileSync("git", ["commit", "-qm", "initial"], { cwd: repo }); + const baseSha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repo, encoding: "utf8" }).trim(); + + const store = new WorkflowStore(stateDir); + const workflow = store.submit({ + definition: definition(["writer"], { access: "workspace_write", workspaceRoot: repo, worktreeRoot, baseSha }), + policy: { version: 1, maxConcurrency: 1 }, + }).workflow; + const identity = { + workflowId: workflow.id, + nodeKey: "writer", + attempt: 1, + claimToken: "retention-claim", + }; + const allocation = await allocateWorkflowWorktree({ store, identity, sourceRoot: repo, worktreeRoot, baseSha }); + preserveWorkflowWorktree(store, identity); + const database = openDatabase(stateDir); + database.sqlite.prepare( + "update workflow_worktrees set retain_until = ? where workflow_run_id = ? and node_key = ? and attempt = ?", + ).run("2020-01-01T00:00:00.000Z", workflow.id, "writer", 1); + database.close(); + + assert.equal(await cleanupExpiredWorkflowWorktrees({ store }), 1); + assert.equal(store.getWorktree(workflow.id, "writer", 1)?.state, "removed"); + assert.equal(existsSync(allocation.path), false); + store.close(); +} + async function testWorktreeCleanupRejectsRootSubstitution(stateDir: string): Promise { const repo = join(stateDir, "repo"); const worktreeRoot = join(stateDir, "managed"); @@ -269,7 +440,7 @@ function definition(keys: string[], overrides: JsonObject): WorkflowDefinitionV1 workspaceRoot: String(overrides.workspaceRoot ?? process.cwd()), worktreeRoot: String(overrides.worktreeRoot ?? ""), baseSha: String(overrides.baseSha ?? ""), - timeoutMs: null, + timeoutMs: overrides.timeoutMs ?? null, effectivePolicy: { version: 1, mode: "workflow", diff --git a/src/workflows/mcp.test.ts b/src/workflows/mcp.test.ts index 3a613ab1..dcdbdbe1 100644 --- a/src/workflows/mcp.test.ts +++ b/src/workflows/mcp.test.ts @@ -7,10 +7,14 @@ import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import type { ServerConfig } from "../config.js"; import { WorkspaceRegistry } from "../workspaces.js"; -import { registerWorkflowTools } from "./mcp.js"; +import { registerWorkflowTools, type WorkflowToolAuditEvent } from "./mcp.js"; import { WorkflowOrchestrator } from "./orchestrator.js"; +import { WorkflowStore } from "./store.js"; +import { runWorkflowSupervisor } from "./supervisor.js"; const root = mkdtempSync(join(tmpdir(), "devspace-workflow-mcp-test-")); +const originalFakeProvider = process.env.DEVSPACE_WORKFLOW_FAKE_PROVIDER; +process.env.DEVSPACE_WORKFLOW_FAKE_PROVIDER = "1"; const stateDir = join(root, "state"); const config = { stateDir, @@ -45,8 +49,40 @@ const workflow = orchestrator.submit({ workspace: { workspaceId: workspace.id, workspaceRoot: workspace.root }, }); +const completionStore = new WorkflowStore(stateDir); +const supervisor = completionStore.acquireSupervisor({ ownerToken: "mcp-test", ownerPid: 1, leaseMs: 5_000 })!; +const claim = completionStore.claimNextAgentNode({ supervisor, claimToken: "mcp-claim", leaseMs: 5_000 })!; +completionStore.completeAgentNode({ + workflowId: workflow.id, + nodeKey: "agent", + attempt: claim.node.attempt, + claimToken: claim.node.claimToken!, + status: "succeeded", + result: { provider: "fake", providerSessionId: "secret-session", finalResponse: "safe final response" }, +}); +completionStore.releaseSupervisor(supervisor); +completionStore.close(); +const audits: WorkflowToolAuditEvent[] = []; +let failSupervisorLaunch = false; const server = new McpServer({ name: "workflow-test", version: "1.0.0" }); -registerWorkflowTools({ server, config, workspaces, orchestrator }); +registerWorkflowTools({ + server, + config, + workspaces, + orchestrator, + launchSupervisor: async () => { + if (failSupervisorLaunch) throw new Error("startup details must stay private"); + await runWorkflowSupervisor(stateDir, { + globalConcurrency: 2, + heartbeatMs: 5, + nodeLeaseMs: 500, + supervisorLeaseMs: 500, + idleMs: 0, + }); + return { requestedWakeGeneration: 1, spawned: true }; + }, + audit: (event) => audits.push(event), +}); const client = new Client({ name: "workflow-test-client", version: "1.0.0" }); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); try { @@ -58,7 +94,7 @@ try { ); const runTool = listed.tools.find((tool) => tool.name === "workflow_run")!; assert.equal(runTool.annotations?.destructiveHint, true); - assert.equal(runTool.annotations?.idempotentHint, true); + assert.equal(runTool.annotations?.idempotentHint, false); const statusTool = listed.tools.find((tool) => tool.name === "workflow_status")!; assert.equal(statusTool.annotations?.readOnlyHint, true); @@ -76,6 +112,43 @@ try { }); assert.equal(ambiguous.isError, true); + const dagRun = await client.callTool({ + name: "workflow_run", + arguments: { + workspaceId: workspace.id, + dag: { + version: 1, + maxConcurrency: 2, + nodes: [ + { key: "first", target: "fake", prompt: "first deterministic task" }, + { key: "second", target: "fake", prompt: "second deterministic task" }, + ], + edges: [{ from: "first", to: "second" }], + }, + idempotencyKey: "mcp-dag-e2e", + }, + }); + assert.equal(dagRun.isError, undefined); + const dagWorkflowId = (dagRun.structuredContent as { workflow: { id: string } }).workflow.id; + const dagWait = await client.callTool({ + name: "workflow_wait", + arguments: { workspaceId: workspace.id, workflowId: dagWorkflowId, timeoutMs: 1_000 }, + }); + const dagResult = dagWait.structuredContent as { + timedOut: boolean; + workflow: { status: string; finalResponse?: string; nodes: Array<{ status: string; finalResponse?: string }> }; + }; + assert.equal(dagResult.timedOut, false); + assert.equal(dagResult.workflow.status, "succeeded"); + assert.deepEqual(dagResult.workflow.nodes.map((node) => node.status), ["succeeded", "succeeded"]); + assert.equal(dagResult.workflow.finalResponse, "fake result"); + const dagEvents = await client.callTool({ + name: "workflow_events", + arguments: { workspaceId: workspace.id, workflowId: dagWorkflowId, after: 0, limit: 100 }, + }); + assert.equal(dagEvents.isError, undefined); + assert.ok(((dagEvents.structuredContent as { events: unknown[] }).events).length > 0); + const result = await client.callTool({ name: "workflow_status", arguments: { workspaceId: workspace.id, workflowId: workflow.id }, @@ -83,11 +156,18 @@ try { const structured = result.structuredContent as { version: number; result: string; - workflow: { id: string; status: string; nodes: Array<{ key: string }> }; + workflow: { + id: string; + status: string; + finalResponse?: string; + nodes: Array<{ key: string; finalResponse?: string }>; + }; }; assert.equal(structured.version, 1); assert.equal(structured.workflow.id, workflow.id); - assert.equal(structured.workflow.status, "queued"); + assert.equal(structured.workflow.status, "succeeded"); + assert.equal(structured.workflow.finalResponse, "safe final response"); + assert.equal(structured.workflow.nodes[0]!.finalResponse, "safe final response"); assert.equal((result.content as Array<{ text: string }>)[0]!.text, structured.result); const serialized = JSON.stringify(structured); assert.doesNotMatch(serialized, /secret prompt|secret profile|workspaceRoot|claimToken|providerSession/); @@ -97,9 +177,39 @@ try { arguments: { workspaceId: "unknown", workflowId: workflow.id }, }); assert.equal(denied.isError, true); + + const cancellable = orchestrator.submit({ + definition: { version: 1, nodes: [{ key: "agent", type: "agent" }], edges: [] }, + workspace: { workspaceId: workspace.id, workspaceRoot: workspace.root }, + }); + failSupervisorLaunch = true; + const cancellation = await client.callTool({ + name: "workflow_cancel", + arguments: { workspaceId: workspace.id, workflowId: cancellable.id }, + }); + assert.equal(cancellation.isError, undefined); + const cancellationResult = cancellation.structuredContent as { + supervisor: { started: boolean; errorCode?: string }; + workflow: { id: string; status: string }; + }; + assert.deepEqual(cancellationResult.supervisor, { + started: false, + errorCode: "supervisor_start_failed", + }); + assert.equal(cancellationResult.workflow.id, cancellable.id); + assert.equal(orchestrator.get(cancellable.id)?.status, "cancelling"); + assert.ok(audits.some((event) => event.tool === "workflow_wait" && event.success)); + assert.ok(audits.some((event) => event.tool === "workflow_events" && event.success)); + assert.ok(audits.some((event) => event.tool === "workflow_status" && event.success)); + assert.ok(audits.some((event) => event.tool === "workflow_status" && !event.success)); + assert.ok(audits.some((event) => event.tool === "workflow_run" && !event.success)); + assert.ok(audits.some((event) => event.tool === "workflow_cancel" && event.success)); + assert.doesNotMatch(JSON.stringify(audits), /secret prompt|secret profile|startup details|secret-session/); } finally { await client.close(); await server.close(); orchestrator.close(); + if (originalFakeProvider === undefined) delete process.env.DEVSPACE_WORKFLOW_FAKE_PROVIDER; + else process.env.DEVSPACE_WORKFLOW_FAKE_PROVIDER = originalFakeProvider; rmSync(root, { recursive: true, force: true }); } diff --git a/src/workflows/mcp.ts b/src/workflows/mcp.ts index eede6470..472e4c72 100644 --- a/src/workflows/mcp.ts +++ b/src/workflows/mcp.ts @@ -46,6 +46,7 @@ const nodeOutputSchema = z.object({ attempt: z.number().int(), completedAt: z.string().optional(), errorCode: z.string().optional(), + finalResponse: z.string().optional(), }); const workflowOutputSchema = z.object({ @@ -57,9 +58,15 @@ const workflowOutputSchema = z.object({ completedAt: z.string().optional(), cancellationRequestedAt: z.string().optional(), errorCode: z.string().optional(), + finalResponse: z.string().optional(), nodes: z.array(nodeOutputSchema), }); +const supervisorOutputSchema = z.object({ + started: z.boolean(), + errorCode: z.string().optional(), +}); + const eventOutputSchema = z.object({ sequence: z.number().int(), type: z.string(), @@ -72,13 +79,25 @@ const commonOutput = { result: z.string(), }; +export interface WorkflowToolAuditEvent { + tool: string; + workspaceId?: string; + workflowId?: string; + success: boolean; + durationMs: number; + errorCode?: string; +} + export function registerWorkflowTools(input: { server: McpServer; config: ServerConfig; workspaces: WorkspaceRegistry; orchestrator: WorkflowOrchestrator; + launchSupervisor?: typeof ensureSupervisor; + audit?: (event: WorkflowToolAuditEvent) => void; }): void { const { server, config, workspaces, orchestrator } = input; + const launchSupervisor = input.launchSupervisor ?? ensureSupervisor; server.registerTool( "workflow_run", @@ -100,16 +119,17 @@ export function registerWorkflowTools(input: { outputSchema: { ...commonOutput, created: z.boolean(), + supervisor: supervisorOutputSchema, workflow: workflowOutputSchema, }, annotations: { readOnlyHint: false, destructiveHint: true, - idempotentHint: true, + idempotentHint: false, openWorldHint: false, }, }, - async (args) => { + async (args) => audited(input.audit, "workflow_run", args.workspaceId, undefined, async () => { const hasSingleFields = [ args.target, args.prompt, @@ -149,13 +169,15 @@ export function registerWorkflowTools(input: { worktreeRoot: config.worktreeRoot, }); const submitted = orchestrator.submitDetailed(request); - await ensureSupervisor({ stateDir: config.stateDir }); + const supervisor = await supervisorStartState(() => launchSupervisor({ stateDir: config.stateDir })); const workflow = publicWorkflow(submitted.workflow); - const result = submitted.created - ? `Workflow ${workflow.id} was submitted with status ${workflow.status}.` - : `Workflow ${workflow.id} already exists for this idempotency key.`; - return response({ version: CONTRACT_VERSION, result, created: submitted.created, workflow }); - }, + const result = supervisor.started + ? submitted.created + ? `Workflow ${workflow.id} was submitted with status ${workflow.status}.` + : `Workflow ${workflow.id} already exists for this idempotency key.` + : `Workflow ${workflow.id} was durably submitted, but its supervisor could not be started.`; + return response({ version: CONTRACT_VERSION, result, created: submitted.created, supervisor, workflow }); + }), ); server.registerTool( @@ -167,11 +189,11 @@ export function registerWorkflowTools(input: { outputSchema: { ...commonOutput, workflow: workflowOutputSchema }, annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, }, - async ({ workspaceId, workflowId }) => { + async ({ workspaceId, workflowId }) => audited(input.audit, "workflow_status", workspaceId, workflowId, async () => { const workflow = publicWorkflow(requireOwned(orchestrator, workflowId, workspaceScope(workspaces, workspaceId))); const result = `Workflow ${workflow.id} is ${workflow.status}.`; return response({ version: CONTRACT_VERSION, result, workflow }); - }, + }), ); server.registerTool( @@ -187,7 +209,7 @@ export function registerWorkflowTools(input: { outputSchema: { ...commonOutput, timedOut: z.boolean(), workflow: workflowOutputSchema }, annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, }, - async ({ workspaceId, workflowId, timeoutMs }) => { + async ({ workspaceId, workflowId, timeoutMs }) => audited(input.audit, "workflow_wait", workspaceId, workflowId, async () => { const scope = workspaceScope(workspaces, workspaceId); const current = requireOwned(orchestrator, workflowId, scope); const waited = await orchestrator.waitForWorkspace(workflowId, scope, { timeoutMs }); @@ -197,7 +219,7 @@ export function registerWorkflowTools(input: { ? `Workflow ${workflow.id} is still ${workflow.status} after the bounded wait.` : `Workflow ${workflow.id} reached ${workflow.status}.`; return response({ version: CONTRACT_VERSION, result, timedOut, workflow }); - }, + }), ); server.registerTool( @@ -219,7 +241,7 @@ export function registerWorkflowTools(input: { }, annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, }, - async ({ workspaceId, workflowId, after, limit }) => { + async ({ workspaceId, workflowId, after, limit }) => audited(input.audit, "workflow_events", workspaceId, workflowId, async () => { const scope = workspaceScope(workspaces, workspaceId); const workflow = requireOwned(orchestrator, workflowId, scope); const page = orchestrator.eventsForWorkspace(workflowId, scope, { after, limit }); @@ -232,7 +254,7 @@ export function registerWorkflowTools(input: { events, cursor: page.nextCursor, }); - }, + }), ); server.registerTool( @@ -241,18 +263,20 @@ export function registerWorkflowTools(input: { title: "Cancel workflow", description: "Request durable cancellation of a workflow and its active local agents.", inputSchema: { workspaceId: z.string().min(1), workflowId: z.string().min(1) }, - outputSchema: { ...commonOutput, workflow: workflowOutputSchema }, + outputSchema: { ...commonOutput, supervisor: supervisorOutputSchema, workflow: workflowOutputSchema }, annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: true, openWorldHint: false }, }, - async ({ workspaceId, workflowId }) => { + async ({ workspaceId, workflowId }) => audited(input.audit, "workflow_cancel", workspaceId, workflowId, async () => { const scope = workspaceScope(workspaces, workspaceId); requireOwned(orchestrator, workflowId, scope); const cancelled = orchestrator.cancelForWorkspace(workflowId, scope); - await ensureSupervisor({ stateDir: config.stateDir }); + const supervisor = await supervisorStartState(() => launchSupervisor({ stateDir: config.stateDir })); const workflow = publicWorkflow(cancelled); - const result = `Cancellation was requested for workflow ${workflow.id}; current status is ${workflow.status}.`; - return response({ version: CONTRACT_VERSION, result, workflow }); - }, + const result = supervisor.started + ? `Cancellation was requested for workflow ${workflow.id}; current status is ${workflow.status}.` + : `Cancellation was durably requested for workflow ${workflow.id}, but its supervisor could not be started.`; + return response({ version: CONTRACT_VERSION, result, supervisor, workflow }); + }), ); } @@ -281,12 +305,14 @@ function publicWorkflow(workflow: WorkflowRunRecord) { completedAt: workflow.completedAt, cancellationRequestedAt: workflow.cancellationRequestedAt, errorCode: stringField(workflow.error, "code"), + finalResponse: stringField(workflow.result, "finalResponse"), nodes: workflow.nodes.map((node) => ({ key: node.key, status: node.status, attempt: node.attempt, completedAt: node.completedAt, errorCode: stringField(node.error, "code"), + finalResponse: stringField(node.result, "finalResponse"), })), }; } @@ -301,11 +327,66 @@ function publicEvent(event: WorkflowEvent, workflow: WorkflowRunRecord) { }; } -function stringField(value: Record | undefined, key: string): string | undefined { - const field = value?.[key]; +function stringField(value: unknown, key: string): string | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const field = (value as Record)[key]; return typeof field === "string" ? field : undefined; } +async function supervisorStartState(start: () => Promise): Promise<{ + started: boolean; + errorCode?: string; +}> { + try { + await start(); + return { started: true }; + } catch { + return { started: false, errorCode: "supervisor_start_failed" }; + } +} + +async function audited( + audit: ((event: WorkflowToolAuditEvent) => void) | undefined, + tool: string, + workspaceId: string | undefined, + workflowId: string | undefined, + operation: () => Promise, +): Promise { + const startedAt = performance.now(); + try { + const result = await operation(); + emitAudit(audit, { + tool, + workspaceId, + workflowId, + success: true, + durationMs: Math.round(performance.now() - startedAt), + }); + return result; + } catch (error) { + emitAudit(audit, { + tool, + workspaceId, + workflowId, + success: false, + durationMs: Math.round(performance.now() - startedAt), + errorCode: error instanceof Error ? error.name : "Error", + }); + throw error; + } +} + +function emitAudit( + audit: ((event: WorkflowToolAuditEvent) => void) | undefined, + event: WorkflowToolAuditEvent, +): void { + try { + audit?.(event); + } catch { + // Audit failures must not change workflow tool behavior. + } +} + function response(structuredContent: T) { return { content: [{ type: "text" as const, text: structuredContent.result }], diff --git a/src/workflows/orchestrator.ts b/src/workflows/orchestrator.ts index bca3df1e..8bc8b250 100644 --- a/src/workflows/orchestrator.ts +++ b/src/workflows/orchestrator.ts @@ -70,8 +70,7 @@ export class WorkflowOrchestrator { options: WorkflowWaitOptions = {}, ): Promise { this.store.requireForWorkspace(workflowId, scope); - const workflow = await this.wait(workflowId, options); - return this.store.requireForWorkspace(workflow.id, scope); + return this.wait(workflowId, options); } events(workflowId: string, options: WorkflowEventReadOptions = {}): WorkflowEventPage { diff --git a/src/workflows/store.ts b/src/workflows/store.ts index f00b0c38..5854614f 100644 --- a/src/workflows/store.ts +++ b/src/workflows/store.ts @@ -226,7 +226,7 @@ export class WorkflowStore { .prepare("select id, request_hash from workflow_runs where idempotency_key = ?") .get(normalized.idempotencyKey) as { id: string; request_hash: string } | undefined; if (existing) { - if (existing.request_hash !== normalized.requestHash) { + if (!normalized.compatibleRequestHashes.has(existing.request_hash)) { throw new WorkflowIdempotencyConflictError(normalized.idempotencyKey); } return { workflowId: existing.id, created: false }; @@ -694,7 +694,10 @@ export class WorkflowStore { where active.workflow_run_id = r.id and active.status = 'running' ) < r.max_concurrency order by case when r.last_dispatched_at is null then 0 else 1 end, - r.last_dispatched_at, r.created_at, n.created_at, n.node_key limit 1`, + r.last_dispatched_at, + (select count(*) from workflow_node_attempts attempts + where attempts.workflow_run_id = r.id), + r.created_at, n.created_at, n.node_key limit 1`, ) .get(now) as WorkflowNodeRow | undefined; if (!candidate) return undefined; @@ -810,14 +813,18 @@ export class WorkflowStore { markNodeCancelling(identity: WorkflowAttemptIdentity): WorkflowNodeAttemptRecord | undefined { const now = new Date().toISOString(); - const result = this.database.sqlite - .prepare( - `update workflow_node_attempts set phase = 'cancelling', cancellation_requested_at = ?, updated_at = ? - where workflow_run_id = ? and node_key = ? and attempt = ? and claim_token = ? - and phase != 'terminal'`, - ) - .run(now, now, identity.workflowId, identity.nodeKey, identity.attempt, identity.claimToken); - return result.changes === 1 ? this.getAttempt(identity) : undefined; + const update = this.database.sqlite.transaction(() => { + const attempt = this.getActiveAttempt(identity, now); + if (!attempt) return undefined; + const result = this.database.sqlite + .prepare( + `update workflow_node_attempts set phase = 'cancelling', cancellation_requested_at = ?, updated_at = ? + where node_id = ? and attempt = ? and claim_token = ? and phase != 'terminal'`, + ) + .run(now, now, attempt.nodeId, attempt.attempt, identity.claimToken); + return result.changes === 1 ? this.getAttempt(identity) : undefined; + }); + return update.immediate(); } recordNodeProviderSession( @@ -825,21 +832,18 @@ export class WorkflowStore { providerSessionId: string, ): WorkflowNodeAttemptRecord | undefined { const now = new Date().toISOString(); - const result = this.database.sqlite - .prepare( - `update workflow_node_attempts set provider_session_id = ?, updated_at = ? - where workflow_run_id = ? and node_key = ? and attempt = ? and claim_token = ? - and phase != 'terminal'`, - ) - .run( - providerSessionId, - now, - identity.workflowId, - identity.nodeKey, - identity.attempt, - identity.claimToken, - ); - return result.changes === 1 ? this.getAttempt(identity) : undefined; + const update = this.database.sqlite.transaction(() => { + const attempt = this.getActiveAttempt(identity, now); + if (!attempt) return undefined; + const result = this.database.sqlite + .prepare( + `update workflow_node_attempts set provider_session_id = ?, updated_at = ? + where node_id = ? and attempt = ? and claim_token = ? and phase != 'terminal'`, + ) + .run(providerSessionId, now, attempt.nodeId, attempt.attempt, identity.claimToken); + return result.changes === 1 ? this.getAttempt(identity) : undefined; + }); + return update.immediate(); } appendNodeExecutionEvent(input: { @@ -850,8 +854,8 @@ export class WorkflowStore { }): { event: WorkflowEvent; created: boolean } { const now = new Date().toISOString(); const append = this.database.sqlite.transaction(() => { - const attempt = this.getAttempt(input.identity); - if (!attempt || attempt.phase === "terminal") return undefined; + const attempt = this.getActiveAttempt(input.identity, now); + if (!attempt) return undefined; const existing = this.database.sqlite .prepare( `select workflow_sequence from workflow_provider_events @@ -1174,6 +1178,18 @@ export class WorkflowStore { return row ? rowToWorktree(row) : undefined; } + listExpiredWorktrees(now = new Date().toISOString()): WorkflowWorktreeRecord[] { + const rows = this.database.sqlite + .prepare( + `select * from workflow_worktrees + where state in ('preserved', 'cleanup_failed') + and retain_until is not null and retain_until <= ? + order by retain_until, created_at`, + ) + .all(now) as WorkflowWorktreeRow[]; + return rows.map(rowToWorktree); + } + updateWorktreeState( workflowId: string, nodeKey: string, @@ -1230,26 +1246,54 @@ export class WorkflowStore { return row ? rowToAttempt(row) : undefined; } - private updateAttemptPhase( + private getActiveAttempt( identity: WorkflowAttemptIdentity, - phase: "dispatching" | "running", + now: string, ): WorkflowNodeAttemptRecord | undefined { - const now = new Date().toISOString(); - const result = this.database.sqlite + const row = this.database.sqlite .prepare( - `update workflow_node_attempts set phase = ?, updated_at = ? - where workflow_run_id = ? and node_key = ? and attempt = ? and claim_token = ? - and phase != 'terminal'`, + `select attempts.* from workflow_node_attempts attempts + join workflow_nodes nodes on nodes.id = attempts.node_id + join workflow_supervisor supervisor on supervisor.id = 1 + where attempts.workflow_run_id = ? and attempts.node_key = ? + and attempts.attempt = ? and attempts.claim_token = ? + and attempts.phase != 'terminal' + and nodes.status = 'running' and nodes.attempt = attempts.attempt + and nodes.claim_token = attempts.claim_token and nodes.claim_expires_at > ? + and nodes.supervisor_owner_token = attempts.supervisor_owner_token + and nodes.supervisor_owner_epoch = attempts.supervisor_owner_epoch + and supervisor.owner_token = attempts.supervisor_owner_token + and supervisor.owner_epoch = attempts.supervisor_owner_epoch + and supervisor.lease_expires_at > ?`, ) - .run( - phase, - now, + .get( identity.workflowId, identity.nodeKey, identity.attempt, identity.claimToken, - ); - return result.changes === 1 ? this.getAttempt(identity) : undefined; + now, + now, + ) as WorkflowAttemptRow | undefined; + return row ? rowToAttempt(row) : undefined; + } + + private updateAttemptPhase( + identity: WorkflowAttemptIdentity, + phase: "dispatching" | "running", + ): WorkflowNodeAttemptRecord | undefined { + const now = new Date().toISOString(); + const update = this.database.sqlite.transaction(() => { + const attempt = this.getActiveAttempt(identity, now); + if (!attempt) return undefined; + const result = this.database.sqlite + .prepare( + `update workflow_node_attempts set phase = ?, updated_at = ? + where node_id = ? and attempt = ? and claim_token = ? and phase != 'terminal'`, + ) + .run(phase, now, attempt.nodeId, attempt.attempt, identity.claimToken); + return result.changes === 1 ? this.getAttempt(identity) : undefined; + }); + return update.immediate(); } private hydrateWorkflow(row: WorkflowRunRow): WorkflowRunRecord { @@ -1434,6 +1478,7 @@ function normalizeSubmission(request: SubmitWorkflowRequest): { workspace?: WorkflowWorkspaceScope; maxConcurrency: number; requestHash: string; + compatibleRequestHashes: ReadonlySet; } { const definition = normalizeDefinition(request.definition); const input = normalizeObject(request.input ?? {}, "Workflow input"); @@ -1452,9 +1497,10 @@ function normalizeSubmission(request: SubmitWorkflowRequest): { } : undefined; const maxConcurrency = readBoundedInteger(policy.maxConcurrency, 1, MAX_RUN_CONCURRENCY, "Workflow maxConcurrency"); - const requestHash = createHash("sha256") - .update(canonicalJson({ definition, input, policy, workspace: workspace ?? null })) - .digest("hex"); + const requestHash = hashWorkflowRequest({ definition, input, policy, workspace: workspace ?? null }); + const compatibleRequestHashes = new Set([requestHash]); + const legacyRequest = legacySingleAgentRequest(definition, input, policy, workspace); + if (legacyRequest) compatibleRequestHashes.add(hashWorkflowRequest(legacyRequest)); return { definition, definitionJson, @@ -1464,6 +1510,52 @@ function normalizeSubmission(request: SubmitWorkflowRequest): { workspace, maxConcurrency, requestHash, + compatibleRequestHashes, + }; +} + +function hashWorkflowRequest(value: unknown): string { + return createHash("sha256").update(canonicalJson(value)).digest("hex"); +} + +function legacySingleAgentRequest( + definition: WorkflowDefinition, + input: JsonObject, + policy: WorkflowPolicy, + workspace: WorkflowWorkspaceScope | undefined, +): JsonObject | undefined { + if ( + input.kind !== "single" || + definition.nodes.length !== 1 || + (definition.edges?.length ?? 0) !== 0 || + policy.maxConcurrency !== 1 || + typeof policy.access !== "string" + ) return undefined; + const node = definition.nodes[0]!; + const config = { ...(node.config ?? {}) } as JsonObject; + const prompt = config.prompt; + const retry = config.retry; + if ( + typeof prompt !== "string" || + !isObject(retry) || + retry.maxAttempts !== 1 || + !Array.isArray(retry.retryOn) || + retry.retryOn.length !== 0 || + retry.backoffMs !== 0 + ) return undefined; + delete config.worktreeRoot; + delete config.baseSha; + delete config.retry; + const legacyDefinition: WorkflowDefinition = { + version: WORKFLOW_DEFINITION_VERSION, + nodes: [{ key: node.key, type: "agent", config }], + edges: [], + }; + return { + definition: legacyDefinition as unknown as JsonObject, + input: { prompt }, + policy: { version: WORKFLOW_POLICY_VERSION, access: policy.access }, + workspace: workspace ? { ...workspace } : null, }; } diff --git a/src/workflows/supervisor.test.ts b/src/workflows/supervisor.test.ts index 09732b8f..5c221022 100644 --- a/src/workflows/supervisor.test.ts +++ b/src/workflows/supervisor.test.ts @@ -16,6 +16,8 @@ try { await testStaleSupervisorProcessRecovery(join(root, "stale-supervisor")); await testAtomicClaimAndLeaseRecovery(join(root, "claims")); await testDispatchHeartbeatsProtectLeases(join(root, "dispatch-heartbeats")); + await testPreparationHeartbeatsProtectLeases(join(root, "preparation-heartbeats")); + await testTimeoutWinsOverLateSuccess(join(root, "timeout-race")); await testCancellationBeforeAndDuringExecution(join(root, "cancellation")); } finally { rmSync(root, { recursive: true, force: true }); @@ -80,7 +82,7 @@ async function testStaleSupervisorProcessRecovery(stateDir: string): Promise stale.ownerEpoch); @@ -112,6 +114,23 @@ async function testAtomicClaimAndLeaseRecovery(stateDir: string): Promise await delay(15); assert.equal(store.reconcileExpiredClaims(), 0); await delay(25); + const expiredIdentity = { + workflowId: first.id, + nodeKey: "agent", + attempt: 1, + claimToken: "first", + }; + assert.equal(store.markNodeDispatching(expiredIdentity), undefined); + assert.equal(store.recordNodeProviderSession(expiredIdentity, "late-session"), undefined); + assert.throws( + () => store.appendNodeExecutionEvent({ + identity: expiredIdentity, + sourceSequence: 1, + type: "provider.output", + payload: { delta: "late" }, + }), + /no longer active/, + ); assert.equal(store.reconcileExpiredClaims(), 1); const failed = store.require(first.id); assert.equal(failed.status, "failed"); @@ -165,6 +184,84 @@ async function testDispatchHeartbeatsProtectLeases(stateDir: string): Promise { + const store = new WorkflowStore(stateDir); + const workflow = store.submit(workflowRequest(executionSnapshot({ + effectivePolicy: { version: 1, mode: "workflow", access: "workspace_write", environment: {} }, + worktreeRoot: join(stateDir, "worktrees"), + baseSha: "base", + }))).workflow; + store.close(); + + const ran = await runWorkflowSupervisor(stateDir, { + idleMs: 20, + heartbeatMs: 5, + supervisorLeaseMs: 40, + nodeLeaseMs: 40, + worktreeAllocator: async ({ store: allocatorStore, identity }) => { + await delay(90); + const record = allocatorStore.recordWorktree({ + workflowId: identity.workflowId, + nodeKey: identity.nodeKey, + attempt: identity.attempt, + path: join(stateDir, "worktrees", "prepared"), + sourceRoot: stateDir, + baseSha: "base", + retainUntil: new Date(Date.now() + 60_000).toISOString(), + }); + return allocatorStore.updateWorktreeState(record.workflowId, record.nodeKey, record.attempt, "active"); + }, + handleFactory: successfulHandleFactory, + }); + assert.equal(ran, true); + + const reader = new WorkflowStore(stateDir); + try { + assert.equal(reader.require(workflow.id).status, "succeeded"); + } finally { + reader.close(); + } +} + +async function testTimeoutWinsOverLateSuccess(stateDir: string): Promise { + const store = new WorkflowStore(stateDir); + const workflow = store.submit(workflowRequest(executionSnapshot({ timeoutMs: 20 }))).workflow; + store.close(); + + await runWorkflowSupervisor(stateDir, { + idleMs: 20, + heartbeatMs: 5, + supervisorLeaseMs: 200, + nodeLeaseMs: 200, + handleFactory: async () => { + const controller = new LocalAgentRunController("fake"); + const timer = setTimeout(() => { + controller.succeed({ + provider: "fake", + providerSessionId: "late-success", + finalResponse: "too late", + items: [], + }); + }, 60); + controller.setLifecycle({ + cancel: () => undefined, + dispose: () => clearTimeout(timer), + }); + return controller; + }, + }); + + const reader = new WorkflowStore(stateDir); + try { + const completed = reader.require(workflow.id); + assert.equal(completed.status, "failed"); + assert.equal((completed.error as JsonObject).code, "timed_out"); + assert.equal(completed.nodes[0]!.status, "failed"); + } finally { + reader.close(); + } +} + async function testCancellationBeforeAndDuringExecution(stateDir: string): Promise { const beforeStore = new WorkflowStore(stateDir); const before = beforeStore.submit(workflowRequest(executionSnapshot())).workflow; diff --git a/src/workflows/supervisor.ts b/src/workflows/supervisor.ts index ef1a24af..8d0acd5c 100644 --- a/src/workflows/supervisor.ts +++ b/src/workflows/supervisor.ts @@ -11,6 +11,7 @@ import { isLocalAgentProvider } from "../local-agent-profiles.js"; import type { EffectiveLocalAgentPolicy } from "./policy.js"; import { allocateWorkflowWorktree, + cleanupExpiredWorkflowWorktrees, cleanupWorkflowWorktree, preserveWorkflowWorktree, } from "./worktrees.js"; @@ -46,6 +47,7 @@ export interface WorkflowSupervisorOptions { ownerPid?: number; globalConcurrency?: number; handleFactory?: WorkflowHandleFactory; + worktreeAllocator?: typeof allocateWorkflowWorktree; } export async function runWorkflowSupervisor( @@ -75,6 +77,10 @@ export async function runWorkflowSupervisor( try { store.reconcileExpiredClaims(); store.convergeCancellations(); + await cleanupExpiredWorkflowWorktrees({ + store, + beforeCleanup: () => store.heartbeatSupervisor(identity, supervisorLeaseMs), + }); while (store.heartbeatSupervisor(identity, supervisorLeaseMs)) { store.reconcileExpiredClaims(); store.convergeCancellations(); @@ -94,6 +100,7 @@ export async function runWorkflowSupervisor( nodeLeaseMs, heartbeatMs, handleFactory: options.handleFactory ?? defaultHandleFactory, + worktreeAllocator: options.worktreeAllocator ?? allocateWorkflowWorktree, }).finally(() => { active.delete(execution); lastWorkAt = Date.now(); @@ -139,6 +146,7 @@ async function executeClaim( nodeLeaseMs: number; heartbeatMs: number; handleFactory: WorkflowHandleFactory; + worktreeAllocator: typeof allocateWorkflowWorktree; }, ): Promise { const identity: WorkflowAttemptIdentity = { @@ -159,13 +167,41 @@ async function executeClaim( let leaseFailure: Error | undefined; let tickRunning = false; + const tick = async () => { + if (tickRunning || leaseFailure) return; + tickRunning = true; + try { + if (!store.heartbeatSupervisor(supervisor, options.supervisorLeaseMs)) { + leaseFailure = new Error("Workflow supervisor lease lost."); + await handle?.cancel(leaseFailure); + return; + } + if (!store.heartbeatNode({ ...identity, leaseMs: options.nodeLeaseMs })) { + leaseFailure = new Error("Workflow node claim lost."); + await handle?.cancel(leaseFailure); + return; + } + if (!cancellationObserved && store.isCancellationRequested(identity.workflowId)) { + cancellationObserved = true; + store.markNodeCancelling(identity); + await handle?.cancel(new Error("Workflow cancellation requested.")); + } + } finally { + tickRunning = false; + } + }; + try { + await tick(); + if (leaseFailure) throw leaseFailure; + heartbeat = setInterval(() => void tick().catch(() => undefined), options.heartbeatMs); + const execution = readExecutionSnapshot(config); const fullPrompt = execution.profileBody ? `${execution.profileBody}\n\nTask:\n${execution.prompt}` : execution.prompt; if (execution.effectivePolicy.access === "workspace_write") { - const allocation = await allocateWorkflowWorktree({ + const allocation = await options.worktreeAllocator({ store, identity, sourceRoot: execution.workspaceRoot, @@ -180,33 +216,6 @@ async function executeClaim( throw new Error("Workflow node attempt was lost before dispatch."); } - const tick = async () => { - if (tickRunning || leaseFailure) return; - tickRunning = true; - try { - if (!store.heartbeatSupervisor(supervisor, options.supervisorLeaseMs)) { - leaseFailure = new Error("Workflow supervisor lease lost."); - await handle?.cancel(leaseFailure); - return; - } - if (!store.heartbeatNode({ ...identity, leaseMs: options.nodeLeaseMs })) { - leaseFailure = new Error("Workflow node claim lost."); - await handle?.cancel(leaseFailure); - return; - } - if (!cancellationObserved && store.isCancellationRequested(identity.workflowId)) { - cancellationObserved = true; - store.markNodeCancelling(identity); - await handle?.cancel(new Error("Workflow cancellation requested.")); - } - } finally { - tickRunning = false; - } - }; - await tick(); - if (leaseFailure) throw leaseFailure; - heartbeat = setInterval(() => void tick().catch(() => undefined), options.heartbeatMs); - handle = await options.handleFactory(execution.provider, { prompt: fullPrompt, workspace: execution.workspaceRoot, @@ -239,6 +248,7 @@ async function executeClaim( } finally { await drain; } + if (timeoutObserved) throw new Error("Workflow node timed out."); if (result.providerSessionId) { store.recordNodeProviderSession(identity, result.providerSessionId); } diff --git a/src/workflows/workflows.test.ts b/src/workflows/workflows.test.ts index 1c4fb475..3ead272d 100644 --- a/src/workflows/workflows.test.ts +++ b/src/workflows/workflows.test.ts @@ -12,7 +12,7 @@ import { WorkflowTransitionError, WorkflowValidationError, } from "./store.js"; -import type { SubmitWorkflowRequest, WorkflowDefinitionV1 } from "./types.js"; +import type { JsonObject, SubmitWorkflowRequest, WorkflowDefinitionV1 } from "./types.js"; interface WorkerSpec { action: "append" | "claim" | "submit"; @@ -37,6 +37,7 @@ async function runTests(): Promise { const root = mkdtempSync(join(tmpdir(), "devspace-workflows-test-")); try { testSubmissionAndIdempotency(join(root, "submission")); + testLegacySingleAgentIdempotencyCompatibility(join(root, "legacy-idempotency")); await testConcurrentSubmission(join(root, "concurrent-submission")); testMonotonicCursorReads(join(root, "events")); await testConcurrentEventSequencing(join(root, "concurrent-events")); @@ -102,6 +103,49 @@ function testSubmissionAndIdempotency(stateDir: string): void { } } +function testLegacySingleAgentIdempotencyCompatibility(stateDir: string): void { + const store = new WorkflowStore(stateDir); + try { + const workspace = { workspaceId: "workspace", workspaceRoot: "/tmp/workspace" }; + const legacyConfig: JsonObject = { + profileBody: "", + profileName: "fake", + profileHash: "hash", + provider: "fake", + model: null, + thinking: null, + effectivePolicy: { version: 1, mode: "workflow", access: "read_only", environment: {} }, + workspaceRoot: workspace.workspaceRoot, + prompt: "same prompt", + timeoutMs: null, + environmentPolicy: {}, + }; + const legacy = store.submit({ + definition: singleAgentDefinition(legacyConfig), + input: { prompt: "same prompt" }, + policy: { version: 1, access: "read_only" }, + idempotencyKey: "legacy-key", + workspace, + }); + const replay = store.submit({ + definition: singleAgentDefinition({ + ...legacyConfig, + worktreeRoot: "/tmp/worktrees", + baseSha: null, + retry: { maxAttempts: 1, retryOn: [], backoffMs: 0 }, + }), + input: { kind: "single" }, + policy: { version: 1, access: "read_only", maxConcurrency: 1 }, + idempotencyKey: "legacy-key", + workspace, + }); + assert.equal(replay.created, false); + assert.equal(replay.workflow.id, legacy.workflow.id); + } finally { + store.close(); + } +} + async function testConcurrentSubmission(stateDir: string): Promise { const same = await runWorkers(stateDir, [ { action: "submit", requestVariant: "same" }, @@ -655,7 +699,7 @@ function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -function singleAgentDefinition(config: Record = {}): WorkflowDefinitionV1 { +function singleAgentDefinition(config: JsonObject = {}): WorkflowDefinitionV1 { return { version: 1, nodes: [{ key: "agent", type: "agent", config }], edges: [] }; } diff --git a/src/workflows/worktrees.ts b/src/workflows/worktrees.ts index 5bd98d35..99dbe773 100644 --- a/src/workflows/worktrees.ts +++ b/src/workflows/worktrees.ts @@ -1,6 +1,6 @@ import { execFile } from "node:child_process"; import { mkdir, realpath } from "node:fs/promises"; -import { basename, join, relative, resolve } from "node:path"; +import { basename, dirname, join, relative, resolve } from "node:path"; import { promisify } from "node:util"; import { WorkflowStore, WorkflowValidationError } from "./store.js"; import type { WorkflowAttemptIdentity, WorkflowWorktreeRecord } from "./types.js"; @@ -82,6 +82,33 @@ export function preserveWorkflowWorktree( return store.updateWorktreeState(identity.workflowId, identity.nodeKey, identity.attempt, "preserved"); } +export async function cleanupExpiredWorkflowWorktrees(input: { + store: WorkflowStore; + now?: string; + beforeCleanup?: () => boolean; +}): Promise { + let removed = 0; + for (const record of input.store.listExpiredWorktrees(input.now)) { + if (input.beforeCleanup && !input.beforeCleanup()) break; + try { + await cleanupWorkflowWorktree({ + store: input.store, + identity: { + workflowId: record.workflowId, + nodeKey: record.nodeKey, + attempt: record.attempt, + claimToken: "retention-cleanup", + }, + worktreeRoot: dirname(record.path), + }); + removed += 1; + } catch { + // cleanupWorkflowWorktree persists the failure for a later retry. + } + } + return removed; +} + export async function cleanupWorkflowWorktree(input: { store: WorkflowStore; identity: WorkflowAttemptIdentity; @@ -95,15 +122,20 @@ export async function cleanupWorkflowWorktree(input: { if (!record || record.state === "removed") return record; try { + const expectedName = `${safeSegment(basename(record.sourceRoot))}-${safeSegment(record.workflowId)}-${safeSegment(record.nodeKey)}-a${record.attempt}`; + if (basename(record.path) !== expectedName) { + throw new WorkflowValidationError("Workflow worktree path does not match persisted ownership"); + } const worktreeRoot = await canonicalDirectory(input.worktreeRoot, "workflow worktree root"); assertContained(record.path, worktreeRoot); const canonicalWorktreePath = await canonicalDirectory(record.path, "workflow worktree path"); assertContained(canonicalWorktreePath, worktreeRoot); const registered = await registeredWorktrees(record.sourceRoot); const match = registered.find((entry) => resolve(entry.path) === resolve(record.path)); - if (!match || match.head !== record.baseSha) { + if (!match) { throw new WorkflowValidationError("Workflow worktree Git registration does not match persisted ownership"); } + await git(["merge-base", "--is-ancestor", record.baseSha, match.head], record.sourceRoot); await git(["worktree", "remove", "--force", record.path], record.sourceRoot); return input.store.updateWorktreeState( record.workflowId, From 7fa59181be232c5d3ff88dbdf73c5a7f8d9fb516 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 22:59:14 +0530 Subject: [PATCH 17/25] fix(process): escalate Windows interrupt cleanup Co-Authored-By: Claude --- src/process-sessions.ts | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/process-sessions.ts b/src/process-sessions.ts index f414df19..5730c3e8 100644 --- a/src/process-sessions.ts +++ b/src/process-sessions.ts @@ -1,5 +1,9 @@ import { spawn } from "node:child_process"; -import { resolveShellCommand, terminateProcessTree } from "./process-platform.js"; +import { + resolveShellCommand, + terminateProcessTree, + terminateProcessTreeGracefully, +} from "./process-platform.js"; const DEFAULT_EXEC_YIELD_MS = 10_000; const DEFAULT_INTERACTIVE_YIELD_MS = 250; @@ -46,7 +50,7 @@ export interface ProcessSnapshot { interface ManagedProcess { write(data: string): void; - kill(signal?: NodeJS.Signals): void; + kill(signal?: NodeJS.Signals): void | Promise; resize?(columns: number, rows: number): void; } @@ -259,7 +263,7 @@ export class ProcessSessionManager { const interruptRequested = chars.includes("\u0003") && session.running; if (interruptRequested) { - session.process?.kill("SIGINT"); + await session.process?.kill("SIGINT"); } const writableChars = chars.replaceAll("\u0003", ""); if (writableChars && session.running) session.process?.write(writableChars); @@ -339,7 +343,16 @@ export class ProcessSessionManager { session.process = { write: (data) => child.stdin.write(data), - kill: (signal = "SIGTERM") => terminateProcessTree(child, signal, detached), + kill: (signal = "SIGTERM") => { + if (process.platform === "win32" && signal === "SIGINT") { + terminateProcessTree(child, "SIGKILL", detached); + return; + } + if (process.platform === "win32" && signal !== "SIGKILL") { + return terminateProcessTreeGracefully(child, detached); + } + terminateProcessTree(child, signal, detached); + }, resize: input.tty ? () => undefined : undefined, }; child.stdout.on("data", (data: Buffer) => this.append(session, data.toString("utf8"))); From d136508f76e3212c387fed484e460090fe404b8f Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 23:11:42 +0530 Subject: [PATCH 18/25] fix(db): close failed workflow database initialization Co-Authored-By: Claude --- src/db/client.ts | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/db/client.ts b/src/db/client.ts index e2264091..88f5253f 100644 --- a/src/db/client.ts +++ b/src/db/client.ts @@ -23,18 +23,23 @@ export function openDatabase(stateDir: string): DatabaseHandle { chmodSync(stateDir, 0o700); const path = databasePath(stateDir); const sqlite = new Database(path); - chmodSync(path, 0o600); - sqlite.pragma("busy_timeout = 5000"); - sqlite.pragma("journal_mode = WAL"); - sqlite.pragma("synchronous = NORMAL"); - sqlite.pragma("foreign_keys = ON"); - migrateDatabase(sqlite); + try { + chmodSync(path, 0o600); + sqlite.pragma("busy_timeout = 5000"); + sqlite.pragma("journal_mode = WAL"); + sqlite.pragma("synchronous = NORMAL"); + sqlite.pragma("foreign_keys = ON"); + migrateDatabase(sqlite); - return { - sqlite, - db: createDrizzleDatabase(sqlite), - close: () => sqlite.close(), - }; + return { + sqlite, + db: createDrizzleDatabase(sqlite), + close: () => sqlite.close(), + }; + } catch (error) { + sqlite.close(); + throw error; + } } function createDrizzleDatabase(sqlite: SqliteDatabase) { From e4566a60bd713ba5412283ce7c3c8810b33bf141 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 23:17:04 +0530 Subject: [PATCH 19/25] test(workflows): use portable loader URL Co-Authored-By: Claude --- src/workflows/cli.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/workflows/cli.test.ts b/src/workflows/cli.test.ts index 43d0e949..9597c4e4 100644 --- a/src/workflows/cli.test.ts +++ b/src/workflows/cli.test.ts @@ -11,7 +11,7 @@ const stateDir = join(root, "state"); const configDir = join(root, "config"); const agentsDir = join(configDir, "agents"); const cliPath = fileURLToPath(new URL("../cli.ts", import.meta.url)); -const loaderPath = fileURLToPath(new URL("../../node_modules/tsx/dist/loader.mjs", import.meta.url)); +const loaderUrl = new URL("../../node_modules/tsx/dist/loader.mjs", import.meta.url).href; mkdirSync(project, { recursive: true }); mkdirSync(stateDir, { recursive: true }); mkdirSync(agentsDir, { recursive: true }); @@ -122,7 +122,7 @@ try { } function runCli(args: string[], env: NodeJS.ProcessEnv) { - return spawnSync(process.execPath, ["--import", loaderPath, cliPath, ...args], { + return spawnSync(process.execPath, ["--import", loaderUrl, cliPath, ...args], { cwd: project, env, encoding: "utf8", From 71e762d1eccd6f52b3d7bc69b46adc81b22e31bc Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 23:22:00 +0530 Subject: [PATCH 20/25] fix(workflows): detach supervisor from caller directory Co-Authored-By: Claude --- src/workflows/supervisor-launch.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/workflows/supervisor-launch.ts b/src/workflows/supervisor-launch.ts index 209bcea6..09de35ae 100644 --- a/src/workflows/supervisor-launch.ts +++ b/src/workflows/supervisor-launch.ts @@ -1,4 +1,5 @@ import { spawn } from "node:child_process"; +import { dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { filterWorkflowEnvironment } from "./policy.js"; import { WorkflowStore } from "./store.js"; @@ -56,6 +57,7 @@ export async function ensureSupervisor(input: { input.stateDir, ], { + cwd: dirname(cliEntrypoint), detached: true, stdio: "ignore", env: environment, From 57b1c22f95ebd3b8d93e8ea7e3b583a9c20fc7a3 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 23:36:37 +0530 Subject: [PATCH 21/25] fix(db): retry concurrent WAL initialization Co-Authored-By: Claude --- src/db/client.ts | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/db/client.ts b/src/db/client.ts index 88f5253f..4eeca0c2 100644 --- a/src/db/client.ts +++ b/src/db/client.ts @@ -5,6 +5,10 @@ import { drizzle } from "drizzle-orm/better-sqlite3"; import * as schema from "./schema.js"; import { migrateDatabase } from "./migrations.js"; +const JOURNAL_MODE_RETRY_TIMEOUT_MS = 5_000; +const JOURNAL_MODE_RETRY_DELAY_MS = 25; +const journalModeRetrySignal = new Int32Array(new SharedArrayBuffer(4)); + export type SqliteDatabase = Database.Database; export type AppDatabase = ReturnType; @@ -26,7 +30,7 @@ export function openDatabase(stateDir: string): DatabaseHandle { try { chmodSync(path, 0o600); sqlite.pragma("busy_timeout = 5000"); - sqlite.pragma("journal_mode = WAL"); + enableWriteAheadLogging(sqlite); sqlite.pragma("synchronous = NORMAL"); sqlite.pragma("foreign_keys = ON"); migrateDatabase(sqlite); @@ -42,6 +46,22 @@ export function openDatabase(stateDir: string): DatabaseHandle { } } +function enableWriteAheadLogging(sqlite: SqliteDatabase): void { + const deadline = Date.now() + JOURNAL_MODE_RETRY_TIMEOUT_MS; + while (true) { + try { + sqlite.pragma("journal_mode = WAL"); + return; + } catch (error) { + const code = (error as { code?: string }).code; + if ((code !== "SQLITE_BUSY" && code !== "SQLITE_LOCKED") || Date.now() >= deadline) { + throw error; + } + Atomics.wait(journalModeRetrySignal, 0, 0, JOURNAL_MODE_RETRY_DELAY_MS); + } + } +} + function createDrizzleDatabase(sqlite: SqliteDatabase) { return drizzle(sqlite, { schema }); } From 7392b939b3a88cde9ed1fc186ba1707fac01a0d7 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 23:48:01 +0530 Subject: [PATCH 22/25] test(workflows): avoid terminal wait race Co-Authored-By: Claude --- src/workflows/cli.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/workflows/cli.test.ts b/src/workflows/cli.test.ts index 9597c4e4..b4d87f7d 100644 --- a/src/workflows/cli.test.ts +++ b/src/workflows/cli.test.ts @@ -43,6 +43,13 @@ try { assert.equal(runEnvelope.version, 1); assert.equal(runEnvelope.ok, true); const workflowId = runEnvelope.workflow!.id; + const timed = runCli([ + "workflows", "wait", workflowId, "--timeout-ms", "0", "--json", + ], baseEnv); + assert.equal(timed.status, 0, timed.stderr); + const timedEnvelope = JSON.parse(timed.stdout) as Envelope; + assert.equal(timedEnvelope.timedOut, true, JSON.stringify(timedEnvelope)); + const replay = runCli([ "workflows", "run", "reviewer", "--prompt", "private task", "--json", "--idempotency-key", "shell-parent", @@ -51,13 +58,6 @@ try { assert.equal((JSON.parse(replay.stdout) as Envelope).created, false); writeProfile(profilePath, "Mutated after durable submission."); - const timed = runCli([ - "workflows", "wait", workflowId, "--timeout-ms", "0", "--json", - ], baseEnv); - assert.equal(timed.status, 0, timed.stderr); - const timedEnvelope = JSON.parse(timed.stdout) as Envelope; - assert.equal(timedEnvelope.timedOut, true, JSON.stringify(timedEnvelope)); - const waited = runCli([ "workflows", "wait", workflowId, "--timeout-ms", "10000", "--after", "0", "--json", ], baseEnv); From f425bc8acae66436484e9676b9971d02f503c953 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 21:12:53 +0530 Subject: [PATCH 23/25] feat(workflows): add restricted JavaScript runtime Co-Authored-By: Claude --- docs/workflow-runtime.md | 75 ++++ package.json | 2 +- src/db/migrations.ts | 61 +++ src/db/schema.ts | 70 ++++ src/oauth-store.test.ts | 1 + src/workflows/cli.test.ts | 40 ++ src/workflows/cli.ts | 72 +++- src/workflows/migrations.test.ts | 13 +- src/workflows/runtime-journal.ts | 377 ++++++++++++++++++ src/workflows/runtime.test.ts | 241 ++++++++++++ src/workflows/runtime.ts | 650 +++++++++++++++++++++++++++++++ 11 files changed, 1597 insertions(+), 5 deletions(-) create mode 100644 docs/workflow-runtime.md create mode 100644 src/workflows/runtime-journal.ts create mode 100644 src/workflows/runtime.test.ts create mode 100644 src/workflows/runtime.ts diff --git a/docs/workflow-runtime.md b/docs/workflow-runtime.md new file mode 100644 index 00000000..56c4c00f --- /dev/null +++ b/docs/workflow-runtime.md @@ -0,0 +1,75 @@ +# Restricted JavaScript workflow runtime + +DevSpace can run a bounded JavaScript workflow that coordinates the same durable local-agent workflows exposed through CLI and MCP. The script is orchestration code only: provider execution, retries, policy enforcement, worktrees, persistence, and cancellation remain owned by the shared workflow store and supervisor. + +## Run a script + +```bash +devspace workflows script ./review.workflow.js \ + --args-json '{"topic":"authentication"}' \ + --idempotency-key review-auth-v1 \ + --json +``` + +The script must be inside the canonical `DEVSPACE_WORKSPACE_ROOT` and use a `.js` or `.mjs` extension. `--args-json` must contain a JSON object. The command emits one versioned JSON envelope and waits for the script to reach a terminal state. + +## Script contract + +```js +// @devspace-workflow {"version":1,"name":"review","maxAgentCalls":4,"maxConcurrency":2,"timeoutMs":600000} +export default async function ({ agent, parallel, pipeline, phase, log, args, budget }) { + const reviews = await phase("review", () => parallel([ + () => agent({ target: "security", prompt: `Review ${args.topic}` }), + () => agent({ target: "tests", prompt: `Test ${args.topic}` }) + ])); + + const summary = await pipeline([ + (items) => agent({ + target: "reviewer", + prompt: `Synthesize: ${JSON.stringify(items)}` + }) + ], reviews); + + log("review complete", { workflowId: summary.workflowId }); + return { reviews, summary }; +} +``` + +The optional first non-empty line is strict JSON metadata. Supported fields are: + +- `version`: must be `1`. +- `name`: at most 128 characters. +- `description`: at most 1,024 characters. +- `maxAgentCalls`: 1–64; default 16. +- `maxConcurrency`: 1–16; default 4. +- `timeoutMs`: 1–86,400,000; default 15 minutes. + +Unknown metadata and `agent()` fields are rejected. + +## Primitives + +- `agent(options)`: submits one durable local-agent workflow and returns `{ workflowId, status, finalResponse }`. Options support `target`, `prompt`, `model`, `thinking`, `access`, `timeoutMs`, and the existing bounded read-only retry policy. +- `parallel(tasks)`: runs task functions concurrently. Actual agent dispatch remains bounded by `maxConcurrency` and supervisor limits. +- `pipeline(tasks, initialValue)`: awaits task functions in sequence, passing each result to the next task. +- `phase(name, task)`: emits durable `phase.started` and `phase.completed` runtime events. +- `log(message, data)`: emits a bounded durable log event. +- `args`: deeply frozen JSON supplied by `--args-json`. +- `budget`: deeply frozen effective `maxAgentCalls`, `maxConcurrency`, and `timeoutMs` values. + +Every `agent()` promise must be awaited, directly or through `parallel`, `pipeline`, or another awaited operation. Returning while calls are outstanding fails the runtime and requests cancellation of active children. + +## Isolation and limits + +The workflow body runs in a separate Node process with the permission model enabled, an empty environment, no inherited stdio, a 64 MiB old-space limit, and a `vm` context that disables string and WebAssembly code generation. The context does not expose Node APIs, imports, `process`, `require`, ambient environment variables, filesystem, network, database handles, or the orchestrator itself. + +Source, arguments, logs, event payloads, results, call counts, concurrency, and wall-clock duration are bounded. Scripts can only cause agent execution through `agent()`. + +This is a defense-in-depth boundary for trusted workspace workflow files, not a general-purpose multi-tenant JavaScript hosting service. Keep Node current because the boundary relies on Node's permission model and VM implementation. + +## Durability and replay + +Runtime runs, events, and each indexed `agent()` call are journaled in SQLite. Reusing an idempotency key with identical source, arguments, metadata, budget, and workspace returns the existing successful result. Reusing it with different input is rejected. + +After a failed or interrupted runtime, invoking the same request re-executes the orchestration function from the beginning. A completed call prefix is replayed only when the call index and normalized request hash match exactly; divergence fails closed. In-flight child workflow IDs are reused rather than submitted twice. + +Replay does not make arbitrary side effects safe. The restricted runtime intentionally offers no raw filesystem, network, shell, or database primitive. `workspace_write` agents use the workflow scheduler's isolated managed-worktree semantics and are not automatically retried. diff --git a/package.json b/package.json index 274635d3..1419833c 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "postinstall": "node scripts/fix-node-pty-permissions.mjs", "start": "node dist/cli.js serve", "test": "tsx src/config.test.ts && tsx src/ui/card-types.test.ts && tsx src/ui/patch-display.test.ts && tsx src/ui/tool-display.test.ts && tsx src/apply-patch.test.ts && tsx src/process-platform.test.ts && tsx src/process-sessions.test.ts && tsx src/mcp-sessions.test.ts && tsx src/server-shutdown.test.ts && tsx src/local-agent-runtime.test.ts && tsx src/local-agent-adapters.test.ts && tsx src/local-agent-availability.test.ts && tsx src/local-agent-profiles.test.ts && tsx src/local-agent-targets.test.ts && tsx src/local-agent-store.test.ts && tsx src/roots.test.ts && tsx src/skills.test.ts && tsx src/workspaces.test.ts && tsx src/review-checkpoints.test.ts && tsx src/oauth-store.test.ts && npm run test:workflows && tsx src/cli.test.ts", - "test:workflows": "tsx src/workflows/migrations.test.ts && tsx src/workflows/policy.test.ts && tsx src/workflows/workflows.test.ts && tsx src/workflows/supervisor.test.ts && tsx src/workflows/dag-scheduler.test.ts && tsx src/workflows/mcp.test.ts && tsx src/workflows/cli.test.ts", + "test:workflows": "tsx src/workflows/migrations.test.ts && tsx src/workflows/policy.test.ts && tsx src/workflows/workflows.test.ts && tsx src/workflows/supervisor.test.ts && tsx src/workflows/dag-scheduler.test.ts && tsx src/workflows/mcp.test.ts && tsx src/workflows/runtime.test.ts && tsx src/workflows/cli.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit" }, "keywords": [], diff --git a/src/db/migrations.ts b/src/db/migrations.ts index 83630ffb..3ff631c1 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -37,6 +37,11 @@ const migrations: Migration[] = [ name: "workflow-dag-scheduler", up: migrateWorkflowDagScheduler, }, + { + version: 7, + name: "workflow-js-runtime", + up: migrateWorkflowJsRuntime, + }, ]; export function migrateDatabase(sqlite: Database.Database): void { @@ -380,6 +385,62 @@ function migrateWorkflowDagScheduler(sqlite: Database.Database): void { `); } +function migrateWorkflowJsRuntime(sqlite: Database.Database): void { + sqlite.exec(` + create table if not exists workflow_runtime_runs ( + id text primary key, + workspace_id text not null, + workspace_root text not null, + source_hash text not null, + args_json text not null, + metadata_json text not null, + budget_json text not null, + idempotency_key text, + request_hash text not null, + status text not null check (status in ('running', 'succeeded', 'failed', 'cancelled')), + result_json text, + error_json text, + created_at text not null, + updated_at text not null, + completed_at text, + unique (workspace_id, workspace_root, idempotency_key) + ); + + create table if not exists workflow_runtime_calls ( + runtime_run_id text not null, + call_index integer not null, + request_hash text not null, + request_json text not null, + workflow_run_id text, + status text not null check (status in ('pending', 'running', 'succeeded', 'failed')), + result_json text, + error_json text, + created_at text not null, + updated_at text not null, + completed_at text, + primary key (runtime_run_id, call_index), + foreign key (runtime_run_id) references workflow_runtime_runs(id) on delete cascade, + foreign key (workflow_run_id) references workflow_runs(id) on delete set null + ); + + create table if not exists workflow_runtime_events ( + runtime_run_id text not null, + sequence integer not null, + event_type text not null, + payload_json text not null, + created_at text not null, + primary key (runtime_run_id, sequence), + foreign key (runtime_run_id) references workflow_runtime_runs(id) on delete cascade + ); + + create index if not exists workflow_runtime_runs_workspace_idx + on workflow_runtime_runs(workspace_id, workspace_root, created_at); + + create index if not exists workflow_runtime_calls_workflow_idx + on workflow_runtime_calls(workflow_run_id); + `); +} + function addColumnIfMissing( sqlite: Database.Database, table: "workspace_sessions" | "local_agent_sessions" | "workflow_runs" | "workflow_nodes", diff --git a/src/db/schema.ts b/src/db/schema.ts index 216edc19..e37afe68 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -275,6 +275,76 @@ export const workflowProviderEvents = sqliteTable( ], ); +export const workflowRuntimeRuns = sqliteTable( + "workflow_runtime_runs", + { + id: text("id").primaryKey(), + workspaceId: text("workspace_id").notNull(), + workspaceRoot: text("workspace_root").notNull(), + sourceHash: text("source_hash").notNull(), + argsJson: text("args_json").notNull(), + metadataJson: text("metadata_json").notNull(), + budgetJson: text("budget_json").notNull(), + idempotencyKey: text("idempotency_key"), + requestHash: text("request_hash").notNull(), + status: text("status").notNull(), + resultJson: text("result_json"), + errorJson: text("error_json"), + createdAt: text("created_at").notNull(), + updatedAt: text("updated_at").notNull(), + completedAt: text("completed_at"), + }, + (table) => [ + uniqueIndex("workflow_runtime_runs_idempotency_idx").on( + table.workspaceId, + table.workspaceRoot, + table.idempotencyKey, + ), + index("workflow_runtime_runs_workspace_idx").on( + table.workspaceId, + table.workspaceRoot, + table.createdAt, + ), + ], +); + +export const workflowRuntimeCalls = sqliteTable( + "workflow_runtime_calls", + { + runtimeRunId: text("runtime_run_id") + .notNull() + .references(() => workflowRuntimeRuns.id, { onDelete: "cascade" }), + callIndex: integer("call_index").notNull(), + requestHash: text("request_hash").notNull(), + requestJson: text("request_json").notNull(), + workflowRunId: text("workflow_run_id").references(() => workflowRuns.id, { onDelete: "set null" }), + status: text("status").notNull(), + resultJson: text("result_json"), + errorJson: text("error_json"), + createdAt: text("created_at").notNull(), + updatedAt: text("updated_at").notNull(), + completedAt: text("completed_at"), + }, + (table) => [ + primaryKey({ columns: [table.runtimeRunId, table.callIndex] }), + index("workflow_runtime_calls_workflow_idx").on(table.workflowRunId), + ], +); + +export const workflowRuntimeEvents = sqliteTable( + "workflow_runtime_events", + { + runtimeRunId: text("runtime_run_id") + .notNull() + .references(() => workflowRuntimeRuns.id, { onDelete: "cascade" }), + sequence: integer("sequence").notNull(), + eventType: text("event_type").notNull(), + payloadJson: text("payload_json").notNull(), + createdAt: text("created_at").notNull(), + }, + (table) => [primaryKey({ columns: [table.runtimeRunId, table.sequence] })], +); + export const workflowWorktrees = sqliteTable( "workflow_worktrees", { diff --git a/src/oauth-store.test.ts b/src/oauth-store.test.ts index 59f414e2..1435d44e 100644 --- a/src/oauth-store.test.ts +++ b/src/oauth-store.test.ts @@ -47,6 +47,7 @@ async function testDatabaseConfiguration(stateDir: string): Promise { { version: 4, name: "durable-workflows" }, { version: 5, name: "workflow-supervisor" }, { version: 6, name: "workflow-dag-scheduler" }, + { version: 7, name: "workflow-js-runtime" }, ]); } finally { database.close(); diff --git a/src/workflows/cli.test.ts b/src/workflows/cli.test.ts index b4d87f7d..db718115 100644 --- a/src/workflows/cli.test.ts +++ b/src/workflows/cli.test.ts @@ -17,7 +17,15 @@ mkdirSync(stateDir, { recursive: true }); mkdirSync(agentsDir, { recursive: true }); const profilePath = join(agentsDir, "reviewer.md"); +const workflowScriptPath = join(project, "example.workflow.js"); writeProfile(profilePath, "Original immutable profile."); +writeFileSync(workflowScriptPath, ` +// @devspace-workflow {"version":1,"name":"cli-test","maxAgentCalls":2,"maxConcurrency":1,"timeoutMs":5000} +export default function ({ args, budget, log }) { + log("script executed", { value: args.value }); + return { value: args.value, maxAgentCalls: budget.maxAgentCalls }; +} +`); const baseEnv = { ...process.env, @@ -117,6 +125,37 @@ try { ], baseEnv); assert.notEqual(conflict.status, 0); assert.equal((JSON.parse(conflict.stdout) as Envelope).error!.code, "idempotency_conflict"); + + const scripted = runCli([ + "workflows", "script", workflowScriptPath, "--args-json", JSON.stringify({ value: "from-cli" }), + "--idempotency-key", "script-shell-parent", "--json", + ], baseEnv); + assert.equal(scripted.status, 0, `${scripted.stderr}\n${scripted.stdout}`); + const scriptEnvelope = JSON.parse(scripted.stdout) as Envelope; + assert.equal(scriptEnvelope.ok, true); + assert.equal(scriptEnvelope.created, true); + assert.equal(scriptEnvelope.runtime!.status, "succeeded"); + assert.deepEqual(scriptEnvelope.runtime!.result, { value: "from-cli", maxAgentCalls: 2 }); + + const scriptReplay = runCli([ + "workflows", "script", workflowScriptPath, "--args-json", JSON.stringify({ value: "from-cli" }), + "--idempotency-key", "script-shell-parent", "--json", + ], baseEnv); + assert.equal(scriptReplay.status, 0, scriptReplay.stderr); + assert.equal((JSON.parse(scriptReplay.stdout) as Envelope).created, false); + + const scriptConflict = runCli([ + "workflows", "script", workflowScriptPath, "--args-json", JSON.stringify({ value: "different" }), + "--idempotency-key", "script-shell-parent", "--json", + ], baseEnv); + assert.notEqual(scriptConflict.status, 0); + assert.equal((JSON.parse(scriptConflict.stdout) as Envelope).error!.code, "idempotency_conflict"); + + const invalidScriptArgs = runCli([ + "workflows", "script", workflowScriptPath, "--args-json", "[]", "--json", + ], baseEnv); + assert.notEqual(invalidScriptArgs.status, 0); + assert.equal((JSON.parse(invalidScriptArgs.stdout) as Envelope).error!.code, "invalid_input"); } finally { rmSync(root, { recursive: true, force: true }); } @@ -157,5 +196,6 @@ interface Envelope { definition: { nodes: Array<{ config: { profileBody: string } }> }; }; events?: Array<{ type: string; sequence: number }>; + runtime?: { status: string; result?: unknown }; error?: { code: string; message: string }; } diff --git a/src/workflows/cli.ts b/src/workflows/cli.ts index e2db7147..f42cfc0b 100644 --- a/src/workflows/cli.ts +++ b/src/workflows/cli.ts @@ -1,4 +1,4 @@ -import { realpath } from "node:fs/promises"; +import { readFile, realpath } from "node:fs/promises"; import { homedir } from "node:os"; import { isAbsolute, join, relative, resolve } from "node:path"; import type { Writable } from "node:stream"; @@ -7,6 +7,7 @@ import { loadLocalAgentProfiles } from "../local-agent-profiles.js"; import { expandHomePath } from "../roots.js"; import { createWorkflowSubmission } from "./submission.js"; import { WorkflowOrchestrator } from "./orchestrator.js"; +import { executeWorkflowRuntime } from "./runtime.js"; import { WorkflowIdempotencyConflictError, WorkflowNotFoundError, @@ -15,6 +16,7 @@ import { import { ensureSupervisor } from "./supervisor-launch.js"; import { runWorkflowSupervisor } from "./supervisor.js"; import type { + JsonObject, WorkflowRunRecord, WorkflowWorkspaceScope, } from "./types.js"; @@ -82,6 +84,8 @@ async function dispatchWorkflowCommand( switch (command) { case "run": return await runCommand(parsed.positionals[0]!, args, context, config, scope, orchestrator); + case "script": + return await scriptCommand(parsed.positionals[0]!, args, context, config, scope, orchestrator); case "status": { const workflowId = parsed.positionals[0]!; return { workflow: orchestrator.getForWorkspace(workflowId, scope) ?? notFound(workflowId) }; @@ -184,6 +188,49 @@ async function runCommand( }; } +async function scriptCommand( + scriptPath: string, + args: string[], + context: WorkflowsCliContext, + config: WorkflowCliConfig, + scope: WorkflowWorkspaceScope, + orchestrator: WorkflowOrchestrator, +): Promise> { + const canonicalScript = await canonicalPath(resolve(context.cwd, scriptPath), "workflow script"); + if (!isInside(canonicalScript, scope.workspaceRoot)) { + throw new WorkspaceDeniedError("Workflow script is outside DEVSPACE_WORKSPACE_ROOT."); + } + if (!canonicalScript.endsWith(".js") && !canonicalScript.endsWith(".mjs")) { + throw new CliInputError("Workflow script must use a .js or .mjs extension."); + } + const source = await readFile(canonicalScript, "utf8"); + const runtimeArgs = parseJsonObjectOption(args, "--args-json"); + const profiles = await loadLocalAgentProfiles(config, scope.workspaceRoot); + const result = await executeWorkflowRuntime({ + stateDir: config.stateDir, + worktreeRoot: config.worktreeRoot, + source, + args: runtimeArgs, + workspace: scope, + profiles, + idempotencyKey: optionalOption(args, "--idempotency-key"), + environment: context.env, + cliEntrypoint: context.cliEntrypoint, + orchestrator, + }); + if (result.run.status !== "succeeded") { + const message = typeof result.run.error?.message === "string" + ? result.run.error.message + : `Workflow runtime ${result.run.id} failed`; + throw new CliRuntimeError(`Workflow runtime ${result.run.id} failed: ${message}`); + } + return { + runtime: result.run, + created: result.created, + replayedCalls: result.replayedCalls, + }; +} + async function resolveWorkspaceScope( context: Pick, allowedRoots: string[], @@ -239,6 +286,10 @@ function validateWorkflowArguments( "--idempotency-key", ]), }, + script: { + positionals: 1, + valueOptions: new Set(["--args-json", "--idempotency-key"]), + }, status: { positionals: 1, valueOptions: new Set() }, events: { positionals: 1, valueOptions: new Set(["--after"]) }, wait: { positionals: 1, valueOptions: new Set(["--timeout-ms", "--after"]) }, @@ -298,6 +349,22 @@ function optionalOption(args: string[], name: string): string | undefined { return undefined; } +function parseJsonObjectOption(args: string[], name: string): JsonObject { + const raw = optionalOption(args, name); + if (raw === undefined) return {}; + try { + const parsed = JSON.parse(raw) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("value must be a JSON object"); + } + return parsed as JsonObject; + } catch (error) { + throw new CliInputError( + `${name} must be valid JSON object: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + function integerOption(args: string[], name: string): number | undefined { const raw = optionalOption(args, name); if (raw === undefined) return undefined; @@ -314,6 +381,7 @@ function normalizeCliError(error: unknown): { code: string; message: string; exi if (error instanceof WorkspaceDeniedError) return { code: "workspace_denied", message: error.message, exitCode: 3 }; if (error instanceof WorkflowNotFoundError) return { code: "not_found", message: error.message, exitCode: 4 }; if (error instanceof WorkflowIdempotencyConflictError) return { code: "idempotency_conflict", message: error.message, exitCode: 5 }; + if (error instanceof CliRuntimeError) return { code: "runtime_failed", message: error.message, exitCode: 6 }; if (error instanceof CliInputError || error instanceof WorkflowValidationError) { return { code: "invalid_input", message: error.message, exitCode: 2 }; } @@ -351,6 +419,7 @@ function workflowsHelp(): string { "", "Usage:", " devspace workflows run --prompt --json [--idempotency-key ]", + " devspace workflows script --json [--args-json ] [--idempotency-key ]", " devspace workflows status --json", " devspace workflows wait --json [--timeout-ms ] [--after ]", " devspace workflows events --json [--after ]", @@ -360,4 +429,5 @@ function workflowsHelp(): string { } class CliInputError extends Error {} +class CliRuntimeError extends Error {} class WorkspaceDeniedError extends Error {} diff --git a/src/workflows/migrations.test.ts b/src/workflows/migrations.test.ts index 6178d546..9ecd888e 100644 --- a/src/workflows/migrations.test.ts +++ b/src/workflows/migrations.test.ts @@ -27,6 +27,7 @@ function testFreshSchema(stateDir: string): void { { version: 4, name: "durable-workflows" }, { version: 5, name: "workflow-supervisor" }, { version: 6, name: "workflow-dag-scheduler" }, + { version: 7, name: "workflow-js-runtime" }, ], ); const tables = database.sqlite @@ -44,6 +45,9 @@ function testFreshSchema(stateDir: string): void { "workflow_nodes", "workflow_provider_events", "workflow_runs", + "workflow_runtime_calls", + "workflow_runtime_events", + "workflow_runtime_runs", "workflow_supervisor", "workflow_worktrees", ]); @@ -100,7 +104,7 @@ function testExistingMigrationCompatibility(stateDir: string): void { try { assert.deepEqual( migrated.sqlite.prepare("select version from devspace_schema_migrations order by version").pluck().all(), - [1, 2, 3, 4, 5, 6], + [1, 2, 3, 4, 5, 6, 7], ); assert.deepEqual(migrated.sqlite.prepare("select * from workspace_sessions").all(), [ { @@ -173,7 +177,7 @@ function testExistingMigrationCompatibility(stateDir: string): void { .prepare("select count(*) from sqlite_master where type = 'table' and name like 'workflow_%'") .pluck() .get(), - 8, + 11, ); } finally { migrated.close(); @@ -224,6 +228,9 @@ function createVersion3Fixture(stateDir: string): void { const initialized = openDatabase(stateDir); try { initialized.sqlite.exec(` + drop table workflow_runtime_events; + drop table workflow_runtime_calls; + drop table workflow_runtime_runs; drop table workflow_worktrees; drop table workflow_provider_events; drop table workflow_node_attempts; @@ -232,7 +239,7 @@ function createVersion3Fixture(stateDir: string): void { drop table workflow_events; drop table workflow_nodes; drop table workflow_runs; - delete from devspace_schema_migrations where version in (4, 5, 6); + delete from devspace_schema_migrations where version in (4, 5, 6, 7); insert into workspace_sessions ( id, root, status, mode, source_root, base_ref, base_sha, managed, created_at, last_used_at diff --git a/src/workflows/runtime-journal.ts b/src/workflows/runtime-journal.ts new file mode 100644 index 00000000..7f9cd825 --- /dev/null +++ b/src/workflows/runtime-journal.ts @@ -0,0 +1,377 @@ +import { createHash, randomUUID } from "node:crypto"; +import { openDatabase, type DatabaseHandle } from "../db/client.js"; +import type { JsonObject, JsonValue, WorkflowWorkspaceScope } from "./types.js"; +import { WorkflowIdempotencyConflictError, WorkflowValidationError } from "./store.js"; + +export type WorkflowRuntimeStatus = "running" | "succeeded" | "failed" | "cancelled"; +export type WorkflowRuntimeCallStatus = "pending" | "running" | "succeeded" | "failed"; + +export interface WorkflowRuntimeMetadata { + version: 1; + name?: string; + description?: string; +} + +export interface WorkflowRuntimeBudget { + maxAgentCalls: number; + maxConcurrency: number; + timeoutMs: number; +} + +export interface WorkflowRuntimeRun { + id: string; + workspaceId: string; + workspaceRoot: string; + sourceHash: string; + args: JsonObject; + metadata: WorkflowRuntimeMetadata; + budget: WorkflowRuntimeBudget; + idempotencyKey?: string; + requestHash: string; + status: WorkflowRuntimeStatus; + result?: JsonValue; + error?: JsonObject; + createdAt: string; + updatedAt: string; + completedAt?: string; +} + +export interface WorkflowRuntimeCall { + runtimeRunId: string; + callIndex: number; + requestHash: string; + request: JsonObject; + workflowRunId?: string; + status: WorkflowRuntimeCallStatus; + result?: JsonValue; + error?: JsonObject; + createdAt: string; + updatedAt: string; + completedAt?: string; +} + +interface RuntimeRunRow { + id: string; + workspace_id: string; + workspace_root: string; + source_hash: string; + args_json: string; + metadata_json: string; + budget_json: string; + idempotency_key: string | null; + request_hash: string; + status: WorkflowRuntimeStatus; + result_json: string | null; + error_json: string | null; + created_at: string; + updated_at: string; + completed_at: string | null; +} + +interface RuntimeCallRow { + runtime_run_id: string; + call_index: number; + request_hash: string; + request_json: string; + workflow_run_id: string | null; + status: WorkflowRuntimeCallStatus; + result_json: string | null; + error_json: string | null; + created_at: string; + updated_at: string; + completed_at: string | null; +} + +export class WorkflowRuntimeJournal { + private readonly database: DatabaseHandle; + + constructor(stateDir: string) { + this.database = openDatabase(stateDir); + } + + submit(input: { + sourceHash: string; + args: JsonObject; + metadata: WorkflowRuntimeMetadata; + budget: WorkflowRuntimeBudget; + workspace: WorkflowWorkspaceScope; + idempotencyKey?: string; + }): { run: WorkflowRuntimeRun; created: boolean } { + const requestHash = hashJson({ + sourceHash: input.sourceHash, + args: input.args, + metadata: input.metadata, + budget: input.budget, + workspace: input.workspace, + }); + const idempotencyKey = input.idempotencyKey?.trim() || undefined; + const submit = this.database.sqlite.transaction(() => { + if (idempotencyKey) { + const existing = this.database.sqlite + .prepare( + `select * from workflow_runtime_runs + where workspace_id = ? and workspace_root = ? and idempotency_key = ?`, + ) + .get(input.workspace.workspaceId, input.workspace.workspaceRoot, idempotencyKey) as RuntimeRunRow | undefined; + if (existing) { + if (existing.request_hash !== requestHash) { + throw new WorkflowIdempotencyConflictError(idempotencyKey); + } + return { run: rowToRun(existing), created: false }; + } + } + + const id = randomUUID(); + const now = new Date().toISOString(); + this.database.sqlite + .prepare( + `insert into workflow_runtime_runs ( + id, workspace_id, workspace_root, source_hash, args_json, metadata_json, + budget_json, idempotency_key, request_hash, status, created_at, updated_at + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, 'running', ?, ?)`, + ) + .run( + id, + input.workspace.workspaceId, + input.workspace.workspaceRoot, + input.sourceHash, + serialize(input.args), + serialize(input.metadata), + serialize(input.budget), + idempotencyKey ?? null, + requestHash, + now, + now, + ); + return { run: this.require(id), created: true }; + }); + const result = submit.immediate(); + if (result.created) { + this.appendEvent(result.run.id, "runtime.started", { sourceHash: input.sourceHash }); + } + return result; + } + + require(runId: string): WorkflowRuntimeRun { + const row = this.database.sqlite + .prepare("select * from workflow_runtime_runs where id = ?") + .get(runId) as RuntimeRunRow | undefined; + if (!row) throw new WorkflowValidationError(`Unknown workflow runtime run: ${runId}`); + return rowToRun(row); + } + + resume(runId: string): WorkflowRuntimeRun { + const current = this.require(runId); + if (current.status === "succeeded") return current; + const now = new Date().toISOString(); + this.database.sqlite + .prepare( + `update workflow_runtime_runs + set status = 'running', result_json = null, error_json = null, + completed_at = null, updated_at = ? where id = ?`, + ) + .run(now, runId); + this.appendEvent(runId, "runtime.resumed", {}); + return this.require(runId); + } + + beginCall(input: { + runId: string; + callIndex: number; + request: JsonObject; + }): { call: WorkflowRuntimeCall; replayed: boolean } { + if (!Number.isSafeInteger(input.callIndex) || input.callIndex < 0) { + throw new WorkflowValidationError("Workflow runtime call index must be a non-negative integer"); + } + const requestHash = hashJson(input.request); + const begin = this.database.sqlite.transaction(() => { + const existing = this.getCallRow(input.runId, input.callIndex); + if (existing) { + if (existing.request_hash !== requestHash) { + throw new WorkflowValidationError( + `Workflow runtime replay diverged at agent call ${input.callIndex}`, + ); + } + return { call: rowToCall(existing), replayed: true }; + } + const now = new Date().toISOString(); + this.database.sqlite + .prepare( + `insert into workflow_runtime_calls ( + runtime_run_id, call_index, request_hash, request_json, status, created_at, updated_at + ) values (?, ?, ?, ?, 'pending', ?, ?)`, + ) + .run(input.runId, input.callIndex, requestHash, serialize(input.request), now, now); + return { call: rowToCall(this.getCallRow(input.runId, input.callIndex)!), replayed: false }; + }); + const result = begin.immediate(); + if (!result.replayed) { + this.appendEvent(input.runId, "agent.requested", { callIndex: input.callIndex, requestHash }); + } + return result; + } + + markCallRunning(runId: string, callIndex: number, workflowRunId: string): WorkflowRuntimeCall { + const now = new Date().toISOString(); + const result = this.database.sqlite + .prepare( + `update workflow_runtime_calls + set status = 'running', workflow_run_id = ?, updated_at = ? + where runtime_run_id = ? and call_index = ? and status in ('pending', 'running')`, + ) + .run(workflowRunId, now, runId, callIndex); + if (result.changes !== 1) throw new WorkflowValidationError("Workflow runtime call is not runnable"); + this.appendEvent(runId, "agent.started", { callIndex, workflowRunId }); + return this.requireCall(runId, callIndex); + } + + completeCall( + runId: string, + callIndex: number, + status: "succeeded" | "failed", + value: JsonValue | JsonObject, + ): WorkflowRuntimeCall { + const now = new Date().toISOString(); + const resultJson = status === "succeeded" ? serialize(value) : null; + const errorJson = status === "failed" ? serialize(value) : null; + const result = this.database.sqlite + .prepare( + `update workflow_runtime_calls + set status = ?, result_json = ?, error_json = ?, updated_at = ?, completed_at = ? + where runtime_run_id = ? and call_index = ? and status in ('pending', 'running')`, + ) + .run(status, resultJson, errorJson, now, now, runId, callIndex); + if (result.changes !== 1) { + const current = this.requireCall(runId, callIndex); + if (current.status === status) return current; + throw new WorkflowValidationError("Workflow runtime call is already terminal"); + } + this.appendEvent(runId, `agent.${status}`, { callIndex }); + return this.requireCall(runId, callIndex); + } + + requireCall(runId: string, callIndex: number): WorkflowRuntimeCall { + const row = this.getCallRow(runId, callIndex); + if (!row) throw new WorkflowValidationError(`Unknown workflow runtime call: ${callIndex}`); + return rowToCall(row); + } + + appendEvent(runId: string, eventType: string, payload: JsonObject): number { + const append = this.database.sqlite.transaction(() => { + this.require(runId); + const row = this.database.sqlite + .prepare( + "select coalesce(max(sequence), 0) + 1 as sequence from workflow_runtime_events where runtime_run_id = ?", + ) + .get(runId) as { sequence: number }; + this.database.sqlite + .prepare( + `insert into workflow_runtime_events + (runtime_run_id, sequence, event_type, payload_json, created_at) + values (?, ?, ?, ?, ?)`, + ) + .run(runId, row.sequence, eventType, serialize(payload), new Date().toISOString()); + return row.sequence; + }); + return append.immediate(); + } + + completeRun( + runId: string, + status: "succeeded" | "failed" | "cancelled", + value: JsonValue | JsonObject, + ): WorkflowRuntimeRun { + const now = new Date().toISOString(); + const resultJson = status === "succeeded" ? serialize(value) : null; + const errorJson = status === "succeeded" ? null : serialize(value); + const updated = this.database.sqlite + .prepare( + `update workflow_runtime_runs + set status = ?, result_json = ?, error_json = ?, updated_at = ?, completed_at = ? + where id = ? and status = 'running'`, + ) + .run(status, resultJson, errorJson, now, now, runId); + if (updated.changes !== 1) { + const current = this.require(runId); + if (current.status === status) return current; + throw new WorkflowValidationError("Workflow runtime run is already terminal"); + } + this.appendEvent(runId, `runtime.${status}`, {}); + return this.require(runId); + } + + close(): void { + this.database.close(); + } + + private getCallRow(runId: string, callIndex: number): RuntimeCallRow | undefined { + return this.database.sqlite + .prepare( + "select * from workflow_runtime_calls where runtime_run_id = ? and call_index = ?", + ) + .get(runId, callIndex) as RuntimeCallRow | undefined; + } +} + +export function workflowRuntimeSourceHash(source: string): string { + return createHash("sha256").update(source).digest("hex"); +} + +function rowToRun(row: RuntimeRunRow): WorkflowRuntimeRun { + return { + id: row.id, + workspaceId: row.workspace_id, + workspaceRoot: row.workspace_root, + sourceHash: row.source_hash, + args: parseJson(row.args_json), + metadata: parseJson(row.metadata_json), + budget: parseJson(row.budget_json), + idempotencyKey: row.idempotency_key ?? undefined, + requestHash: row.request_hash, + status: row.status, + result: parseOptional(row.result_json), + error: parseOptional(row.error_json) as JsonObject | undefined, + createdAt: row.created_at, + updatedAt: row.updated_at, + completedAt: row.completed_at ?? undefined, + }; +} + +function rowToCall(row: RuntimeCallRow): WorkflowRuntimeCall { + return { + runtimeRunId: row.runtime_run_id, + callIndex: row.call_index, + requestHash: row.request_hash, + request: parseJson(row.request_json), + workflowRunId: row.workflow_run_id ?? undefined, + status: row.status, + result: parseOptional(row.result_json), + error: parseOptional(row.error_json) as JsonObject | undefined, + createdAt: row.created_at, + updatedAt: row.updated_at, + completedAt: row.completed_at ?? undefined, + }; +} + +function hashJson(value: unknown): string { + return createHash("sha256").update(canonicalJson(value)).digest("hex"); +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + const record = value as Record; + return `{${Object.keys(record).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(",")}}`; +} + +function serialize(value: unknown): string { + return JSON.stringify(value); +} + +function parseJson(value: string): T { + return JSON.parse(value) as T; +} + +function parseOptional(value: string | null): JsonValue | undefined { + return value === null ? undefined : parseJson(value); +} diff --git a/src/workflows/runtime.test.ts b/src/workflows/runtime.test.ts new file mode 100644 index 00000000..ff15d70e --- /dev/null +++ b/src/workflows/runtime.test.ts @@ -0,0 +1,241 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { LocalAgentRunController, type LocalAgentRunHandle } from "../local-agent-runtime.js"; +import { WorkflowOrchestrator } from "./orchestrator.js"; +import { executeWorkflowRuntime, parseWorkflowRuntimeSource } from "./runtime.js"; +import { runWorkflowSupervisor } from "./supervisor.js"; + +const root = mkdtempSync(join(tmpdir(), "devspace-workflow-runtime-test-")); +try { + await testParallelPipelineRuntime(join(root, "parallel")); + await testPrefixReplay(join(root, "replay")); + await testRestrictedGlobalsAndBudget(join(root, "sandbox")); + await testUnawaitedAgentFails(join(root, "unawaited")); + testMetadataValidation(); +} finally { + rmSync(root, { recursive: true, force: true }); +} + +async function testParallelPipelineRuntime(stateDir: string): Promise { + const orchestrator = new WorkflowOrchestrator(stateDir); + let active = 0; + let maximum = 0; + const prompts: string[] = []; + const handleFactory = async (_provider: string, input: { prompt: string }): Promise => { + active += 1; + maximum = Math.max(maximum, active); + prompts.push(input.prompt); + const controller = new LocalAgentRunController("fake"); + const timer = setTimeout(() => { + active -= 1; + controller.succeed({ + provider: "fake", + providerSessionId: `runtime-${input.prompt}`, + finalResponse: `result:${input.prompt}`, + items: [], + }); + }, 30); + controller.setLifecycle({ + cancel: () => { + clearTimeout(timer); + active = Math.max(0, active - 1); + }, + dispose: () => clearTimeout(timer), + }); + return controller; + }; + const wakeSupervisor = async () => { + await runWorkflowSupervisor(stateDir, { + handleFactory, + globalConcurrency: 4, + heartbeatMs: 5, + nodeLeaseMs: 500, + supervisorLeaseMs: 500, + idleMs: 0, + }); + }; + const source = ` +// @devspace-workflow {"version":1,"name":"parallel-test","maxAgentCalls":3,"maxConcurrency":2,"timeoutMs":5000} +export default async function ({ agent, parallel, pipeline, phase, log, args }) { + const pair = await phase("research", () => parallel([ + () => agent({ target: "fake", prompt: args.first }), + () => agent({ target: "fake", prompt: args.second }) + ])); + log("research complete", { count: pair.length }); + const final = await pipeline([ + (value) => agent({ target: "fake", prompt: value[0].finalResponse + "+summary" }) + ], pair); + return { pair, final }; +} +`; + try { + const result = await executeWorkflowRuntime({ + stateDir, + worktreeRoot: join(stateDir, "worktrees"), + source, + args: { first: "alpha", second: "beta" }, + workspace: { workspaceId: "workspace", workspaceRoot: root }, + profiles: [], + idempotencyKey: "parallel-runtime", + environment: { ...process.env, DEVSPACE_WORKFLOW_FAKE_PROVIDER: "1" }, + orchestrator, + wakeSupervisor, + }); + assert.equal(result.run.status, "succeeded", JSON.stringify(result.run.error)); + assert.equal(maximum, 2); + assert.deepEqual(prompts.slice(0, 2).sort(), ["alpha", "beta"]); + assert.equal(prompts[2], "result:alpha+summary"); + const output = result.run.result as { final: { finalResponse: string } }; + assert.equal(output.final.finalResponse, "result:result:alpha+summary"); + } finally { + orchestrator.close(); + } +} + +async function testPrefixReplay(stateDir: string): Promise { + const orchestrator = new WorkflowOrchestrator(stateDir); + let starts = 0; + const handleFactory = async (_provider: string, input: { prompt: string }): Promise => { + starts += 1; + const controller = new LocalAgentRunController("fake"); + queueMicrotask(() => controller.succeed({ + provider: "fake", + providerSessionId: `replay-${starts}`, + finalResponse: input.prompt, + items: [], + })); + return controller; + }; + const wakeSupervisor = async () => { + await runWorkflowSupervisor(stateDir, { + handleFactory, + heartbeatMs: 5, + nodeLeaseMs: 500, + supervisorLeaseMs: 500, + idleMs: 0, + }); + }; + const source = ` +// @devspace-workflow {"version":1,"maxAgentCalls":1,"timeoutMs":5000} +export default async function ({ agent }) { + await agent({ target: "fake", prompt: "once" }); + throw new Error("intentional failure after durable prefix"); +} +`; + try { + const input = { + stateDir, + worktreeRoot: join(stateDir, "worktrees"), + source, + args: {}, + workspace: { workspaceId: "workspace", workspaceRoot: root }, + profiles: [], + idempotencyKey: "prefix-replay", + environment: { ...process.env, DEVSPACE_WORKFLOW_FAKE_PROVIDER: "1" }, + orchestrator, + wakeSupervisor, + }; + const first = await executeWorkflowRuntime(input); + assert.equal(first.run.status, "failed"); + assert.equal(starts, 1); + const replay = await executeWorkflowRuntime(input); + assert.equal(replay.run.status, "failed"); + assert.equal(replay.replayedCalls, 1); + assert.equal(starts, 1, "completed prefix must not execute the provider twice"); + } finally { + orchestrator.close(); + } +} + +async function testRestrictedGlobalsAndBudget(stateDir: string): Promise { + const source = ` +// @devspace-workflow {"version":1,"maxAgentCalls":7,"maxConcurrency":3,"timeoutMs":5000} +export default function ({ args, budget }) { + return { + args, + budget, + globals: { + process: typeof process, + require: typeof require, + fetch: typeof fetch + } + }; +} +`; + const result = await executeWorkflowRuntime({ + stateDir, + worktreeRoot: join(stateDir, "worktrees"), + source, + args: { value: "safe" }, + workspace: { workspaceId: "workspace", workspaceRoot: root }, + profiles: [], + }); + assert.equal(result.run.status, "succeeded", JSON.stringify(result.run.error)); + assert.deepEqual(result.run.result, { + args: { value: "safe" }, + budget: { maxAgentCalls: 7, maxConcurrency: 3, timeoutMs: 5000 }, + globals: { process: "undefined", require: "undefined", fetch: "undefined" }, + }); +} + +async function testUnawaitedAgentFails(stateDir: string): Promise { + const orchestrator = new WorkflowOrchestrator(stateDir); + const handleFactory = async (): Promise => { + const controller = new LocalAgentRunController("fake"); + const timer = setTimeout(() => controller.succeed({ + provider: "fake", + providerSessionId: "late-session", + finalResponse: "late", + items: [], + }), 5_000); + controller.setLifecycle({ + cancel: () => { + clearTimeout(timer); + }, + dispose: () => clearTimeout(timer), + }); + return controller; + }; + try { + const result = await executeWorkflowRuntime({ + stateDir, + worktreeRoot: join(stateDir, "worktrees"), + source: ` +// @devspace-workflow {"version":1,"maxAgentCalls":1,"timeoutMs":5000} +export default function ({ agent }) { + agent({ target: "fake", prompt: "must await" }); + return { premature: true }; +} +`, + args: {}, + workspace: { workspaceId: "workspace", workspaceRoot: root }, + profiles: [], + environment: { ...process.env, DEVSPACE_WORKFLOW_FAKE_PROVIDER: "1" }, + orchestrator, + wakeSupervisor: async () => runWorkflowSupervisor(stateDir, { + handleFactory, + heartbeatMs: 5, + nodeLeaseMs: 500, + supervisorLeaseMs: 500, + idleMs: 0, + }), + }); + assert.equal(result.run.status, "failed"); + assert.match(String(result.run.error?.message), /before all agent\(\) calls were awaited/); + } finally { + orchestrator.close(); + } +} + +function testMetadataValidation(): void { + assert.throws( + () => parseWorkflowRuntimeSource(`// @devspace-workflow {"version":1,"unknown":true}\nexport default function () {}`), + /Unknown workflow metadata field/, + ); + assert.throws( + () => parseWorkflowRuntimeSource("export const workflow = () => null"), + /default function declaration/, + ); +} diff --git a/src/workflows/runtime.ts b/src/workflows/runtime.ts new file mode 100644 index 00000000..7fd0b234 --- /dev/null +++ b/src/workflows/runtime.ts @@ -0,0 +1,650 @@ +import { spawn, type ChildProcess } from "node:child_process"; +import type { LocalAgentProfile } from "../local-agent-profiles.js"; +import { createWorkflowSubmission, type WorkflowAgentIntent } from "./submission.js"; +import { WorkflowOrchestrator } from "./orchestrator.js"; +import { ensureSupervisor } from "./supervisor-launch.js"; +import { + WorkflowRuntimeJournal, + workflowRuntimeSourceHash, + type WorkflowRuntimeBudget, + type WorkflowRuntimeMetadata, + type WorkflowRuntimeRun, +} from "./runtime-journal.js"; +import { WorkflowValidationError } from "./store.js"; +import type { JsonObject, JsonValue, WorkflowRunRecord, WorkflowWorkspaceScope } from "./types.js"; + +const MAX_SOURCE_BYTES = 256 * 1024; +const MAX_ARGS_BYTES = 64 * 1024; +const DEFAULT_MAX_AGENT_CALLS = 16; +const MAX_AGENT_CALLS = 64; +const DEFAULT_MAX_CONCURRENCY = 4; +const MAX_CONCURRENCY = 16; +const DEFAULT_RUNTIME_TIMEOUT_MS = 15 * 60_000; +const MAX_RUNTIME_TIMEOUT_MS = 24 * 60 * 60_000; +const MAX_LOG_CHARS = 16_384; +const MAX_EVENT_DATA_BYTES = 64 * 1024; +const CHILD_WORKFLOW_WAIT_MS = 300_000; + +export interface ParsedWorkflowRuntimeSource { + metadata: WorkflowRuntimeMetadata; + budget: WorkflowRuntimeBudget; + code: string; +} + +export interface ExecuteWorkflowRuntimeInput { + stateDir: string; + worktreeRoot: string; + source: string; + args: JsonObject; + workspace: WorkflowWorkspaceScope; + profiles: LocalAgentProfile[]; + idempotencyKey?: string; + environment?: NodeJS.ProcessEnv; + cliEntrypoint?: string; + orchestrator?: WorkflowOrchestrator; + wakeSupervisor?: () => Promise; +} + +export interface ExecuteWorkflowRuntimeResult { + run: WorkflowRuntimeRun; + created: boolean; + replayedCalls: number; +} + +interface RuntimeAgentMessage { + type: "agent"; + requestId: number; + callIndex: number; + request: unknown; +} + +interface RuntimeEventMessage { + type: "event"; + eventType: unknown; + payload: unknown; +} + +interface RuntimeResultMessage { + type: "result"; + result: unknown; +} + +interface RuntimeFailureMessage { + type: "failure"; + error: unknown; +} + +type RuntimeChildMessage = RuntimeAgentMessage | RuntimeEventMessage | RuntimeResultMessage | RuntimeFailureMessage; + +export function parseWorkflowRuntimeSource(source: string): ParsedWorkflowRuntimeSource { + if (Buffer.byteLength(source, "utf8") > MAX_SOURCE_BYTES) { + throw new WorkflowValidationError(`Workflow script exceeds ${MAX_SOURCE_BYTES} bytes`); + } + const lines = source.replace(/^/, "").split(/\r?\n/); + const firstContentIndex = lines.findIndex((line) => line.trim().length > 0); + let header: Record = {}; + if (firstContentIndex >= 0 && lines[firstContentIndex]!.trim().startsWith("// @devspace-workflow")) { + const line = lines[firstContentIndex]!.trim(); + const raw = line.slice("// @devspace-workflow".length).trim(); + if (!raw) throw new WorkflowValidationError("Workflow metadata comment must contain JSON"); + try { + const parsed = JSON.parse(raw) as unknown; + if (!isPlainObject(parsed)) throw new Error("metadata must be an object"); + header = parsed; + } catch (error) { + throw new WorkflowValidationError( + `Invalid workflow metadata JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } + lines.splice(firstContentIndex, 1); + } + + const allowed = new Set([ + "version", + "name", + "description", + "maxAgentCalls", + "maxConcurrency", + "timeoutMs", + ]); + for (const key of Object.keys(header)) { + if (!allowed.has(key)) throw new WorkflowValidationError(`Unknown workflow metadata field: ${key}`); + } + const version = header.version ?? 1; + if (version !== 1) throw new WorkflowValidationError("Workflow metadata version must be 1"); + const name = optionalBoundedString(header.name, "Workflow name", 128); + const description = optionalBoundedString(header.description, "Workflow description", 1_024); + const metadata: WorkflowRuntimeMetadata = { + version: 1, + ...(name ? { name } : {}), + ...(description ? { description } : {}), + }; + const budget: WorkflowRuntimeBudget = { + maxAgentCalls: boundedInteger( + header.maxAgentCalls, + DEFAULT_MAX_AGENT_CALLS, + 1, + MAX_AGENT_CALLS, + "Workflow maxAgentCalls", + ), + maxConcurrency: boundedInteger( + header.maxConcurrency, + DEFAULT_MAX_CONCURRENCY, + 1, + MAX_CONCURRENCY, + "Workflow maxConcurrency", + ), + timeoutMs: boundedInteger( + header.timeoutMs, + DEFAULT_RUNTIME_TIMEOUT_MS, + 1, + MAX_RUNTIME_TIMEOUT_MS, + "Workflow timeoutMs", + ), + }; + const code = lines.join("\n").trim(); + if (!/^export\s+default\s+(?:async\s+)?function\b/.test(code)) { + throw new WorkflowValidationError( + "Workflow script must export a default function declaration", + ); + } + return { metadata, budget, code }; +} + +export async function executeWorkflowRuntime( + input: ExecuteWorkflowRuntimeInput, +): Promise { + validateJsonObject(input.args, "Workflow args", MAX_ARGS_BYTES); + const parsed = parseWorkflowRuntimeSource(input.source); + const journal = new WorkflowRuntimeJournal(input.stateDir); + const orchestrator = input.orchestrator ?? new WorkflowOrchestrator(input.stateDir); + const ownsOrchestrator = !input.orchestrator; + const submitted = journal.submit({ + sourceHash: workflowRuntimeSourceHash(parsed.code), + args: input.args, + metadata: parsed.metadata, + budget: parsed.budget, + workspace: input.workspace, + idempotencyKey: input.idempotencyKey, + }); + if (!submitted.created && submitted.run.status === "succeeded") { + journal.close(); + if (ownsOrchestrator) orchestrator.close(); + return { run: submitted.run, created: false, replayedCalls: 0 }; + } + const run = submitted.created ? submitted.run : journal.resume(submitted.run.id); + const activeWorkflowIds = new Set(); + const scheduledOperations = new Set>(); + let aborting = false; + let replayedCalls = 0; + let child: ChildProcess | undefined; + let timeout: NodeJS.Timeout | undefined; + try { + const finalResult = await new Promise((resolve, reject) => { + let settled = false; + let activeAgents = 0; + const queuedAgents: Array<() => void> = []; + const settle = (action: () => void) => { + if (settled) return; + settled = true; + if (timeout) clearTimeout(timeout); + action(); + }; + const runQueued = () => { + while (!settled && activeAgents < parsed.budget.maxConcurrency && queuedAgents.length > 0) { + activeAgents += 1; + queuedAgents.shift()!(); + } + }; + const scheduleAgent = (operation: () => Promise) => { + queuedAgents.push(() => { + const scheduled = operation().finally(() => { + activeAgents -= 1; + scheduledOperations.delete(scheduled); + runQueued(); + }); + scheduledOperations.add(scheduled); + void scheduled; + }); + runQueued(); + }; + + child = spawnRuntimeChild(); + child.once("error", (error) => settle(() => reject(error))); + child.once("exit", (code, signal) => { + if (!settled) { + settle(() => reject(new Error( + `Workflow runtime child exited before returning a result (${signal ? `signal ${signal}` : `exit code ${code ?? "unknown"}`})`, + ))); + } + }); + child.on("message", (raw: unknown) => { + void handleChildMessage(raw).catch((error) => settle(() => reject(error))); + }); + timeout = setTimeout(() => { + settle(() => reject(new WorkflowValidationError( + `Workflow runtime exceeded ${parsed.budget.timeoutMs} milliseconds`, + ))); + }, parsed.budget.timeoutMs); + + child.send({ + type: "start", + source: parsed.code, + args: input.args, + budget: parsed.budget, + }); + + async function handleChildMessage(raw: unknown): Promise { + if (!isPlainObject(raw) || typeof raw.type !== "string") { + throw new WorkflowValidationError("Workflow runtime child sent an invalid message"); + } + const message = raw as unknown as RuntimeChildMessage; + if (message.type === "agent") { + if (!Number.isSafeInteger(message.requestId) || Number(message.requestId) < 0) { + throw new WorkflowValidationError("Workflow runtime request ID is invalid"); + } + if (!Number.isSafeInteger(message.callIndex) || Number(message.callIndex) < 0) { + throw new WorkflowValidationError("Workflow runtime call index is invalid"); + } + if (message.callIndex >= parsed.budget.maxAgentCalls) { + sendChild(child, { + type: "agent_result", + requestId: message.requestId, + ok: false, + error: { code: "budget_exceeded", message: "Workflow agent-call budget was exceeded" }, + }); + return; + } + scheduleAgent(async () => { + try { + const result = await executeAgentCall(message.callIndex, message.request); + sendChild(child, { type: "agent_result", requestId: message.requestId, ok: true, result }); + } catch (error) { + sendChild(child, { + type: "agent_result", + requestId: message.requestId, + ok: false, + error: normalizeError(error), + }); + } + }); + return; + } + if (message.type === "event") { + const eventType = requiredBoundedString(message.eventType, "Runtime event type", 64); + if (eventType !== "phase.started" && eventType !== "phase.completed" && eventType !== "log") { + throw new WorkflowValidationError(`Unsupported workflow runtime event: ${eventType}`); + } + const payload = normalizeEventPayload(message.payload); + journal.appendEvent(run.id, eventType, payload); + return; + } + if (message.type === "result") { + const result = normalizeJsonValue(message.result, "Workflow result"); + settle(() => resolve(result)); + return; + } + if (message.type === "failure") { + const error = normalizeChildError(message.error); + settle(() => reject(new WorkflowValidationError(error.message))); + return; + } + throw new WorkflowValidationError("Workflow runtime child sent an unsupported message"); + } + + async function executeAgentCall(callIndex: number, rawRequest: unknown): Promise { + const request = normalizeAgentRequest(rawRequest); + const journaled = journal.beginCall({ runId: run.id, callIndex, request: request as unknown as JsonObject }); + if (journaled.replayed) replayedCalls += 1; + if (journaled.call.status === "succeeded") return journaled.call.result ?? null; + if (journaled.call.status === "failed") { + throw new WorkflowValidationError( + typeof journaled.call.error?.message === "string" + ? journaled.call.error.message + : `Workflow agent call ${callIndex} previously failed`, + ); + } + + let workflow: WorkflowRunRecord; + if (journaled.call.workflowRunId) { + workflow = orchestrator.getForWorkspace(journaled.call.workflowRunId, input.workspace) + ?? (() => { throw new WorkflowValidationError("Journaled child workflow is unavailable"); })(); + } else { + const submission = await createWorkflowSubmission({ + intent: { + single: request, + idempotencyKey: `runtime:${run.id}:agent:${callIndex}`, + }, + workspace: input.workspace, + profiles: input.profiles, + worktreeRoot: input.worktreeRoot, + environment: input.environment, + }); + const childWorkflow = orchestrator.submitDetailed(submission).workflow; + journal.markCallRunning(run.id, callIndex, childWorkflow.id); + workflow = childWorkflow; + await wakeSupervisor(input); + } + activeWorkflowIds.add(workflow.id); + if (aborting && !isTerminal(workflow)) { + orchestrator.cancelForWorkspace(workflow.id, input.workspace); + await wakeSupervisor(input); + } + while (!isTerminal(workflow)) { + workflow = await orchestrator.waitForWorkspace(workflow.id, input.workspace, { + timeoutMs: Math.min(CHILD_WORKFLOW_WAIT_MS, parsed.budget.timeoutMs), + }); + } + activeWorkflowIds.delete(workflow.id); + if (workflow.status !== "succeeded") { + const error = { + code: stringField(workflow.error, "code") ?? workflow.status, + message: stringField(workflow.error, "message") ?? `Child workflow ${workflow.id} ${workflow.status}`, + workflowId: workflow.id, + } satisfies JsonObject; + journal.completeCall(run.id, callIndex, "failed", error); + throw new WorkflowValidationError(String(error.message)); + } + const result = { + workflowId: workflow.id, + status: workflow.status, + finalResponse: stringField(workflow.result as JsonObject | undefined, "finalResponse") ?? "", + } satisfies JsonObject; + journal.completeCall(run.id, callIndex, "succeeded", result); + return result; + } + }); + child?.kill("SIGTERM"); + return { + run: journal.completeRun(run.id, "succeeded", finalResult), + created: submitted.created, + replayedCalls, + }; + } catch (error) { + aborting = true; + child?.kill("SIGKILL"); + for (const workflowId of activeWorkflowIds) { + try { + orchestrator.cancelForWorkspace(workflowId, input.workspace); + } catch { + // The child may have terminalized while the runtime was aborting. + } + } + if (activeWorkflowIds.size > 0 || scheduledOperations.size > 0) { + await wakeSupervisor(input).catch(() => undefined); + } + await Promise.allSettled([...scheduledOperations]); + const normalized = normalizeError(error); + return { + run: journal.completeRun(run.id, "failed", normalized), + created: submitted.created, + replayedCalls, + }; + } finally { + if (timeout) clearTimeout(timeout); + journal.close(); + if (ownsOrchestrator) orchestrator.close(); + } +} + +async function wakeSupervisor(input: ExecuteWorkflowRuntimeInput): Promise { + if (input.wakeSupervisor) return input.wakeSupervisor(); + return ensureSupervisor({ + stateDir: input.stateDir, + cliEntrypoint: input.cliEntrypoint, + env: input.environment, + }); +} + +function spawnRuntimeChild(): ChildProcess { + return spawn( + process.execPath, + [ + "--permission", + "--max-old-space-size=64", + "--input-type=module", + "--eval", + RUNTIME_CHILD_SOURCE, + ], + { + stdio: ["ignore", "ignore", "ignore", "ipc"], + env: {}, + shell: false, + windowsHide: true, + }, + ); +} + +function sendChild(child: ChildProcess | undefined, message: JsonObject): void { + if (!child?.connected) return; + child.send(message); +} + +function normalizeAgentRequest(value: unknown): WorkflowAgentIntent { + if (!isPlainObject(value)) throw new WorkflowValidationError("agent() requires an options object"); + const allowed = new Set(["target", "prompt", "model", "thinking", "access", "timeoutMs", "retry"]); + for (const key of Object.keys(value)) { + if (!allowed.has(key)) throw new WorkflowValidationError(`Unknown agent() option: ${key}`); + } + const target = requiredBoundedString(value.target, "agent target", 128); + const prompt = requiredBoundedString(value.prompt, "agent prompt", 200_000); + const model = optionalBoundedString(value.model, "agent model", 256); + const thinking = optionalBoundedString(value.thinking, "agent thinking", 64); + const access = value.access === undefined ? undefined : requiredBoundedString(value.access, "agent access", 32); + if (access !== undefined && access !== "read_only" && access !== "workspace_write") { + throw new WorkflowValidationError("agent access must be read_only or workspace_write"); + } + const timeoutMs = value.timeoutMs === undefined + ? undefined + : boundedInteger(value.timeoutMs, 1, 1, MAX_RUNTIME_TIMEOUT_MS, "agent timeoutMs"); + let retry: WorkflowAgentIntent["retry"]; + if (value.retry !== undefined) { + if (!isPlainObject(value.retry)) throw new WorkflowValidationError("agent retry must be an object"); + const retryAllowed = new Set(["maxAttempts", "retryOn", "backoffMs"]); + for (const key of Object.keys(value.retry)) { + if (!retryAllowed.has(key)) throw new WorkflowValidationError(`Unknown agent retry option: ${key}`); + } + const retryOn = value.retry.retryOn; + if (retryOn !== undefined && (!Array.isArray(retryOn) || retryOn.some( + (entry) => entry !== "provider_failed" && entry !== "timed_out", + ))) { + throw new WorkflowValidationError("agent retryOn contains an unsupported failure class"); + } + retry = { + maxAttempts: value.retry.maxAttempts === undefined + ? undefined + : boundedInteger(value.retry.maxAttempts, 1, 1, 10, "agent retry maxAttempts"), + retryOn: retryOn as Array<"provider_failed" | "timed_out"> | undefined, + backoffMs: value.retry.backoffMs === undefined + ? undefined + : boundedInteger(value.retry.backoffMs, 0, 0, 60_000, "agent retry backoffMs"), + }; + } + return { target, prompt, model, thinking, access, timeoutMs, retry }; +} + +function normalizeEventPayload(value: unknown): JsonObject { + if (!isPlainObject(value)) throw new WorkflowValidationError("Workflow runtime event payload must be an object"); + const serialized = JSON.stringify(value); + if (Buffer.byteLength(serialized, "utf8") > MAX_EVENT_DATA_BYTES) { + throw new WorkflowValidationError("Workflow runtime event payload is too large"); + } + if (typeof value.message === "string" && value.message.length > MAX_LOG_CHARS) { + throw new WorkflowValidationError(`Workflow runtime log exceeds ${MAX_LOG_CHARS} characters`); + } + return JSON.parse(serialized) as JsonObject; +} + +function normalizeJsonValue(value: unknown, label: string): JsonValue { + const serialized = JSON.stringify(value); + if (serialized === undefined) return null; + if (Buffer.byteLength(serialized, "utf8") > MAX_EVENT_DATA_BYTES) { + throw new WorkflowValidationError(`${label} exceeds ${MAX_EVENT_DATA_BYTES} bytes`); + } + return JSON.parse(serialized) as JsonValue; +} + +function validateJsonObject(value: JsonObject, label: string, maxBytes: number): void { + if (!isPlainObject(value)) throw new WorkflowValidationError(`${label} must be an object`); + const serialized = JSON.stringify(value); + if (Buffer.byteLength(serialized, "utf8") > maxBytes) { + throw new WorkflowValidationError(`${label} exceeds ${maxBytes} bytes`); + } +} + +function normalizeError(error: unknown): JsonObject { + return { + code: error instanceof WorkflowValidationError ? "invalid_workflow" : "runtime_failed", + message: error instanceof Error ? error.message.slice(0, MAX_LOG_CHARS) : String(error).slice(0, MAX_LOG_CHARS), + }; +} + +function normalizeChildError(error: unknown): { message: string } { + if (!isPlainObject(error) || typeof error.message !== "string") { + return { message: "Workflow script failed" }; + } + return { message: error.message.slice(0, MAX_LOG_CHARS) }; +} + +function requiredBoundedString(value: unknown, label: string, maximum: number): string { + if (typeof value !== "string" || !value.trim()) throw new WorkflowValidationError(`${label} is required`); + const normalized = value.trim(); + if (normalized.length > maximum) throw new WorkflowValidationError(`${label} exceeds ${maximum} characters`); + return normalized; +} + +function optionalBoundedString(value: unknown, label: string, maximum: number): string | undefined { + if (value === undefined || value === null) return undefined; + return requiredBoundedString(value, label, maximum); +} + +function boundedInteger( + value: unknown, + fallback: number, + minimum: number, + maximum: number, + label: string, +): number { + if (value === undefined || value === null) return fallback; + if (!Number.isSafeInteger(value) || Number(value) < minimum || Number(value) > maximum) { + throw new WorkflowValidationError(`${label} must be between ${minimum} and ${maximum}`); + } + return Number(value); +} + +function isPlainObject(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function isTerminal(workflow: WorkflowRunRecord): boolean { + return workflow.status === "succeeded" || workflow.status === "failed" || workflow.status === "cancelled"; +} + +function stringField(value: JsonObject | undefined, key: string): string | undefined { + const field = value?.[key]; + return typeof field === "string" ? field : undefined; +} + +const RUNTIME_CHILD_SOURCE = String.raw` +import vm from "node:vm"; + +let nextRequestId = 0; +let nextCallIndex = 0; +const pending = new Map(); + +process.on("message", (message) => { + if (!message || typeof message !== "object") return; + if (message.type === "agent_result") { + const entry = pending.get(message.requestId); + if (!entry) return; + pending.delete(message.requestId); + if (message.ok) entry.resolve(message.result); + else entry.reject(Object.assign(new Error(message.error?.message || "Agent call failed"), message.error)); + return; + } + if (message.type === "start") void start(message); +}); + +function rpcAgent(request) { + const requestId = nextRequestId++; + const callIndex = nextCallIndex++; + return new Promise((resolve, reject) => { + pending.set(requestId, { resolve, reject }); + process.send({ type: "agent", requestId, callIndex, request }); + }); +} + +function emit(eventType, payload) { + process.send({ type: "event", eventType, payload }); +} + +async function start(message) { + try { + const transformed = message.source.replace( + /^export\s+default\s+/, + "globalThis.__devspaceWorkflow = ", + ); + const sandbox = Object.create(null); + const context = vm.createContext(sandbox, { + name: "devspace-workflow-runtime", + codeGeneration: { strings: false, wasm: false }, + }); + const script = new vm.Script('"use strict";\n' + transformed, { + filename: "workflow.js", + }); + script.runInContext(context, { timeout: 1_000 }); + const workflow = context.__devspaceWorkflow; + if (typeof workflow !== "function") throw new Error("Default export must be a function"); + + const api = Object.freeze({ + args: deepFreeze(message.args), + budget: deepFreeze(message.budget), + agent: (request) => rpcAgent(request), + parallel: async (tasks) => { + if (!Array.isArray(tasks)) throw new Error("parallel() requires an array of functions"); + return Promise.all(tasks.map((task) => { + if (typeof task !== "function") throw new Error("parallel() entries must be functions"); + return task(); + })); + }, + pipeline: async (tasks, initialValue) => { + if (!Array.isArray(tasks)) throw new Error("pipeline() requires an array of functions"); + let value = initialValue; + for (const task of tasks) { + if (typeof task !== "function") throw new Error("pipeline() entries must be functions"); + value = await task(value); + } + return value; + }, + phase: async (name, task) => { + if (typeof name !== "string" || !name.trim()) throw new Error("phase() requires a name"); + if (typeof task !== "function") throw new Error("phase() requires a function"); + emit("phase.started", { name: name.trim() }); + const result = await task(); + emit("phase.completed", { name: name.trim() }); + return result; + }, + log: (message, data = null) => { + if (typeof message !== "string") throw new Error("log() requires a string message"); + emit("log", { message, data }); + }, + }); + const result = await workflow(api); + if (pending.size > 0) { + throw new Error("Workflow returned before all agent() calls were awaited"); + } + process.send({ type: "result", result: result === undefined ? null : result }); + } catch (error) { + process.send({ + type: "failure", + error: { name: error?.name || "Error", message: error?.message || String(error) }, + }); + } +} + +function deepFreeze(value) { + if (!value || typeof value !== "object") return value; + for (const nested of Object.values(value)) deepFreeze(nested); + return Object.freeze(value); +} +`; From 298e9a98fa2432f37b86adf5114dd4169949a3ae Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 22:55:58 +0530 Subject: [PATCH 24/25] fix(workflows): confine JavaScript runtime with SES Co-Authored-By: Claude --- docs/workflow-runtime.md | 6 +-- package-lock.json | 30 +++++++++++ package.json | 1 + src/workflows/runtime.test.ts | 65 ++++++++++++++++++++--- src/workflows/runtime.ts | 99 ++++++++++++++++++++++++++--------- 5 files changed, 165 insertions(+), 36 deletions(-) diff --git a/docs/workflow-runtime.md b/docs/workflow-runtime.md index 56c4c00f..efbdf95b 100644 --- a/docs/workflow-runtime.md +++ b/docs/workflow-runtime.md @@ -60,11 +60,11 @@ Every `agent()` promise must be awaited, directly or through `parallel`, `pipeli ## Isolation and limits -The workflow body runs in a separate Node process with the permission model enabled, an empty environment, no inherited stdio, a 64 MiB old-space limit, and a `vm` context that disables string and WebAssembly code generation. The context does not expose Node APIs, imports, `process`, `require`, ambient environment variables, filesystem, network, database handles, or the orchestrator itself. +The workflow body runs in a separate Node process with the permission model enabled, an empty environment, no inherited stdio, and a 64 MiB old-space limit. Before evaluating workflow code, the child applies SES `lockdown()` and creates a fresh authority-free `Compartment`. Only a hardened orchestration API and bounded JSON values cross into that compartment; Node APIs, imports, `process`, `require`, ambient environment variables, filesystem, network, database handles, and the orchestrator are not endowed. -Source, arguments, logs, event payloads, results, call counts, concurrency, and wall-clock duration are bounded. Scripts can only cause agent execution through `agent()`. +The child can read only the installed SES runtime packages. Node permissions provide a second boundary around filesystem, subprocess, worker, addon, and—where supported by the running Node release—network access. Source, arguments, logs, event payloads, results, call counts, concurrency, and wall-clock duration are bounded. Scripts can only cause agent execution through `agent()`. -This is a defense-in-depth boundary for trusted workspace workflow files, not a general-purpose multi-tenant JavaScript hosting service. Keep Node current because the boundary relies on Node's permission model and VM implementation. +Keep Node and the pinned SES dependency current. For hostile multi-tenant execution requiring protection beyond the JavaScript object-capability boundary, run DevSpace inside an operator-managed OS or container sandbox as an additional layer. ## Durability and replay diff --git a/package-lock.json b/package-lock.json index 0d7decc4..d9214e34 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "react": "^19.2.6", "react-dom": "^19.2.6", "semver": "^7.8.4", + "ses": "^2.2.0", "yaml": "^2.9.0", "zod": "^4.4.3" }, @@ -2101,6 +2102,24 @@ "tslib": "^2.4.0" } }, + "node_modules/@endo/cache-map": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@endo/cache-map/-/cache-map-1.1.0.tgz", + "integrity": "sha512-owFGshs/97PDw9oguZqU/px8Lv1d0KjAUtDUiPwKHNXRVUE/jyettEbRoTbNJR1OaI8biMn6bHr9kVJsOh6dXw==", + "license": "Apache-2.0" + }, + "node_modules/@endo/env-options": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/@endo/env-options/-/env-options-1.1.11.tgz", + "integrity": "sha512-p9OnAPsdqoX4YJsE98e3NBVhIr2iW9gNZxHhAI2/Ul5TdRfoOViItzHzTqrgUVopw6XxA1u1uS6CykLMDUxarA==", + "license": "Apache-2.0" + }, + "node_modules/@endo/immutable-arraybuffer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@endo/immutable-arraybuffer/-/immutable-arraybuffer-1.1.2.tgz", + "integrity": "sha512-u+NaYB2aqEugQ3u7w3c5QNkPogf8q/xGgsPaqdY6pUiGWtYiTiFspKFcha6+oeZhWXWQ23rf0KrUq0kfuzqYyQ==", + "license": "Apache-2.0" + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", @@ -5534,6 +5553,17 @@ "url": "https://opencollective.com/express" } }, + "node_modules/ses": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ses/-/ses-2.2.0.tgz", + "integrity": "sha512-mXZfO9O2bhE9E3INX5Dbqq+eo1Dj6Yeo+r7jas7GYTXRS6fzcZFYf0UbSHcXO7xJuPbhztBskSuSqJLMMoYMVQ==", + "license": "Apache-2.0", + "dependencies": { + "@endo/cache-map": "^1.1.0", + "@endo/env-options": "^1.1.11", + "@endo/immutable-arraybuffer": "^1.1.2" + } + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", diff --git a/package.json b/package.json index 1419833c..061618e1 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "react": "^19.2.6", "react-dom": "^19.2.6", "semver": "^7.8.4", + "ses": "^2.2.0", "yaml": "^2.9.0", "zod": "^4.4.3" }, diff --git a/src/workflows/runtime.test.ts b/src/workflows/runtime.test.ts index ff15d70e..daae3d46 100644 --- a/src/workflows/runtime.test.ts +++ b/src/workflows/runtime.test.ts @@ -12,6 +12,7 @@ try { await testParallelPipelineRuntime(join(root, "parallel")); await testPrefixReplay(join(root, "replay")); await testRestrictedGlobalsAndBudget(join(root, "sandbox")); + await testDynamicImportRejected(join(root, "dynamic-import")); await testUnawaitedAgentFails(join(root, "unawaited")); testMetadataValidation(); } finally { @@ -152,14 +153,32 @@ export default async function ({ agent }) { async function testRestrictedGlobalsAndBudget(stateDir: string): Promise { const source = ` // @devspace-workflow {"version":1,"maxAgentCalls":7,"maxConcurrency":3,"timeoutMs":5000} -export default function ({ args, budget }) { +export default function (api) { + const probes = [ + () => api.agent.constructor("return typeof process")(), + () => api.agent.constructor.constructor("return typeof process")(), + () => api.constructor.constructor("return typeof process")(), + () => Object.getPrototypeOf(api.agent).constructor("return typeof process")(), + () => (async function () {}).constructor("return typeof process")(), + () => (function* () {}).constructor("return typeof process")(), + () => (0, eval)("typeof process") + ].map((probe) => { + try { + return probe(); + } catch { + return "blocked"; + } + }); return { - args, - budget, + args: api.args, + budget: api.budget, + probes, globals: { process: typeof process, require: typeof require, - fetch: typeof fetch + fetch: typeof fetch, + Buffer: typeof Buffer, + WebSocket: typeof WebSocket } }; } @@ -173,11 +192,41 @@ export default function ({ args, budget }) { profiles: [], }); assert.equal(result.run.status, "succeeded", JSON.stringify(result.run.error)); - assert.deepEqual(result.run.result, { - args: { value: "safe" }, - budget: { maxAgentCalls: 7, maxConcurrency: 3, timeoutMs: 5000 }, - globals: { process: "undefined", require: "undefined", fetch: "undefined" }, + const output = result.run.result as { + args: { value: string }; + budget: { maxAgentCalls: number; maxConcurrency: number; timeoutMs: number }; + probes: string[]; + globals: Record; + }; + assert.deepEqual(output.args, { value: "safe" }); + assert.deepEqual(output.budget, { maxAgentCalls: 7, maxConcurrency: 3, timeoutMs: 5000 }); + assert.ok(output.probes.every((probe) => probe === "undefined" || probe === "blocked")); + assert.deepEqual(output.globals, { + process: "undefined", + require: "undefined", + fetch: "undefined", + Buffer: "undefined", + WebSocket: "undefined", + }); +} + +async function testDynamicImportRejected(stateDir: string): Promise { + const result = await executeWorkflowRuntime({ + stateDir, + worktreeRoot: join(stateDir, "worktrees"), + source: ` +// @devspace-workflow {"version":1,"timeoutMs":5000} +export default async function () { + await import("node:net"); + return "unreachable"; +} +`, + args: {}, + workspace: { workspaceId: "workspace", workspaceRoot: root }, + profiles: [], }); + assert.equal(result.run.status, "failed"); + assert.match(String(result.run.error?.message), /import expression rejected/i); } async function testUnawaitedAgentFails(stateDir: string): Promise { diff --git a/src/workflows/runtime.ts b/src/workflows/runtime.ts index 7fd0b234..f211d651 100644 --- a/src/workflows/runtime.ts +++ b/src/workflows/runtime.ts @@ -1,4 +1,7 @@ import { spawn, type ChildProcess } from "node:child_process"; +import { readFileSync, realpathSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; import type { LocalAgentProfile } from "../local-agent-profiles.js"; import { createWorkflowSubmission, type WorkflowAgentIntent } from "./submission.js"; import { WorkflowOrchestrator } from "./orchestrator.js"; @@ -397,14 +400,21 @@ async function wakeSupervisor(input: ExecuteWorkflowRuntimeInput): Promise `--allow-fs-read=${root}`); + const sesEntrypoint = pathToFileURL(realpathSync(fileURLToPath(import.meta.resolve("ses")))).href; + const childSource = RUNTIME_CHILD_SOURCE.replace( + "DEVSPACE_SES_ENTRYPOINT", + JSON.stringify(sesEntrypoint), + ); return spawn( process.execPath, [ "--permission", + ...permissionArgs, "--max-old-space-size=64", "--input-type=module", "--eval", - RUNTIME_CHILD_SOURCE, + childSource, ], { stdio: ["ignore", "ignore", "ignore", "ipc"], @@ -415,6 +425,32 @@ function spawnRuntimeChild(): ChildProcess { ); } +function runtimeDependencyRoots(): string[] { + return [ + "ses", + "@endo/cache-map", + "@endo/env-options", + "@endo/immutable-arraybuffer", + ].map(findPackageRoot); +} + +function findPackageRoot(specifier: string): string { + let current = dirname(fileURLToPath(import.meta.resolve(specifier))); + while (true) { + try { + const manifest = JSON.parse(readFileSync(join(current, "package.json"), "utf8")) as { + name?: unknown; + }; + if (manifest.name === specifier) return realpathSync(current); + } catch { + // Continue through package-internal directories until the package root is found. + } + const parent = dirname(current); + if (parent === current) throw new Error(`Unable to resolve runtime dependency '${specifier}'.`); + current = parent; + } +} + function sendChild(child: ChildProcess | undefined, message: JsonObject): void { if (!child?.connected) return; child.send(message); @@ -546,7 +582,12 @@ function stringField(value: JsonObject | undefined, key: string): string | undef } const RUNTIME_CHILD_SOURCE = String.raw` -import vm from "node:vm"; +import DEVSPACE_SES_ENTRYPOINT; + +lockdown(); +if (typeof Compartment !== "function" || typeof harden !== "function") { + throw new Error("SES lockdown did not initialize the workflow runtime"); +} let nextRequestId = 0; let nextCallIndex = 0; @@ -558,8 +599,13 @@ process.on("message", (message) => { const entry = pending.get(message.requestId); if (!entry) return; pending.delete(message.requestId); - if (message.ok) entry.resolve(message.result); - else entry.reject(Object.assign(new Error(message.error?.message || "Agent call failed"), message.error)); + if (message.ok) { + entry.resolve(copyJson(message.result, "Agent result")); + } else { + entry.reject(new Error( + typeof message.error?.message === "string" ? message.error.message : "Agent call failed", + )); + } return; } if (message.type === "start") void start(message); @@ -568,14 +614,15 @@ process.on("message", (message) => { function rpcAgent(request) { const requestId = nextRequestId++; const callIndex = nextCallIndex++; + const safeRequest = copyJson(request, "Agent request"); return new Promise((resolve, reject) => { pending.set(requestId, { resolve, reject }); - process.send({ type: "agent", requestId, callIndex, request }); + process.send({ type: "agent", requestId, callIndex, request: safeRequest }); }); } function emit(eventType, payload) { - process.send({ type: "event", eventType, payload }); + process.send({ type: "event", eventType, payload: copyJson(payload, "Runtime event") }); } async function start(message) { @@ -584,21 +631,15 @@ async function start(message) { /^export\s+default\s+/, "globalThis.__devspaceWorkflow = ", ); - const sandbox = Object.create(null); - const context = vm.createContext(sandbox, { - name: "devspace-workflow-runtime", - codeGeneration: { strings: false, wasm: false }, - }); - const script = new vm.Script('"use strict";\n' + transformed, { - filename: "workflow.js", - }); - script.runInContext(context, { timeout: 1_000 }); - const workflow = context.__devspaceWorkflow; + const compartment = new Compartment(); + compartment.evaluate('"use strict";\n' + transformed); + const workflow = compartment.globalThis.__devspaceWorkflow; + delete compartment.globalThis.__devspaceWorkflow; if (typeof workflow !== "function") throw new Error("Default export must be a function"); - const api = Object.freeze({ - args: deepFreeze(message.args), - budget: deepFreeze(message.budget), + const api = harden({ + args: copyJson(message.args, "Workflow args"), + budget: copyJson(message.budget, "Workflow budget"), agent: (request) => rpcAgent(request), parallel: async (tasks) => { if (!Array.isArray(tasks)) throw new Error("parallel() requires an array of functions"); @@ -633,18 +674,26 @@ async function start(message) { if (pending.size > 0) { throw new Error("Workflow returned before all agent() calls were awaited"); } - process.send({ type: "result", result: result === undefined ? null : result }); + process.send({ type: "result", result: copyJson(result === undefined ? null : result, "Workflow result") }); } catch (error) { process.send({ type: "failure", - error: { name: error?.name || "Error", message: error?.message || String(error) }, + error: { + name: typeof error?.name === "string" ? error.name : "Error", + message: typeof error?.message === "string" ? error.message : String(error), + }, }); } } -function deepFreeze(value) { - if (!value || typeof value !== "object") return value; - for (const nested of Object.values(value)) deepFreeze(nested); - return Object.freeze(value); +function copyJson(value, label) { + let serialized; + try { + serialized = JSON.stringify(value); + } catch { + throw new Error(label + " must be JSON serializable"); + } + if (serialized === undefined) return null; + return harden(JSON.parse(serialized)); } `; From 0052715a56f9b55e375ccaa20a6e3574c40d5fd7 Mon Sep 17 00:00:00 2001 From: Waishnav Date: Fri, 17 Jul 2026 23:30:07 +0530 Subject: [PATCH 25/25] test(workflows): retry detached supervisor cleanup Co-Authored-By: Claude --- src/workflows/cli.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workflows/cli.test.ts b/src/workflows/cli.test.ts index db718115..b7c475fc 100644 --- a/src/workflows/cli.test.ts +++ b/src/workflows/cli.test.ts @@ -157,7 +157,7 @@ try { assert.notEqual(invalidScriptArgs.status, 0); assert.equal((JSON.parse(invalidScriptArgs.stdout) as Envelope).error!.code, "invalid_input"); } finally { - rmSync(root, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true, maxRetries: 50, retryDelay: 100 }); } function runCli(args: string[], env: NodeJS.ProcessEnv) {