diff --git a/.changeset/backwards-compat.md b/.changeset/backwards-compat.md new file mode 100644 index 0000000000..09d2a5a05c --- /dev/null +++ b/.changeset/backwards-compat.md @@ -0,0 +1,15 @@ +--- +"@workflow/world": minor +"@workflow/world-local": patch +"@workflow/world-postgres": patch +"@workflow/errors": patch +--- + +Add backwards compatibility for runs created with different spec versions + +- Add `RunNotSupportedError` for runs requiring newer world versions +- Add semver-based version comparison utilities +- Legacy runs (< 4.1): route to legacy handlers +- `run_cancelled`: skip event storage, directly update run +- `wait_completed`: store event only (no entity mutation) +- Unknown legacy events: throw error diff --git a/.changeset/cli-event-sourced.md b/.changeset/cli-event-sourced.md new file mode 100644 index 0000000000..998ff49ea6 --- /dev/null +++ b/.changeset/cli-event-sourced.md @@ -0,0 +1,5 @@ +--- +"@workflow/cli": patch +--- + +Use `events.create()` for run cancellation diff --git a/.changeset/core-event-sourced.md b/.changeset/core-event-sourced.md new file mode 100644 index 0000000000..0da5ab5e3c --- /dev/null +++ b/.changeset/core-event-sourced.md @@ -0,0 +1,9 @@ +--- +"@workflow/core": patch +--- + +Runtime uses event-sourced entity creation + +- Suspension handler creates entities via `events.create()` +- Track `hasCreatedEvent` flag to avoid duplicate event creation on replay +- Handle `hook_conflict` events during replay to reject duplicate token hooks diff --git a/.changeset/event-sourced-storage.md b/.changeset/event-sourced-storage.md new file mode 100644 index 0000000000..4b8787fed0 --- /dev/null +++ b/.changeset/event-sourced-storage.md @@ -0,0 +1,12 @@ +--- +"@workflow/world": minor +--- + +**BREAKING**: Storage interface is now read-only; all mutations go through `events.create()` + +- Remove `cancel`, `pause`, `resume` from `runs` +- Remove `create`, `update` from `runs`, `steps`, `hooks` +- Add run lifecycle events: `run_created`, `run_started`, `run_completed`, `run_failed`, `run_cancelled` +- Add `step_created` event type +- Remove `fatal` field from `step_failed` (terminal failure is now implicit) +- Add `step_retrying` event with error info for retriable failures diff --git a/.changeset/hook-conflict-event.md b/.changeset/hook-conflict-event.md new file mode 100644 index 0000000000..c91af4bb0b --- /dev/null +++ b/.changeset/hook-conflict-event.md @@ -0,0 +1,9 @@ +--- +"@workflow/world": patch +"@workflow/errors": patch +--- + +Add `hook_conflict` event type for duplicate token detection + +- World returns `hook_conflict` event when `hook_created` uses an existing token +- Add `HOOK_CONFLICT` error slug diff --git a/.changeset/remove-deprecated-workflow-events.md b/.changeset/remove-deprecated-workflow-events.md new file mode 100644 index 0000000000..517b648adf --- /dev/null +++ b/.changeset/remove-deprecated-workflow-events.md @@ -0,0 +1,8 @@ +--- +"@workflow/world": patch +"@workflow/world-local": patch +"@workflow/world-postgres": patch +"@workflow/web-shared": patch +--- + +Remove deprecated `workflow_completed`, `workflow_failed`, and `workflow_started` events in favor of `run_completed`, `run_failed`, and `run_started` events. diff --git a/.changeset/spec-version.md b/.changeset/spec-version.md new file mode 100644 index 0000000000..5f75bab1dd --- /dev/null +++ b/.changeset/spec-version.md @@ -0,0 +1,13 @@ +--- +"@workflow/world": patch +"@workflow/world-local": patch +"@workflow/world-postgres": patch +"@workflow/world-vercel": patch +"@workflow/web-shared": patch +--- + +Add `specVersion` property to World interface + +- All worlds expose `@workflow/world` package version for protocol compatibility +- Stored in `run_created` event and `WorkflowRun` schema +- Displayed in observability UI diff --git a/.changeset/world-local-event-sourced.md b/.changeset/world-local-event-sourced.md new file mode 100644 index 0000000000..d6fca429cb --- /dev/null +++ b/.changeset/world-local-event-sourced.md @@ -0,0 +1,9 @@ +--- +"@workflow/world-local": patch +--- + +Implement event-sourced entity creation in `events.create()` + +- Atomically create run/step/hook entities when processing corresponding events +- Return `hook_conflict` event when hook token already exists +- Remove direct entity mutation methods from storage diff --git a/.changeset/world-postgres-event-sourced.md b/.changeset/world-postgres-event-sourced.md new file mode 100644 index 0000000000..e51c92f0b7 --- /dev/null +++ b/.changeset/world-postgres-event-sourced.md @@ -0,0 +1,9 @@ +--- +"@workflow/world-postgres": patch +--- + +Implement event-sourced entity creation in `events.create()` + +- Atomically create run/step/hook entities when processing corresponding events +- Return `hook_conflict` event when hook token already exists +- Add `spec_version` column to runs table diff --git a/.changeset/world-vercel-event-sourced.md b/.changeset/world-vercel-event-sourced.md new file mode 100644 index 0000000000..b57b9bb784 --- /dev/null +++ b/.changeset/world-vercel-event-sourced.md @@ -0,0 +1,8 @@ +--- +"@workflow/world-vercel": patch +--- + +Route entity mutations through v2 events API + +- `events.create()` calls v2 endpoint for atomic entity creation +- Remove `cancel`, `pause`, `resume` from storage interface diff --git a/docs/content/docs/errors/hook-conflict.mdx b/docs/content/docs/errors/hook-conflict.mdx new file mode 100644 index 0000000000..375a54021e --- /dev/null +++ b/docs/content/docs/errors/hook-conflict.mdx @@ -0,0 +1,111 @@ +--- +title: hook-conflict +--- + +This error occurs when you try to create a hook with a token that is already in use by another active workflow run. Hook tokens must be unique across all running workflows in your project. + +## Error Message + +``` +Hook token conflict: Hook with token already exists for this project +``` + +## Why This Happens + +Hooks use tokens to identify incoming webhook payloads. When you create a hook with `createHook({ token: "my-token" })`, the Workflow runtime reserves that token for your workflow run. If another workflow run is already using that token, a conflict occurs. + +This typically happens when: + +1. **Two workflows start simultaneously** with the same hardcoded token +2. **A previous workflow run is still waiting** for a hook when a new run tries to use the same token + +## Common Causes + +### Hardcoded Token Values + +{/* @skip-typecheck: incomplete code sample */} +```typescript lineNumbers +// Error - multiple concurrent runs will conflict +export async function processPayment() { + "use workflow"; + + const hook = createHook({ token: "payment-hook" }); // [!code highlight] + // If another run is already waiting on "payment-hook", this will fail + const payment = await hook; +} +``` + +**Solution:** Use unique tokens that include the run ID or other unique identifiers. + +```typescript lineNumbers +export async function processPayment(orderId: string) { + "use workflow"; + + // Include unique identifier in token + const hook = createHook({ token: `payment-${orderId}` }); // [!code highlight] + const payment = await hook; +} +``` + +### Omitting the Token (Auto-generated) + +The safest approach is to let the Workflow runtime generate a unique token automatically: + +```typescript lineNumbers +export async function processPayment() { + "use workflow"; + + const hook = createHook(); // Auto-generated unique token // [!code highlight] + console.log(`Send webhook to token: ${hook.token}`); + const payment = await hook; +} +``` + +## Handling Hook Conflicts in Your Workflow + +When a hook conflict occurs, awaiting the hook will throw a `WorkflowRuntimeError`. You can catch this error to handle the conflict gracefully: + +```typescript lineNumbers +import { WorkflowRuntimeError } from "@workflow/errors"; + +export async function processPayment(orderId: string) { + "use workflow"; + + const hook = createHook({ token: `payment-${orderId}` }); + + try { + const payment = await hook; // [!code highlight] + return { success: true, payment }; + } catch (error) { + if (error instanceof WorkflowRuntimeError && error.slug === "hook-conflict") { // [!code highlight] + // Another workflow is already processing this order + return { success: false, reason: "duplicate-processing" }; + } + throw error; // Re-throw other errors + } +} +``` + +This pattern is useful when you want to detect and handle duplicate processing attempts instead of letting the workflow fail. + +## When Hook Tokens Are Released + +Hook tokens are automatically released when: + +- The workflow run **completes** (successfully or with an error) +- The workflow run is **cancelled** +- The hook is explicitly **disposed** + +After a workflow completes, its hook tokens become available for reuse by other workflows. + +## Best Practices + +1. **Use auto-generated tokens** when possible - they are guaranteed to be unique +2. **Include unique identifiers** if you need custom tokens (order ID, user ID, etc.) +3. **Avoid reusing the same token** across multiple concurrent workflow runs +4. **Consider using webhooks** (`createWebhook`) if you need a fixed, predictable URL that can receive multiple payloads + +## Related + +- [Hooks](/docs/foundations/hooks) - Learn more about using hooks in workflows +- [createWebhook](/docs/api-reference/workflow/create-webhook) - Alternative for fixed webhook URLs diff --git a/docs/content/docs/errors/index.mdx b/docs/content/docs/errors/index.mdx index 53ec75b72b..e65fbf3e23 100644 --- a/docs/content/docs/errors/index.mdx +++ b/docs/content/docs/errors/index.mdx @@ -8,6 +8,9 @@ Fix common mistakes when creating and executing workflows in the **Workflow DevK Learn how to use fetch in workflow functions. + + Learn how to handle hook token conflicts between workflows. + Learn how to use Node.js modules in workflows. diff --git a/docs/content/docs/foundations/streaming.mdx b/docs/content/docs/foundations/streaming.mdx index 99636f0e76..7971cf3683 100644 --- a/docs/content/docs/foundations/streaming.mdx +++ b/docs/content/docs/foundations/streaming.mdx @@ -83,7 +83,7 @@ This allows clients to reconnect and continue receiving data from where they lef [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream) and [`WritableStream`](https://developer.mozilla.org/en-US/docs/Web/API/WritableStream) are standard Web Streams API types that Workflow DevKit makes serializable. These are not custom types - they follow the web standard - but Workflow DevKit adds the ability to pass them between functions while maintaining their streaming capabilities. -Unlike regular values that are fully serialized to the event log, streams maintain their streaming capabilities when passed between functions. +Unlike regular values that are fully serialized to the [event log](/docs/how-it-works/event-sourcing), streams maintain their streaming capabilities when passed between functions. **Key properties:** - Stream references can be passed between workflow and step functions @@ -151,7 +151,7 @@ async function processInputStream(input: ReadableStream) { You cannot read from or write to streams directly within a workflow function. All stream operations must happen in step functions. -Workflow functions must be deterministic to support replay. Since streams bypass the event log for performance, reading stream data in a workflow would break determinism - each replay could see different data. By requiring all stream operations to happen in steps, the framework ensures consistent behavior. +Workflow functions must be deterministic to support replay. Since streams bypass the [event log](/docs/how-it-works/event-sourcing) for performance, reading stream data in a workflow would break determinism - each replay could see different data. By requiring all stream operations to happen in steps, the framework ensures consistent behavior. For more on determinism and replay, see [Workflows and Steps](/docs/foundations/workflows-and-steps). diff --git a/docs/content/docs/foundations/workflows-and-steps.mdx b/docs/content/docs/foundations/workflows-and-steps.mdx index 29304b4141..620faeac02 100644 --- a/docs/content/docs/foundations/workflows-and-steps.mdx +++ b/docs/content/docs/foundations/workflows-and-steps.mdx @@ -36,10 +36,10 @@ export async function processOrderWorkflow(orderId: string) { **Key Characteristics:** - Runs in a sandboxed environment without full Node.js access -- All step results are persisted to the event log +- All step results are persisted to the [event log](/docs/how-it-works/event-sourcing) - Must be **deterministic** to allow resuming after failures -Determinism in the workflow is required to resume the workflow from a suspension. Essentially, the workflow code gets re-run multiple times during its lifecycle, each time using an event log to resume the workflow to the correct spot. +Determinism in the workflow is required to resume the workflow from a suspension. Essentially, the workflow code gets re-run multiple times during its lifecycle, each time using the [event log](/docs/how-it-works/event-sourcing) to resume the workflow to the correct spot. The sandboxed environment that workflows run in already ensures determinism. For instance, `Math.random` and `Date` constructors are fixed in workflow runs, so you are safe to use them, and the framework ensures that the values don't change across replays. @@ -112,7 +112,7 @@ Keep in mind that calling a step function outside of a workflow function will no ### Suspension and Resumption -Workflow functions have the ability to automatically suspend while they wait on asynchronous work. While suspended, the workflow's state is stored via the event log and no compute resources are used until the workflow resumes execution. +Workflow functions have the ability to automatically suspend while they wait on asynchronous work. While suspended, the workflow's state is stored via the [event log](/docs/how-it-works/event-sourcing) and no compute resources are used until the workflow resumes execution. There are multiple ways a workflow can suspend: diff --git a/docs/content/docs/how-it-works/code-transform.mdx b/docs/content/docs/how-it-works/code-transform.mdx index 892ca6936c..42d60e2d3d 100644 --- a/docs/content/docs/how-it-works/code-transform.mdx +++ b/docs/content/docs/how-it-works/code-transform.mdx @@ -145,7 +145,7 @@ handleUserSignup.workflowId = "workflow//workflows/user.js//handleUserSignup"; / - The workflow function gets a `workflowId` property for runtime identification - The `"use workflow"` directive is removed -**Why this transformation?** When a workflow executes, it needs to replay past steps from the event log rather than re-executing them. The `WORKFLOW_USE_STEP` symbol is a special runtime hook that: +**Why this transformation?** When a workflow executes, it needs to replay past steps from the [event log](/docs/how-it-works/event-sourcing) rather than re-executing them. The `WORKFLOW_USE_STEP` symbol is a special runtime hook that: 1. Checks if the step has already been executed (in the event log) 2. If yes: Returns the cached result @@ -290,7 +290,7 @@ Because workflow functions are deterministic and have no side effects, they can - Can make API calls, database queries, etc. - Have full access to Node.js runtime and APIs -- Results are cached in the event log after first execution +- Results are cached in the [event log](/docs/how-it-works/event-sourcing) after first execution Learn more about [Workflows and Steps](/docs/foundations/workflows-and-steps). diff --git a/docs/content/docs/how-it-works/event-sourcing.mdx b/docs/content/docs/how-it-works/event-sourcing.mdx new file mode 100644 index 0000000000..db7e0aa82e --- /dev/null +++ b/docs/content/docs/how-it-works/event-sourcing.mdx @@ -0,0 +1,254 @@ +--- +title: Event Sourcing +--- + + +This guide explores how the Workflow DevKit uses event sourcing internally. Understanding these concepts is helpful for debugging and building observability tools, but is not required to use workflows. For getting started with workflows, see the [getting started](/docs/getting-started) guides for your framework. + + +The Workflow DevKit uses event sourcing to track all state changes in workflow executions. Every mutation creates an event that is persisted to the event log, and entity state is derived by replaying these events. + +This page explains the event sourcing model and entity lifecycles. + +## Event Sourcing Overview + +Event sourcing is a persistence pattern where state changes are stored as a sequence of events rather than by updating records in place. The current state of any entity is reconstructed by replaying its events from the beginning. + +**Benefits for durable workflows:** + +- **Complete audit trail**: Every state change is recorded with its timestamp and context +- **Debugging**: Replay the exact sequence of events that led to any state +- **Consistency**: Events provide a single source of truth for all entity state +- **Recoverability**: State can be reconstructed from the event log after failures + +In the Workflow DevKit, the following entity types are managed through events: + +- **Runs**: Workflow execution instances (materialized in storage) +- **Steps**: Individual atomic operations within a workflow (materialized in storage) +- **Hooks**: Suspension points that can receive external data (materialized in storage) +- **Waits**: Sleep or delay operations (tracked via events only, not materialized) + +## Entity Lifecycles + +Each entity type follows a specific lifecycle defined by the events that can affect it. Events transition entities between states, and certain states are terminal—once reached, no further transitions are possible. + + +In the diagrams below, purple nodes indicate terminal states that cannot be transitioned out of. + + +### Run Lifecycle + +A run represents a single execution of a workflow function. Runs begin in `pending` state when created, transition to `running` when execution starts, and end in one of three terminal states. + +```mermaid +flowchart TD + A["(start)"] -->|"run_created"| B["pending"] + B -->|"run_started"| C["running"] + C -->|"run_completed"| D["completed"] + C -->|"run_failed"| E["failed"] + C -->|"run_cancelled"| F["cancelled"] + B -->|"run_cancelled"| F + + style D fill:#a78bfa,stroke:#8b5cf6,color:#000 + style E fill:#a78bfa,stroke:#8b5cf6,color:#000 + style F fill:#a78bfa,stroke:#8b5cf6,color:#000 +``` + +**Run states:** + +- `pending`: Created but not yet executing +- `running`: Actively executing workflow code +- `completed`: Finished successfully with an output value +- `failed`: Terminated due to an unrecoverable error +- `cancelled`: Explicitly cancelled by the user or system + +### Step Lifecycle + +A step represents a single invocation of a step function. Steps can retry on failure, either transitioning back to `pending` via `step_retrying` or being re-executed directly with another `step_started` event. + +```mermaid +flowchart TD + A["(start)"] -->|"step_created"| B["pending"] + B -->|"step_started"| C["running"] + C -->|"step_completed"| D["completed"] + C -->|"step_failed"| E["failed"] + C -.->|"step_retrying"| B + + style D fill:#a78bfa,stroke:#8b5cf6,color:#000 + style E fill:#a78bfa,stroke:#8b5cf6,color:#000 +``` + +**Step states:** + +- `pending`: Created but not yet executing, or waiting to retry +- `running`: Actively executing step code +- `completed`: Finished successfully with a result value +- `failed`: Terminated after exhausting all retry attempts +- `cancelled`: Reserved for future use (not currently emitted) + + +The `step_retrying` event is optional. Steps can retry without it - the retry mechanism works regardless of whether this event is emitted. You may see back-to-back `step_started` events in logs when a step retries after a timeout or when the error is not explicitly captured. See [Errors and Retries](/docs/foundations/errors-and-retries) for more on how retries work. + + +When present, the `step_retrying` event moves a step back to `pending` state and records the error that caused the retry. This provides two benefits: + +- **Cleaner observability**: The event log explicitly shows retry transitions rather than consecutive `step_started` events +- **Error history**: The error that triggered the retry is preserved for debugging + +### Hook Lifecycle + +A hook represents a suspension point that can receive external data, created by [`createHook()`](/docs/api-reference/workflow/create-hook). Hooks enable workflows to pause and wait for external events, user interactions, or HTTP requests. Webhooks (created with [`createWebhook()`](/docs/api-reference/workflow/create-webhook)) are a higher-level abstraction built on hooks that adds automatic HTTP request/response handling. + +Hooks can receive multiple payloads while active and are disposed when no longer needed. + +```mermaid +flowchart TD + A["(start)"] -->|"hook_created"| B["active"] + A -->|"hook_conflict"| D["conflicted"] + B -->|"hook_received"| B + B -->|"hook_disposed"| C["disposed"] + + style C fill:#a78bfa,stroke:#8b5cf6,color:#000 + style D fill:#a78bfa,stroke:#8b5cf6,color:#000 +``` + +**Hook states:** + +- `active`: Ready to receive payloads (hook exists in storage) +- `disposed`: No longer accepting payloads (hook is deleted from storage) +- `conflicted`: Hook creation failed because the token is already in use by another workflow + +Unlike other entities, hooks don't have a `status` field—the states above are conceptual. An "active" hook is one that exists in storage, while "disposed" means the hook has been deleted. When a `hook_disposed` event is created, the hook record is removed rather than updated. + +While a hook is active, its token is reserved and cannot be used by other workflows. If a workflow attempts to create a hook with a token that is already in use by another active hook, a `hook_conflict` event is recorded instead of `hook_created`. This causes the hook's promise to reject with a `WorkflowRuntimeError`, failing the workflow gracefully. See the [hook-conflict error](/docs/errors/hook-conflict) documentation for more details. + +When a hook is disposed (either explicitly or when its workflow completes), the token is released and can be claimed by future workflows. Hooks are automatically disposed when a workflow reaches a terminal state (`completed`, `failed`, or `cancelled`). The `hook_disposed` event is only needed for explicit disposal before workflow completion. + +See [Hooks & Webhooks](/docs/foundations/hooks) for more on how hooks and webhooks work. + +### Wait Lifecycle + +A wait represents a sleep operation created by [`sleep()`](/docs/api-reference/workflow/sleep). Waits track when a delay period has elapsed. + +```mermaid +flowchart TD + A["(start)"] -->|"wait_created"| B["waiting"] + B -->|"wait_completed"| C["completed"] + + style C fill:#a78bfa,stroke:#8b5cf6,color:#000 +``` + +**Wait states:** + +- `waiting`: Delay period has not yet elapsed +- `completed`: Delay period has elapsed, workflow can resume + + +Unlike Runs, Steps, and Hooks, waits are conceptual entities tracked only through events. There is no separate "Wait" record in storage that can be queried—the wait state is derived entirely from the `wait_created` and `wait_completed` events in the event log. + + +## Event Types Reference + +Events are categorized by the entity type they affect. Each event contains metadata including a timestamp and a `correlationId` that links the event to a specific entity: + +- Step events use the `stepId` as the correlation ID +- Hook events use the `hookId` as the correlation ID +- Wait events use the `waitId` as the correlation ID +- Run events do not require a correlation ID since the `runId` itself identifies the entity + +### Run Events + +| Event | Description | +|-------|-------------| +| `run_created` | Creates a new workflow run in `pending` state. Contains the deployment ID, workflow name, input arguments, and optional execution context. | +| `run_started` | Transitions the run to `running` state when execution begins. | +| `run_completed` | Transitions the run to `completed` state with the workflow's return value. | +| `run_failed` | Transitions the run to `failed` state with error details and optional error code. | +| `run_cancelled` | Transitions the run to `cancelled` state. Can be triggered from `pending` or `running` states. | + +### Step Events + +| Event | Description | +|-------|-------------| +| `step_created` | Creates a new step in `pending` state. Contains the step name and serialized input arguments. | +| `step_started` | Transitions the step to `running` state. Includes the current attempt number for retries. | +| `step_completed` | Transitions the step to `completed` state with the step's return value. | +| `step_failed` | Transitions the step to `failed` state with error details. The step will not be retried. | +| `step_retrying` | (Optional) Transitions the step back to `pending` state for retry. Contains the error that caused the retry and optional delay before the next attempt. When not emitted, retries appear as consecutive `step_started` events. | + +### Hook Events + +| Event | Description | +|-------|-------------| +| `hook_created` | Creates a new hook in `active` state. Contains the hook token and optional metadata. | +| `hook_conflict` | Records that hook creation failed because the token is already in use by another active hook. The hook is not created, and the workflow will fail with a `WorkflowRuntimeError` when the hook is awaited. | +| `hook_received` | Records that a payload was delivered to the hook. The hook remains `active` and can receive more payloads. | +| `hook_disposed` | Deletes the hook from storage (conceptually transitioning to `disposed` state). The token is released for reuse by future workflows. | + +### Wait Events + +| Event | Description | +|-------|-------------| +| `wait_created` | Creates a new wait in `waiting` state. Contains the timestamp when the wait should complete. | +| `wait_completed` | Transitions the wait to `completed` state when the delay period has elapsed. | + +## Terminal States + +Terminal states represent the end of an entity's lifecycle. Once an entity reaches a terminal state, no further events can transition it to another state. + +**Run terminal states:** + +- `completed`: Workflow finished successfully +- `failed`: Workflow encountered an unrecoverable error +- `cancelled`: Workflow was explicitly cancelled + +**Step terminal states:** + +- `completed`: Step finished successfully +- `failed`: Step failed after all retry attempts + +**Hook terminal states:** + +- `disposed`: Hook has been deleted from storage and is no longer active +- `conflicted`: Hook creation failed due to token conflict (hook was never created) + +**Wait terminal states:** + +- `completed`: Delay period has elapsed + +Attempting to create an event that would transition an entity out of a terminal state will result in an error. This prevents inconsistent state and ensures the integrity of the event log. + +## Event Correlation + +Events use a `correlationId` to link related events together. For step, hook, and wait events, the correlation ID identifies the specific entity instance: + +- Step events share the same `correlationId` (the step ID) across all events for that step execution +- Hook events share the same `correlationId` (the hook ID) across all events for that hook +- Wait events share the same `correlationId` (the wait ID) across creation and completion + +Run events do not require a correlation ID since the `runId` itself provides the correlation. + +This correlation enables: + +- Querying all events for a specific step, hook, or wait +- Building timelines of entity lifecycle transitions +- Debugging by tracing the complete history of any entity + +## Entity IDs + +All entities in the Workflow DevKit use a consistent ID format: a 4-character prefix followed by an underscore and a [ULID](https://github.com/ulid/spec) (Universally Unique Lexicographically Sortable Identifier). + +| Entity | Prefix | Example | +|--------|--------|---------| +| Run | `wrun_` | `wrun_01HXYZ123ABC456DEF789GHJ` | +| Step | `step_` | `step_01HXYZ123ABC456DEF789GHJ` | +| Hook | `hook_` | `hook_01HXYZ123ABC456DEF789GHJ` | +| Wait | `wait_` | `wait_01HXYZ123ABC456DEF789GHJ` | +| Event | `evnt_` | `evnt_01HXYZ123ABC456DEF789GHJ` | +| Stream | `strm_` | `strm_01HXYZ123ABC456DEF789GHJ` | + +**Why this format?** + +- **Prefixes enable introspection**: Given any ID, you can immediately identify what type of entity it refers to. This makes debugging, logging, and cross-referencing entities across the system straightforward. + +- **ULIDs enable chronological ordering**: Unlike UUIDs, ULIDs encode a timestamp in their first 48 bits, making them lexicographically sortable by creation time. This property is essential for the event log—events are always stored and retrieved in the correct chronological order simply by sorting their IDs. diff --git a/docs/content/docs/how-it-works/meta.json b/docs/content/docs/how-it-works/meta.json index 09d452d826..a69c654e44 100644 --- a/docs/content/docs/how-it-works/meta.json +++ b/docs/content/docs/how-it-works/meta.json @@ -3,7 +3,8 @@ "pages": [ "understanding-directives", "code-transform", - "framework-integrations" + "framework-integrations", + "event-sourcing" ], "defaultOpen": false } diff --git a/docs/content/docs/how-it-works/understanding-directives.mdx b/docs/content/docs/how-it-works/understanding-directives.mdx index c69174d0be..72bb93ef7a 100644 --- a/docs/content/docs/how-it-works/understanding-directives.mdx +++ b/docs/content/docs/how-it-works/understanding-directives.mdx @@ -48,7 +48,7 @@ export async function onboardUser(userId: string) { } ``` -**The key insight:** Workflows resume from suspension by replaying their code using cached step results from the event log. When a step like `await fetchUserData(userId)` is called: +**The key insight:** Workflows resume from suspension by replaying their code using cached step results from the [event log](/docs/how-it-works/event-sourcing). When a step like `await fetchUserData(userId)` is called: - **If already executed:** Returns the cached result immediately from the event log - **If not yet executed:** Suspends the workflow, enqueues the step for background execution, and resumes later with the result diff --git a/docs/content/docs/observability/index.mdx b/docs/content/docs/observability/index.mdx index 4daf9b867f..841a8edef0 100644 --- a/docs/content/docs/observability/index.mdx +++ b/docs/content/docs/observability/index.mdx @@ -2,7 +2,7 @@ title: Observability --- -Workflow DevKit provides powerful tools to inspect, monitor, and debug your workflows through the CLI and Web UI. These tools allow you to inspect workflow runs, steps, webhooks, events, and stream output. +Workflow DevKit provides powerful tools to inspect, monitor, and debug your workflows through the CLI and Web UI. These tools allow you to inspect workflow runs, steps, webhooks, [events](/docs/how-it-works/event-sourcing), and stream output. ## Quick Start diff --git a/docs/content/docs/worlds/index.mdx b/docs/content/docs/worlds/index.mdx index 67c4d7c7e9..5549ae93df 100644 --- a/docs/content/docs/worlds/index.mdx +++ b/docs/content/docs/worlds/index.mdx @@ -14,7 +14,7 @@ The Workflow `World` is an interface that abstracts how workflows and steps comm ## What is a World? A World implementation handles: -- **Workflow Storage**: Persisting workflow state and event logs +- **Workflow Storage**: Persisting workflow state and [event logs](/docs/how-it-works/event-sourcing) - **Step Execution**: Managing step function invocations - **Message Passing**: Communication between workflow orchestrator and step functions diff --git a/packages/cli/src/lib/inspect/run.ts b/packages/cli/src/lib/inspect/run.ts index 8a4f0f2f28..6827e3aa88 100644 --- a/packages/cli/src/lib/inspect/run.ts +++ b/packages/cli/src/lib/inspect/run.ts @@ -68,6 +68,6 @@ export const startRun = async ( }; export const cancelRun = async (world: World, runId: string) => { - await world.runs.cancel(runId); + await world.events.create(runId, { eventType: 'run_cancelled' }); logger.log(chalk.green(`Cancel signal sent to run ${runId}`)); }; diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index ae01d9d618..f58d127eb0 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -960,6 +960,65 @@ describe('e2e', () => { } ); + test( + 'concurrent hook token conflict - two workflows cannot use the same hook token simultaneously', + { timeout: 60_000 }, + async () => { + const token = Math.random().toString(36).slice(2); + const customData = Math.random().toString(36).slice(2); + + // Start first workflow - it will create a hook and wait for a payload + const run1 = await triggerWorkflow('hookCleanupTestWorkflow', [ + token, + customData, + ]); + + // Wait for the hook to be registered by workflow 1 + await new Promise((resolve) => setTimeout(resolve, 5_000)); + + // Start second workflow with the SAME token while first is still running + // This should fail because the hook token is already in use + const run2 = await triggerWorkflow('hookCleanupTestWorkflow', [ + token, + customData, + ]); + + // The second workflow should fail with a hook token conflict error + const run2Result = await getWorkflowReturnValue(run2.runId); + expect(run2Result.name).toBe('WorkflowRunFailedError'); + expect(run2Result.cause.message).toContain( + 'already in use by another workflow' + ); + + // Verify workflow 2 failed + const { json: run2Data } = await cliInspectJson(`runs ${run2.runId}`); + expect(run2Data.status).toBe('failed'); + + // Now send a payload to complete workflow 1 + const hookUrl = new URL('/api/hook', deploymentUrl); + const res = await fetch(hookUrl, { + method: 'POST', + headers: getProtectionBypassHeaders(), + body: JSON.stringify({ + token, + data: { message: 'test-concurrent', customData }, + }), + }); + expect(res.status).toBe(200); + + // Verify workflow 1 completed successfully + const run1Result = await getWorkflowReturnValue(run1.runId); + expect(run1Result).toMatchObject({ + message: 'test-concurrent', + customData, + hookCleanupTestData: 'workflow_completed', + }); + + const { json: run1Data } = await cliInspectJson(`runs ${run1.runId}`); + expect(run1Data.status).toBe('completed'); + } + ); + test( 'stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars)', { timeout: 60_000 }, diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index f38d7fbd64..221d111fa3 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -78,5 +78,10 @@ export class EventsConsumer { return; } } + + // If we reach here, all callbacks returned NotConsumed. + // We do NOT auto-advance - every event must have a consumer. + // With proper consumers for run_created/run_started/step_created, + // this should not cause events to get stuck. }; } diff --git a/packages/core/src/global.ts b/packages/core/src/global.ts index 3e5e84e08e..3002f41303 100644 --- a/packages/core/src/global.ts +++ b/packages/core/src/global.ts @@ -8,6 +8,7 @@ export interface StepInvocationQueueItem { args: Serializable[]; closureVars?: Record; thisVal?: Serializable; + hasCreatedEvent?: boolean; } export interface HookInvocationQueueItem { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 09ee35871f..9ef2762e7b 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -2,9 +2,11 @@ import { WorkflowRunCancelledError, WorkflowRunFailedError, WorkflowRunNotCompletedError, + WorkflowRuntimeError, } from '@workflow/errors'; import { type Event, + SPEC_VERSION_CURRENT, WorkflowInvokePayloadSchema, type WorkflowRun, type WorkflowRunStatus, @@ -110,7 +112,10 @@ export class Run { * Cancels the workflow run. */ async cancel(): Promise { - await this.world.runs.cancel(this.runId); + await this.world.events.create(this.runId, { + eventType: 'run_cancelled', + specVersion: SPEC_VERSION_CURRENT, + }); } /** @@ -288,16 +293,24 @@ export function workflowEntrypoint( let workflowRun = await world.runs.get(runId); if (workflowRun.status === 'pending') { - workflowRun = await world.runs.update(runId, { - // This sets the `startedAt` timestamp at the database level - status: 'running', + // Transition run to 'running' via event (event-sourced architecture) + const result = await world.events.create(runId, { + eventType: 'run_started', + specVersion: SPEC_VERSION_CURRENT, }); + // Use the run entity from the event response (no extra get call needed) + if (!result.run) { + throw new WorkflowRuntimeError( + `Event creation for 'run_started' did not return the run entity for run \"${runId}\"` + ); + } + workflowRun = result.run; } // At this point, the workflow is "running" and `startedAt` should // definitely be set. if (!workflowRun.startedAt) { - throw new Error( + throw new WorkflowRuntimeError( `Workflow run "${runId}" has no "startedAt" timestamp` ); } @@ -328,25 +341,34 @@ export function workflowEntrypoint( // Check for any elapsed waits and create wait_completed events const now = Date.now(); - for (const event of events) { - if (event.eventType === 'wait_created') { - const resumeAt = event.eventData.resumeAt as Date; - const hasCompleted = events.some( - (e) => - e.eventType === 'wait_completed' && - e.correlationId === event.correlationId - ); - // If wait has elapsed and hasn't been completed yet - if (!hasCompleted && now >= resumeAt.getTime()) { - const completedEvent = await world.events.create(runId, { - eventType: 'wait_completed', - correlationId: event.correlationId, - }); - // Add the event to the events array so the workflow can see it - events.push(completedEvent); - } - } + // Pre-compute completed correlation IDs for O(n) lookup instead of O(n²) + const completedWaitIds = new Set( + events + .filter((e) => e.eventType === 'wait_completed') + .map((e) => e.correlationId) + ); + + // Collect all waits that need completion + const waitsToComplete = events + .filter( + (e): e is typeof e & { correlationId: string } => + e.eventType === 'wait_created' && + e.correlationId !== undefined && + !completedWaitIds.has(e.correlationId) && + now >= (e.eventData.resumeAt as Date).getTime() + ) + .map((e) => ({ + eventType: 'wait_completed' as const, + specVersion: SPEC_VERSION_CURRENT, + correlationId: e.correlationId, + })); + + // Create all wait_completed events + for (const waitEvent of waitsToComplete) { + const result = await world.events.create(runId, waitEvent); + // Add the event to the events array so the workflow can see it + events.push(result.event!); } const result = await runWorkflow( @@ -355,10 +377,13 @@ export function workflowEntrypoint( events ); - // Update the workflow run with the result - await world.runs.update(runId, { - status: 'completed', - output: result as Serializable, + // Complete the workflow run via event (event-sourced architecture) + await world.events.create(runId, { + eventType: 'run_completed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + output: result as Serializable, + }, }); span?.setAttributes({ @@ -390,6 +415,10 @@ export function workflowEntrypoint( return { timeoutSeconds: result.timeoutSeconds }; } } else { + // NOTE: this error could be an error thrown in user code, or could also be a WorkflowRuntimeError + // (for instance when the event log is corrupted, this is thrown by the event consumer). We could + // specially handle these if needed. + const errorName = getErrorName(err); const errorMessage = err instanceof Error ? err.message : String(err); @@ -409,14 +438,19 @@ export function workflowEntrypoint( console.error( `${errorName} while running "${runId}" workflow:\n\n${errorStack}` ); - await world.runs.update(runId, { - status: 'failed', - error: { - message: errorMessage, - stack: errorStack, + // Fail the workflow run via event (event-sourced architecture) + await world.events.create(runId, { + eventType: 'run_failed', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + error: { + message: errorMessage, + stack: errorStack, + }, // TODO: include error codes when we define them }, }); + span?.setAttributes({ ...Attribute.WorkflowRunStatus('failed'), ...Attribute.WorkflowErrorName(errorName), diff --git a/packages/core/src/runtime/resume-hook.ts b/packages/core/src/runtime/resume-hook.ts index b2021ec05e..3f2827d794 100644 --- a/packages/core/src/runtime/resume-hook.ts +++ b/packages/core/src/runtime/resume-hook.ts @@ -1,6 +1,10 @@ import { waitUntil } from '@vercel/functions'; import { ERROR_SLUGS, WorkflowRuntimeError } from '@workflow/errors'; -import type { Hook, WorkflowInvokePayload } from '@workflow/world'; +import { + type Hook, + SPEC_VERSION_CURRENT, + type WorkflowInvokePayload, +} from '@workflow/world'; import { dehydrateStepReturnValue, hydrateStepArguments, @@ -93,6 +97,7 @@ export async function resumeHook( // Create a hook_received event with the payload await world.events.create(hook.runId, { eventType: 'hook_received', + specVersion: SPEC_VERSION_CURRENT, correlationId: hook.hookId, eventData: { payload: dehydratedPayload, diff --git a/packages/core/src/runtime/start.ts b/packages/core/src/runtime/start.ts index 2042afdc2f..2db3940742 100644 --- a/packages/core/src/runtime/start.ts +++ b/packages/core/src/runtime/start.ts @@ -2,6 +2,7 @@ import { waitUntil } from '@vercel/functions'; import { WorkflowRuntimeError } from '@workflow/errors'; import { withResolvers } from '@workflow/utils'; import type { WorkflowInvokePayload, World } from '@workflow/world'; +import { SPEC_VERSION_CURRENT } from '@workflow/world'; import { Run } from '../runtime.js'; import type { Serializable } from '../schemas.js'; import { dehydrateWorkflowArguments } from '../serialization.js'; @@ -98,22 +99,37 @@ export async function start( const { promise: runIdPromise, resolve: resolveRunId } = withResolvers(); + // Serialize current trace context to propagate across queue boundary + const traceCarrier = await serializeTraceCarrier(); + + // Create run via run_created event (event-sourced architecture) + // Pass null for runId - the server generates it and returns it in the response const workflowArguments = dehydrateWorkflowArguments( args, ops, runIdPromise ); - // Serialize current trace context to propagate across queue boundary - const traceCarrier = await serializeTraceCarrier(); - const runResponse = await world.runs.create({ - deploymentId: deploymentId, - workflowName: workflowName, - input: workflowArguments, - executionContext: { traceCarrier }, + const result = await world.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: { + deploymentId: deploymentId, + workflowName: workflowName, + input: workflowArguments, + executionContext: { traceCarrier }, + }, }); - resolveRunId(runResponse.runId); + // Assert that the run was created + if (!result.run) { + throw new WorkflowRuntimeError( + "Missing 'run' in server response for 'run_created' event" + ); + } + + const runId = result.run.runId; + resolveRunId(runId); waitUntil( Promise.all(ops).catch((err) => { @@ -125,15 +141,15 @@ export async function start( ); span?.setAttributes({ - ...Attribute.WorkflowRunId(runResponse.runId), - ...Attribute.WorkflowRunStatus(runResponse.status), + ...Attribute.WorkflowRunId(runId), + ...Attribute.WorkflowRunStatus(result.run.status), ...Attribute.DeploymentId(deploymentId), }); await world.queue( `__wkf_workflow_${workflowName}`, { - runId: runResponse.runId, + runId, traceCarrier, } satisfies WorkflowInvokePayload, { @@ -141,7 +157,7 @@ export async function start( } ); - return new Run(runResponse.runId); + return new Run(runId); }); }); } diff --git a/packages/core/src/runtime/step-handler.ts b/packages/core/src/runtime/step-handler.ts index 59ddafed30..abbb2fadae 100644 --- a/packages/core/src/runtime/step-handler.ts +++ b/packages/core/src/runtime/step-handler.ts @@ -7,7 +7,7 @@ import { } from '@workflow/errors'; import { pluralize } from '@workflow/utils'; import { getPort } from '@workflow/utils/get-port'; -import { StepInvokePayloadSchema } from '@workflow/world'; +import { SPEC_VERSION_CURRENT, StepInvokePayloadSchema } from '@workflow/world'; import { runtimeLogger } from '../logger.js'; import { getStepFunction } from '../private.js'; import type { Serializable } from '../schemas.js'; @@ -131,34 +131,28 @@ const stepHandler = getWorldHandlers().createQueueHandler( } let result: unknown; - const attempt = step.attempt + 1; // Check max retries FIRST before any state changes. + // step.attempt tracks how many times step_started has been called. + // If step.attempt >= maxRetries, we've already tried maxRetries times. // This handles edge cases where the step handler is invoked after max retries have been exceeded - // (e.g., when the step repeatedly times out or fails before reaching the catch handler at line 822). + // (e.g., when the step repeatedly times out or fails before reaching the catch handler). // Without this check, the step would retry forever. // Note: maxRetries is the number of RETRIES after the first attempt, so total attempts = maxRetries + 1 // Use > here (not >=) because this guards against re-invocation AFTER all attempts are used. // The post-failure check uses >= to decide whether to retry after a failure. - if (attempt > maxRetries + 1) { - const retryCount = attempt - 1; + if (step.attempt > maxRetries + 1) { + const retryCount = step.attempt - 1; const errorMessage = `Step "${stepName}" exceeded max retries (${retryCount} ${pluralize('retry', 'retries', retryCount)})`; console.error(`[Workflows] "${workflowRunId}" - ${errorMessage}`); - // Update step status first (idempotent), then create event - await world.steps.update(workflowRunId, stepId, { - status: 'failed', - error: { - message: errorMessage, - stack: undefined, - }, - }); + // Fail the step via event (event-sourced architecture) await world.events.create(workflowRunId, { eventType: 'step_failed', + specVersion: SPEC_VERSION_CURRENT, correlationId: stepId, eventData: { error: errorMessage, stack: step.error?.stack, - fatal: true, }, }); @@ -214,15 +208,24 @@ const stepHandler = getWorldHandlers().createQueueHandler( return; } - await world.events.create(workflowRunId, { - eventType: 'step_started', // TODO: Replace with 'step_retrying' + // Start the step via event (event-sourced architecture) + // step_started increments the attempt counter in the World implementation + const startResult = await world.events.create(workflowRunId, { + eventType: 'step_started', + specVersion: SPEC_VERSION_CURRENT, correlationId: stepId, }); - step = await world.steps.update(workflowRunId, stepId, { - attempt, - status: 'running', - }); + // Use the step entity from the event response (no extra get call needed) + if (!startResult.step) { + throw new WorkflowRuntimeError( + `step_started event for "${stepId}" did not return step entity` + ); + } + step = startResult.step; + + // step.attempt is now the current attempt number (after increment) + const attempt = step.attempt; if (!step.startedAt) { throw new WorkflowRuntimeError( @@ -280,18 +283,11 @@ const stepHandler = getWorldHandlers().createQueueHandler( }) ); - // Mark the step as completed first. This order is important. If a concurrent - // execution marked the step as complete, this request should throw, and - // this prevent the step_completed event in the event log - // TODO: this should really be atomic and handled by the world - await world.steps.update(workflowRunId, stepId, { - status: 'completed', - output: result as Serializable, - }); - - // Then, append the event log with the step result + // Complete the step via event (event-sourced architecture) + // The event creation atomically updates the step entity await world.events.create(workflowRunId, { eventType: 'step_completed', + specVersion: SPEC_VERSION_CURRENT, correlationId: stepId, eventData: { result: result as Serializable, @@ -324,22 +320,14 @@ const stepHandler = getWorldHandlers().createQueueHandler( console.error( `[Workflows] "${workflowRunId}" - Encountered \`FatalError\` while executing step "${stepName}":\n > ${stackLines.join('\n > ')}\n\nBubbling up error to parent workflow` ); - // Fatal error - store the error in the event log and re-invoke the workflow + // Fail the step via event (event-sourced architecture) await world.events.create(workflowRunId, { eventType: 'step_failed', + specVersion: SPEC_VERSION_CURRENT, correlationId: stepId, eventData: { error: String(err), stack: errorStack, - fatal: true, - }, - }); - await world.steps.update(workflowRunId, stepId, { - status: 'failed', - error: { - message: err.message || String(err), - stack: errorStack, - // TODO: include error codes when we define them }, }); @@ -349,36 +337,32 @@ const stepHandler = getWorldHandlers().createQueueHandler( }); } else { const maxRetries = stepFn.maxRetries ?? DEFAULT_STEP_MAX_RETRIES; + // step.attempt was incremented by step_started, use it here + const currentAttempt = step.attempt; span?.setAttributes({ - ...Attribute.StepAttempt(attempt), + ...Attribute.StepAttempt(currentAttempt), ...Attribute.StepMaxRetries(maxRetries), }); // Note: maxRetries is the number of RETRIES after the first attempt, so total attempts = maxRetries + 1 - if (attempt >= maxRetries + 1) { + if (currentAttempt >= maxRetries + 1) { // Max retries reached const errorStack = getErrorStack(err); const stackLines = errorStack.split('\n').slice(0, 4); - const retryCount = attempt - 1; + const retryCount = step.attempt - 1; console.error( - `[Workflows] "${workflowRunId}" - Encountered \`Error\` while executing step "${stepName}" (attempt ${attempt}, ${retryCount} ${pluralize('retry', 'retries', retryCount)}):\n > ${stackLines.join('\n > ')}\n\n Max retries reached\n Bubbling error to parent workflow` + `[Workflows] "${workflowRunId}" - Encountered \`Error\` while executing step "${stepName}" (attempt ${step.attempt}, ${retryCount} ${pluralize('retry', 'retries', retryCount)}):\n > ${stackLines.join('\n > ')}\n\n Max retries reached\n Bubbling error to parent workflow` ); const errorMessage = `Step "${stepName}" failed after ${maxRetries} ${pluralize('retry', 'retries', maxRetries)}: ${String(err)}`; + // Fail the step via event (event-sourced architecture) await world.events.create(workflowRunId, { eventType: 'step_failed', + specVersion: SPEC_VERSION_CURRENT, correlationId: stepId, eventData: { error: errorMessage, stack: errorStack, - fatal: true, - }, - }); - await world.steps.update(workflowRunId, stepId, { - status: 'failed', - error: { - message: errorMessage, - stack: errorStack, }, }); @@ -390,30 +374,30 @@ const stepHandler = getWorldHandlers().createQueueHandler( // Not at max retries yet - log as a retryable error if (RetryableError.is(err)) { console.warn( - `[Workflows] "${workflowRunId}" - Encountered \`RetryableError\` while executing step "${stepName}" (attempt ${attempt}):\n > ${String(err.message)}\n\n This step has failed but will be retried` + `[Workflows] "${workflowRunId}" - Encountered \`RetryableError\` while executing step "${stepName}" (attempt ${currentAttempt}):\n > ${String(err.message)}\n\n This step has failed but will be retried` ); } else { const stackLines = getErrorStack(err).split('\n').slice(0, 4); console.error( - `[Workflows] "${workflowRunId}" - Encountered \`Error\` while executing step "${stepName}" (attempt ${attempt}):\n > ${stackLines.join('\n > ')}\n\n This step has failed but will be retried` + `[Workflows] "${workflowRunId}" - Encountered \`Error\` while executing step "${stepName}" (attempt ${currentAttempt}):\n > ${stackLines.join('\n > ')}\n\n This step has failed but will be retried` ); } + // Set step to pending for retry via event (event-sourced architecture) + // step_retrying records the error and sets status to pending + const errorStack = getErrorStack(err); await world.events.create(workflowRunId, { - eventType: 'step_failed', + eventType: 'step_retrying', + specVersion: SPEC_VERSION_CURRENT, correlationId: stepId, eventData: { error: String(err), - stack: getErrorStack(err), + stack: errorStack, + ...(RetryableError.is(err) && { + retryAfter: err.retryAfter, + }), }, }); - await world.steps.update(workflowRunId, stepId, { - status: 'pending', // TODO: Should be "retrying" once we have that status - ...(RetryableError.is(err) && { - retryAfter: err.retryAfter, - }), - }); - const timeoutSeconds = Math.max( 1, RetryableError.is(err) diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index aa3f13ce61..aa5de91cc1 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -1,6 +1,11 @@ import type { Span } from '@opentelemetry/api'; +import { waitUntil } from '@vercel/functions'; import { WorkflowAPIError } from '@workflow/errors'; -import type { World } from '@workflow/world'; +import { + type CreateEventRequest, + SPEC_VERSION_CURRENT, + type World, +} from '@workflow/world'; import type { HookInvocationQueueItem, StepInvocationQueueItem, @@ -26,188 +31,14 @@ export interface SuspensionHandlerResult { timeoutSeconds?: number; } -interface ProcessHookParams { - queueItem: HookInvocationQueueItem; - world: World; - runId: string; - global: typeof globalThis; -} - -/** - * Processes a single hook by creating it in the database and event log. - */ -async function processHook({ - queueItem, - world, - runId, - global, -}: ProcessHookParams): Promise { - try { - // Create hook in database - const hookMetadata = - typeof queueItem.metadata === 'undefined' - ? undefined - : dehydrateStepArguments(queueItem.metadata, global); - await world.hooks.create(runId, { - hookId: queueItem.correlationId, - token: queueItem.token, - metadata: hookMetadata, - }); - - // Create hook_created event in event log - await world.events.create(runId, { - eventType: 'hook_created', - correlationId: queueItem.correlationId, - }); - } catch (err) { - if (WorkflowAPIError.is(err)) { - if (err.status === 409) { - // Hook already exists (duplicate hook_id constraint), so we can skip it - console.warn( - `Hook with correlation ID "${queueItem.correlationId}" already exists, skipping: ${err.message}` - ); - return; - } else if (err.status === 410) { - // Workflow has already completed, so no-op - console.warn( - `Workflow run "${runId}" has already completed, skipping hook "${queueItem.correlationId}": ${err.message}` - ); - return; - } - } - throw err; - } -} - -interface ProcessStepParams { - queueItem: StepInvocationQueueItem; - world: World; - runId: string; - workflowName: string; - workflowStartedAt: number; - global: typeof globalThis; -} - -/** - * Processes a single step by creating it in the database and queueing execution. - * - * IMPORTANT: The queue write MUST always happen, even if the step already exists. - * This handles the case where: - * 1. Step is written to workflow database - * 2. Process crashes, times out, or fails before queue write completes - * 3. Upstream retry occurs - * 4. Step already exists in database (409 conflict) - * - * If we skipped the queue write on 409, the step would sit "pending" forever - * with 0 attempts. The queue write uses an idempotency key (correlation ID), - * so duplicate queue writes are safely deduplicated by the queue service. - */ -async function processStep({ - queueItem, - world, - runId, - workflowName, - workflowStartedAt, - global, -}: ProcessStepParams): Promise { - const dehydratedInput = dehydrateStepArguments( - { - args: queueItem.args, - closureVars: queueItem.closureVars, - thisVal: queueItem.thisVal, - }, - global - ); - - // The stepId to use for the queue message. This will be the correlation ID - // regardless of whether we created a new step or the step already existed. - const stepId = queueItem.correlationId; - - try { - await world.steps.create(runId, { - stepId: queueItem.correlationId, - stepName: queueItem.stepName, - input: dehydratedInput as Serializable, - }); - } catch (err) { - if (WorkflowAPIError.is(err) && err.status === 409) { - // Step already exists - this is expected on retries. We still need to - // proceed with the queue write below to ensure the step gets executed. - // See function comment above for details on why this is critical. - console.warn( - `Step "${queueItem.stepName}" with correlation ID "${queueItem.correlationId}" already exists, proceeding with queue write` - ); - } else { - throw err; - } - } - - // Always write to queue, even if step already existed. The idempotency key - // ensures duplicate queue writes are safely deduplicated by the queue service. - await queueMessage( - world, - `__wkf_step_${queueItem.stepName}`, - { - workflowName, - workflowRunId: runId, - workflowStartedAt, - stepId, - traceCarrier: await serializeTraceCarrier(), - requestedAt: new Date(), - }, - { - idempotencyKey: queueItem.correlationId, - } - ); -} - -interface ProcessWaitParams { - queueItem: WaitInvocationQueueItem; - world: World; - runId: string; -} - -/** - * Processes a single wait by creating the event and calculating timeout. - * @returns The timeout in seconds, or null if the wait already exists. - */ -async function processWait({ - queueItem, - world, - runId, -}: ProcessWaitParams): Promise { - try { - // Only create wait_created event if it hasn't been created yet - if (!queueItem.hasCreatedEvent) { - await world.events.create(runId, { - eventType: 'wait_created', - correlationId: queueItem.correlationId, - eventData: { - resumeAt: queueItem.resumeAt, - }, - }); - } - - // Calculate how long to wait before resuming - const now = Date.now(); - const resumeAtMs = queueItem.resumeAt.getTime(); - const delayMs = Math.max(1000, resumeAtMs - now); - return Math.ceil(delayMs / 1000); - } catch (err) { - if (WorkflowAPIError.is(err) && err.status === 409) { - // Wait already exists, so we can skip it - console.warn( - `Wait with correlation ID "${queueItem.correlationId}" already exists, skipping: ${err.message}` - ); - return null; - } - throw err; - } -} - /** * Handles a workflow suspension by processing all pending operations (hooks, steps, waits). - * Hooks are processed first to prevent race conditions, then steps and waits in parallel. + * Uses an event-sourced architecture where entities (steps, hooks) are created atomically + * with their corresponding events via events.create(). + * + * Processing order: + * 1. Hooks are processed first to prevent race conditions with webhook receivers + * 2. Steps and waits are processed in parallel after hooks complete */ export async function handleSuspension({ suspension, @@ -217,7 +48,7 @@ export async function handleSuspension({ workflowStartedAt, span, }: SuspensionHandlerParams): Promise { - // Separate queue items by type for parallel processing + // Separate queue items by type const stepItems = suspension.steps.filter( (item): item is StepInvocationQueueItem => item.type === 'step' ); @@ -228,49 +59,169 @@ export async function handleSuspension({ (item): item is WaitInvocationQueueItem => item.type === 'wait' ); - // Process all hooks first to prevent race conditions - await Promise.all( - hookItems.map((queueItem) => - processHook({ - queueItem, - world, - runId, - global: suspension.globalThis, + // Build hook_created events (World will atomically create hook entities) + const hookEvents: CreateEventRequest[] = hookItems.map((queueItem) => { + const hookMetadata = + typeof queueItem.metadata === 'undefined' + ? undefined + : dehydrateStepArguments(queueItem.metadata, suspension.globalThis); + return { + eventType: 'hook_created' as const, + specVersion: SPEC_VERSION_CURRENT, + correlationId: queueItem.correlationId, + eventData: { + token: queueItem.token, + metadata: hookMetadata, + }, + }; + }); + + // Process hooks first to prevent race conditions with webhook receivers + // All hook creations run in parallel + // Track any hook conflicts that occur - these will be handled by re-enqueueing the workflow + let hasHookConflict = false; + + if (hookEvents.length > 0) { + await Promise.all( + hookEvents.map(async (hookEvent) => { + try { + const result = await world.events.create(runId, hookEvent); + // Check if the world returned a hook_conflict event instead of hook_created + // The hook_conflict event is stored in the event log and will be replayed + // on the next workflow invocation, causing the hook's promise to reject + // Note: hook events always create an event (legacy runs throw, not return undefined) + if (result.event!.eventType === 'hook_conflict') { + hasHookConflict = true; + } + } catch (err) { + if (WorkflowAPIError.is(err)) { + if (err.status === 410) { + console.warn( + `Workflow run "${runId}" has already completed, skipping hook: ${err.message}` + ); + } else { + throw err; + } + } else { + throw err; + } + } }) - ) + ); + } + + // Build a map of stepId -> step event for steps that need creation + const stepsNeedingCreation = new Set( + stepItems + .filter((queueItem) => !queueItem.hasCreatedEvent) + .map((queueItem) => queueItem.correlationId) ); - // Then process steps and waits in parallel - const [, waitTimeouts] = await Promise.all([ - Promise.all( - stepItems.map((queueItem) => - processStep({ - queueItem, - world, - runId, - workflowName, - workflowStartedAt, - global: suspension.globalThis, - }) - ) - ), - Promise.all( - waitItems.map((queueItem) => - processWait({ - queueItem, + // Process steps and waits in parallel + // Each step: create event (if needed) -> queue message + // Each wait: create event (if needed) + const ops: Promise[] = []; + + // Steps: create event then queue message, all in parallel + for (const queueItem of stepItems) { + ops.push( + (async () => { + // Create step event if not already created + if (stepsNeedingCreation.has(queueItem.correlationId)) { + const dehydratedInput = dehydrateStepArguments( + { + args: queueItem.args, + closureVars: queueItem.closureVars, + thisVal: queueItem.thisVal, + }, + suspension.globalThis + ); + const stepEvent: CreateEventRequest = { + eventType: 'step_created' as const, + specVersion: SPEC_VERSION_CURRENT, + correlationId: queueItem.correlationId, + eventData: { + stepName: queueItem.stepName, + input: dehydratedInput as Serializable, + }, + }; + try { + await world.events.create(runId, stepEvent); + } catch (err) { + if (WorkflowAPIError.is(err) && err.status === 409) { + console.warn(`Step already exists, continuing: ${err.message}`); + } else { + throw err; + } + } + } + + // Queue step execution message + await queueMessage( world, - runId, - }) - ) - ), - ]); + `__wkf_step_${queueItem.stepName}`, + { + workflowName, + workflowRunId: runId, + workflowStartedAt, + stepId: queueItem.correlationId, + traceCarrier: await serializeTraceCarrier(), + requestedAt: new Date(), + }, + { + idempotencyKey: queueItem.correlationId, + } + ); + })() + ); + } + + // Waits: create events in parallel (no queueing needed for waits) + for (const queueItem of waitItems) { + if (!queueItem.hasCreatedEvent) { + ops.push( + (async () => { + const waitEvent: CreateEventRequest = { + eventType: 'wait_created' as const, + specVersion: SPEC_VERSION_CURRENT, + correlationId: queueItem.correlationId, + eventData: { + resumeAt: queueItem.resumeAt, + }, + }; + try { + await world.events.create(runId, waitEvent); + } catch (err) { + if (WorkflowAPIError.is(err) && err.status === 409) { + console.warn(`Wait already exists, continuing: ${err.message}`); + } else { + throw err; + } + } + })() + ); + } + } - // Find minimum timeout from waits - const minTimeoutSeconds = waitTimeouts.reduce( - (min, timeout) => { - if (timeout === null) return min; - if (min === null) return timeout; - return Math.min(min, timeout); + // Wait for all step and wait operations to complete + waitUntil( + Promise.all(ops).catch((opErr) => { + const isAbortError = + opErr?.name === 'AbortError' || opErr?.name === 'ResponseAborted'; + if (!isAbortError) throw opErr; + }) + ); + await Promise.all(ops); + + // Calculate minimum timeout from waits + const now = Date.now(); + const minTimeoutSeconds = waitItems.reduce( + (min, queueItem) => { + const resumeAtMs = queueItem.resumeAt.getTime(); + const delayMs = Math.max(1000, resumeAtMs - now); + const timeoutSeconds = Math.ceil(delayMs / 1000); + if (min === null) return timeoutSeconds; + return Math.min(min, timeoutSeconds); }, null ); @@ -282,7 +233,15 @@ export async function handleSuspension({ ...Attribute.WorkflowWaitsCreated(waitItems.length), }); - // If we encountered any waits, return the minimum timeout + // If any hook conflicts occurred, re-enqueue the workflow immediately + // On the next iteration, the hook consumer will see the hook_conflict event + // and reject the promise with a WorkflowRuntimeError + // We do this after processing all other operations (steps, waits) to ensure + // they are recorded in the event log before the re-execution + if (hasHookConflict) { + return { timeoutSeconds: 1 }; + } + if (minTimeoutSeconds !== null) { return { timeoutSeconds: minTimeoutSeconds }; } diff --git a/packages/core/src/step.test.ts b/packages/core/src/step.test.ts index cd49e9c6c6..d659fff09a 100644 --- a/packages/core/src/step.test.ts +++ b/packages/core/src/step.test.ts @@ -1,4 +1,4 @@ -import { FatalError } from '@workflow/errors'; +import { FatalError, WorkflowRuntimeError } from '@workflow/errors'; import type { Event } from '@workflow/world'; import * as nanoid from 'nanoid'; import { monotonicFactory } from 'ulid'; @@ -59,7 +59,6 @@ describe('createUseStep', () => { correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', eventData: { error: 'test', - fatal: true, }, createdAt: new Date(), }, @@ -207,18 +206,15 @@ describe('createUseStep', () => { }); it('should capture closure variables when provided', async () => { - const ctx = setupWorkflowContext([ - { - eventId: 'evnt_0', - runId: 'wrun_123', - eventType: 'step_completed', - correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', - eventData: { - result: ['Result: 42'], - }, - createdAt: new Date(), - }, - ]); + // Use empty events to check queue state before step completes + const ctx = setupWorkflowContext([]); + let workflowErrorReject: (err: Error) => void; + const workflowErrorPromise = new Promise((_, reject) => { + workflowErrorReject = reject; + }); + ctx.onWorkflowError = (err) => { + workflowErrorReject(err); + }; const useStep = createUseStep(ctx); const count = 42; @@ -227,11 +223,16 @@ describe('createUseStep', () => { // Create step with closure variables function const calculate = useStep('calculate', () => ({ count, prefix })); - // Call the step - const result = await calculate(); + // Call the step - will suspend since no events + let error: Error | undefined; + try { + await Promise.race([calculate(), workflowErrorPromise]); + } catch (err_) { + error = err_ as Error; + } - // Verify result - expect(result).toBe('Result: 42'); + // Verify suspension happened + expect(error).toBeInstanceOf(WorkflowSuspension); // Verify closure variables were added to invocation queue expect(ctx.invocationsQueue.size).toBe(1); @@ -245,37 +246,312 @@ describe('createUseStep', () => { }); it('should handle empty closure variables', async () => { + // Use empty events to check queue state before step completes + const ctx = setupWorkflowContext([]); + let workflowErrorReject: (err: Error) => void; + const workflowErrorPromise = new Promise((_, reject) => { + workflowErrorReject = reject; + }); + ctx.onWorkflowError = (err) => { + workflowErrorReject(err); + }; + + const useStep = createUseStep(ctx); + + // Create step without closure variables + const add = useStep('add'); + + // Call the step - will suspend since no events + let error: Error | undefined; + try { + await Promise.race([add(2, 3), workflowErrorPromise]); + } catch (err_) { + error = err_ as Error; + } + + // Verify suspension happened + expect(error).toBeInstanceOf(WorkflowSuspension); + + // Verify queue item was added with correct structure (no closureVars when not provided) + expect(ctx.invocationsQueue.size).toBe(1); + const queueItem = [...ctx.invocationsQueue.values()][0]; + expect(queueItem).toMatchObject({ + type: 'step', + stepName: 'add', + args: [2, 3], + }); + }); + + it('should mark hasCreatedEvent when step_created event is received', async () => { + // step_created marks the queue item but doesn't complete the step const ctx = setupWorkflowContext([ { eventId: 'evnt_0', runId: 'wrun_123', - eventType: 'step_completed', + eventType: 'step_created', correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', - eventData: { - result: [5], - }, + eventData: {}, createdAt: new Date(), }, ]); - const useStep = createUseStep(ctx); + let workflowErrorReject: (err: Error) => void; + const workflowErrorPromise = new Promise((_, reject) => { + workflowErrorReject = reject; + }); + ctx.onWorkflowError = (err) => { + workflowErrorReject(err); + }; - // Create step without closure variables + const useStep = createUseStep(ctx); const add = useStep('add'); - // Call the step - const result = await add(2, 3); + // Call the step - will suspend after processing step_created + let error: Error | undefined; + try { + await Promise.race([add(1, 2), workflowErrorPromise]); + } catch (err_) { + error = err_ as Error; + } - // Verify result - expect(result).toBe(5); + expect(error).toBeInstanceOf(WorkflowSuspension); - // Verify empty closure variables were added to invocation queue + // Queue item should still exist with hasCreatedEvent = true expect(ctx.invocationsQueue.size).toBe(1); const queueItem = [...ctx.invocationsQueue.values()][0]; expect(queueItem).toMatchObject({ type: 'step', stepName: 'add', - args: [2, 3], + hasCreatedEvent: true, + }); + }); + + it('should consume step_started without removing from queue', async () => { + // step_started is consumed but item stays in queue for potential re-enqueue + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'step_started', + correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: {}, + createdAt: new Date(), + }, + ]); + + let workflowErrorReject: (err: Error) => void; + const workflowErrorPromise = new Promise((_, reject) => { + workflowErrorReject = reject; }); + ctx.onWorkflowError = (err) => { + workflowErrorReject(err); + }; + + const useStep = createUseStep(ctx); + const add = useStep('add'); + + // Call the step - will suspend after processing step_started + let error: Error | undefined; + try { + await Promise.race([add(1, 2), workflowErrorPromise]); + } catch (err_) { + error = err_ as Error; + } + + expect(error).toBeInstanceOf(WorkflowSuspension); + + // Queue item should still exist (step_started doesn't remove it) + expect(ctx.invocationsQueue.size).toBe(1); + }); + + it('should consume step_retrying event and continue waiting', async () => { + // step_retrying is just consumed, step continues to wait for next events + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'step_retrying', + correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: {}, + createdAt: new Date(), + }, + ]); + + let workflowErrorReject: (err: Error) => void; + const workflowErrorPromise = new Promise((_, reject) => { + workflowErrorReject = reject; + }); + ctx.onWorkflowError = (err) => { + workflowErrorReject(err); + }; + + const useStep = createUseStep(ctx); + const add = useStep('add'); + + // Call the step - will suspend after processing step_retrying + let error: Error | undefined; + try { + await Promise.race([add(1, 2), workflowErrorPromise]); + } catch (err_) { + error = err_ as Error; + } + + expect(error).toBeInstanceOf(WorkflowSuspension); + expect(ctx.invocationsQueue.size).toBe(1); + }); + + it('should remove queue item when step_completed (terminal state)', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'step_completed', + correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + result: [42], + }, + createdAt: new Date(), + }, + ]); + + const useStep = createUseStep(ctx); + const add = useStep('add'); + + const result = await add(1, 2); + + expect(result).toBe(42); + // Queue should be empty after completion (terminal state) + expect(ctx.invocationsQueue.size).toBe(0); + }); + + it('should remove queue item when step_failed (terminal state)', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'step_failed', + correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + error: 'test error', + }, + createdAt: new Date(), + }, + ]); + + const useStep = createUseStep(ctx); + const add = useStep('add'); + + let error: Error | undefined; + try { + await add(1, 2); + } catch (err_) { + error = err_ as Error; + } + + expect(error).toBeInstanceOf(FatalError); + // Queue should be empty after failure (terminal state) + expect(ctx.invocationsQueue.size).toBe(0); + }); + + it('should extract message and stack from object error in step_failed', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'step_failed', + correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + error: { + message: 'Custom error message', + stack: + 'Error: Custom error message\n at someFunction (file.js:10:5)', + }, + }, + createdAt: new Date(), + }, + ]); + + const useStep = createUseStep(ctx); + const add = useStep('add'); + + let error: Error | undefined; + try { + await add(1, 2); + } catch (err_) { + error = err_ as Error; + } + + expect(error).toBeInstanceOf(FatalError); + expect(error?.message).toBe('Custom error message'); + expect(error?.stack).toContain('someFunction'); + expect(error?.stack).toContain('file.js:10:5'); + }); + + it('should fallback to eventData.stack when error object has no stack', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'step_failed', + correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + error: { + message: 'Error without stack', + }, + stack: + 'Fallback stack trace\n at fallbackFunction (fallback.js:20:10)', + }, + createdAt: new Date(), + }, + ]); + + const useStep = createUseStep(ctx); + const add = useStep('add'); + + let error: Error | undefined; + try { + await add(1, 2); + } catch (err_) { + error = err_ as Error; + } + + expect(error).toBeInstanceOf(FatalError); + expect(error?.message).toBe('Error without stack'); + expect(error?.stack).toContain('fallbackFunction'); + }); + + it('should invoke workflow error handler with WorkflowRuntimeError for unexpected event type', async () => { + // Simulate a corrupted event log where a step receives an unexpected event type + // (e.g., a wait_completed event when expecting step_completed/step_failed) + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'wait_completed', // Wrong event type for a step! + correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: {}, + createdAt: new Date(), + }, + ]); + + let workflowError: Error | undefined; + ctx.onWorkflowError = (err) => { + workflowError = err; + }; + + const useStep = createUseStep(ctx); + const add = useStep('add'); + + // Start the step - it will process the event asynchronously + const stepPromise = add(1, 2); + + // Wait for the error handler to be called + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(workflowError).toBeInstanceOf(WorkflowRuntimeError); + expect(workflowError?.message).toContain('Unexpected event type for step'); + expect(workflowError?.message).toContain('step_01K11TFZ62YS0YYFDQ3E8B9YCV'); + expect(workflowError?.message).toContain('add'); + expect(workflowError?.message).toContain('wait_completed'); }); }); diff --git a/packages/core/src/step.ts b/packages/core/src/step.ts index df5a5c6a84..caec871107 100644 --- a/packages/core/src/step.ts +++ b/packages/core/src/step.ts @@ -42,11 +42,6 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { ctx.invocationsQueue.set(correlationId, queueItem); - // Track whether we've already seen a "step_started" event for this step. - // This is important because after a retryable failure, the step moves back to - // "pending" status which causes another "step_started" event to be emitted. - let hasSeenStepStarted = false; - stepLogger.debug('Step consumer setup', { correlationId, stepName, @@ -76,53 +71,77 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { }); if (event.correlationId !== correlationId) { - // We're not interested in this event - the correlationId belongs to a different step + // We're not interested in this event - the correlationId belongs to a different entity return EventConsumerResult.NotConsumed; } - if (event.eventType === 'step_started') { - // Step has started - so remove from the invocations queue (only on the first "step_started" event) - if (!hasSeenStepStarted) { - // O(1) lookup and delete using Map - if (ctx.invocationsQueue.has(correlationId)) { - ctx.invocationsQueue.delete(correlationId); - } else { - setTimeout(() => { - reject( - new WorkflowRuntimeError( - `Corrupted event log: step ${correlationId} (${stepName}) started but not found in invocation queue` - ) - ); - }, 0); - return EventConsumerResult.Finished; - } - hasSeenStepStarted = true; + if (event.eventType === 'step_created') { + // Step has been created (registered for execution) - mark as having event + // but keep in queue so suspension handler knows to queue execution without + // creating a duplicate step_created event + const queueItem = ctx.invocationsQueue.get(correlationId); + if (!queueItem || queueItem.type !== 'step') { + // This indicates event log corruption - step_created received + // but the step was never invoked in the workflow during replay. + setTimeout(() => { + reject( + new WorkflowRuntimeError( + `Corrupted event log: step ${correlationId} (${stepName}) created but not found in invocation queue` + ) + ); + }, 0); + return EventConsumerResult.Finished; } - // If this is a subsequent "step_started" event (after a retry), we just consume it - // without trying to remove from the queue again or logging a warning + queueItem.hasCreatedEvent = true; + // Continue waiting for step_started/step_completed/step_failed events + return EventConsumerResult.Consumed; + } + + if (event.eventType === 'step_started') { + // Step was started - don't do anything. The step is left in the invocationQueue which + // will allow it to be re-enqueued. We rely on the queue's idempotency to prevent it from + // actually being over enqueued. + return EventConsumerResult.Consumed; + } + + if (event.eventType === 'step_retrying') { + // Step is being retried - just consume the event and wait for next step_started return EventConsumerResult.Consumed; } if (event.eventType === 'step_failed') { + // Terminal state - we can remove the invocationQueue item + ctx.invocationsQueue.delete(event.correlationId); // Step failed - bubble up to workflow - if (event.eventData.fatal) { - setTimeout(() => { - const error = new FatalError(event.eventData.error); - // Preserve the original stack trace from the step execution - // This ensures that deeply nested errors show the full call chain - if (event.eventData.stack) { - error.stack = event.eventData.stack; - } - reject(error); - }, 0); - return EventConsumerResult.Finished; - } else { - // This is a retryable error, so nothing to do here, - // but we will consume the event - return EventConsumerResult.Consumed; - } - } else if (event.eventType === 'step_completed') { - // Step has already completed, so resolve the Promise with the cached result + setTimeout(() => { + const errorData = event.eventData.error; + const isErrorObject = + typeof errorData === 'object' && errorData !== null; + + const errorMessage = isErrorObject + ? (errorData.message ?? 'Unknown error') + : typeof errorData === 'string' + ? errorData + : 'Unknown error'; + + const errorStack = + (isErrorObject ? errorData.stack : undefined) ?? + event.eventData.stack; + + const error = new FatalError(errorMessage); + if (errorStack) { + error.stack = errorStack; + } + reject(error); + }, 0); + return EventConsumerResult.Finished; + } + + if (event.eventType === 'step_completed') { + // Terminal state - we can remove the invocationQueue item + ctx.invocationsQueue.delete(event.correlationId); + + // Step has completed, so resolve the Promise with the cached result const hydratedResult = hydrateStepReturnValue( event.eventData.result, ctx.globalThis @@ -131,17 +150,17 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { resolve(hydratedResult); }, 0); return EventConsumerResult.Finished; - } else { - // An unexpected event type has been received, but it does belong to this step (matching `correlationId`) - setTimeout(() => { - reject( - new WorkflowRuntimeError( - `Unexpected event type: "${event.eventType}"` - ) - ); - }, 0); - return EventConsumerResult.Finished; } + + // An unexpected event type has been received, this event log looks corrupted. Let's fail immediately. + setTimeout(() => { + ctx.onWorkflowError( + new WorkflowRuntimeError( + `Unexpected event type for step ${correlationId} (name: ${stepName}) "${event.eventType}"` + ) + ); + }, 0); + return EventConsumerResult.Finished; }); return promise; diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 6e7b537875..32159ea6e8 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -1,4 +1,5 @@ import { types } from 'node:util'; +import { WorkflowRuntimeError } from '@workflow/errors'; import type { Event, WorkflowRun } from '@workflow/world'; import { assert, describe, expect, it } from 'vitest'; import type { WorkflowSuspension } from './global.js'; @@ -144,6 +145,7 @@ describe('runWorkflow', () => { expect(hydrateWorkflowReturnValue(result as any, ops)).toEqual(3); }); + // Test that timestamps update correctly as events are consumed it('should update the timestamp in the vm context as events are replayed', async () => { const ops: Promise[] = []; const workflowRunId = 'wrun_123'; @@ -158,7 +160,27 @@ describe('runWorkflow', () => { deploymentId: 'test-deployment', }; + // Events now include run_created, run_started, and step_created for proper consumption const events: Event[] = [ + { + eventId: 'event-run-created', + runId: workflowRunId, + eventType: 'run_created', + createdAt: new Date('2024-01-01T00:00:00.000Z'), + }, + { + eventId: 'event-run-started', + runId: workflowRunId, + eventType: 'run_started', + createdAt: new Date('2024-01-01T00:00:00.500Z'), + }, + { + eventId: 'event-step1-created', + runId: workflowRunId, + eventType: 'step_created', + correlationId: 'step_01HK153X00Y11PCQTCHQRK34HF', + createdAt: new Date('2024-01-01T00:00:00.600Z'), + }, { eventId: 'event-0', runId: workflowRunId, @@ -176,6 +198,13 @@ describe('runWorkflow', () => { }, createdAt: new Date('2024-01-01T00:00:02.000Z'), }, + { + eventId: 'event-step2-created', + runId: workflowRunId, + eventType: 'step_created', + correlationId: 'step_01HK153X00Y11PCQTCHQRK34HG', + createdAt: new Date('2024-01-01T00:00:02.500Z'), + }, { eventId: 'event-2', runId: workflowRunId, @@ -193,6 +222,13 @@ describe('runWorkflow', () => { }, createdAt: new Date('2024-01-01T00:00:04.000Z'), }, + { + eventId: 'event-step3-created', + runId: workflowRunId, + eventType: 'step_created', + correlationId: 'step_01HK153X00Y11PCQTCHQRK34HH', + createdAt: new Date('2024-01-01T00:00:04.500Z'), + }, { eventId: 'event-4', runId: workflowRunId, @@ -228,10 +264,15 @@ describe('runWorkflow', () => { workflowRun, events ); + // Timestamps: + // - Initial: 0s (from startedAt) + // - After step 1 completes (at 2s), timestamp advances to step2_created (2.5s) + // - After step 2 completes (at 4s), timestamp advances to step3_created (4.5s) + // - After step 3 completes: 6s expect(hydrateWorkflowReturnValue(result as any, ops)).toEqual([ new Date('2024-01-01T00:00:00.000Z'), - 1704067203000, - 1704067205000, + 1704067202500, // 2.5s (step2_created timestamp) + 1704067204500, // 4.5s (step3_created timestamp) new Date('2024-01-01T00:00:06.000Z'), ]); }); @@ -855,8 +896,9 @@ describe('runWorkflow', () => { } assert(error); expect(error.name).toEqual('WorkflowSuspension'); - expect(error.message).toEqual('0 steps have not been run yet'); - expect((error as WorkflowSuspension).steps).toEqual([]); + // step_started no longer removes from queue - step stays in queue for re-enqueueing + expect(error.message).toEqual('1 step has not been run yet'); + expect((error as WorkflowSuspension).steps).toHaveLength(1); }); it('should throw `WorkflowSuspension` for multiple steps with `Promise.all()`', async () => { @@ -1599,6 +1641,103 @@ describe('runWorkflow', () => { result: 'success', }); }); + + it('should reject with WorkflowRuntimeError when hook_conflict event is received', async () => { + const ops: Promise[] = []; + const workflowRun: WorkflowRun = { + runId: 'test-run-123', + workflowName: 'workflow', + status: 'running', + input: dehydrateWorkflowArguments([], ops), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + + const events: Event[] = [ + { + eventId: 'event-0', + runId: workflowRun.runId, + eventType: 'hook_conflict', + correlationId: 'hook_01HK153X008RT6YEW43G8QX6JX', + eventData: { + token: 'my-duplicate-token', + }, + createdAt: new Date(), + }, + ]; + + let error: Error | undefined; + try { + await runWorkflow( + `const createHook = globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]; + async function workflow() { + const hook = createHook({ token: 'my-duplicate-token' }); + const payload = await hook; + return payload; + }${getWorkflowTransformCode('workflow')}`, + workflowRun, + events + ); + } catch (err) { + error = err as Error; + } + + expect(error).toBeInstanceOf(WorkflowRuntimeError); + expect(error?.message).toContain('already in use by another workflow'); + expect(error?.message).toContain('my-duplicate-token'); + }); + + it('should reject multiple awaits when hook_conflict is received (iterator pattern)', async () => { + const ops: Promise[] = []; + const workflowRun: WorkflowRun = { + runId: 'test-run-123', + workflowName: 'workflow', + status: 'running', + input: dehydrateWorkflowArguments([], ops), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + + const events: Event[] = [ + { + eventId: 'event-0', + runId: workflowRun.runId, + eventType: 'hook_conflict', + correlationId: 'hook_01HK153X008RT6YEW43G8QX6JX', + eventData: { + token: 'conflicting-token', + }, + createdAt: new Date(), + }, + ]; + + let error: Error | undefined; + try { + await runWorkflow( + `const createHook = globalThis[Symbol.for("WORKFLOW_CREATE_HOOK")]; + async function workflow() { + const hook = createHook({ token: 'conflicting-token' }); + const results = []; + for await (const payload of hook) { + results.push(payload); + if (results.length >= 2) break; + } + return results; + }${getWorkflowTransformCode('workflow')}`, + workflowRun, + events + ); + } catch (err) { + error = err as Error; + } + + expect(error).toBeInstanceOf(WorkflowRuntimeError); + expect(error?.message).toContain('already in use by another workflow'); + }); }); describe('Response', () => { diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 6ecfba7fd1..bd466d302f 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -90,6 +90,27 @@ export async function runWorkflow( return EventConsumerResult.NotConsumed; }); + // Consume run lifecycle events - these are structural events that don't + // need special handling in the workflow, but must be consumed to advance + // past them in the event log + workflowContext.eventsConsumer.subscribe((event) => { + if (!event) { + return EventConsumerResult.NotConsumed; + } + + // Consume run_created - every run has exactly one + if (event.eventType === 'run_created') { + return EventConsumerResult.Consumed; + } + + // Consume run_started - every run has exactly one + if (event.eventType === 'run_started') { + return EventConsumerResult.Consumed; + } + + return EventConsumerResult.NotConsumed; + }); + const useStep = createUseStep(workflowContext); const createHook = createCreateHook(workflowContext); const sleep = createSleep(workflowContext); diff --git a/packages/core/src/workflow/hook.test.ts b/packages/core/src/workflow/hook.test.ts new file mode 100644 index 0000000000..a8d2d884ba --- /dev/null +++ b/packages/core/src/workflow/hook.test.ts @@ -0,0 +1,341 @@ +import { WorkflowRuntimeError } from '@workflow/errors'; +import type { Event } from '@workflow/world'; +import * as nanoid from 'nanoid'; +import { monotonicFactory } from 'ulid'; +import { describe, expect, it, vi } from 'vitest'; +import { EventsConsumer } from '../events-consumer.js'; +import { WorkflowSuspension } from '../global.js'; +import type { WorkflowOrchestratorContext } from '../private.js'; +import { dehydrateStepReturnValue } from '../serialization.js'; +import { createContext } from '../vm/index.js'; +import { createCreateHook } from './hook.js'; + +// Helper to setup context to simulate a workflow run +function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { + const context = createContext({ + seed: 'test', + fixedTimestamp: 1753481739458, + }); + const ulid = monotonicFactory(() => context.globalThis.Math.random()); + const workflowStartedAt = context.globalThis.Date.now(); + return { + globalThis: context.globalThis, + eventsConsumer: new EventsConsumer(events), + invocationsQueue: new Map(), + generateUlid: () => ulid(workflowStartedAt), + generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => + new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) + ), + onWorkflowError: vi.fn(), + }; +} + +describe('createCreateHook', () => { + it('should resolve with payload when hook_received event is received', async () => { + const ops: Promise[] = []; + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_received', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + payload: dehydrateStepReturnValue({ message: 'hello' }, ops), + }, + createdAt: new Date(), + }, + ]); + const createHook = createCreateHook(ctx); + const hook = createHook(); + const result = await hook; + expect(result).toEqual({ message: 'hello' }); + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + }); + + it('should throw WorkflowSuspension when no events are available', async () => { + const ctx = setupWorkflowContext([]); + + let workflowError: Error | undefined; + ctx.onWorkflowError = (err) => { + workflowError = err; + }; + + const createHook = createCreateHook(ctx); + const hook = createHook(); + + // Start awaiting the hook - it will process events asynchronously + const hookPromise = hook.then((v) => v); + + // Wait for the error handler to be called + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(workflowError).toBeInstanceOf(WorkflowSuspension); + }); + + it('should invoke workflow error handler with WorkflowRuntimeError for unexpected event type', async () => { + // Simulate a corrupted event log where a hook receives an unexpected event type + // (e.g., a step_completed event when expecting hook_created/hook_received/hook_disposed) + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'step_completed', // Wrong event type for a hook! + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + result: ['test'], + }, + createdAt: new Date(), + }, + ]); + + let workflowError: Error | undefined; + ctx.onWorkflowError = (err) => { + workflowError = err; + }; + + const createHook = createCreateHook(ctx); + const hook = createHook(); + + // Start awaiting the hook - it will process events asynchronously + const hookPromise = hook.then((v) => v); + + // Wait for the error handler to be called + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(workflowError).toBeInstanceOf(WorkflowRuntimeError); + expect(workflowError?.message).toContain('Unexpected event type for hook'); + expect(workflowError?.message).toContain('hook_01K11TFZ62YS0YYFDQ3E8B9YCV'); + expect(workflowError?.message).toContain('step_completed'); + }); + + it('should consume hook_created event and remove from invocations queue', async () => { + const ops: Promise[] = []; + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_created', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: {}, + createdAt: new Date(), + }, + { + eventId: 'evnt_1', + runId: 'wrun_123', + eventType: 'hook_received', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + payload: dehydrateStepReturnValue({ data: 'test' }, ops), + }, + createdAt: new Date(), + }, + ]); + + const createHook = createCreateHook(ctx); + const hook = createHook(); + + // After creating the hook, it should be in the queue + expect(ctx.invocationsQueue.size).toBe(1); + + const result = await hook; + + // After hook_created is processed, the hook should be removed from the queue + expect(ctx.invocationsQueue.size).toBe(0); + expect(result).toEqual({ data: 'test' }); + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + }); + + it('should finish processing when hook_disposed event is received', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_disposed', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: {}, + createdAt: new Date(), + }, + ]); + + const createHook = createCreateHook(ctx); + const hook = createHook(); + + // Wait for event processing + await new Promise((resolve) => setTimeout(resolve, 10)); + + // The hook consumer should have finished (returned EventConsumerResult.Finished) + // and should not have called onWorkflowError with a RuntimeError + const calls = (ctx.onWorkflowError as ReturnType).mock.calls; + const runtimeErrors = calls.filter( + ([err]) => err instanceof WorkflowRuntimeError + ); + expect(runtimeErrors).toHaveLength(0); + }); + + it('should handle multiple hook_received events with iterator', async () => { + const ops: Promise[] = []; + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_created', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: {}, + createdAt: new Date(), + }, + { + eventId: 'evnt_1', + runId: 'wrun_123', + eventType: 'hook_received', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + payload: dehydrateStepReturnValue({ message: 'first' }, ops), + }, + createdAt: new Date(), + }, + { + eventId: 'evnt_2', + runId: 'wrun_123', + eventType: 'hook_received', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + payload: dehydrateStepReturnValue({ message: 'second' }, ops), + }, + createdAt: new Date(), + }, + { + eventId: 'evnt_3', + runId: 'wrun_123', + eventType: 'hook_disposed', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: {}, + createdAt: new Date(), + }, + ]); + + const createHook = createCreateHook(ctx); + const hook = createHook<{ message: string }>(); + + const payloads: { message: string }[] = []; + for await (const payload of hook) { + payloads.push(payload); + if (payloads.length >= 2) break; + } + + expect(payloads).toHaveLength(2); + expect(payloads[0]).toEqual({ message: 'first' }); + expect(payloads[1]).toEqual({ message: 'second' }); + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + }); + + it('should include token in error message for unexpected event type', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'step_completed', // Wrong event type + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + result: ['test'], + }, + createdAt: new Date(), + }, + ]); + + let workflowError: Error | undefined; + ctx.onWorkflowError = (err) => { + workflowError = err; + }; + + const createHook = createCreateHook(ctx); + // Create hook with a specific token + const hook = createHook({ token: 'my-custom-token' }); + + // Start awaiting the hook + const hookPromise = hook.then((v) => v); + + // Wait for the error handler to be called + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(workflowError).toBeInstanceOf(WorkflowRuntimeError); + expect(workflowError?.message).toContain('my-custom-token'); + }); + + it('should reject with WorkflowRuntimeError when hook_conflict event is received', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_conflict', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + token: 'my-conflicting-token', + }, + createdAt: new Date(), + }, + ]); + + const createHook = createCreateHook(ctx); + const hook = createHook({ token: 'my-conflicting-token' }); + + // Await should reject with WorkflowRuntimeError + await expect(hook).rejects.toThrow(WorkflowRuntimeError); + await expect(hook).rejects.toThrow(/hook-conflict/); + }); + + it('should reject multiple awaits when hook_conflict event is received (iterator case)', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_conflict', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + token: 'my-conflicting-token', + }, + createdAt: new Date(), + }, + ]); + + const createHook = createCreateHook(ctx); + const hook = createHook({ token: 'my-conflicting-token' }); + + // First await should reject + await expect(hook).rejects.toThrow(WorkflowRuntimeError); + + // Subsequent awaits should also reject (simulating iterator pattern) + await expect(hook).rejects.toThrow(WorkflowRuntimeError); + await expect(hook).rejects.toThrow(WorkflowRuntimeError); + }); + + it('should remove hook from invocations queue when hook_conflict event is received', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_conflict', + correlationId: 'hook_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + token: 'my-conflicting-token', + }, + createdAt: new Date(), + }, + ]); + + const createHook = createCreateHook(ctx); + const hook = createHook({ token: 'my-conflicting-token' }); + + // Hook should initially be in the queue + expect(ctx.invocationsQueue.size).toBe(1); + + // Try to await (will reject) + try { + await hook; + } catch { + // Expected to throw + } + + // After processing conflict event, hook should be removed from queue + expect(ctx.invocationsQueue.size).toBe(0); + }); +}); diff --git a/packages/core/src/workflow/hook.ts b/packages/core/src/workflow/hook.ts index 5219c6d583..70763e87d0 100644 --- a/packages/core/src/workflow/hook.ts +++ b/packages/core/src/workflow/hook.ts @@ -1,11 +1,12 @@ import { type PromiseWithResolvers, withResolvers } from '@workflow/utils'; -import type { HookReceivedEvent } from '@workflow/world'; +import type { HookConflictEvent, HookReceivedEvent } from '@workflow/world'; import type { Hook, HookOptions } from '../create-hook.js'; import { EventConsumerResult } from '../events-consumer.js'; import { WorkflowSuspension } from '../global.js'; import { webhookLogger } from '../logger.js'; import type { WorkflowOrchestratorContext } from '../private.js'; import { hydrateStepReturnValue } from '../serialization.js'; +import { ERROR_SLUGS, WorkflowRuntimeError } from '@workflow/errors'; export function createCreateHook(ctx: WorkflowOrchestratorContext) { return function createHookImpl(options: HookOptions = {}): Hook { @@ -29,6 +30,10 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { let eventLogEmpty = false; + // Track if we have a conflict so we can reject future awaits + let hasConflict = false; + let conflictErrorRef: WorkflowRuntimeError | null = null; + webhookLogger.debug('Hook consumer setup', { correlationId, token }); ctx.eventsConsumer.subscribe((event) => { // If there are no events and there are promises waiting, @@ -43,24 +48,48 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { new WorkflowSuspension(ctx.invocationsQueue, ctx.globalThis) ); }, 0); - return EventConsumerResult.Finished; } + return EventConsumerResult.NotConsumed; + } + + if (event.correlationId !== correlationId) { + // We're not interested in this event - the correlationId belongs to a different entity + return EventConsumerResult.NotConsumed; } // Check for hook_created event to remove this hook from the queue if it was already created - if ( - event?.eventType === 'hook_created' && - event.correlationId === correlationId - ) { + if (event.eventType === 'hook_created') { // Remove this hook from the invocations queue (O(1) delete using Map) ctx.invocationsQueue.delete(correlationId); return EventConsumerResult.Consumed; } - if ( - event?.eventType === 'hook_received' && - event.correlationId === correlationId - ) { + // Handle hook_conflict event - another workflow is using this token + if (event.eventType === 'hook_conflict') { + // Remove this hook from the invocations queue + ctx.invocationsQueue.delete(correlationId); + + // Store the conflict event so we can reject any awaited promises + const conflictEvent = event as HookConflictEvent; + const conflictError = new WorkflowRuntimeError( + `Hook token "${conflictEvent.eventData.token}" is already in use by another workflow`, + { slug: ERROR_SLUGS.HOOK_CONFLICT } + ); + + // Reject any pending promises + for (const resolver of promises) { + resolver.reject(conflictError); + } + promises.length = 0; + + // Mark that we have a conflict so future awaits also reject + hasConflict = true; + conflictErrorRef = conflictError; + + return EventConsumerResult.Consumed; + } + + if (event.eventType === 'hook_received') { if (promises.length > 0) { const next = promises.shift(); if (next) { @@ -78,12 +107,34 @@ export function createCreateHook(ctx: WorkflowOrchestratorContext) { return EventConsumerResult.Consumed; } - return EventConsumerResult.NotConsumed; + if (event.eventType === 'hook_disposed') { + // If a hook is explicitly disposed, we're done processing any more + // events for it + return EventConsumerResult.Finished; + } + + // An unexpected event type has been received, this event log looks corrupted. Let's fail immediately. + setTimeout(() => { + ctx.onWorkflowError( + new WorkflowRuntimeError( + `Unexpected event type for hook ${correlationId} (token: ${token}) "${event.eventType}"` + ) + ); + }, 0); + return EventConsumerResult.Finished; }); // Helper function to create a new promise that waits for the next hook payload function createHookPromise(): Promise { const resolvers = withResolvers(); + + // If we have a conflict, reject immediately + // This handles the iterator case where each await should reject + if (hasConflict && conflictErrorRef) { + resolvers.reject(conflictErrorRef); + return resolvers.promise; + } + if (payloadsQueue.length > 0) { const nextPayload = payloadsQueue.shift(); if (nextPayload) { diff --git a/packages/core/src/workflow/sleep.test.ts b/packages/core/src/workflow/sleep.test.ts new file mode 100644 index 0000000000..0bed156458 --- /dev/null +++ b/packages/core/src/workflow/sleep.test.ts @@ -0,0 +1,278 @@ +import { WorkflowRuntimeError } from '@workflow/errors'; +import type { Event } from '@workflow/world'; +import * as nanoid from 'nanoid'; +import { monotonicFactory } from 'ulid'; +import { describe, expect, it, vi } from 'vitest'; +import { EventsConsumer } from '../events-consumer.js'; +import { WorkflowSuspension } from '../global.js'; +import type { WorkflowOrchestratorContext } from '../private.js'; +import { createContext } from '../vm/index.js'; +import { createSleep } from './sleep.js'; + +// Helper to setup context to simulate a workflow run +function setupWorkflowContext(events: Event[]): WorkflowOrchestratorContext { + const context = createContext({ + seed: 'test', + fixedTimestamp: 1753481739458, + }); + const ulid = monotonicFactory(() => context.globalThis.Math.random()); + const workflowStartedAt = context.globalThis.Date.now(); + return { + globalThis: context.globalThis, + eventsConsumer: new EventsConsumer(events), + invocationsQueue: new Map(), + generateUlid: () => ulid(workflowStartedAt), + generateNanoid: nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => + new Uint8Array(size).map(() => 256 * context.globalThis.Math.random()) + ), + onWorkflowError: vi.fn(), + }; +} + +describe('createSleep', () => { + it('should resolve when wait_completed event is received', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'wait_created', + correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + resumeAt: new Date('2024-01-01T00:00:01.000Z'), + }, + createdAt: new Date(), + }, + { + eventId: 'evnt_1', + runId: 'wrun_123', + eventType: 'wait_completed', + correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: {}, + createdAt: new Date(), + }, + ]); + + const sleep = createSleep(ctx); + await sleep('1s'); + + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + expect(ctx.invocationsQueue.size).toBe(0); + }); + + it('should throw WorkflowSuspension when no events are available', async () => { + const ctx = setupWorkflowContext([]); + + let workflowError: Error | undefined; + ctx.onWorkflowError = (err) => { + workflowError = err; + }; + + const sleep = createSleep(ctx); + + // Start the sleep - it will process events asynchronously + const sleepPromise = sleep('1s'); + + // Wait for the error handler to be called + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(workflowError).toBeInstanceOf(WorkflowSuspension); + }); + + it('should invoke workflow error handler with WorkflowRuntimeError for unexpected event type', async () => { + // Simulate a corrupted event log where a sleep/wait receives an unexpected event type + // (e.g., a step_completed event when expecting wait_created/wait_completed) + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'step_completed', // Wrong event type for a wait! + correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + result: ['test'], + }, + createdAt: new Date(), + }, + ]); + + let workflowError: Error | undefined; + ctx.onWorkflowError = (err) => { + workflowError = err; + }; + + const sleep = createSleep(ctx); + + // Start the sleep - it will process events asynchronously + const sleepPromise = sleep('1s'); + + // Wait for the error handler to be called + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(workflowError).toBeInstanceOf(WorkflowRuntimeError); + expect(workflowError?.message).toContain('Unexpected event type for wait'); + expect(workflowError?.message).toContain('wait_01K11TFZ62YS0YYFDQ3E8B9YCV'); + expect(workflowError?.message).toContain('step_completed'); + }); + + it('should mark wait as having created event when wait_created is received', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'wait_created', + correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + resumeAt: new Date('2024-01-01T00:00:05.000Z'), + }, + createdAt: new Date(), + }, + ]); + + let workflowError: Error | undefined; + ctx.onWorkflowError = (err) => { + workflowError = err; + }; + + const sleep = createSleep(ctx); + + // Start the sleep - it will process events asynchronously + const sleepPromise = sleep('5s'); + + // Wait for event processing + await new Promise((resolve) => setTimeout(resolve, 10)); + + // Check that the wait item has been updated with hasCreatedEvent + const waitItem = ctx.invocationsQueue.get( + 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV' + ); + expect(waitItem).toBeDefined(); + expect(waitItem?.type).toBe('wait'); + if (waitItem?.type === 'wait') { + expect(waitItem.hasCreatedEvent).toBe(true); + } + + // Should suspend since wait_completed is not yet received + expect(workflowError).toBeInstanceOf(WorkflowSuspension); + }); + + it('should handle hook_received as unexpected event type for wait', async () => { + // Test with a different unexpected event type to ensure all non-wait events are caught + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'hook_received', // Wrong event type for a wait! + correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + payload: { data: 'test' }, + }, + createdAt: new Date(), + }, + ]); + + let workflowError: Error | undefined; + ctx.onWorkflowError = (err) => { + workflowError = err; + }; + + const sleep = createSleep(ctx); + const sleepPromise = sleep('1s'); + + // Wait for the error handler to be called + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(workflowError).toBeInstanceOf(WorkflowRuntimeError); + expect(workflowError?.message).toContain('Unexpected event type for wait'); + expect(workflowError?.message).toContain('hook_received'); + }); + + it('should keep queue item after wait_created (not terminal)', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'wait_created', + correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + resumeAt: new Date('2024-01-01T00:00:05.000Z'), + }, + createdAt: new Date(), + }, + ]); + + let workflowError: Error | undefined; + ctx.onWorkflowError = (err) => { + workflowError = err; + }; + + const sleep = createSleep(ctx); + const sleepPromise = sleep('5s'); + + // Wait for event processing + await new Promise((resolve) => setTimeout(resolve, 10)); + + // Queue item should still exist (wait_created is not terminal) + expect(ctx.invocationsQueue.size).toBe(1); + const waitItem = ctx.invocationsQueue.get( + 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV' + ); + expect(waitItem).toBeDefined(); + expect(waitItem?.type).toBe('wait'); + + // Should suspend since wait_completed is not yet received + expect(workflowError).toBeInstanceOf(WorkflowSuspension); + }); + + it('should remove queue item when wait_completed (terminal state)', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'wait_created', + correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: { + resumeAt: new Date('2024-01-01T00:00:01.000Z'), + }, + createdAt: new Date(), + }, + { + eventId: 'evnt_1', + runId: 'wrun_123', + eventType: 'wait_completed', + correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: {}, + createdAt: new Date(), + }, + ]); + + const sleep = createSleep(ctx); + + // Before sleep completes, queue should have the item + expect(ctx.invocationsQueue.size).toBe(0); // Not added yet + + await sleep('1s'); + + // Queue should be empty after completion (terminal state) + expect(ctx.invocationsQueue.size).toBe(0); + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + }); + + it('should resolve with void when wait_completed', async () => { + const ctx = setupWorkflowContext([ + { + eventId: 'evnt_0', + runId: 'wrun_123', + eventType: 'wait_completed', + correlationId: 'wait_01K11TFZ62YS0YYFDQ3E8B9YCV', + eventData: {}, + createdAt: new Date(), + }, + ]); + + const sleep = createSleep(ctx); + const result = await sleep('1s'); + + // sleep() should resolve with void/undefined + expect(result).toBeUndefined(); + expect(ctx.onWorkflowError).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/core/src/workflow/sleep.ts b/packages/core/src/workflow/sleep.ts index dc9df77343..9a51e69385 100644 --- a/packages/core/src/workflow/sleep.ts +++ b/packages/core/src/workflow/sleep.ts @@ -3,6 +3,7 @@ import type { StringValue } from 'ms'; import { EventConsumerResult } from '../events-consumer.js'; import { type WaitInvocationQueueItem, WorkflowSuspension } from '../global.js'; import type { WorkflowOrchestratorContext } from '../private.js'; +import { WorkflowRuntimeError } from '@workflow/errors'; export function createSleep(ctx: WorkflowOrchestratorContext) { return async function sleepImpl( @@ -34,11 +35,13 @@ export function createSleep(ctx: WorkflowOrchestratorContext) { return EventConsumerResult.NotConsumed; } + if (event.correlationId !== correlationId) { + // We're not interested in this event - the correlationId belongs to a different entity + return EventConsumerResult.NotConsumed; + } + // Check for wait_created event to mark this wait as having the event created - if ( - event?.eventType === 'wait_created' && - event.correlationId === correlationId - ) { + if (event.eventType === 'wait_created') { // Mark this wait as having the created event, but keep it in the queue // O(1) lookup using Map const queueItem = ctx.invocationsQueue.get(correlationId); @@ -50,10 +53,7 @@ export function createSleep(ctx: WorkflowOrchestratorContext) { } // Check for wait_completed event - if ( - event?.eventType === 'wait_completed' && - event.correlationId === correlationId - ) { + if (event.eventType === 'wait_completed') { // Remove this wait from the invocations queue (O(1) delete using Map) ctx.invocationsQueue.delete(correlationId); @@ -64,7 +64,15 @@ export function createSleep(ctx: WorkflowOrchestratorContext) { return EventConsumerResult.Finished; } - return EventConsumerResult.NotConsumed; + // An unexpected event type has been received, this event log looks corrupted. Let's fail immediately. + setTimeout(() => { + ctx.onWorkflowError( + new WorkflowRuntimeError( + `Unexpected event type for wait ${correlationId} "${event.eventType}"` + ) + ); + }, 0); + return EventConsumerResult.Finished; }); return promise; diff --git a/packages/errors/src/index.ts b/packages/errors/src/index.ts index 071fc359d1..59017b9be1 100644 --- a/packages/errors/src/index.ts +++ b/packages/errors/src/index.ts @@ -31,6 +31,7 @@ export const ERROR_SLUGS = { WEBHOOK_RESPONSE_NOT_SENT: 'webhook-response-not-sent', FETCH_IN_WORKFLOW_FUNCTION: 'fetch-in-workflow', TIMEOUT_FUNCTIONS_IN_WORKFLOW: 'timeout-in-workflow', + HOOK_CONFLICT: 'hook-conflict', } as const; type ErrorSlug = (typeof ERROR_SLUGS)[keyof typeof ERROR_SLUGS]; @@ -231,6 +232,31 @@ export class WorkflowRunCancelledError extends WorkflowError { } } +/** + * Thrown when attempting to operate on a workflow run that requires a newer World version. + * + * This error occurs when a run was created with a newer spec version than the + * current World implementation supports. Users should upgrade their @workflow packages. + */ +export class RunNotSupportedError extends WorkflowError { + readonly runSpecVersion: number; + readonly worldSpecVersion: number; + + constructor(runSpecVersion: number, worldSpecVersion: number) { + super( + `Run requires spec version ${runSpecVersion}, but world supports version ${worldSpecVersion}. ` + + `Please upgrade 'workflow' package.` + ); + this.name = 'RunNotSupportedError'; + this.runSpecVersion = runSpecVersion; + this.worldSpecVersion = worldSpecVersion; + } + + static is(value: unknown): value is RunNotSupportedError { + return isError(value) && value.name === 'RunNotSupportedError'; + } +} + /** * A fatal error is an error that cannot be retried. * It will cause the step to fail and the error will diff --git a/packages/web-shared/src/api/workflow-server-actions.ts b/packages/web-shared/src/api/workflow-server-actions.ts index 10941fd25d..42a13591dc 100644 --- a/packages/web-shared/src/api/workflow-server-actions.ts +++ b/packages/web-shared/src/api/workflow-server-actions.ts @@ -819,10 +819,12 @@ export async function cancelRun( ): Promise> { try { const world = await getWorldFromEnv(worldEnv); - await world.runs.cancel(runId); + await world.events.create(runId, { eventType: 'run_cancelled' }); return createResponse(undefined); } catch (error) { - return createServerActionError(error, 'world.runs.cancel', { runId }); + return createServerActionError(error, 'world.events.create', { + runId, + }); } } diff --git a/packages/web-shared/src/sidebar/attribute-panel.tsx b/packages/web-shared/src/sidebar/attribute-panel.tsx index 24930ebdd6..cff424c461 100644 --- a/packages/web-shared/src/sidebar/attribute-panel.tsx +++ b/packages/web-shared/src/sidebar/attribute-panel.tsx @@ -496,6 +496,7 @@ const attributeOrder: AttributeKey[] = [ 'correlationId', 'eventType', 'deploymentId', + 'specVersion', 'ownerId', 'projectId', 'environment', @@ -571,6 +572,7 @@ const attributeToDisplayFn: Record< correlationId: (value: unknown) => String(value), // Project details deploymentId: (value: unknown) => String(value), + specVersion: (value: unknown) => String(value), // Tenancy (we don't show these) ownerId: (_value: unknown) => null, projectId: (_value: unknown) => null, diff --git a/packages/web-shared/src/workflow-traces/event-colors.ts b/packages/web-shared/src/workflow-traces/event-colors.ts index df8eb7d39b..5f7c070254 100644 --- a/packages/web-shared/src/workflow-traces/event-colors.ts +++ b/packages/web-shared/src/workflow-traces/event-colors.ts @@ -19,7 +19,7 @@ export interface EventColorPalette { /** * Get the color palette for an event based on its type - * - Red for failures (step_failed, workflow_failed) + * - Red for failures (step_failed, run_failed) * - Orange/yellow for retries (step_retrying) * - Purple for webhook-related events * - Blue otherwise (default) @@ -28,7 +28,7 @@ export function getEventColor( eventType: Event['eventType'] ): EventColorPalette { // Failures - Red - if (eventType === 'step_failed' || eventType === 'workflow_failed') { + if (eventType === 'step_failed' || eventType === 'run_failed') { return { color: 'var(--ds-red-600)', background: 'var(--ds-red-100)', diff --git a/packages/web-shared/src/workflow-traces/trace-colors.ts b/packages/web-shared/src/workflow-traces/trace-colors.ts index aa1f2e0172..c56ea5856e 100644 --- a/packages/web-shared/src/workflow-traces/trace-colors.ts +++ b/packages/web-shared/src/workflow-traces/trace-colors.ts @@ -89,7 +89,7 @@ export const getCustomSpanEventClassName = ( const eventName = spanEvent.event.name; // Failure events - Red - if (eventName === 'step_failed' || eventName === 'workflow_failed') { + if (eventName === 'step_failed' || eventName === 'run_failed') { return styles.eventFailed; } diff --git a/packages/web-shared/src/workflow-traces/trace-span-construction.ts b/packages/web-shared/src/workflow-traces/trace-span-construction.ts index 53e4e92d1e..e118b400c7 100644 --- a/packages/web-shared/src/workflow-traces/trace-span-construction.ts +++ b/packages/web-shared/src/workflow-traces/trace-span-construction.ts @@ -23,7 +23,7 @@ const MARKER_EVENT_TYPES: Set = new Set([ 'step_started', 'step_retrying', 'step_failed', - 'workflow_failed', + 'run_failed', 'wait_created', 'wait_completed', ]); diff --git a/packages/world-local/src/storage.test.ts b/packages/world-local/src/storage.test.ts index c2426fb735..4afcc118b3 100644 --- a/packages/world-local/src/storage.test.ts +++ b/packages/world-local/src/storage.test.ts @@ -2,15 +2,17 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import type { Storage } from '@workflow/world'; -import { - EventSchema, - HookSchema, - StepSchema, - WorkflowRunSchema, -} from '@workflow/world'; import { monotonicFactory } from 'ulid'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { createStorage } from './storage.js'; +import { + createHook, + createRun, + createStep, + disposeHook, + updateRun, + updateStep, +} from './test-helpers.js'; describe('Storage', () => { let testDir: string; @@ -41,7 +43,7 @@ describe('Storage', () => { input: ['arg1', 'arg2'], }; - const run = await storage.runs.create(runData); + const run = await createRun(storage, runData); expect(run.runId).toMatch(/^wrun_/); expect(run.deploymentId).toBe('deployment-123'); @@ -72,37 +74,16 @@ describe('Storage', () => { input: [], }; - const run = await storage.runs.create(runData); + const run = await createRun(storage, runData); expect(run.executionContext).toBeUndefined(); expect(run.input).toEqual([]); }); - - it('should validate run against schema before writing', async () => { - const parseSpy = vi.spyOn(WorkflowRunSchema, 'parse'); - - await storage.runs.create({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - input: [], - }); - - expect(parseSpy).toHaveBeenCalledTimes(1); - expect(parseSpy).toHaveBeenCalledWith( - expect.objectContaining({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - status: 'pending', - }) - ); - - parseSpy.mockRestore(); - }); }); describe('get', () => { it('should retrieve an existing run', async () => { - const created = await storage.runs.create({ + const created = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -120,9 +101,9 @@ describe('Storage', () => { }); }); - describe('update', () => { - it('should update run status to running', async () => { - const created = await storage.runs.create({ + describe('update via events', () => { + it('should update run status to running via run_started event', async () => { + const created = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -131,9 +112,7 @@ describe('Storage', () => { // Small delay to ensure different timestamps await new Promise((resolve) => setTimeout(resolve, 1)); - const updated = await storage.runs.update(created.runId, { - status: 'running', - }); + const updated = await updateRun(storage, created.runId, 'run_started'); expect(updated.status).toBe('running'); expect(updated.startedAt).toBeInstanceOf(Date); @@ -142,56 +121,47 @@ describe('Storage', () => { ); }); - it('should update run status to completed', async () => { - const created = await storage.runs.create({ + it('should update run status to completed via run_completed event', async () => { + const created = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], }); - const updated = await storage.runs.update(created.runId, { - status: 'completed', - output: { result: 'success' }, - }); + const updated = await updateRun( + storage, + created.runId, + 'run_completed', + { + output: { result: 'success' }, + } + ); expect(updated.status).toBe('completed'); expect(updated.output).toEqual({ result: 'success' }); expect(updated.completedAt).toBeInstanceOf(Date); }); - it('should update run status to failed', async () => { - const created = await storage.runs.create({ + it('should update run status to failed via run_failed event', async () => { + const created = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], }); - const updated = await storage.runs.update(created.runId, { - status: 'failed', - error: { - message: 'Something went wrong', - code: 'ERR_001', - }, + const updated = await updateRun(storage, created.runId, 'run_failed', { + error: 'Something went wrong', }); expect(updated.status).toBe('failed'); - expect(updated.error).toEqual({ - message: 'Something went wrong', - code: 'ERR_001', - }); + expect(updated.error?.message).toBe('Something went wrong'); expect(updated.completedAt).toBeInstanceOf(Date); }); - - it('should throw error for non-existent run', async () => { - await expect( - storage.runs.update('wrun_nonexistent', { status: 'running' }) - ).rejects.toThrow('Workflow run "wrun_nonexistent" not found'); - }); }); describe('list', () => { it('should list all runs', async () => { - const run1 = await storage.runs.create({ + const run1 = await createRun(storage, { deploymentId: 'deployment-1', workflowName: 'workflow-1', input: [], @@ -200,7 +170,7 @@ describe('Storage', () => { // Small delay to ensure different timestamps in ULIDs await new Promise((resolve) => setTimeout(resolve, 2)); - const run2 = await storage.runs.create({ + const run2 = await createRun(storage, { deploymentId: 'deployment-2', workflowName: 'workflow-2', input: [], @@ -218,12 +188,12 @@ describe('Storage', () => { }); it('should filter runs by workflowName', async () => { - await storage.runs.create({ + await createRun(storage, { deploymentId: 'deployment-1', workflowName: 'workflow-1', input: [], }); - const run2 = await storage.runs.create({ + const run2 = await createRun(storage, { deploymentId: 'deployment-2', workflowName: 'workflow-2', input: [], @@ -238,7 +208,7 @@ describe('Storage', () => { it('should support pagination', async () => { // Create multiple runs for (let i = 0; i < 5; i++) { - await storage.runs.create({ + await createRun(storage, { deploymentId: `deployment-${i}`, workflowName: `workflow-${i}`, input: [], @@ -260,28 +230,13 @@ describe('Storage', () => { expect(page2.data[0].runId).not.toBe(page1.data[0].runId); }); }); - - describe('cancel', () => { - it('should cancel a run', async () => { - const created = await storage.runs.create({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - input: [], - }); - - const cancelled = await storage.runs.cancel(created.runId); - - expect(cancelled.status).toBe('cancelled'); - expect(cancelled.completedAt).toBeInstanceOf(Date); - }); - }); }); describe('steps', () => { let testRunId: string; beforeEach(async () => { - const run = await storage.runs.create({ + const run = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -297,7 +252,7 @@ describe('Storage', () => { input: ['input1', 'input2'], }; - const step = await storage.steps.create(testRunId, stepData); + const step = await createStep(storage, testRunId, stepData); expect(step.runId).toBe(testRunId); expect(step.stepId).toBe('step_123'); @@ -324,33 +279,11 @@ describe('Storage', () => { .catch(() => false); expect(fileExists).toBe(true); }); - - it('should validate step against schema before writing', async () => { - const parseSpy = vi.spyOn(StepSchema, 'parse'); - - await storage.steps.create(testRunId, { - stepId: 'step_validated', - stepName: 'validated-step', - input: ['arg1'], - }); - - expect(parseSpy).toHaveBeenCalledTimes(1); - expect(parseSpy).toHaveBeenCalledWith( - expect.objectContaining({ - runId: testRunId, - stepId: 'step_validated', - stepName: 'validated-step', - status: 'pending', - }) - ); - - parseSpy.mockRestore(); - }); }); describe('get', () => { it('should retrieve a step with runId and stepId', async () => { - const created = await storage.steps.create(testRunId, { + const created = await createStep(storage, testRunId, { stepId: 'step_123', stepName: 'test-step', input: ['input1'], @@ -362,7 +295,7 @@ describe('Storage', () => { }); it('should retrieve a step with only stepId', async () => { - const created = await storage.steps.create(testRunId, { + const created = await createStep(storage, testRunId, { stepId: 'unique_step_123', stepName: 'test-step', input: ['input1'], @@ -380,83 +313,76 @@ describe('Storage', () => { }); }); - describe('update', () => { - it('should update step status to running', async () => { - await storage.steps.create(testRunId, { + describe('update via events', () => { + it('should update step status to running via step_started event', async () => { + await createStep(storage, testRunId, { stepId: 'step_123', stepName: 'test-step', input: ['input1'], }); - const updated = await storage.steps.update(testRunId, 'step_123', { - status: 'running', - }); + const updated = await updateStep( + storage, + testRunId, + 'step_123', + 'step_started', + {} // step_started no longer needs attempt in eventData - World increments it + ); expect(updated.status).toBe('running'); expect(updated.startedAt).toBeInstanceOf(Date); + expect(updated.attempt).toBe(1); // Incremented by step_started }); - it('should update step status to completed', async () => { - await storage.steps.create(testRunId, { + it('should update step status to completed via step_completed event', async () => { + await createStep(storage, testRunId, { stepId: 'step_123', stepName: 'test-step', input: ['input1'], }); - const updated = await storage.steps.update(testRunId, 'step_123', { - status: 'completed', - output: { result: 'done' }, - }); + const updated = await updateStep( + storage, + testRunId, + 'step_123', + 'step_completed', + { result: { result: 'done' } } + ); expect(updated.status).toBe('completed'); expect(updated.output).toEqual({ result: 'done' }); expect(updated.completedAt).toBeInstanceOf(Date); }); - it('should update step status to failed', async () => { - await storage.steps.create(testRunId, { + it('should update step status to failed via step_failed event', async () => { + await createStep(storage, testRunId, { stepId: 'step_123', stepName: 'test-step', input: ['input1'], }); - const updated = await storage.steps.update(testRunId, 'step_123', { - status: 'failed', - error: { - message: 'Step failed', - code: 'STEP_ERR', - }, - }); + const updated = await updateStep( + storage, + testRunId, + 'step_123', + 'step_failed', + { error: 'Step failed' } + ); expect(updated.status).toBe('failed'); expect(updated.error?.message).toBe('Step failed'); - expect(updated.error?.code).toBe('STEP_ERR'); expect(updated.completedAt).toBeInstanceOf(Date); }); - - it('should update attempt count', async () => { - await storage.steps.create(testRunId, { - stepId: 'step_123', - stepName: 'test-step', - input: ['input1'], - }); - - const updated = await storage.steps.update(testRunId, 'step_123', { - attempt: 2, - }); - - expect(updated.attempt).toBe(2); - }); }); describe('list', () => { it('should list all steps for a run', async () => { - const step1 = await storage.steps.create(testRunId, { + const step1 = await createStep(storage, testRunId, { stepId: 'step_1', stepName: 'first-step', input: [], }); - const step2 = await storage.steps.create(testRunId, { + const step2 = await createStep(storage, testRunId, { stepId: 'step_2', stepName: 'second-step', input: [], @@ -478,7 +404,7 @@ describe('Storage', () => { it('should support pagination', async () => { // Create multiple steps for (let i = 0; i < 5; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -505,7 +431,7 @@ describe('Storage', () => { it('should handle pagination when new items are created after getting a cursor', async () => { // Create initial set of items (4 items) for (let i = 0; i < 4; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -525,7 +451,7 @@ describe('Storage', () => { // Now create 4 more items (total: 8 items) for (let i = 4; i < 8; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -565,7 +491,7 @@ describe('Storage', () => { it('should handle pagination with cursor after items are added mid-pagination', async () => { // Create initial 4 items for (let i = 0; i < 4; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -597,7 +523,7 @@ describe('Storage', () => { // Now add 4 more items (total: 8) for (let i = 4; i < 8; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -638,7 +564,7 @@ describe('Storage', () => { // Start with X items (4 items) for (let i = 0; i < 4; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -660,7 +586,7 @@ describe('Storage', () => { // Create new items (total becomes 2X = 8 items) for (let i = 4; i < 8; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -712,7 +638,7 @@ describe('Storage', () => { let testRunId: string; beforeEach(async () => { - const run = await storage.runs.create({ + const run = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -722,12 +648,19 @@ describe('Storage', () => { describe('create', () => { it('should create a new event', async () => { + // Create step first (required for step events) + await createStep(storage, testRunId, { + stepId: 'corr_123', + stepName: 'test-step', + input: [], + }); + const eventData = { eventType: 'step_started' as const, correlationId: 'corr_123', }; - const event = await storage.events.create(testRunId, eventData); + const { event } = await storage.events.create(testRunId, eventData); expect(event.runId).toBe(testRunId); expect(event.eventId).toMatch(/^evnt_/); @@ -748,48 +681,39 @@ describe('Storage', () => { expect(fileExists).toBe(true); }); - it('should handle workflow completed events', async () => { + it('should handle run completed events', async () => { const eventData = { - eventType: 'workflow_completed' as const, + eventType: 'run_completed' as const, + eventData: { output: { result: 'done' } }, }; - const event = await storage.events.create(testRunId, eventData); + const { event } = await storage.events.create(testRunId, eventData); - expect(event.eventType).toBe('workflow_completed'); + expect(event.eventType).toBe('run_completed'); expect(event.correlationId).toBeUndefined(); }); - - it('should validate event against schema before writing', async () => { - const parseSpy = vi.spyOn(EventSchema, 'parse'); - - await storage.events.create(testRunId, { - eventType: 'step_started' as const, - correlationId: 'corr_validated', - }); - - expect(parseSpy).toHaveBeenCalledTimes(1); - expect(parseSpy).toHaveBeenCalledWith( - expect.objectContaining({ - runId: testRunId, - eventType: 'step_started', - correlationId: 'corr_validated', - }) - ); - - parseSpy.mockRestore(); - }); }); describe('list', () => { it('should list all events for a run', async () => { - const event1 = await storage.events.create(testRunId, { - eventType: 'workflow_started' as const, + // Note: testRunId was created via createRun which creates a run_created event + const { event: event1 } = await storage.events.create(testRunId, { + eventType: 'run_started' as const, }); // Small delay to ensure different timestamps in event IDs await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(testRunId, { + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: 'corr_step_1', + stepName: 'test-step', + input: [], + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + + const { event: event2 } = await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId: 'corr_step_1', }); @@ -799,24 +723,37 @@ describe('Storage', () => { pagination: { sortOrder: 'asc' }, // Explicitly request ascending order }); - expect(result.data).toHaveLength(2); + // 4 events: run_created (from createRun), run_started, step_created, step_started + expect(result.data).toHaveLength(4); // Should be in chronological order (oldest first) - expect(result.data[0].eventId).toBe(event1.eventId); - expect(result.data[1].eventId).toBe(event2.eventId); - expect(result.data[1].createdAt.getTime()).toBeGreaterThanOrEqual( - result.data[0].createdAt.getTime() + expect(result.data[0].eventType).toBe('run_created'); + expect(result.data[1].eventId).toBe(event1.eventId); + expect(result.data[2].eventType).toBe('step_created'); + expect(result.data[3].eventId).toBe(event2.eventId); + expect(result.data[3].createdAt.getTime()).toBeGreaterThanOrEqual( + result.data[2].createdAt.getTime() ); }); it('should list events in descending order when explicitly requested (newest first)', async () => { - const event1 = await storage.events.create(testRunId, { - eventType: 'workflow_started' as const, + // Note: testRunId was created via createRun which creates a run_created event + const { event: event1 } = await storage.events.create(testRunId, { + eventType: 'run_started' as const, }); // Small delay to ensure different timestamps in event IDs await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(testRunId, { + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: 'corr_step_1', + stepName: 'test-step', + input: [], + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + + const { event: event2 } = await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId: 'corr_step_1', }); @@ -826,18 +763,26 @@ describe('Storage', () => { pagination: { sortOrder: 'desc' }, }); - expect(result.data).toHaveLength(2); + // 4 events: run_created (from createRun), run_started, step_created, step_started + expect(result.data).toHaveLength(4); // Should be in reverse chronological order (newest first) expect(result.data[0].eventId).toBe(event2.eventId); - expect(result.data[1].eventId).toBe(event1.eventId); + expect(result.data[1].eventType).toBe('step_created'); + expect(result.data[2].eventId).toBe(event1.eventId); + expect(result.data[3].eventType).toBe('run_created'); expect(result.data[0].createdAt.getTime()).toBeGreaterThanOrEqual( result.data[1].createdAt.getTime() ); }); it('should support pagination', async () => { - // Create multiple events + // Create steps first, then create step_completed events for (let i = 0; i < 5; i++) { + await createStep(storage, testRunId, { + stepId: `corr_${i}`, + stepName: `step-${i}`, + input: [], + }); await storage.events.create(testRunId, { eventType: 'step_completed' as const, correlationId: `corr_${i}`, @@ -867,15 +812,29 @@ describe('Storage', () => { it('should list all events with a specific correlation ID', async () => { const correlationId = 'step-abc123'; + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + + // Create step for the different correlation ID too + await createStep(storage, testRunId, { + stepId: 'different-step', + stepName: 'different-step', + input: [], + }); + // Create events with the target correlation ID - const event1 = await storage.events.create(testRunId, { + const { event: event1 } = await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(testRunId, { + const { event: event2 } = await storage.events.create(testRunId, { eventType: 'step_completed' as const, correlationId, eventData: { result: 'success' }, @@ -887,7 +846,8 @@ describe('Storage', () => { correlationId: 'different-step', }); await storage.events.create(testRunId, { - eventType: 'workflow_completed' as const, + eventType: 'run_completed' as const, + eventData: { output: { result: 'done' } }, }); const result = await storage.events.listByCorrelationId({ @@ -895,32 +855,37 @@ describe('Storage', () => { pagination: {}, }); - expect(result.data).toHaveLength(2); - expect(result.data[0].eventId).toBe(event1.eventId); + // step_created + step_started + step_completed = 3 events + expect(result.data).toHaveLength(3); + // First event is step_created from createStep + expect(result.data[0].eventType).toBe('step_created'); expect(result.data[0].correlationId).toBe(correlationId); - expect(result.data[1].eventId).toBe(event2.eventId); + expect(result.data[1].eventId).toBe(event1.eventId); expect(result.data[1].correlationId).toBe(correlationId); + expect(result.data[2].eventId).toBe(event2.eventId); + expect(result.data[2].correlationId).toBe(correlationId); }); it('should list events across multiple runs with same correlation ID', async () => { const correlationId = 'hook-xyz789'; // Create another run - const run2 = await storage.runs.create({ + const run2 = await createRun(storage, { deploymentId: 'deployment-456', workflowName: 'test-workflow-2', input: [], }); // Create events in both runs with same correlation ID - const event1 = await storage.events.create(testRunId, { + const { event: event1 } = await storage.events.create(testRunId, { eventType: 'hook_created' as const, correlationId, + eventData: { token: `test-token-${correlationId}`, metadata: {} }, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(run2.runId, { + const { event: event2 } = await storage.events.create(run2.runId, { eventType: 'hook_received' as const, correlationId, eventData: { payload: { data: 'test' } }, @@ -928,7 +893,7 @@ describe('Storage', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const event3 = await storage.events.create(testRunId, { + const { event: event3 } = await storage.events.create(testRunId, { eventType: 'hook_disposed' as const, correlationId, }); @@ -948,6 +913,13 @@ describe('Storage', () => { }); it('should return empty list for non-existent correlation ID', async () => { + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: 'existing-step', + stepName: 'existing-step', + input: [], + }); + await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId: 'existing-step', @@ -966,6 +938,13 @@ describe('Storage', () => { it('should respect pagination parameters', async () => { const correlationId = 'step-paginated'; + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + // Create multiple events await storage.events.create(testRunId, { eventType: 'step_started' as const, @@ -977,7 +956,14 @@ describe('Storage', () => { await storage.events.create(testRunId, { eventType: 'step_retrying' as const, correlationId, - eventData: { attempt: 1 }, + eventData: { error: 'retry error' }, + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + + await storage.events.create(testRunId, { + eventType: 'step_started' as const, + correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); @@ -988,7 +974,7 @@ describe('Storage', () => { eventData: { result: 'success' }, }); - // Get first page + // Get first page (step_created + step_started = 2) const page1 = await storage.events.listByCorrelationId({ correlationId, pagination: { limit: 2 }, @@ -998,19 +984,26 @@ describe('Storage', () => { expect(page1.hasMore).toBe(true); expect(page1.cursor).toBeDefined(); - // Get second page + // Get second page (step_retrying + step_started + step_completed = 3) const page2 = await storage.events.listByCorrelationId({ correlationId, - pagination: { limit: 2, cursor: page1.cursor || undefined }, + pagination: { limit: 3, cursor: page1.cursor || undefined }, }); - expect(page2.data).toHaveLength(1); + expect(page2.data).toHaveLength(3); expect(page2.hasMore).toBe(false); }); it('should filter event data when resolveData is "none"', async () => { const correlationId = 'step-with-data'; + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + await storage.events.create(testRunId, { eventType: 'step_completed' as const, correlationId, @@ -1023,23 +1016,34 @@ describe('Storage', () => { resolveData: 'none', }); - expect(result.data).toHaveLength(1); + // step_created + step_completed = 2 events + expect(result.data).toHaveLength(2); expect((result.data[0] as any).eventData).toBeUndefined(); + expect((result.data[1] as any).eventData).toBeUndefined(); expect(result.data[0].correlationId).toBe(correlationId); }); it('should return events in ascending order by default', async () => { const correlationId = 'step-ordering'; + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + // Create events with slight delays to ensure different timestamps - const event1 = await storage.events.create(testRunId, { + const { event: event1 } = await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(testRunId, { + const { event: event2 } = await storage.events.create(testRunId, { eventType: 'step_completed' as const, correlationId, eventData: { result: 'success' }, @@ -1050,9 +1054,12 @@ describe('Storage', () => { pagination: {}, }); - expect(result.data).toHaveLength(2); - expect(result.data[0].eventId).toBe(event1.eventId); - expect(result.data[1].eventId).toBe(event2.eventId); + // step_created + step_started + step_completed = 3 events + expect(result.data).toHaveLength(3); + // Verify order: step_created, step_started, step_completed + expect(result.data[0].eventType).toBe('step_created'); + expect(result.data[1].eventId).toBe(event1.eventId); + expect(result.data[2].eventId).toBe(event2.eventId); expect(result.data[0].createdAt.getTime()).toBeLessThanOrEqual( result.data[1].createdAt.getTime() ); @@ -1061,14 +1068,23 @@ describe('Storage', () => { it('should support descending order', async () => { const correlationId = 'step-desc-order'; - const event1 = await storage.events.create(testRunId, { + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + + const { event: event1 } = await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(testRunId, { + const { event: event2 } = await storage.events.create(testRunId, { eventType: 'step_completed' as const, correlationId, eventData: { result: 'success' }, @@ -1079,9 +1095,12 @@ describe('Storage', () => { pagination: { sortOrder: 'desc' }, }); - expect(result.data).toHaveLength(2); + // step_created + step_started + step_completed = 3 events + expect(result.data).toHaveLength(3); + // Verify order: step_completed, step_started, step_created (descending) expect(result.data[0].eventId).toBe(event2.eventId); expect(result.data[1].eventId).toBe(event1.eventId); + expect(result.data[2].eventType).toBe('step_created'); expect(result.data[0].createdAt.getTime()).toBeGreaterThanOrEqual( result.data[1].createdAt.getTime() ); @@ -1091,14 +1110,15 @@ describe('Storage', () => { const hookId = 'hook_test123'; // Create a typical hook lifecycle - const created = await storage.events.create(testRunId, { + const { event: created } = await storage.events.create(testRunId, { eventType: 'hook_created' as const, correlationId: hookId, + eventData: { token: `test-token-${hookId}`, metadata: {} }, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const received1 = await storage.events.create(testRunId, { + const { event: received1 } = await storage.events.create(testRunId, { eventType: 'hook_received' as const, correlationId: hookId, eventData: { payload: { request: 1 } }, @@ -1106,7 +1126,7 @@ describe('Storage', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const received2 = await storage.events.create(testRunId, { + const { event: received2 } = await storage.events.create(testRunId, { eventType: 'hook_received' as const, correlationId: hookId, eventData: { payload: { request: 2 } }, @@ -1114,7 +1134,7 @@ describe('Storage', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const disposed = await storage.events.create(testRunId, { + const { event: disposed } = await storage.events.create(testRunId, { eventType: 'hook_disposed' as const, correlationId: hookId, }); @@ -1141,7 +1161,7 @@ describe('Storage', () => { let testRunId: string; beforeEach(async () => { - const run = await storage.runs.create({ + const run = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -1156,14 +1176,11 @@ describe('Storage', () => { token: 'my-hook-token', }; - const hook = await storage.hooks.create(testRunId, hookData); + const hook = await createHook(storage, testRunId, hookData); expect(hook.runId).toBe(testRunId); expect(hook.hookId).toBe('hook_123'); expect(hook.token).toBe('my-hook-token'); - expect(hook.ownerId).toBe('local-owner'); - expect(hook.projectId).toBe('local-project'); - expect(hook.environment).toBe('local'); expect(hook.createdAt).toBeInstanceOf(Date); // Verify file was created @@ -1175,35 +1192,37 @@ describe('Storage', () => { expect(fileExists).toBe(true); }); - it('should throw error when creating a hook with a duplicate token', async () => { + it('should return hook_conflict event when creating a hook with a duplicate token', async () => { // Create first hook with a token const hookData = { hookId: 'hook_1', token: 'duplicate-test-token', }; - await storage.hooks.create(testRunId, hookData); + await createHook(storage, testRunId, hookData); - // Try to create another hook with the same token - const duplicateHookData = { - hookId: 'hook_2', - token: 'duplicate-test-token', - }; + // Try to create another hook with the same token - should return hook_conflict event + const result = await storage.events.create(testRunId, { + eventType: 'hook_created', + correlationId: 'hook_2', + eventData: { token: 'duplicate-test-token' }, + }); - await expect( - storage.hooks.create(testRunId, duplicateHookData) - ).rejects.toThrow( - 'Hook with token duplicate-test-token already exists for this project' + expect(result.event.eventType).toBe('hook_conflict'); + expect(result.event.correlationId).toBe('hook_2'); + expect((result.event as any).eventData.token).toBe( + 'duplicate-test-token' ); + expect(result.hook).toBeUndefined(); }); it('should allow multiple hooks with different tokens for the same run', async () => { - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token: 'token-1', }); - const hook2 = await storage.hooks.create(testRunId, { + const hook2 = await createHook(storage, testRunId, { hookId: 'hook_2', token: 'token-2', }); @@ -1216,28 +1235,28 @@ describe('Storage', () => { const token = 'reusable-token'; // Create first hook - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token, }); expect(hook1.token).toBe(token); - // Try to create another hook with the same token - should fail - await expect( - storage.hooks.create(testRunId, { - hookId: 'hook_2', - token, - }) - ).rejects.toThrow( - `Hook with token ${token} already exists for this project` - ); + // Try to create another hook with the same token - should return hook_conflict + const conflictResult = await storage.events.create(testRunId, { + eventType: 'hook_created', + correlationId: 'hook_2', + eventData: { token }, + }); + + expect(conflictResult.event.eventType).toBe('hook_conflict'); + expect(conflictResult.hook).toBeUndefined(); - // Dispose the first hook - await storage.hooks.dispose('hook_1'); + // Dispose the first hook via hook_disposed event + await disposeHook(storage, testRunId, 'hook_1'); // Now we should be able to create a new hook with the same token - const hook2 = await storage.hooks.create(testRunId, { + const hook2 = await createHook(storage, testRunId, { hookId: 'hook_2', token, }); @@ -1248,7 +1267,7 @@ describe('Storage', () => { it('should enforce token uniqueness across different runs within the same project', async () => { // Create a second run - const run2 = await storage.runs.create({ + const run2 = await createRun(storage, { deploymentId: 'deployment-456', workflowName: 'another-workflow', input: [], @@ -1257,48 +1276,29 @@ describe('Storage', () => { const token = 'shared-token-across-runs'; // Create hook in first run - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token, }); expect(hook1.token).toBe(token); - // Try to create hook with same token in second run - should fail - await expect( - storage.hooks.create(run2.runId, { - hookId: 'hook_2', - token, - }) - ).rejects.toThrow( - `Hook with token ${token} already exists for this project` - ); - }); - - it('should validate hook against schema before writing', async () => { - const parseSpy = vi.spyOn(HookSchema, 'parse'); - - await storage.hooks.create(testRunId, { - hookId: 'hook_validated', - token: 'validated-token', + // Try to create hook with same token in second run - should return hook_conflict + const result = await storage.events.create(run2.runId, { + eventType: 'hook_created', + correlationId: 'hook_2', + eventData: { token }, }); - expect(parseSpy).toHaveBeenCalledTimes(1); - expect(parseSpy).toHaveBeenCalledWith( - expect.objectContaining({ - runId: testRunId, - hookId: 'hook_validated', - token: 'validated-token', - }) - ); - - parseSpy.mockRestore(); + expect(result.event.eventType).toBe('hook_conflict'); + expect((result.event as any).eventData.token).toBe(token); + expect(result.hook).toBeUndefined(); }); }); describe('get', () => { it('should retrieve an existing hook by hookId', async () => { - const created = await storage.hooks.create(testRunId, { + const created = await createHook(storage, testRunId, { hookId: 'hook_123', token: 'test-token-123', }); @@ -1315,7 +1315,7 @@ describe('Storage', () => { }); it('should respect resolveData option', async () => { - const created = await storage.hooks.create(testRunId, { + const created = await createHook(storage, testRunId, { hookId: 'hook_with_response', token: 'test-token', }); @@ -1337,7 +1337,7 @@ describe('Storage', () => { describe('getByToken', () => { it('should retrieve an existing hook by token', async () => { - const created = await storage.hooks.create(testRunId, { + const created = await createHook(storage, testRunId, { hookId: 'hook_123', token: 'test-token-123', }); @@ -1354,15 +1354,15 @@ describe('Storage', () => { }); it('should find the correct hook when multiple hooks exist', async () => { - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token: 'token-1', }); - await storage.hooks.create(testRunId, { + await createHook(storage, testRunId, { hookId: 'hook_2', token: 'token-2', }); - await storage.hooks.create(testRunId, { + await createHook(storage, testRunId, { hookId: 'hook_3', token: 'token-3', }); @@ -1376,7 +1376,7 @@ describe('Storage', () => { describe('list', () => { it('should list all hooks', async () => { - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token: 'token-1', }); @@ -1384,7 +1384,7 @@ describe('Storage', () => { // Small delay to ensure different timestamps await new Promise((resolve) => setTimeout(resolve, 2)); - const hook2 = await storage.hooks.create(testRunId, { + const hook2 = await createHook(storage, testRunId, { hookId: 'hook_2', token: 'token-2', }); @@ -1402,17 +1402,17 @@ describe('Storage', () => { it('should filter hooks by runId', async () => { // Create a second run - const run2 = await storage.runs.create({ + const run2 = await createRun(storage, { deploymentId: 'deployment-456', workflowName: 'test-workflow-2', input: [], }); - await storage.hooks.create(testRunId, { + await createHook(storage, testRunId, { hookId: 'hook_run1', token: 'token-run1', }); - const hook2 = await storage.hooks.create(run2.runId, { + const hook2 = await createHook(storage, run2.runId, { hookId: 'hook_run2', token: 'token-run2', }); @@ -1427,7 +1427,7 @@ describe('Storage', () => { it('should support pagination', async () => { // Create multiple hooks for (let i = 0; i < 5; i++) { - await storage.hooks.create(testRunId, { + await createHook(storage, testRunId, { hookId: `hook_${i}`, token: `token-${i}`, }); @@ -1450,14 +1450,14 @@ describe('Storage', () => { }); it('should support ascending sort order', async () => { - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token: 'token-1', }); await new Promise((resolve) => setTimeout(resolve, 2)); - const hook2 = await storage.hooks.create(testRunId, { + const hook2 = await createHook(storage, testRunId, { hookId: 'hook_2', token: 'token-2', }); @@ -1473,7 +1473,7 @@ describe('Storage', () => { }); it('should respect resolveData option', async () => { - await storage.hooks.create(testRunId, { + await createHook(storage, testRunId, { hookId: 'hook_with_response', token: 'token-with-response', }); @@ -1501,28 +1501,1002 @@ describe('Storage', () => { expect(result.hasMore).toBe(false); }); }); + }); + + describe('step terminal state validation', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); - describe('dispose', () => { - it('should delete an existing hook', async () => { - const created = await storage.hooks.create(testRunId, { - hookId: 'hook_to_delete', - token: 'token-to-delete', + describe('completed step', () => { + it('should reject step_started on completed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_terminal_1', + stepName: 'test-step', + input: [], }); + await updateStep( + storage, + testRunId, + 'step_terminal_1', + 'step_completed', + { + result: 'done', + } + ); - const disposed = await storage.hooks.dispose('hook_to_delete'); + await expect( + updateStep(storage, testRunId, 'step_terminal_1', 'step_started') + ).rejects.toThrow(/terminal/i); + }); - expect(disposed).toEqual(created); + it('should reject step_completed on already completed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_terminal_2', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + testRunId, + 'step_terminal_2', + 'step_completed', + { + result: 'done', + } + ); - // Verify file was deleted await expect( - storage.hooks.getByToken('token-to-delete') - ).rejects.toThrow('Hook with token token-to-delete not found'); + updateStep(storage, testRunId, 'step_terminal_2', 'step_completed', { + result: 'done again', + }) + ).rejects.toThrow(/terminal/i); }); - it('should throw error for non-existent hook', async () => { - await expect(storage.hooks.dispose('hook_nonexistent')).rejects.toThrow( - 'Hook hook_nonexistent not found' + it('should reject step_failed on completed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_terminal_3', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + testRunId, + 'step_terminal_3', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep(storage, testRunId, 'step_terminal_3', 'step_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('failed step', () => { + it('should reject step_started on failed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_failed_1', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, testRunId, 'step_failed_1', 'step_failed', { + error: 'Failed permanently', + }); + + await expect( + updateStep(storage, testRunId, 'step_failed_1', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_completed on failed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_failed_2', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, testRunId, 'step_failed_2', 'step_failed', { + error: 'Failed permanently', + }); + + await expect( + updateStep(storage, testRunId, 'step_failed_2', 'step_completed', { + result: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_failed on already failed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_failed_3', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, testRunId, 'step_failed_3', 'step_failed', { + error: 'Failed once', + }); + + await expect( + updateStep(storage, testRunId, 'step_failed_3', 'step_failed', { + error: 'Failed again', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_retrying on failed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_failed_retry', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + testRunId, + 'step_failed_retry', + 'step_failed', + { + error: 'Failed permanently', + } ); + + await expect( + updateStep(storage, testRunId, 'step_failed_retry', 'step_retrying', { + error: 'Retry attempt', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('step_retrying validation', () => { + it('should reject step_retrying on completed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_completed_retry', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + testRunId, + 'step_completed_retry', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep( + storage, + testRunId, + 'step_completed_retry', + 'step_retrying', + { + error: 'Retry attempt', + } + ) + ).rejects.toThrow(/terminal/i); + }); + }); + }); + + describe('run terminal state validation', () => { + describe('completed run', () => { + it('should reject run_started on completed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_completed', { + output: 'done', + }); + + await expect( + updateRun(storage, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_failed on completed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_completed', { + output: 'done', + }); + + await expect( + updateRun(storage, run.runId, 'run_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_cancelled on completed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_completed', { + output: 'done', + }); + + await expect( + storage.events.create(run.runId, { eventType: 'run_cancelled' }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('failed run', () => { + it('should reject run_started on failed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + updateRun(storage, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_completed on failed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + updateRun(storage, run.runId, 'run_completed', { + output: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_cancelled on failed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + storage.events.create(run.runId, { eventType: 'run_cancelled' }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('cancelled run', () => { + it('should reject run_started on cancelled run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(storage, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_completed on cancelled run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(storage, run.runId, 'run_completed', { + output: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_failed on cancelled run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(storage, run.runId, 'run_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + }); + + describe('allowed operations on terminal runs', () => { + it('should allow step_completed on completed run for in-progress step', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step (making it in-progress) + await createStep(storage, run.runId, { + stepId: 'step_in_progress', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, run.runId, 'step_in_progress', 'step_started'); + + // Complete the run while step is still running + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + // Should succeed - completing an in-progress step on a terminal run is allowed + const result = await updateStep( + storage, + run.runId, + 'step_in_progress', + 'step_completed', + { result: 'step done' } + ); + expect(result.status).toBe('completed'); + }); + + it('should allow step_failed on completed run for in-progress step', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step + await createStep(storage, run.runId, { + stepId: 'step_in_progress_fail', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + run.runId, + 'step_in_progress_fail', + 'step_started' + ); + + // Complete the run + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + // Should succeed - failing an in-progress step on a terminal run is allowed + const result = await updateStep( + storage, + run.runId, + 'step_in_progress_fail', + 'step_failed', + { error: 'step failed' } + ); + expect(result.status).toBe('failed'); + }); + + it('should auto-delete hooks when run completes (world-local specific behavior)', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a hook + await createHook(storage, run.runId, { + hookId: 'hook_auto_delete', + token: 'test-token-auto-delete', + }); + + // Verify hook exists before completion + const hookBefore = await storage.hooks.get('hook_auto_delete'); + expect(hookBefore).toBeDefined(); + + // Complete the run - this auto-deletes hooks in world-local + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + // Hook should be auto-deleted + await expect(storage.hooks.get('hook_auto_delete')).rejects.toThrow( + /not found/i + ); + }); + }); + + describe('disallowed operations on terminal runs', () => { + it('should reject step_created on completed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + await expect( + createStep(storage, run.runId, { + stepId: 'new_step', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_started on completed run for pending step', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a step but don't start it + await createStep(storage, run.runId, { + stepId: 'pending_step', + stepName: 'test-step', + input: [], + }); + + // Complete the run + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + // Should reject - cannot start a pending step on a terminal run + await expect( + updateStep(storage, run.runId, 'pending_step', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on completed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + await expect( + createHook(storage, run.runId, { + hookId: 'new_hook', + token: 'new-token', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_created on failed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + createStep(storage, run.runId, { + stepId: 'new_step_failed', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_created on cancelled run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createStep(storage, run.runId, { + stepId: 'new_step_cancelled', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on failed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + createHook(storage, run.runId, { + hookId: 'new_hook_failed', + token: 'new-token-failed', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on cancelled run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createHook(storage, run.runId, { + hookId: 'new_hook_cancelled', + token: 'new-token-cancelled', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('idempotent operations', () => { + it('should allow run_cancelled on already cancelled run (idempotent)', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should succeed - idempotent operation + const result = await storage.events.create(run.runId, { + eventType: 'run_cancelled', + }); + expect(result.run?.status).toBe('cancelled'); + }); + }); + + describe('step_retrying event handling', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); + + it('should set step status to pending and record error', async () => { + await createStep(storage, testRunId, { + stepId: 'step_retry_1', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, testRunId, 'step_retry_1', 'step_started'); + + const result = await storage.events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_1', + eventData: { + error: 'Temporary failure', + retryAfter: new Date(Date.now() + 5000), + }, + }); + + expect(result.step?.status).toBe('pending'); + expect(result.step?.error?.message).toBe('Temporary failure'); + expect(result.step?.retryAfter).toBeInstanceOf(Date); + }); + + it('should increment attempt when step_started is called after step_retrying', async () => { + await createStep(storage, testRunId, { + stepId: 'step_retry_2', + stepName: 'test-step', + input: [], + }); + + // First attempt + const started1 = await updateStep( + storage, + testRunId, + 'step_retry_2', + 'step_started' + ); + expect(started1.attempt).toBe(1); + + // Retry + await storage.events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_2', + eventData: { error: 'Temporary failure' }, + }); + + // Second attempt + const started2 = await updateStep( + storage, + testRunId, + 'step_retry_2', + 'step_started' + ); + expect(started2.attempt).toBe(2); + }); + + it('should reject step_retrying on completed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_retry_completed', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + testRunId, + 'step_retry_completed', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + storage.events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_completed', + eventData: { error: 'Should not work' }, + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_retrying on failed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_retry_failed', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, testRunId, 'step_retry_failed', 'step_failed', { + error: 'Permanent failure', + }); + + await expect( + storage.events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_failed', + eventData: { error: 'Should not work' }, + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('run cancellation with in-flight entities', () => { + it('should allow in-progress step to complete after run cancelled', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step + await createStep(storage, run.runId, { + stepId: 'step_in_flight', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, run.runId, 'step_in_flight', 'step_started'); + + // Cancel the run + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should succeed - completing an in-progress step is allowed + const result = await updateStep( + storage, + run.runId, + 'step_in_flight', + 'step_completed', + { result: 'done' } + ); + expect(result.status).toBe('completed'); + }); + + it('should reject step_created after run cancelled', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createStep(storage, run.runId, { + stepId: 'new_step_after_cancel', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_started for pending step after run cancelled', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a step but don't start it + await createStep(storage, run.runId, { + stepId: 'pending_after_cancel', + stepName: 'test-step', + input: [], + }); + + // Cancel the run + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should reject - cannot start a pending step on a cancelled run + await expect( + updateStep(storage, run.runId, 'pending_after_cancel', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('event ordering validation', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); + + it('should reject step_completed before step_created', async () => { + await expect( + storage.events.create(testRunId, { + eventType: 'step_completed', + correlationId: 'nonexistent_step', + eventData: { result: 'done' }, + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject step_started before step_created', async () => { + await expect( + storage.events.create(testRunId, { + eventType: 'step_started', + correlationId: 'nonexistent_step_started', + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject step_failed before step_created', async () => { + await expect( + storage.events.create(testRunId, { + eventType: 'step_failed', + correlationId: 'nonexistent_step_failed', + eventData: { error: 'Failed' }, + }) + ).rejects.toThrow(/not found/i); + }); + + it('should allow step_completed without step_started (instant completion)', async () => { + await createStep(storage, testRunId, { + stepId: 'instant_complete', + stepName: 'test-step', + input: [], + }); + + // Should succeed - instant completion without starting + const result = await updateStep( + storage, + testRunId, + 'instant_complete', + 'step_completed', + { result: 'instant' } + ); + expect(result.status).toBe('completed'); + }); + + it('should reject hook_disposed before hook_created', async () => { + await expect( + storage.events.create(testRunId, { + eventType: 'hook_disposed', + correlationId: 'nonexistent_hook', + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject hook_received before hook_created', async () => { + await expect( + storage.events.create(testRunId, { + eventType: 'hook_received', + correlationId: 'nonexistent_hook_received', + eventData: { payload: {} }, + }) + ).rejects.toThrow(/not found/i); + }); + }); + + describe('legacy/backwards compatibility', () => { + // Helper to create a legacy run directly on disk (bypassing events.create) + async function createLegacyRun( + runId: string, + specVersion: number | undefined + ) { + const runsDir = path.join(testDir, 'runs'); + await fs.mkdir(runsDir, { recursive: true }); + const run = { + runId, + deploymentId: 'legacy-deployment', + workflowName: 'legacy-workflow', + specVersion, + status: 'running', + input: [], + createdAt: new Date(), + updatedAt: new Date(), + }; + await fs.writeFile( + path.join(runsDir, `${runId}.json`), + JSON.stringify(run, null, 2) + ); + return run; + } + + describe('legacy runs (specVersion < 2 or undefined)', () => { + it('should handle run_cancelled on legacy run with specVersion=1', async () => { + const runId = 'wrun_legacy_v1'; + await createLegacyRun(runId, 1); + + const result = await storage.events.create(runId, { + eventType: 'run_cancelled', + }); + + // Legacy behavior: run is updated but event is not stored + expect(result.run?.status).toBe('cancelled'); + expect(result.event).toBeUndefined(); + }); + + it('should handle run_cancelled on legacy run with specVersion=undefined', async () => { + const runId = 'wrun_legacy_undefined'; + await createLegacyRun(runId, undefined); + + const result = await storage.events.create(runId, { + eventType: 'run_cancelled', + }); + + // Legacy behavior: run is updated but event is not stored + expect(result.run?.status).toBe('cancelled'); + expect(result.event).toBeUndefined(); + }); + + it('should handle wait_completed on legacy run', async () => { + const runId = 'wrun_legacy_wait'; + await createLegacyRun(runId, 1); + + const result = await storage.events.create(runId, { + eventType: 'wait_completed', + correlationId: 'wait_123', + eventData: { result: 'waited' }, + } as any); + + // Legacy behavior: event is stored but no entity mutation + expect(result.event).toBeDefined(); + expect(result.event?.eventType).toBe('wait_completed'); + expect(result.run).toBeUndefined(); + }); + + it('should handle hook_received on legacy run', async () => { + const runId = 'wrun_legacy_hook_received'; + await createLegacyRun(runId, 1); + + const result = await storage.events.create(runId, { + eventType: 'hook_received', + correlationId: 'hook_123', + eventData: { payload: { data: 'test' } }, + } as any); + + // Legacy behavior: event is stored but no entity mutation + // (hooks exist via old system, not via events) + expect(result.event).toBeDefined(); + expect(result.event?.eventType).toBe('hook_received'); + expect(result.event?.correlationId).toBe('hook_123'); + expect(result.hook).toBeUndefined(); + }); + + it('should reject unsupported events on legacy runs', async () => { + const runId = 'wrun_legacy_unsupported'; + await createLegacyRun(runId, 1); + + // run_started is not supported for legacy runs + await expect( + storage.events.create(runId, { eventType: 'run_started' }) + ).rejects.toThrow(/not supported for legacy runs/i); + + // run_completed is not supported for legacy runs + await expect( + storage.events.create(runId, { + eventType: 'run_completed', + eventData: { output: 'done' }, + }) + ).rejects.toThrow(/not supported for legacy runs/i); + + // run_failed is not supported for legacy runs + await expect( + storage.events.create(runId, { + eventType: 'run_failed', + eventData: { error: 'failed' }, + }) + ).rejects.toThrow(/not supported for legacy runs/i); + }); + + it('should delete hooks when legacy run is cancelled', async () => { + const runId = 'wrun_legacy_hooks'; + await createLegacyRun(runId, 1); + + // Create a hook for this run (hooks can be created on legacy runs) + const hooksDir = path.join(testDir, 'hooks'); + await fs.mkdir(hooksDir, { recursive: true }); + await fs.writeFile( + path.join(hooksDir, 'hook_legacy.json'), + JSON.stringify({ + hookId: 'hook_legacy', + runId, + token: 'legacy-token', + ownerId: 'test-owner', + projectId: 'test-project', + environment: 'test', + createdAt: new Date(), + }) + ); + + // Verify hook exists + const hookBefore = await storage.hooks.get('hook_legacy'); + expect(hookBefore).toBeDefined(); + + // Cancel the legacy run + await storage.events.create(runId, { eventType: 'run_cancelled' }); + + // Hook should be deleted + await expect(storage.hooks.get('hook_legacy')).rejects.toThrow( + /not found/i + ); + }); + }); + + describe('newer runs (specVersion > current)', () => { + it('should reject events on runs with newer specVersion', async () => { + const runId = 'wrun_future'; + // Create a run with a future spec version (higher than current) + await createLegacyRun(runId, 999); + + await expect( + storage.events.create(runId, { eventType: 'run_started' }) + ).rejects.toThrow(/requires spec version 999/i); + }); + }); + + describe('current version runs', () => { + it('should process events normally for current specVersion runs', async () => { + // Create run via events.create (gets current specVersion) + const run = await createRun(storage, { + deploymentId: 'current-deployment', + workflowName: 'current-workflow', + input: [], + }); + + // Should work normally + const result = await storage.events.create(run.runId, { + eventType: 'run_started', + }); + + expect(result.run?.status).toBe('running'); + expect(result.event?.eventType).toBe('run_started'); }); }); }); diff --git a/packages/world-local/src/storage.ts b/packages/world-local/src/storage.ts index 7c0225fa26..d207f3fc9c 100644 --- a/packages/world-local/src/storage.ts +++ b/packages/world-local/src/storage.ts @@ -1,576 +1,11 @@ -import path from 'node:path'; -import { WorkflowRunNotFoundError } from '@workflow/errors'; -import { - type CreateHookRequest, - type Event, - EventSchema, - type GetHookParams, - type Hook, - HookSchema, - type ListHooksParams, - type PaginatedResponse, - type Step, - StepSchema, - type Storage, - type WorkflowRun, - WorkflowRunSchema, -} from '@workflow/world'; -import { monotonicFactory } from 'ulid'; -import { DEFAULT_RESOLVE_DATA_OPTION } from './config.js'; -import { - deleteJSON, - listJSONFiles, - paginatedFileSystemQuery, - readJSON, - ulidToDate, - writeJSON, -} from './fs.js'; - -// Create a monotonic ULID factory that ensures ULIDs are always increasing -// even when generated within the same millisecond -const monotonicUlid = monotonicFactory(() => Math.random()); - -// Helper functions to filter data based on resolveData setting -function filterRunData( - run: WorkflowRun, - resolveData: 'none' | 'all' -): WorkflowRun { - if (resolveData === 'none') { - return { - ...run, - input: [], - output: undefined, - }; - } - return run; -} - -function filterStepData(step: Step, resolveData: 'none' | 'all'): Step { - if (resolveData === 'none') { - return { - ...step, - input: [], - output: undefined, - }; - } - return step; -} - -function filterEventData(event: Event, resolveData: 'none' | 'all'): Event { - if (resolveData === 'none') { - const { eventData: _eventData, ...rest } = event as any; - return rest; - } - return event; -} - -function filterHookData(hook: Hook, resolveData: 'none' | 'all'): Hook { - if (resolveData === 'none') { - const { metadata: _metadata, ...rest } = hook as any; - return rest; - } - return hook; -} - -const getObjectCreatedAt = - (idPrefix: string) => - (filename: string): Date | null => { - const replaceRegex = new RegExp(`^${idPrefix}_`, 'g'); - const dashIndex = filename.indexOf('-'); - - if (dashIndex === -1) { - // No dash - extract ULID from the filename (e.g., wrun_ULID.json, evnt_ULID.json) - const ulid = filename.replace(/\.json$/, '').replace(replaceRegex, ''); - return ulidToDate(ulid); - } - - // For composite keys like {runId}-{stepId}, extract from the appropriate part - if (idPrefix === 'step') { - // Steps use sequential IDs (step_0, step_1, etc.) - no timestamp in filename. - // Return null to skip filename-based optimization and defer to JSON-based filtering. - return null; - } - - // For events: wrun_ULID-evnt_ULID.json - extract from the eventId part - const id = filename.substring(dashIndex + 1).replace(/\.json$/, ''); - const ulid = id.replace(replaceRegex, ''); - return ulidToDate(ulid); - }; - /** - * Creates a hooks storage implementation using the filesystem. - * Implements the Storage['hooks'] interface with hook CRUD operations. + * Filesystem-based storage implementation for workflow data. + * + * This module provides a complete Storage implementation that persists + * workflow runs, steps, events, and hooks to the local filesystem. + * + * @module */ -function createHooksStorage(basedir: string): Storage['hooks'] { - // Helper function to find a hook by token (shared between create and getByToken) - async function findHookByToken(token: string): Promise { - const hooksDir = path.join(basedir, 'hooks'); - const files = await listJSONFiles(hooksDir); - - for (const file of files) { - const hookPath = path.join(hooksDir, `${file}.json`); - const hook = await readJSON(hookPath, HookSchema); - if (hook && hook.token === token) { - return hook; - } - } - - return null; - } - - async function create(runId: string, data: CreateHookRequest): Promise { - // Check if a hook with the same token already exists - // Token uniqueness is enforced globally per local environment - const existingHook = await findHookByToken(data.token); - if (existingHook) { - throw new Error( - `Hook with token ${data.token} already exists for this project` - ); - } - - const now = new Date(); - - const result = { - runId, - hookId: data.hookId, - token: data.token, - metadata: data.metadata, - ownerId: 'local-owner', - projectId: 'local-project', - environment: 'local', - createdAt: now, - } as Hook; - - const hookPath = path.join(basedir, 'hooks', `${data.hookId}.json`); - HookSchema.parse(result); - await writeJSON(hookPath, result); - return result; - } - - async function get(hookId: string, params?: GetHookParams): Promise { - const hookPath = path.join(basedir, 'hooks', `${hookId}.json`); - const hook = await readJSON(hookPath, HookSchema); - if (!hook) { - throw new Error(`Hook ${hookId} not found`); - } - const resolveData = params?.resolveData || DEFAULT_RESOLVE_DATA_OPTION; - return filterHookData(hook, resolveData); - } - - async function getByToken(token: string): Promise { - const hook = await findHookByToken(token); - if (!hook) { - throw new Error(`Hook with token ${token} not found`); - } - return hook; - } - - async function list( - params: ListHooksParams - ): Promise> { - const hooksDir = path.join(basedir, 'hooks'); - const resolveData = params.resolveData || DEFAULT_RESOLVE_DATA_OPTION; - - const result = await paginatedFileSystemQuery({ - directory: hooksDir, - schema: HookSchema, - sortOrder: params.pagination?.sortOrder, - limit: params.pagination?.limit, - cursor: params.pagination?.cursor, - filePrefix: undefined, // Hooks don't have ULIDs, so we can't optimize by filename - filter: (hook) => { - // Filter by runId if provided - if (params.runId && hook.runId !== params.runId) { - return false; - } - return true; - }, - getCreatedAt: () => { - // Hook files don't have ULID timestamps in filename - // We need to read the file to get createdAt, but that's inefficient - // So we return the hook's createdAt directly (item.createdAt will be used for sorting) - // Return a dummy date to pass the null check, actual sorting uses item.createdAt - return new Date(0); - }, - getId: (hook) => hook.hookId, - }); - - // Transform the data after pagination - return { - ...result, - data: result.data.map((hook) => filterHookData(hook, resolveData)), - }; - } - - async function dispose(hookId: string): Promise { - const hookPath = path.join(basedir, 'hooks', `${hookId}.json`); - const hook = await readJSON(hookPath, HookSchema); - if (!hook) { - throw new Error(`Hook ${hookId} not found`); - } - await deleteJSON(hookPath); - return hook; - } - - return { create, get, getByToken, list, dispose }; -} - -/** - * Helper function to delete all hooks associated with a workflow run - */ -async function deleteAllHooksForRun( - basedir: string, - runId: string -): Promise { - const hooksDir = path.join(basedir, 'hooks'); - const files = await listJSONFiles(hooksDir); - - for (const file of files) { - const hookPath = path.join(hooksDir, `${file}.json`); - const hook = await readJSON(hookPath, HookSchema); - if (hook && hook.runId === runId) { - await deleteJSON(hookPath); - } - } -} - -export function createStorage(basedir: string): Storage { - return { - runs: { - async create(data) { - const runId = `wrun_${monotonicUlid()}`; - const now = new Date(); - - const result: WorkflowRun = { - runId, - deploymentId: data.deploymentId, - status: 'pending', - workflowName: data.workflowName, - executionContext: data.executionContext as - | Record - | undefined, - input: (data.input as any[]) || [], - output: undefined, - error: undefined, - startedAt: undefined, - completedAt: undefined, - createdAt: now, - updatedAt: now, - }; - - const runPath = path.join(basedir, 'runs', `${runId}.json`); - WorkflowRunSchema.parse(result); - await writeJSON(runPath, result); - return result; - }, - - async get(id, params) { - const runPath = path.join(basedir, 'runs', `${id}.json`); - const run = await readJSON(runPath, WorkflowRunSchema); - if (!run) { - throw new WorkflowRunNotFoundError(id); - } - const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - return filterRunData(run, resolveData); - }, - - /** - * Updates a workflow run. - * - * Note: This operation is not atomic. Concurrent updates from multiple - * processes may result in lost updates (last writer wins). This is an - * inherent limitation of filesystem-based storage without locking. - * For the local world, this is acceptable as it's typically - * used in single-process scenarios. - */ - async update(id, data) { - const runPath = path.join(basedir, 'runs', `${id}.json`); - const run = await readJSON(runPath, WorkflowRunSchema); - if (!run) { - throw new WorkflowRunNotFoundError(id); - } - - const now = new Date(); - const updatedRun = { - ...run, - ...data, - updatedAt: now, - } as WorkflowRun; - - // Only set startedAt the first time the run transitions to 'running' - if (data.status === 'running' && !updatedRun.startedAt) { - updatedRun.startedAt = now; - } - - const isBecomingTerminal = - data.status === 'completed' || - data.status === 'failed' || - data.status === 'cancelled'; - - if (isBecomingTerminal) { - updatedRun.completedAt = now; - } - - WorkflowRunSchema.parse(updatedRun); - await writeJSON(runPath, updatedRun, { overwrite: true }); - - // If transitioning to a terminal status, clean up all hooks for this run - if (isBecomingTerminal) { - await deleteAllHooksForRun(basedir, id); - } - - return updatedRun; - }, - - async list(params) { - const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - const result = await paginatedFileSystemQuery({ - directory: path.join(basedir, 'runs'), - schema: WorkflowRunSchema, - filter: (run) => { - if ( - params?.workflowName && - run.workflowName !== params.workflowName - ) { - return false; - } - if (params?.status && run.status !== params.status) { - return false; - } - return true; - }, - sortOrder: params?.pagination?.sortOrder ?? 'desc', - limit: params?.pagination?.limit, - cursor: params?.pagination?.cursor, - getCreatedAt: getObjectCreatedAt('wrun'), - getId: (run) => run.runId, - }); - - // If resolveData is "none", replace input/output with empty data - if (resolveData === 'none') { - return { - ...result, - data: result.data.map((run) => ({ - ...run, - input: [], - output: undefined, - })), - }; - } - - return result; - }, - - async cancel(id, params) { - // This will call update which triggers hook cleanup automatically - const run = await this.update(id, { status: 'cancelled' }); - const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - return filterRunData(run, resolveData); - }, - }, - - steps: { - async create(runId, data) { - const now = new Date(); - - const result: Step = { - runId, - stepId: data.stepId, - stepName: data.stepName, - status: 'pending', - input: data.input as any[], - output: undefined, - error: undefined, - attempt: 0, - startedAt: undefined, - completedAt: undefined, - createdAt: now, - updatedAt: now, - }; - - const compositeKey = `${runId}-${data.stepId}`; - const stepPath = path.join(basedir, 'steps', `${compositeKey}.json`); - StepSchema.parse(result); - await writeJSON(stepPath, result); - - return result; - }, - - async get( - runId: string | undefined, - stepId: string, - params - ): Promise { - if (!runId) { - const fileIds = await listJSONFiles(path.join(basedir, 'steps')); - const fileId = fileIds.find((fileId) => - fileId.endsWith(`-${stepId}`) - ); - if (!fileId) { - throw new Error(`Step ${stepId} not found`); - } - runId = fileId.split('-')[0]; - } - const compositeKey = `${runId}-${stepId}`; - const stepPath = path.join(basedir, 'steps', `${compositeKey}.json`); - const step = await readJSON(stepPath, StepSchema); - if (!step) { - throw new Error(`Step ${stepId} in run ${runId} not found`); - } - const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - return filterStepData(step, resolveData); - }, - - /** - * Updates a step. - * - * Note: This operation is not atomic. Concurrent updates from multiple - * processes may result in lost updates (last writer wins). This is an - * inherent limitation of filesystem-based storage without locking. - */ - async update(runId, stepId, data) { - const compositeKey = `${runId}-${stepId}`; - const stepPath = path.join(basedir, 'steps', `${compositeKey}.json`); - const step = await readJSON(stepPath, StepSchema); - if (!step) { - throw new Error(`Step ${stepId} in run ${runId} not found`); - } - - const now = new Date(); - const updatedStep: Step = { - ...step, - ...data, - updatedAt: now, - }; - - // Only set startedAt the first time the step transitions to 'running' - if (data.status === 'running' && !updatedStep.startedAt) { - updatedStep.startedAt = now; - } - if (data.status === 'completed' || data.status === 'failed') { - updatedStep.completedAt = now; - } - - StepSchema.parse(updatedStep); - await writeJSON(stepPath, updatedStep, { overwrite: true }); - return updatedStep; - }, - - async list(params) { - const resolveData = params.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - const result = await paginatedFileSystemQuery({ - directory: path.join(basedir, 'steps'), - schema: StepSchema, - filePrefix: `${params.runId}-`, - sortOrder: params.pagination?.sortOrder ?? 'desc', - limit: params.pagination?.limit, - cursor: params.pagination?.cursor, - getCreatedAt: getObjectCreatedAt('step'), - getId: (step) => step.stepId, - }); - - // If resolveData is "none", replace input/output with empty data - if (resolveData === 'none') { - return { - ...result, - data: result.data.map((step) => ({ - ...step, - input: [], - output: undefined, - })), - }; - } - - return result; - }, - }, - - // Events - filesystem-backed storage - events: { - async create(runId, data, params) { - const eventId = `evnt_${monotonicUlid()}`; - const now = new Date(); - - const result: Event = { - ...data, - runId, - eventId, - createdAt: now, - }; - - // Store event using composite key {runId}-{eventId} - const compositeKey = `${runId}-${eventId}`; - const eventPath = path.join(basedir, 'events', `${compositeKey}.json`); - EventSchema.parse(result); - await writeJSON(eventPath, result); - - const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - return filterEventData(result, resolveData); - }, - - async list(params) { - const { runId } = params; - const resolveData = params.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - const result = await paginatedFileSystemQuery({ - directory: path.join(basedir, 'events'), - schema: EventSchema, - filePrefix: `${runId}-`, - // Events in chronological order (oldest first) by default, - // different from the default for other list calls. - sortOrder: params.pagination?.sortOrder ?? 'asc', - limit: params.pagination?.limit, - cursor: params.pagination?.cursor, - getCreatedAt: getObjectCreatedAt('evnt'), - getId: (event) => event.eventId, - }); - - // If resolveData is "none", remove eventData from events - if (resolveData === 'none') { - return { - ...result, - data: result.data.map((event) => { - const { eventData: _eventData, ...rest } = event as any; - return rest; - }), - }; - } - - return result; - }, - - async listByCorrelationId(params) { - const correlationId = params.correlationId; - const resolveData = params.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - const result = await paginatedFileSystemQuery({ - directory: path.join(basedir, 'events'), - schema: EventSchema, - // No filePrefix - search all events - filter: (event) => event.correlationId === correlationId, - // Events in chronological order (oldest first) by default, - // different from the default for other list calls. - sortOrder: params.pagination?.sortOrder ?? 'asc', - limit: params.pagination?.limit, - cursor: params.pagination?.cursor, - getCreatedAt: getObjectCreatedAt('evnt'), - getId: (event) => event.eventId, - }); - - // If resolveData is "none", remove eventData from events - if (resolveData === 'none') { - return { - ...result, - data: result.data.map((event) => { - const { eventData: _eventData, ...rest } = event as any; - return rest; - }), - }; - } - - return result; - }, - }, - // Hooks - hooks: createHooksStorage(basedir), - }; -} +// Re-export from the modular storage implementation +export { createStorage } from './storage/index.js'; diff --git a/packages/world-local/src/storage/events-storage.ts b/packages/world-local/src/storage/events-storage.ts new file mode 100644 index 0000000000..a44c13461b --- /dev/null +++ b/packages/world-local/src/storage/events-storage.ts @@ -0,0 +1,676 @@ +import path from 'node:path'; +import { RunNotSupportedError, WorkflowAPIError } from '@workflow/errors'; +import type { + Event, + EventResult, + Hook, + Step, + Storage, + WorkflowRun, +} from '@workflow/world'; +import { + EventSchema, + HookSchema, + isLegacySpecVersion, + requiresNewerWorld, + SPEC_VERSION_CURRENT, + StepSchema, + WorkflowRunSchema, +} from '@workflow/world'; +import { DEFAULT_RESOLVE_DATA_OPTION } from '../config.js'; +import { + deleteJSON, + listJSONFiles, + paginatedFileSystemQuery, + readJSON, + writeJSON, +} from '../fs.js'; +import { filterEventData } from './filters.js'; +import { getObjectCreatedAt, monotonicUlid } from './helpers.js'; +import { deleteAllHooksForRun } from './hooks-storage.js'; +import { handleLegacyEvent } from './legacy.js'; + +/** + * Creates the events storage implementation using the filesystem. + * Implements the Storage['events'] interface with create, list, and listByCorrelationId operations. + */ +export function createEventsStorage(basedir: string): Storage['events'] { + return { + async create(runId, data, params): Promise { + const eventId = `evnt_${monotonicUlid()}`; + const now = new Date(); + + // For run_created events, generate runId server-side if null or empty + let effectiveRunId: string; + if (data.eventType === 'run_created' && (!runId || runId === '')) { + effectiveRunId = `wrun_${monotonicUlid()}`; + } else if (!runId) { + throw new Error('runId is required for non-run_created events'); + } else { + effectiveRunId = runId; + } + + // specVersion is always sent by the runtime, but we provide a fallback for safety + const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + + // Helper to check if run is in terminal state + const isRunTerminal = (status: string) => + ['completed', 'failed', 'cancelled'].includes(status); + + // Helper to check if step is in terminal state + const isStepTerminal = (status: string) => + ['completed', 'failed'].includes(status); + + // Get current run state for validation (if not creating a new run) + // Skip run validation for step_completed and step_retrying - they only operate + // on running steps, and running steps are always allowed to modify regardless + // of run state. This optimization saves filesystem reads per step event. + let currentRun: WorkflowRun | null = null; + const skipRunValidationEvents = ['step_completed', 'step_retrying']; + if ( + data.eventType !== 'run_created' && + !skipRunValidationEvents.includes(data.eventType) + ) { + const runPath = path.join(basedir, 'runs', `${effectiveRunId}.json`); + currentRun = await readJSON(runPath, WorkflowRunSchema); + } + + // ============================================================ + // VERSION COMPATIBILITY: Check run spec version + // ============================================================ + // For events that have fetched the run, check version compatibility. + // Skip for run_created (no existing run) and runtime events (step_completed, step_retrying). + if (currentRun) { + // Check if run requires a newer world version + if (requiresNewerWorld(currentRun.specVersion)) { + throw new RunNotSupportedError( + currentRun.specVersion!, + SPEC_VERSION_CURRENT + ); + } + + // Route to legacy handler for pre-event-sourcing runs + if (isLegacySpecVersion(currentRun.specVersion)) { + return handleLegacyEvent( + basedir, + effectiveRunId, + data, + currentRun, + params + ); + } + } + + // ============================================================ + // VALIDATION: Terminal state and event ordering checks + // ============================================================ + + // Run terminal state validation + if (currentRun && isRunTerminal(currentRun.status)) { + const runTerminalEvents = [ + 'run_started', + 'run_completed', + 'run_failed', + ]; + + // Idempotent operation: run_cancelled on already cancelled run is allowed + if ( + data.eventType === 'run_cancelled' && + currentRun.status === 'cancelled' + ) { + // Return existing state (idempotent) + const event: Event = { + ...data, + runId: effectiveRunId, + eventId, + createdAt: now, + specVersion: effectiveSpecVersion, + }; + const compositeKey = `${effectiveRunId}-${eventId}`; + const eventPath = path.join( + basedir, + 'events', + `${compositeKey}.json` + ); + await writeJSON(eventPath, event); + const resolveData = + params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + return { + event: filterEventData(event, resolveData), + run: currentRun, + }; + } + + // Run state transitions are not allowed on terminal runs + if ( + runTerminalEvents.includes(data.eventType) || + data.eventType === 'run_cancelled' + ) { + throw new WorkflowAPIError( + `Cannot transition run from terminal state "${currentRun.status}"`, + { status: 410 } + ); + } + + // Creating new entities on terminal runs is not allowed + if ( + data.eventType === 'step_created' || + data.eventType === 'hook_created' + ) { + throw new WorkflowAPIError( + `Cannot create new entities on run in terminal state "${currentRun.status}"`, + { status: 410 } + ); + } + } + + // Step-related event validation (ordering and terminal state) + // Store existingStep so we can reuse it later (avoid double read) + let validatedStep: Step | null = null; + const stepEvents = [ + 'step_started', + 'step_completed', + 'step_failed', + 'step_retrying', + ]; + if (stepEvents.includes(data.eventType) && data.correlationId) { + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + validatedStep = await readJSON(stepPath, StepSchema); + + // Event ordering: step must exist before these events + if (!validatedStep) { + throw new WorkflowAPIError(`Step "${data.correlationId}" not found`, { + status: 404, + }); + } + + // Step terminal state validation + if (isStepTerminal(validatedStep.status)) { + throw new WorkflowAPIError( + `Cannot modify step in terminal state "${validatedStep.status}"`, + { status: 410 } + ); + } + + // On terminal runs: only allow completing/failing in-progress steps + if (currentRun && isRunTerminal(currentRun.status)) { + if (validatedStep.status !== 'running') { + throw new WorkflowAPIError( + `Cannot modify non-running step on run in terminal state "${currentRun.status}"`, + { status: 410 } + ); + } + } + } + + // Hook-related event validation (ordering) + const hookEventsRequiringExistence = ['hook_disposed', 'hook_received']; + if ( + hookEventsRequiringExistence.includes(data.eventType) && + data.correlationId + ) { + const hookPath = path.join( + basedir, + 'hooks', + `${data.correlationId}.json` + ); + const existingHook = await readJSON(hookPath, HookSchema); + + if (!existingHook) { + throw new WorkflowAPIError(`Hook "${data.correlationId}" not found`, { + status: 404, + }); + } + } + const event: Event = { + ...data, + runId: effectiveRunId, + eventId, + createdAt: now, + specVersion: effectiveSpecVersion, + }; + + // Track entity created/updated for EventResult + let run: WorkflowRun | undefined; + let step: Step | undefined; + let hook: Hook | undefined; + + // Create/update entity based on event type (event-sourced architecture) + // Run lifecycle events + if (data.eventType === 'run_created' && 'eventData' in data) { + const runData = data.eventData as { + deploymentId: string; + workflowName: string; + input: any[]; + executionContext?: Record; + }; + run = { + runId: effectiveRunId, + deploymentId: runData.deploymentId, + status: 'pending', + workflowName: runData.workflowName, + // Propagate specVersion from the event to the run entity + specVersion: effectiveSpecVersion, + executionContext: runData.executionContext, + input: runData.input || [], + output: undefined, + error: undefined, + startedAt: undefined, + completedAt: undefined, + createdAt: now, + updatedAt: now, + }; + const runPath = path.join(basedir, 'runs', `${effectiveRunId}.json`); + await writeJSON(runPath, run); + } else if (data.eventType === 'run_started') { + // Reuse currentRun from validation (already read above) + if (currentRun) { + const runPath = path.join(basedir, 'runs', `${effectiveRunId}.json`); + run = { + runId: currentRun.runId, + deploymentId: currentRun.deploymentId, + workflowName: currentRun.workflowName, + specVersion: currentRun.specVersion, + executionContext: currentRun.executionContext, + input: currentRun.input, + createdAt: currentRun.createdAt, + expiredAt: currentRun.expiredAt, + status: 'running', + output: undefined, + error: undefined, + completedAt: undefined, + startedAt: currentRun.startedAt ?? now, + updatedAt: now, + }; + await writeJSON(runPath, run, { overwrite: true }); + } + } else if (data.eventType === 'run_completed' && 'eventData' in data) { + const completedData = data.eventData as { output?: any }; + // Reuse currentRun from validation (already read above) + if (currentRun) { + const runPath = path.join(basedir, 'runs', `${effectiveRunId}.json`); + run = { + runId: currentRun.runId, + deploymentId: currentRun.deploymentId, + workflowName: currentRun.workflowName, + specVersion: currentRun.specVersion, + executionContext: currentRun.executionContext, + input: currentRun.input, + createdAt: currentRun.createdAt, + expiredAt: currentRun.expiredAt, + startedAt: currentRun.startedAt, + status: 'completed', + output: completedData.output, + error: undefined, + completedAt: now, + updatedAt: now, + }; + await writeJSON(runPath, run, { overwrite: true }); + await deleteAllHooksForRun(basedir, effectiveRunId); + } + } else if (data.eventType === 'run_failed' && 'eventData' in data) { + const failedData = data.eventData as { + error: any; + errorCode?: string; + }; + // Reuse currentRun from validation (already read above) + if (currentRun) { + const runPath = path.join(basedir, 'runs', `${effectiveRunId}.json`); + run = { + runId: currentRun.runId, + deploymentId: currentRun.deploymentId, + workflowName: currentRun.workflowName, + specVersion: currentRun.specVersion, + executionContext: currentRun.executionContext, + input: currentRun.input, + createdAt: currentRun.createdAt, + expiredAt: currentRun.expiredAt, + startedAt: currentRun.startedAt, + status: 'failed', + output: undefined, + error: { + message: + typeof failedData.error === 'string' + ? failedData.error + : (failedData.error?.message ?? 'Unknown error'), + stack: failedData.error?.stack, + code: failedData.errorCode, + }, + completedAt: now, + updatedAt: now, + }; + await writeJSON(runPath, run, { overwrite: true }); + await deleteAllHooksForRun(basedir, effectiveRunId); + } + } else if (data.eventType === 'run_cancelled') { + // Reuse currentRun from validation (already read above) + if (currentRun) { + const runPath = path.join(basedir, 'runs', `${effectiveRunId}.json`); + run = { + runId: currentRun.runId, + deploymentId: currentRun.deploymentId, + workflowName: currentRun.workflowName, + specVersion: currentRun.specVersion, + executionContext: currentRun.executionContext, + input: currentRun.input, + createdAt: currentRun.createdAt, + expiredAt: currentRun.expiredAt, + startedAt: currentRun.startedAt, + status: 'cancelled', + output: undefined, + error: undefined, + completedAt: now, + updatedAt: now, + }; + await writeJSON(runPath, run, { overwrite: true }); + await deleteAllHooksForRun(basedir, effectiveRunId); + } + } else if ( + // Step lifecycle events + data.eventType === 'step_created' && + 'eventData' in data + ) { + // step_created: Creates step entity with status 'pending', attempt=0, createdAt set + const stepData = data.eventData as { + stepName: string; + input: any; + }; + step = { + runId: effectiveRunId, + stepId: data.correlationId, + stepName: stepData.stepName, + status: 'pending', + input: stepData.input, + output: undefined, + error: undefined, + attempt: 0, + startedAt: undefined, + completedAt: undefined, + createdAt: now, + updatedAt: now, + // Propagate specVersion from the event to the step entity + specVersion: effectiveSpecVersion, + }; + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + await writeJSON(stepPath, step); + } else if (data.eventType === 'step_started') { + // step_started: Increments attempt, sets status to 'running' + // Sets startedAt only on the first start (not updated on retries) + // Reuse validatedStep from validation (already read above) + if (validatedStep) { + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + step = { + ...validatedStep, + status: 'running', + // Only set startedAt on the first start + startedAt: validatedStep.startedAt ?? now, + // Increment attempt counter on every start + attempt: validatedStep.attempt + 1, + updatedAt: now, + }; + await writeJSON(stepPath, step, { overwrite: true }); + } + } else if (data.eventType === 'step_completed' && 'eventData' in data) { + // step_completed: Terminal state with output + // Reuse validatedStep from validation (already read above) + const completedData = data.eventData as { result: any }; + if (validatedStep) { + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + step = { + ...validatedStep, + status: 'completed', + output: completedData.result, + completedAt: now, + updatedAt: now, + }; + await writeJSON(stepPath, step, { overwrite: true }); + } + } else if (data.eventType === 'step_failed' && 'eventData' in data) { + // step_failed: Terminal state with error + // Reuse validatedStep from validation (already read above) + const failedData = data.eventData as { + error: any; + stack?: string; + }; + if (validatedStep) { + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + const error = { + message: + typeof failedData.error === 'string' + ? failedData.error + : (failedData.error?.message ?? 'Unknown error'), + stack: failedData.stack, + }; + step = { + ...validatedStep, + status: 'failed', + error, + completedAt: now, + updatedAt: now, + }; + await writeJSON(stepPath, step, { overwrite: true }); + } + } else if (data.eventType === 'step_retrying' && 'eventData' in data) { + // step_retrying: Sets status back to 'pending', records error + // Reuse validatedStep from validation (already read above) + const retryData = data.eventData as { + error: any; + stack?: string; + retryAfter?: Date; + }; + if (validatedStep) { + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + step = { + ...validatedStep, + status: 'pending', + error: { + message: + typeof retryData.error === 'string' + ? retryData.error + : (retryData.error?.message ?? 'Unknown error'), + stack: retryData.stack, + }, + retryAfter: retryData.retryAfter, + updatedAt: now, + }; + await writeJSON(stepPath, step, { overwrite: true }); + } + } else if ( + // Hook lifecycle events + data.eventType === 'hook_created' && + 'eventData' in data + ) { + const hookData = data.eventData as { + token: string; + metadata?: any; + }; + + // Check for duplicate token before creating hook + const hooksDir = path.join(basedir, 'hooks'); + const hookFiles = await listJSONFiles(hooksDir); + let hasConflict = false; + for (const file of hookFiles) { + const existingHookPath = path.join(hooksDir, `${file}.json`); + const existingHook = await readJSON(existingHookPath, HookSchema); + if (existingHook && existingHook.token === hookData.token) { + hasConflict = true; + break; + } + } + + if (hasConflict) { + // Create hook_conflict event instead of hook_created + // This allows the workflow to continue and fail gracefully when the hook is awaited + const conflictEvent: Event = { + eventType: 'hook_conflict', + correlationId: data.correlationId, + eventData: { + token: hookData.token, + }, + runId: effectiveRunId, + eventId, + createdAt: now, + specVersion: effectiveSpecVersion, + }; + + // Store the conflict event + const compositeKey = `${effectiveRunId}-${eventId}`; + const eventPath = path.join( + basedir, + 'events', + `${compositeKey}.json` + ); + await writeJSON(eventPath, conflictEvent); + + const resolveData = + params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const filteredEvent = filterEventData(conflictEvent, resolveData); + + // Return EventResult with conflict event (no hook entity created) + return { + event: filteredEvent, + run, + step, + hook: undefined, + }; + } + + hook = { + runId: effectiveRunId, + hookId: data.correlationId, + token: hookData.token, + metadata: hookData.metadata, + ownerId: 'local-owner', + projectId: 'local-project', + environment: 'local', + createdAt: now, + // Propagate specVersion from the event to the hook entity + specVersion: effectiveSpecVersion, + }; + const hookPath = path.join( + basedir, + 'hooks', + `${data.correlationId}.json` + ); + await writeJSON(hookPath, hook); + } else if (data.eventType === 'hook_disposed') { + // Delete the hook when disposed + const hookPath = path.join( + basedir, + 'hooks', + `${data.correlationId}.json` + ); + await deleteJSON(hookPath); + } + // Note: hook_received events are stored in the event log but don't + // modify the Hook entity (which doesn't have a payload field) + + // Store event using composite key {runId}-{eventId} + const compositeKey = `${effectiveRunId}-${eventId}`; + const eventPath = path.join(basedir, 'events', `${compositeKey}.json`); + await writeJSON(eventPath, event); + + const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const filteredEvent = filterEventData(event, resolveData); + + // Return EventResult with event and any created/updated entity + return { + event: filteredEvent, + run, + step, + hook, + }; + }, + + async list(params) { + const { runId } = params; + const resolveData = params.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const result = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'events'), + schema: EventSchema, + filePrefix: `${runId}-`, + // Events in chronological order (oldest first) by default, + // different from the default for other list calls. + sortOrder: params.pagination?.sortOrder ?? 'asc', + limit: params.pagination?.limit, + cursor: params.pagination?.cursor, + getCreatedAt: getObjectCreatedAt('evnt'), + getId: (event) => event.eventId, + }); + + // If resolveData is "none", remove eventData from events + if (resolveData === 'none') { + return { + ...result, + data: result.data.map((event) => { + const { eventData: _eventData, ...rest } = event as any; + return rest; + }), + }; + } + + return result; + }, + + async listByCorrelationId(params) { + const correlationId = params.correlationId; + const resolveData = params.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const result = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'events'), + schema: EventSchema, + // No filePrefix - search all events + filter: (event) => event.correlationId === correlationId, + // Events in chronological order (oldest first) by default, + // different from the default for other list calls. + sortOrder: params.pagination?.sortOrder ?? 'asc', + limit: params.pagination?.limit, + cursor: params.pagination?.cursor, + getCreatedAt: getObjectCreatedAt('evnt'), + getId: (event) => event.eventId, + }); + + // If resolveData is "none", remove eventData from events + if (resolveData === 'none') { + return { + ...result, + data: result.data.map((event) => { + const { eventData: _eventData, ...rest } = event as any; + return rest; + }), + }; + } + + return result; + }, + }; +} diff --git a/packages/world-local/src/storage/filters.ts b/packages/world-local/src/storage/filters.ts new file mode 100644 index 0000000000..92df88aec6 --- /dev/null +++ b/packages/world-local/src/storage/filters.ts @@ -0,0 +1,61 @@ +import type { Event, Hook, Step, WorkflowRun } from '@workflow/world'; + +/** + * Filter run data based on resolveData setting. + * When resolveData is 'none', strips input/output to reduce payload size. + */ +export function filterRunData( + run: WorkflowRun, + resolveData: 'none' | 'all' +): WorkflowRun { + if (resolveData === 'none') { + return { + ...run, + input: [], + output: undefined, + }; + } + return run; +} + +/** + * Filter step data based on resolveData setting. + * When resolveData is 'none', strips input/output to reduce payload size. + */ +export function filterStepData(step: Step, resolveData: 'none' | 'all'): Step { + if (resolveData === 'none') { + return { + ...step, + input: [], + output: undefined, + }; + } + return step; +} + +/** + * Filter event data based on resolveData setting. + * When resolveData is 'none', strips eventData to reduce payload size. + */ +export function filterEventData( + event: Event, + resolveData: 'none' | 'all' +): Event { + if (resolveData === 'none') { + const { eventData: _eventData, ...rest } = event as any; + return rest; + } + return event; +} + +/** + * Filter hook data based on resolveData setting. + * When resolveData is 'none', strips metadata to reduce payload size. + */ +export function filterHookData(hook: Hook, resolveData: 'none' | 'all'): Hook { + if (resolveData === 'none') { + const { metadata: _metadata, ...rest } = hook as any; + return rest; + } + return hook; +} diff --git a/packages/world-local/src/storage/helpers.ts b/packages/world-local/src/storage/helpers.ts new file mode 100644 index 0000000000..34fd450b0a --- /dev/null +++ b/packages/world-local/src/storage/helpers.ts @@ -0,0 +1,40 @@ +import { monotonicFactory } from 'ulid'; +import { ulidToDate } from '../fs.js'; + +/** + * Create a monotonic ULID factory that ensures ULIDs are always increasing + * even when generated within the same millisecond. + */ +export const monotonicUlid = monotonicFactory(() => Math.random()); + +/** + * Creates a function to extract createdAt date from a filename based on ULID. + * Used for efficient pagination without reading file contents. + * + * @param idPrefix - The prefix to strip from filenames (e.g., 'wrun', 'evnt', 'step') + * @returns A function that extracts Date from filename, or null if not extractable + */ +export const getObjectCreatedAt = + (idPrefix: string) => + (filename: string): Date | null => { + const replaceRegex = new RegExp(`^${idPrefix}_`, 'g'); + const dashIndex = filename.indexOf('-'); + + if (dashIndex === -1) { + // No dash - extract ULID from the filename (e.g., wrun_ULID.json, evnt_ULID.json) + const ulid = filename.replace(/\.json$/, '').replace(replaceRegex, ''); + return ulidToDate(ulid); + } + + // For composite keys like {runId}-{stepId}, extract from the appropriate part + if (idPrefix === 'step') { + // Steps use sequential IDs (step_0, step_1, etc.) - no timestamp in filename. + // Return null to skip filename-based optimization and defer to JSON-based filtering. + return null; + } + + // For events: wrun_ULID-evnt_ULID.json - extract from the eventId part + const id = filename.substring(dashIndex + 1).replace(/\.json$/, ''); + const ulid = id.replace(replaceRegex, ''); + return ulidToDate(ulid); + }; diff --git a/packages/world-local/src/storage/hooks-storage.ts b/packages/world-local/src/storage/hooks-storage.ts new file mode 100644 index 0000000000..95522dd349 --- /dev/null +++ b/packages/world-local/src/storage/hooks-storage.ts @@ -0,0 +1,116 @@ +import path from 'node:path'; +import type { + GetHookParams, + Hook, + ListHooksParams, + PaginatedResponse, + Storage, +} from '@workflow/world'; +import { HookSchema } from '@workflow/world'; +import { DEFAULT_RESOLVE_DATA_OPTION } from '../config.js'; +import { + deleteJSON, + listJSONFiles, + paginatedFileSystemQuery, + readJSON, +} from '../fs.js'; +import { filterHookData } from './filters.js'; + +/** + * Creates a hooks storage implementation using the filesystem. + * Implements the Storage['hooks'] interface with hook CRUD operations. + */ +export function createHooksStorage(basedir: string): Storage['hooks'] { + // Helper function to find a hook by token (shared between getByToken) + async function findHookByToken(token: string): Promise { + const hooksDir = path.join(basedir, 'hooks'); + const files = await listJSONFiles(hooksDir); + + for (const file of files) { + const hookPath = path.join(hooksDir, `${file}.json`); + const hook = await readJSON(hookPath, HookSchema); + if (hook && hook.token === token) { + return hook; + } + } + + return null; + } + + async function get(hookId: string, params?: GetHookParams): Promise { + const hookPath = path.join(basedir, 'hooks', `${hookId}.json`); + const hook = await readJSON(hookPath, HookSchema); + if (!hook) { + throw new Error(`Hook ${hookId} not found`); + } + const resolveData = params?.resolveData || DEFAULT_RESOLVE_DATA_OPTION; + return filterHookData(hook, resolveData); + } + + async function getByToken(token: string): Promise { + const hook = await findHookByToken(token); + if (!hook) { + throw new Error(`Hook with token ${token} not found`); + } + return hook; + } + + async function list( + params: ListHooksParams + ): Promise> { + const hooksDir = path.join(basedir, 'hooks'); + const resolveData = params.resolveData || DEFAULT_RESOLVE_DATA_OPTION; + + const result = await paginatedFileSystemQuery({ + directory: hooksDir, + schema: HookSchema, + sortOrder: params.pagination?.sortOrder, + limit: params.pagination?.limit, + cursor: params.pagination?.cursor, + filePrefix: undefined, // Hooks don't have ULIDs, so we can't optimize by filename + filter: (hook) => { + // Filter by runId if provided + if (params.runId && hook.runId !== params.runId) { + return false; + } + return true; + }, + getCreatedAt: () => { + // Hook files don't have ULID timestamps in filename + // We need to read the file to get createdAt, but that's inefficient + // So we return the hook's createdAt directly (item.createdAt will be used for sorting) + // Return a dummy date to pass the null check, actual sorting uses item.createdAt + return new Date(0); + }, + getId: (hook) => hook.hookId, + }); + + // Transform the data after pagination + return { + ...result, + data: result.data.map((hook) => filterHookData(hook, resolveData)), + }; + } + + return { get, getByToken, list }; +} + +/** + * Helper function to delete all hooks associated with a workflow run. + * Called when a run reaches a terminal state. + */ +export async function deleteAllHooksForRun( + basedir: string, + runId: string +): Promise { + const hooksDir = path.join(basedir, 'hooks'); + const files = await listJSONFiles(hooksDir); + + for (const file of files) { + const hookPath = path.join(hooksDir, `${file}.json`); + const hook = await readJSON(hookPath, HookSchema); + if (hook && hook.runId === runId) { + await deleteJSON(hookPath); + } + } +} diff --git a/packages/world-local/src/storage/index.ts b/packages/world-local/src/storage/index.ts new file mode 100644 index 0000000000..937bb4356a --- /dev/null +++ b/packages/world-local/src/storage/index.ts @@ -0,0 +1,21 @@ +import type { Storage } from '@workflow/world'; +import { createEventsStorage } from './events-storage.js'; +import { createHooksStorage } from './hooks-storage.js'; +import { createRunsStorage } from './runs-storage.js'; +import { createStepsStorage } from './steps-storage.js'; + +/** + * Creates a complete storage implementation using the filesystem. + * This is the main entry point that composes all storage implementations. + * + * @param basedir - The base directory for storing workflow data + * @returns A complete Storage implementation + */ +export function createStorage(basedir: string): Storage { + return { + runs: createRunsStorage(basedir), + steps: createStepsStorage(basedir), + events: createEventsStorage(basedir), + hooks: createHooksStorage(basedir), + }; +} diff --git a/packages/world-local/src/storage/legacy.ts b/packages/world-local/src/storage/legacy.ts new file mode 100644 index 0000000000..74c17cdb6e --- /dev/null +++ b/packages/world-local/src/storage/legacy.ts @@ -0,0 +1,81 @@ +import path from 'node:path'; +import type { Event, EventResult, WorkflowRun } from '@workflow/world'; +import { SPEC_VERSION_CURRENT } from '@workflow/world'; +import { DEFAULT_RESOLVE_DATA_OPTION } from '../config.js'; +import { writeJSON } from '../fs.js'; +import { filterEventData, filterRunData } from './filters.js'; +import { monotonicUlid } from './helpers.js'; +import { deleteAllHooksForRun } from './hooks-storage.js'; + +/** + * Handle events for legacy runs (pre-event-sourcing, specVersion < 2). + * Legacy runs use different behavior: + * - run_cancelled: Skip event storage, directly update run + * - wait_completed: Store event only (no entity mutation) + * - hook_received: Store event only (hooks exist via old system, no entity mutation) + * - Other events: Throw error (not supported for legacy runs) + */ +export async function handleLegacyEvent( + basedir: string, + runId: string, + data: any, + currentRun: WorkflowRun, + params?: { resolveData?: 'none' | 'all' } +): Promise { + const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + + switch (data.eventType) { + case 'run_cancelled': { + // Legacy: Skip event storage, directly update run to cancelled + const now = new Date(); + const run: WorkflowRun = { + runId: currentRun.runId, + deploymentId: currentRun.deploymentId, + workflowName: currentRun.workflowName, + specVersion: currentRun.specVersion, + executionContext: currentRun.executionContext, + input: currentRun.input, + createdAt: currentRun.createdAt, + expiredAt: currentRun.expiredAt, + startedAt: currentRun.startedAt, + status: 'cancelled', + output: undefined, + error: undefined, + completedAt: now, + updatedAt: now, + }; + const runPath = path.join(basedir, 'runs', `${runId}.json`); + await writeJSON(runPath, run, { overwrite: true }); + await deleteAllHooksForRun(basedir, runId); + // Return without event (legacy behavior skips event storage) + return { event: undefined, run: filterRunData(run, resolveData) }; + } + + case 'wait_completed': + case 'hook_received': { + // Legacy: Store event only (no entity mutation) + // - wait_completed: for replay purposes + // - hook_received: hooks exist via old system, just record the event + const eventId = `evnt_${monotonicUlid()}`; + const now = new Date(); + const event: Event = { + ...data, + runId, + eventId, + createdAt: now, + specVersion: SPEC_VERSION_CURRENT, + }; + const compositeKey = `${runId}-${eventId}`; + const eventPath = path.join(basedir, 'events', `${compositeKey}.json`); + await writeJSON(eventPath, event); + return { event: filterEventData(event, resolveData) }; + } + + default: + throw new Error( + `Event type '${data.eventType}' not supported for legacy runs ` + + `(specVersion: ${currentRun.specVersion || 'undefined'}). ` + + `Please upgrade 'workflow' package.` + ); + } +} diff --git a/packages/world-local/src/storage/runs-storage.ts b/packages/world-local/src/storage/runs-storage.ts new file mode 100644 index 0000000000..5783f50cd5 --- /dev/null +++ b/packages/world-local/src/storage/runs-storage.ts @@ -0,0 +1,65 @@ +import path from 'node:path'; +import { WorkflowRunNotFoundError } from '@workflow/errors'; +import type { Storage } from '@workflow/world'; +import { WorkflowRunSchema } from '@workflow/world'; +import { DEFAULT_RESOLVE_DATA_OPTION } from '../config.js'; +import { paginatedFileSystemQuery, readJSON } from '../fs.js'; +import { filterRunData } from './filters.js'; +import { getObjectCreatedAt } from './helpers.js'; + +/** + * Creates the runs storage implementation using the filesystem. + * Implements the Storage['runs'] interface with get and list operations. + */ +export function createRunsStorage(basedir: string): Storage['runs'] { + return { + async get(id, params) { + const runPath = path.join(basedir, 'runs', `${id}.json`); + const run = await readJSON(runPath, WorkflowRunSchema); + if (!run) { + throw new WorkflowRunNotFoundError(id); + } + const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + return filterRunData(run, resolveData); + }, + + async list(params) { + const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const result = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'runs'), + schema: WorkflowRunSchema, + filter: (run) => { + if ( + params?.workflowName && + run.workflowName !== params.workflowName + ) { + return false; + } + if (params?.status && run.status !== params.status) { + return false; + } + return true; + }, + sortOrder: params?.pagination?.sortOrder ?? 'desc', + limit: params?.pagination?.limit, + cursor: params?.pagination?.cursor, + getCreatedAt: getObjectCreatedAt('wrun'), + getId: (run) => run.runId, + }); + + // If resolveData is "none", replace input/output with empty data + if (resolveData === 'none') { + return { + ...result, + data: result.data.map((run) => ({ + ...run, + input: [], + output: undefined, + })), + }; + } + + return result; + }, + }; +} diff --git a/packages/world-local/src/storage/steps-storage.ts b/packages/world-local/src/storage/steps-storage.ts new file mode 100644 index 0000000000..b33c7813fe --- /dev/null +++ b/packages/world-local/src/storage/steps-storage.ts @@ -0,0 +1,66 @@ +import path from 'node:path'; +import type { Step, Storage } from '@workflow/world'; +import { StepSchema } from '@workflow/world'; +import { DEFAULT_RESOLVE_DATA_OPTION } from '../config.js'; +import { listJSONFiles, paginatedFileSystemQuery, readJSON } from '../fs.js'; +import { filterStepData } from './filters.js'; +import { getObjectCreatedAt } from './helpers.js'; + +/** + * Creates the steps storage implementation using the filesystem. + * Implements the Storage['steps'] interface with get and list operations. + */ +export function createStepsStorage(basedir: string): Storage['steps'] { + return { + async get( + runId: string | undefined, + stepId: string, + params + ): Promise { + if (!runId) { + const fileIds = await listJSONFiles(path.join(basedir, 'steps')); + const fileId = fileIds.find((fileId) => fileId.endsWith(`-${stepId}`)); + if (!fileId) { + throw new Error(`Step ${stepId} not found`); + } + runId = fileId.split('-')[0]; + } + const compositeKey = `${runId}-${stepId}`; + const stepPath = path.join(basedir, 'steps', `${compositeKey}.json`); + const step = await readJSON(stepPath, StepSchema); + if (!step) { + throw new Error(`Step ${stepId} in run ${runId} not found`); + } + const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + return filterStepData(step, resolveData); + }, + + async list(params) { + const resolveData = params.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + const result = await paginatedFileSystemQuery({ + directory: path.join(basedir, 'steps'), + schema: StepSchema, + filePrefix: `${params.runId}-`, + sortOrder: params.pagination?.sortOrder ?? 'desc', + limit: params.pagination?.limit, + cursor: params.pagination?.cursor, + getCreatedAt: getObjectCreatedAt('step'), + getId: (step) => step.stepId, + }); + + // If resolveData is "none", replace input/output with empty data + if (resolveData === 'none') { + return { + ...result, + data: result.data.map((step) => ({ + ...step, + input: [], + output: undefined, + })), + }; + } + + return result; + }, + }; +} diff --git a/packages/world-local/src/test-helpers.ts b/packages/world-local/src/test-helpers.ts new file mode 100644 index 0000000000..c8f92fe188 --- /dev/null +++ b/packages/world-local/src/test-helpers.ts @@ -0,0 +1,135 @@ +import type { Hook, Step, Storage, WorkflowRun } from '@workflow/world'; +import { SPEC_VERSION_CURRENT } from '@workflow/world'; + +/** + * Test helper functions for creating and updating storage entities through events. + * These helpers simplify test setup by providing a convenient API for common operations. + */ + +/** + * Create a new workflow run through the run_created event. + */ +export async function createRun( + storage: Storage, + data: { + deploymentId: string; + workflowName: string; + input: unknown[]; + executionContext?: Record; + } +): Promise { + const result = await storage.events.create(null, { + eventType: 'run_created', + specVersion: SPEC_VERSION_CURRENT, + eventData: data, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run; +} + +/** + * Update a workflow run's status through lifecycle events. + */ +export async function updateRun( + storage: Storage, + runId: string, + eventType: 'run_started' | 'run_completed' | 'run_failed', + eventData?: Record +): Promise { + const result = await storage.events.create(runId, { + eventType, + specVersion: SPEC_VERSION_CURRENT, + eventData, + } as any); + if (!result.run) { + throw new Error('Expected run to be updated'); + } + return result.run; +} + +/** + * Create a new step through the step_created event. + */ +export async function createStep( + storage: Storage, + runId: string, + data: { + stepId: string; + stepName: string; + input: unknown[]; + } +): Promise { + const result = await storage.events.create(runId, { + eventType: 'step_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: data.stepId, + eventData: { stepName: data.stepName, input: data.input }, + }); + if (!result.step) { + throw new Error('Expected step to be created'); + } + return result.step; +} + +/** + * Update a step's status through lifecycle events. + */ +export async function updateStep( + storage: Storage, + runId: string, + stepId: string, + eventType: 'step_started' | 'step_completed' | 'step_failed', + eventData?: Record +): Promise { + const result = await storage.events.create(runId, { + eventType, + specVersion: SPEC_VERSION_CURRENT, + correlationId: stepId, + eventData, + } as any); + if (!result.step) { + throw new Error('Expected step to be updated'); + } + return result.step; +} + +/** + * Create a new hook through the hook_created event. + */ +export async function createHook( + storage: Storage, + runId: string, + data: { + hookId: string; + token: string; + metadata?: unknown; + } +): Promise { + const result = await storage.events.create(runId, { + eventType: 'hook_created', + specVersion: SPEC_VERSION_CURRENT, + correlationId: data.hookId, + eventData: { token: data.token, metadata: data.metadata }, + }); + if (!result.hook) { + throw new Error('Expected hook to be created'); + } + return result.hook; +} + +/** + * Dispose a hook through the hook_disposed event. + */ +export async function disposeHook( + storage: Storage, + runId: string, + hookId: string +): Promise { + await storage.events.create(runId, { + eventType: 'hook_disposed', + specVersion: SPEC_VERSION_CURRENT, + correlationId: hookId, + }); +} diff --git a/packages/world-postgres/src/drizzle/migrations/0005_add_spec_version.sql b/packages/world-postgres/src/drizzle/migrations/0005_add_spec_version.sql new file mode 100644 index 0000000000..392ce50041 --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0005_add_spec_version.sql @@ -0,0 +1,5 @@ +ALTER TABLE "workflow"."workflow_runs" ALTER COLUMN "status" SET DATA TYPE text;--> statement-breakpoint +DROP TYPE "public"."status";--> statement-breakpoint +CREATE TYPE "public"."status" AS ENUM('pending', 'running', 'completed', 'failed', 'cancelled');--> statement-breakpoint +ALTER TABLE "workflow"."workflow_runs" ALTER COLUMN "status" SET DATA TYPE "public"."status" USING "status"::"public"."status";--> statement-breakpoint +ALTER TABLE "workflow"."workflow_runs" ADD COLUMN "spec_version" varchar; \ No newline at end of file diff --git a/packages/world-postgres/src/drizzle/migrations/0006_add_error_cbor.sql b/packages/world-postgres/src/drizzle/migrations/0006_add_error_cbor.sql new file mode 100644 index 0000000000..2d3845d2f4 --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/0006_add_error_cbor.sql @@ -0,0 +1,5 @@ +ALTER TABLE "workflow"."workflow_runs" ADD COLUMN "error_cbor" bytea;--> statement-breakpoint +ALTER TABLE "workflow"."workflow_steps" ADD COLUMN "error_cbor" bytea;--> statement-breakpoint +ALTER TABLE "workflow"."workflow_steps" ADD COLUMN "spec_version" integer;--> statement-breakpoint +ALTER TABLE "workflow"."workflow_hooks" ADD COLUMN "spec_version" integer;--> statement-breakpoint +ALTER TABLE "workflow"."workflow_events" ADD COLUMN "spec_version" integer; diff --git a/packages/world-postgres/src/drizzle/migrations/meta/0005_snapshot.json b/packages/world-postgres/src/drizzle/migrations/meta/0005_snapshot.json new file mode 100644 index 0000000000..0c2878bcd8 --- /dev/null +++ b/packages/world-postgres/src/drizzle/migrations/meta/0005_snapshot.json @@ -0,0 +1,575 @@ +{ + "id": "7adbbd35-ca90-4353-bb34-3d1b2435a027", + "prevId": "b9c4d5e6-f7a8-9012-3456-78901bcdef01", + "version": "7", + "dialect": "postgresql", + "tables": { + "workflow.workflow_events": { + "name": "workflow_events", + "schema": "workflow", + "columns": { + "id": { + "name": "id", + "type": "varchar", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "correlation_id": { + "name": "correlation_id", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "run_id": { + "name": "run_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "payload_cbor": { + "name": "payload_cbor", + "type": "bytea", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_events_run_id_index": { + "name": "workflow_events_run_id_index", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_events_correlation_id_index": { + "name": "workflow_events_correlation_id_index", + "columns": [ + { + "expression": "correlation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "workflow.workflow_hooks": { + "name": "workflow_hooks", + "schema": "workflow", + "columns": { + "run_id": { + "name": "run_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "hook_id": { + "name": "hook_id", + "type": "varchar", + "primaryKey": true, + "notNull": true + }, + "token": { + "name": "token", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "owner_id": { + "name": "owner_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "environment": { + "name": "environment", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "metadata_cbor": { + "name": "metadata_cbor", + "type": "bytea", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_hooks_run_id_index": { + "name": "workflow_hooks_run_id_index", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_hooks_token_index": { + "name": "workflow_hooks_token_index", + "columns": [ + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "workflow.workflow_runs": { + "name": "workflow_runs", + "schema": "workflow", + "columns": { + "id": { + "name": "id", + "type": "varchar", + "primaryKey": true, + "notNull": true + }, + "output": { + "name": "output", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "output_cbor": { + "name": "output_cbor", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "deployment_id": { + "name": "deployment_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "spec_version": { + "name": "spec_version", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "execution_context": { + "name": "execution_context", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "execution_context_cbor": { + "name": "execution_context_cbor", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "input": { + "name": "input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "input_cbor": { + "name": "input_cbor", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "expired_at": { + "name": "expired_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_runs_name_index": { + "name": "workflow_runs_name_index", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_runs_status_index": { + "name": "workflow_runs_status_index", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "workflow.workflow_steps": { + "name": "workflow_steps", + "schema": "workflow", + "columns": { + "run_id": { + "name": "run_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "step_id": { + "name": "step_id", + "type": "varchar", + "primaryKey": true, + "notNull": true + }, + "step_name": { + "name": "step_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "step_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "input": { + "name": "input", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "input_cbor": { + "name": "input_cbor", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "output": { + "name": "output", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "output_cbor": { + "name": "output_cbor", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "retry_after": { + "name": "retry_after", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "workflow_steps_run_id_index": { + "name": "workflow_steps_run_id_index", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "workflow_steps_status_index": { + "name": "workflow_steps_status_index", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "workflow.workflow_stream_chunks": { + "name": "workflow_stream_chunks", + "schema": "workflow", + "columns": { + "id": { + "name": "id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "stream_id": { + "name": "stream_id", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "varchar", + "primaryKey": false, + "notNull": false + }, + "data": { + "name": "data", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "eof": { + "name": "eof", + "type": "boolean", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "workflow_stream_chunks_run_id_index": { + "name": "workflow_stream_chunks_run_id_index", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "workflow_stream_chunks_stream_id_id_pk": { + "name": "workflow_stream_chunks_stream_id_id_pk", + "columns": ["stream_id", "id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.step_status": { + "name": "step_status", + "schema": "public", + "values": ["pending", "running", "completed", "failed", "cancelled"] + }, + "public.status": { + "name": "status", + "schema": "public", + "values": ["pending", "running", "completed", "failed", "cancelled"] + } + }, + "schemas": { + "workflow": "workflow" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json index 8fdd359537..bd208f909f 100644 --- a/packages/world-postgres/src/drizzle/migrations/meta/_journal.json +++ b/packages/world-postgres/src/drizzle/migrations/meta/_journal.json @@ -36,6 +36,20 @@ "when": 1765900000001, "tag": "0004_remove_run_pause_status", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1767723210726, + "tag": "0005_add_spec_version", + "breakpoints": true + }, + { + "idx": 6, + "version": "7", + "when": 1768500000000, + "tag": "0006_add_error_cbor", + "breakpoints": true } ] } diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 8a6bce004e..219068c7fa 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -3,6 +3,7 @@ import { type Hook, type Step, StepStatusSchema, + type StructuredError, type WorkflowRun, WorkflowRunStatusSchema, } from '@workflow/world'; @@ -63,6 +64,7 @@ export const runs = schema.table( deploymentId: varchar('deployment_id').notNull(), status: workflowRunStatus('status').notNull(), workflowName: varchar('name').notNull(), + specVersion: integer('spec_version'), /** @deprecated */ executionContextJson: jsonb('execution_context').$type>(), @@ -70,7 +72,9 @@ export const runs = schema.table( /** @deprecated */ inputJson: jsonb('input').$type(), input: Cbor()('input_cbor'), - error: text('error'), + /** @deprecated - use error instead */ + errorJson: text('error'), + error: Cbor()('error_cbor'), createdAt: timestamp('created_at').defaultNow().notNull(), updatedAt: timestamp('updated_at') .defaultNow() @@ -82,7 +86,7 @@ export const runs = schema.table( } satisfies DrizzlishOfType< Cborized< Omit & { input?: unknown }, - 'input' | 'output' | 'executionContext' + 'input' | 'output' | 'executionContext' | 'error' > >, (tb) => [index().on(tb.workflowName), index().on(tb.status)] @@ -99,6 +103,7 @@ export const events = schema.table( /** @deprecated */ eventDataJson: jsonb('payload'), eventData: Cbor()('payload_cbor'), + specVersion: integer('spec_version'), } satisfies DrizzlishOfType< Cborized >, @@ -118,8 +123,11 @@ export const steps = schema.table( /** @deprecated we stream binary data */ outputJson: jsonb('output').$type(), output: Cbor()('output_cbor'), - error: text('error'), + /** @deprecated - use error instead */ + errorJson: text('error'), + error: Cbor()('error_cbor'), attempt: integer('attempt').notNull(), + /** Maps to startedAt in Step interface */ startedAt: timestamp('started_at'), completedAt: timestamp('completed_at'), createdAt: timestamp('created_at').defaultNow().notNull(), @@ -128,8 +136,14 @@ export const steps = schema.table( .$onUpdateFn(() => new Date()) .notNull(), retryAfter: timestamp('retry_after'), + specVersion: integer('spec_version'), } satisfies DrizzlishOfType< - Cborized & { input?: unknown }, 'output' | 'input'> + Cborized< + Omit & { + input?: unknown; + }, + 'output' | 'input' | 'error' + > >, (tb) => [index().on(tb.runId), index().on(tb.status)] ); @@ -147,6 +161,7 @@ export const hooks = schema.table( /** @deprecated */ metadataJson: jsonb('metadata').$type(), metadata: Cbor()('metadata_cbor'), + specVersion: integer('spec_version'), } satisfies DrizzlishOfType>, (tb) => [index().on(tb.runId), index().on(tb.token)] ); diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index 89f4838360..c3de61e3f9 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -1,6 +1,7 @@ -import { WorkflowAPIError } from '@workflow/errors'; +import { RunNotSupportedError, WorkflowAPIError } from '@workflow/errors'; import type { Event, + EventResult, Hook, ListEventsParams, ListHooksParams, @@ -8,144 +9,86 @@ import type { ResolveData, Step, Storage, - UpdateStepRequest, - UpdateWorkflowRunRequest, + StructuredError, WorkflowRun, } from '@workflow/world'; import { EventSchema, HookSchema, + isLegacySpecVersion, + requiresNewerWorld, + SPEC_VERSION_CURRENT, StepSchema, WorkflowRunSchema, } from '@workflow/world'; -import { and, desc, eq, gt, lt, sql } from 'drizzle-orm'; +import { and, desc, eq, gt, lt, notInArray, sql } from 'drizzle-orm'; import { monotonicFactory } from 'ulid'; import { type Drizzle, Schema } from './drizzle/index.js'; import type { SerializedContent } from './drizzle/schema.js'; import { compact } from './util.js'; /** - * Serialize a StructuredError object into a JSON string + * Parse legacy errorJson (text column with JSON-stringified StructuredError). + * Used for backwards compatibility when reading from deprecated error column. */ -function serializeRunError(data: UpdateWorkflowRunRequest): any { - if (!data.error) { - return data; +function parseErrorJson(errorJson: string | null): StructuredError | null { + if (!errorJson) return null; + try { + const parsed = JSON.parse(errorJson); + if (typeof parsed === 'object' && parsed.message !== undefined) { + return { + message: parsed.message, + stack: parsed.stack, + code: parsed.code, + }; + } + // Not a structured error object, treat as plain string + return { message: String(parsed) }; + } catch { + // Not JSON, treat as plain string error message + return { message: errorJson }; } - - const { error, ...rest } = data; - return { - ...rest, - error: JSON.stringify({ - message: error.message, - stack: error.stack, - code: error.code, - }), - }; } /** - * Deserialize error JSON string (or legacy flat fields) into a StructuredError object - * Handles backwards compatibility: - * - If error is a JSON string with {message, stack, code} → parse into StructuredError - * - If error is a plain string → treat as error message - * - If errorStack/errorCode exist (legacy) → combine into StructuredError + * Deserialize run data, handling legacy error fields. + * The error field should already be deserialized from CBOR or fallback to errorJson. + * This function only handles very old legacy fields (errorStack, errorCode). */ function deserializeRunError(run: any): WorkflowRun { - const { error, errorStack, errorCode, ...rest } = run; + const { errorStack, errorCode, ...rest } = run; - if (!error && !errorStack && !errorCode) { - return run as WorkflowRun; + // If no legacy fields, return as-is (error is already a StructuredError or undefined) + if (!errorStack && !errorCode) { + return rest as WorkflowRun; } - // Try to parse as structured error JSON - if (error) { - try { - const parsed = JSON.parse(error); - if (typeof parsed === 'object' && parsed.message !== undefined) { - return { - ...rest, - error: { - message: parsed.message, - stack: parsed.stack, - code: parsed.code, - }, - } as WorkflowRun; - } - } catch { - // Not JSON, treat as plain string - } - } - - // Backwards compatibility: handle legacy separate fields or plain string error + // Very old legacy: separate errorStack/errorCode fields + const existingError = rest.error as StructuredError | undefined; return { ...rest, error: { - message: error || '', - stack: errorStack, - code: errorCode, + message: existingError?.message || '', + stack: existingError?.stack || errorStack, + code: existingError?.code || errorCode, }, } as WorkflowRun; } /** - * Serialize a StructuredError object into a JSON string for steps - */ -function serializeStepError(data: UpdateStepRequest): any { - if (!data.error) { - return data; - } - - const { error, ...rest } = data; - return { - ...rest, - error: JSON.stringify({ - message: error.message, - stack: error.stack, - code: error.code, - }), - }; -} - -/** - * Deserialize error JSON string (or legacy flat fields) into a StructuredError object for steps + * Deserialize step data, mapping DB columns to interface fields. + * The error field should already be deserialized from CBOR or fallback to errorJson. */ function deserializeStepError(step: any): Step { - const { error, ...rest } = step; - - if (!error) { - return step as Step; - } + const { startedAt, ...rest } = step; - // Try to parse as structured error JSON - if (error) { - try { - const parsed = JSON.parse(error); - if (typeof parsed === 'object' && parsed.message !== undefined) { - return { - ...rest, - error: { - message: parsed.message, - stack: parsed.stack, - code: parsed.code, - }, - } as Step; - } - } catch { - // Not JSON, treat as plain string - } - } - - // Backwards compatibility: handle legacy separate fields or plain string error return { ...rest, - error: { - message: error || '', - }, + startedAt, } as Step; } export function createRunsStorage(drizzle: Drizzle): Storage['runs'] { - const ulid = monotonicFactory(); const { runs } = Schema; const get = drizzle .select() @@ -163,25 +106,7 @@ export function createRunsStorage(drizzle: Drizzle): Storage['runs'] { value.output ||= value.outputJson; value.input ||= value.inputJson; value.executionContext ||= value.executionContextJson; - const deserialized = deserializeRunError(compact(value)); - const parsed = WorkflowRunSchema.parse(deserialized); - const resolveData = params?.resolveData ?? 'all'; - return filterRunData(parsed, resolveData); - }, - async cancel(id, params) { - // TODO: we might want to guard this for only specific statuses - const [value] = await drizzle - .update(Schema.runs) - .set({ status: 'cancelled', completedAt: sql`now()` }) - .where(eq(runs.runId, id)) - .returning(); - if (!value) { - throw new WorkflowAPIError(`Run not found: ${id}`, { status: 404 }); - } - - // Clean up all hooks for this run when cancelling - await drizzle.delete(Schema.hooks).where(eq(Schema.hooks.runId, id)); - + value.error ||= parseErrorJson(value.errorJson); const deserialized = deserializeRunError(compact(value)); const parsed = WorkflowRunSchema.parse(deserialized); const resolveData = params?.resolveData ?? 'all'; @@ -209,6 +134,10 @@ export function createRunsStorage(drizzle: Drizzle): Storage['runs'] { const resolveData = params?.resolveData ?? 'all'; return { data: values.map((v) => { + v.output ||= v.outputJson; + v.input ||= v.inputJson; + v.executionContext ||= v.executionContextJson; + v.error ||= parseErrorJson(v.errorJson); const deserialized = deserializeRunError(compact(v)); const parsed = WorkflowRunSchema.parse(deserialized); return filterRunData(parsed, resolveData); @@ -217,102 +146,744 @@ export function createRunsStorage(drizzle: Drizzle): Storage['runs'] { cursor: values.at(-1)?.runId ?? null, }; }, - async create(data) { - const runId = `wrun_${ulid()}`; - const [value] = await drizzle - .insert(runs) + }; +} + +function map(obj: T | null | undefined, fn: (v: T) => R): undefined | R { + return obj ? fn(obj) : undefined; +} + +/** + * Handle events for legacy runs (pre-event-sourcing, specVersion < 2). + * Legacy runs use different behavior: + * - run_cancelled: Skip event storage, directly update run + * - wait_completed: Store event only (no entity mutation) + * - hook_received: Store event only (hooks exist via old system, no entity mutation) + * - Other events: Throw error (not supported for legacy runs) + */ +async function handleLegacyEventPostgres( + drizzle: Drizzle, + runId: string, + eventId: string, + data: any, + currentRun: { status: string; specVersion: number | null }, + params?: { resolveData?: ResolveData } +): Promise { + const resolveData = params?.resolveData ?? 'all'; + + switch (data.eventType) { + case 'run_cancelled': { + // Legacy: Skip event storage, directly update run to cancelled + const now = new Date(); + + // Update run status to cancelled + await drizzle + .update(Schema.runs) + .set({ + status: 'cancelled', + completedAt: now, + updatedAt: now, + }) + .where(eq(Schema.runs.runId, runId)); + + // Delete all hooks for this run + await drizzle.delete(Schema.hooks).where(eq(Schema.hooks.runId, runId)); + + // Fetch updated run for return value + const [updatedRun] = await drizzle + .select() + .from(Schema.runs) + .where(eq(Schema.runs.runId, runId)) + .limit(1); + + // Return without event (legacy behavior skips event storage) + return { + run: updatedRun + ? filterRunData(deserializeRunError(compact(updatedRun)), resolveData) + : undefined, + }; + } + + case 'wait_completed': + case 'hook_received': { + // Legacy: Store event only (no entity mutation) + // - wait_completed: for replay purposes + // - hook_received: hooks exist via old system, just record the event + const [insertedEvent] = await drizzle + .insert(Schema.events) .values({ runId, - input: data.input, - executionContext: data.executionContext as Record< - string, - unknown - > | null, - deploymentId: data.deploymentId, - status: 'pending', - workflowName: data.workflowName, + eventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: 'eventData' in data ? data.eventData : undefined, + specVersion: SPEC_VERSION_CURRENT, }) - .onConflictDoNothing() - .returning(); - if (!value) { - throw new WorkflowAPIError(`Run ${runId} already exists`, { - status: 409, + .returning({ createdAt: Schema.events.createdAt }); + + const event = EventSchema.parse({ + ...data, + ...insertedEvent, + runId, + eventId, + }); + return { event: filterEventData(event, resolveData) }; + } + + default: + throw new Error( + `Event type '${data.eventType}' not supported for legacy runs ` + + `(specVersion: ${currentRun.specVersion || 'undefined'}). ` + + `Please upgrade @workflow packages.` + ); + } +} + +export function createEventsStorage(drizzle: Drizzle): Storage['events'] { + const ulid = monotonicFactory(); + const { events } = Schema; + + // Prepared statements for validation queries (performance optimization) + const getRunForValidation = drizzle + .select({ + status: Schema.runs.status, + specVersion: Schema.runs.specVersion, + }) + .from(Schema.runs) + .where(eq(Schema.runs.runId, sql.placeholder('runId'))) + .limit(1) + .prepare('events_get_run_for_validation'); + + const getStepForValidation = drizzle + .select({ + status: Schema.steps.status, + startedAt: Schema.steps.startedAt, + }) + .from(Schema.steps) + .where( + and( + eq(Schema.steps.runId, sql.placeholder('runId')), + eq(Schema.steps.stepId, sql.placeholder('stepId')) + ) + ) + .limit(1) + .prepare('events_get_step_for_validation'); + + const getHookByToken = drizzle + .select({ hookId: Schema.hooks.hookId }) + .from(Schema.hooks) + .where(eq(Schema.hooks.token, sql.placeholder('token'))) + .limit(1) + .prepare('events_get_hook_by_token'); + + return { + async create(runId, data, params): Promise { + const eventId = `wevt_${ulid()}`; + + // For run_created events, generate runId server-side if null or empty + let effectiveRunId: string; + if (data.eventType === 'run_created' && (!runId || runId === '')) { + effectiveRunId = `wrun_${ulid()}`; + } else if (!runId) { + throw new Error('runId is required for non-run_created events'); + } else { + effectiveRunId = runId; + } + + // specVersion is always sent by the runtime, but we provide a fallback for safety + const effectiveSpecVersion = data.specVersion ?? SPEC_VERSION_CURRENT; + + // Track entity created/updated for EventResult + let run: WorkflowRun | undefined; + let step: Step | undefined; + let hook: Hook | undefined; + const now = new Date(); + + // Helper to check if run is in terminal state + const isRunTerminal = (status: string) => + ['completed', 'failed', 'cancelled'].includes(status); + + // Helper to check if step is in terminal state + const isStepTerminal = (status: string) => + ['completed', 'failed'].includes(status); + + // ============================================================ + // VALIDATION: Terminal state and event ordering checks + // ============================================================ + + // Get current run state for validation (if not creating a new run) + // Skip run validation for step_completed and step_retrying - they only operate + // on running steps, and running steps are always allowed to modify regardless + // of run state. This optimization saves database queries per step event. + let currentRun: { status: string; specVersion: number | null } | null = + null; + const skipRunValidationEvents = ['step_completed', 'step_retrying']; + if ( + data.eventType !== 'run_created' && + !skipRunValidationEvents.includes(data.eventType) + ) { + // Use prepared statement for better performance + const [runValue] = await getRunForValidation.execute({ + runId: effectiveRunId, }); + currentRun = runValue ?? null; } - return deserializeRunError(compact(value)); - }, - async update(id, data) { - // Fetch current run to check if startedAt is already set - const [currentRun] = await drizzle - .select() - .from(runs) - .where(eq(runs.runId, id)) - .limit(1); - if (!currentRun) { - throw new WorkflowAPIError(`Run not found: ${id}`, { status: 404 }); + // ============================================================ + // VERSION COMPATIBILITY: Check run spec version + // ============================================================ + // For events that have fetched the run, check version compatibility. + // Skip for run_created (no existing run) and runtime events (step_completed, step_retrying). + if (currentRun) { + // Check if run requires a newer world version + if (requiresNewerWorld(currentRun.specVersion)) { + throw new RunNotSupportedError( + currentRun.specVersion!, + SPEC_VERSION_CURRENT + ); + } + + // Route to legacy handler for pre-event-sourcing runs + if (isLegacySpecVersion(currentRun.specVersion)) { + return handleLegacyEventPostgres( + drizzle, + effectiveRunId, + eventId, + data, + currentRun, + params + ); + } } - // Serialize the error field if present - const serialized = serializeRunError(data); + // Run terminal state validation + if (currentRun && isRunTerminal(currentRun.status)) { + const runTerminalEvents = [ + 'run_started', + 'run_completed', + 'run_failed', + ]; - const updates: Partial = { - ...serialized, - output: data.output as SerializedContent, - }; + // Idempotent operation: run_cancelled on already cancelled run is allowed + if ( + data.eventType === 'run_cancelled' && + currentRun.status === 'cancelled' + ) { + // Get full run for return value + const [fullRun] = await drizzle + .select() + .from(Schema.runs) + .where(eq(Schema.runs.runId, effectiveRunId)) + .limit(1); - // Only set startedAt the first time transitioning to 'running' - if (data.status === 'running' && !currentRun.startedAt) { - updates.startedAt = new Date(); + // Create the event (still record it) + const [value] = await drizzle + .insert(Schema.events) + .values({ + runId: effectiveRunId, + eventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: 'eventData' in data ? data.eventData : undefined, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: Schema.events.createdAt }); + + const result = { ...data, ...value, runId: effectiveRunId, eventId }; + const parsed = EventSchema.parse(result); + const resolveData = params?.resolveData ?? 'all'; + return { + event: filterEventData(parsed, resolveData), + run: fullRun ? deserializeRunError(compact(fullRun)) : undefined, + }; + } + + // Run state transitions are not allowed on terminal runs + if ( + runTerminalEvents.includes(data.eventType) || + data.eventType === 'run_cancelled' + ) { + throw new WorkflowAPIError( + `Cannot transition run from terminal state "${currentRun.status}"`, + { status: 410 } + ); + } + + // Creating new entities on terminal runs is not allowed + if ( + data.eventType === 'step_created' || + data.eventType === 'hook_created' + ) { + throw new WorkflowAPIError( + `Cannot create new entities on run in terminal state "${currentRun.status}"`, + { status: 410 } + ); + } } - const isBecomingTerminal = - data.status === 'completed' || - data.status === 'failed' || - data.status === 'cancelled'; + // Step-related event validation (ordering and terminal state) + // Fetch status + startedAt so we can reuse for step_started (avoid double read) + // Skip validation for step_completed/step_failed - use conditional UPDATE instead + let validatedStep: { status: string; startedAt: Date | null } | null = + null; + const stepEventsNeedingValidation = ['step_started', 'step_retrying']; + if ( + stepEventsNeedingValidation.includes(data.eventType) && + data.correlationId + ) { + // Use prepared statement for better performance + const [existingStep] = await getStepForValidation.execute({ + runId: effectiveRunId, + stepId: data.correlationId, + }); + + validatedStep = existingStep ?? null; + + // Event ordering: step must exist before these events + if (!validatedStep) { + throw new WorkflowAPIError(`Step "${data.correlationId}" not found`, { + status: 404, + }); + } - if (isBecomingTerminal) { - updates.completedAt = new Date(); + // Step terminal state validation + if (isStepTerminal(validatedStep.status)) { + throw new WorkflowAPIError( + `Cannot modify step in terminal state "${validatedStep.status}"`, + { status: 410 } + ); + } + + // On terminal runs: only allow completing/failing in-progress steps + if (currentRun && isRunTerminal(currentRun.status)) { + if (validatedStep.status !== 'running') { + throw new WorkflowAPIError( + `Cannot modify non-running step on run in terminal state "${currentRun.status}"`, + { status: 410 } + ); + } + } } - const [value] = await drizzle - .update(runs) - .set(updates) - .where(eq(runs.runId, id)) - .returning(); - if (!value) { - throw new WorkflowAPIError(`Run not found: ${id}`, { status: 404 }); + // Hook-related event validation (ordering) + const hookEventsRequiringExistence = ['hook_disposed', 'hook_received']; + if ( + hookEventsRequiringExistence.includes(data.eventType) && + data.correlationId + ) { + const [existingHook] = await drizzle + .select({ hookId: Schema.hooks.hookId }) + .from(Schema.hooks) + .where(eq(Schema.hooks.hookId, data.correlationId)) + .limit(1); + + if (!existingHook) { + throw new WorkflowAPIError(`Hook "${data.correlationId}" not found`, { + status: 404, + }); + } } - // If transitioning to a terminal status, clean up all hooks for this run - if (isBecomingTerminal) { - await drizzle.delete(Schema.hooks).where(eq(Schema.hooks.runId, id)); + // ============================================================ + // Entity creation/updates based on event type + // ============================================================ + + // Handle run_created event: create the run entity atomically + if (data.eventType === 'run_created') { + const eventData = (data as any).eventData as { + deploymentId: string; + workflowName: string; + input: any[]; + executionContext?: Record; + }; + const [runValue] = await drizzle + .insert(Schema.runs) + .values({ + runId: effectiveRunId, + deploymentId: eventData.deploymentId, + workflowName: eventData.workflowName, + // Propagate specVersion from the event to the run entity + specVersion: effectiveSpecVersion, + input: eventData.input as SerializedContent, + executionContext: eventData.executionContext as + | SerializedContent + | undefined, + status: 'pending', + }) + .onConflictDoNothing() + .returning(); + if (runValue) { + run = deserializeRunError(compact(runValue)); + } } - return deserializeRunError(compact(value)); - }, - }; -} + // Handle run_started event: update run status + if (data.eventType === 'run_started') { + const [runValue] = await drizzle + .update(Schema.runs) + .set({ + status: 'running', + startedAt: now, + updatedAt: now, + }) + .where(eq(Schema.runs.runId, effectiveRunId)) + .returning(); + if (runValue) { + run = deserializeRunError(compact(runValue)); + } + } -function map(obj: T | null | undefined, fn: (v: T) => R): undefined | R { - return obj ? fn(obj) : undefined; -} + // Handle run_completed event: update run status and cleanup hooks + if (data.eventType === 'run_completed') { + const eventData = (data as any).eventData as { output?: any }; + const [runValue] = await drizzle + .update(Schema.runs) + .set({ + status: 'completed', + output: eventData.output as SerializedContent | undefined, + completedAt: now, + updatedAt: now, + }) + .where(eq(Schema.runs.runId, effectiveRunId)) + .returning(); + if (runValue) { + run = deserializeRunError(compact(runValue)); + } + // Delete all hooks for this run to allow token reuse + await drizzle + .delete(Schema.hooks) + .where(eq(Schema.hooks.runId, effectiveRunId)); + } -export function createEventsStorage(drizzle: Drizzle): Storage['events'] { - const ulid = monotonicFactory(); - const { events } = Schema; + // Handle run_failed event: update run status and cleanup hooks + if (data.eventType === 'run_failed') { + const eventData = (data as any).eventData as { + error: any; + errorCode?: string; + }; + const errorMessage = + typeof eventData.error === 'string' + ? eventData.error + : (eventData.error?.message ?? 'Unknown error'); + const [runValue] = await drizzle + .update(Schema.runs) + .set({ + status: 'failed', + error: { + message: errorMessage, + stack: eventData.error?.stack, + code: eventData.errorCode, + }, + completedAt: now, + updatedAt: now, + }) + .where(eq(Schema.runs.runId, effectiveRunId)) + .returning(); + if (runValue) { + run = deserializeRunError(compact(runValue)); + } + // Delete all hooks for this run to allow token reuse + await drizzle + .delete(Schema.hooks) + .where(eq(Schema.hooks.runId, effectiveRunId)); + } + + // Handle run_cancelled event: update run status and cleanup hooks + if (data.eventType === 'run_cancelled') { + const [runValue] = await drizzle + .update(Schema.runs) + .set({ + status: 'cancelled', + completedAt: now, + updatedAt: now, + }) + .where(eq(Schema.runs.runId, effectiveRunId)) + .returning(); + if (runValue) { + run = deserializeRunError(compact(runValue)); + } + // Delete all hooks for this run to allow token reuse + await drizzle + .delete(Schema.hooks) + .where(eq(Schema.hooks.runId, effectiveRunId)); + } + + // Handle step_created event: create step entity + if (data.eventType === 'step_created') { + const eventData = (data as any).eventData as { + stepName: string; + input: any; + }; + const [stepValue] = await drizzle + .insert(Schema.steps) + .values({ + runId: effectiveRunId, + stepId: data.correlationId!, + stepName: eventData.stepName, + input: eventData.input as SerializedContent, + status: 'pending', + attempt: 0, + // Propagate specVersion from the event to the step entity + specVersion: effectiveSpecVersion, + }) + .onConflictDoNothing() + .returning(); + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } + } + + // Handle step_started event: increment attempt, set status to 'running' + // Sets startedAt (maps to startedAt) only on first start + // Reuse validatedStep from validation (already read above) + if (data.eventType === 'step_started') { + const isFirstStart = !validatedStep?.startedAt; + + const [stepValue] = await drizzle + .update(Schema.steps) + .set({ + status: 'running', + // Increment attempt counter using SQL + attempt: sql`${Schema.steps.attempt} + 1`, + // Only set startedAt on first start (not updated on retries) + ...(isFirstStart ? { startedAt: now } : {}), + }) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!) + ) + ) + .returning(); + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } + } + + // Handle step_completed event: update step status + // Uses conditional UPDATE to skip validation query (performance optimization) + if (data.eventType === 'step_completed') { + const eventData = (data as any).eventData as { result?: any }; + const [stepValue] = await drizzle + .update(Schema.steps) + .set({ + status: 'completed', + output: eventData.result as SerializedContent | undefined, + completedAt: now, + }) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!), + // Only update if not already in terminal state (validation in WHERE clause) + notInArray(Schema.steps.status, ['completed', 'failed']) + ) + ) + .returning(); + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } else { + // Step not updated - check if it exists and why + const [existing] = await getStepForValidation.execute({ + runId: effectiveRunId, + stepId: data.correlationId!, + }); + if (!existing) { + throw new WorkflowAPIError( + `Step "${data.correlationId}" not found`, + { status: 404 } + ); + } + if (['completed', 'failed'].includes(existing.status)) { + throw new WorkflowAPIError( + `Cannot modify step in terminal state "${existing.status}"`, + { status: 410 } + ); + } + } + } + + // Handle step_failed event: terminal state with error + // Uses conditional UPDATE to skip validation query (performance optimization) + if (data.eventType === 'step_failed') { + const eventData = (data as any).eventData as { + error?: any; + stack?: string; + }; + const errorMessage = + typeof eventData.error === 'string' + ? eventData.error + : (eventData.error?.message ?? 'Unknown error'); + + const [stepValue] = await drizzle + .update(Schema.steps) + .set({ + status: 'failed', + error: { + message: errorMessage, + stack: eventData.stack, + }, + completedAt: now, + }) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!), + // Only update if not already in terminal state (validation in WHERE clause) + notInArray(Schema.steps.status, ['completed', 'failed']) + ) + ) + .returning(); + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } else { + // Step not updated - check if it exists and why + const [existing] = await getStepForValidation.execute({ + runId: effectiveRunId, + stepId: data.correlationId!, + }); + if (!existing) { + throw new WorkflowAPIError( + `Step "${data.correlationId}" not found`, + { status: 404 } + ); + } + if (['completed', 'failed'].includes(existing.status)) { + throw new WorkflowAPIError( + `Cannot modify step in terminal state "${existing.status}"`, + { status: 410 } + ); + } + } + } + + // Handle step_retrying event: sets status back to 'pending', records error + if (data.eventType === 'step_retrying') { + const eventData = (data as any).eventData as { + error?: any; + stack?: string; + retryAfter?: Date; + }; + const errorMessage = + typeof eventData.error === 'string' + ? eventData.error + : (eventData.error?.message ?? 'Unknown error'); + + const [stepValue] = await drizzle + .update(Schema.steps) + .set({ + status: 'pending', + error: { + message: errorMessage, + stack: eventData.stack, + }, + retryAfter: eventData.retryAfter, + }) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!) + ) + ) + .returning(); + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } + } + + // Handle hook_created event: create hook entity + // Uses prepared statement for token uniqueness check (performance optimization) + if (data.eventType === 'hook_created') { + const eventData = (data as any).eventData as { + token: string; + metadata?: any; + }; + + // Check for duplicate token using prepared statement + const [existingHook] = await getHookByToken.execute({ + token: eventData.token, + }); + if (existingHook) { + // Create hook_conflict event instead of throwing 409 + // This allows the workflow to continue and fail gracefully when the hook is awaited + const conflictEventData = { + token: eventData.token, + }; + + const [conflictValue] = await drizzle + .insert(events) + .values({ + runId: effectiveRunId, + eventId, + correlationId: data.correlationId, + eventType: 'hook_conflict', + eventData: conflictEventData, + specVersion: effectiveSpecVersion, + }) + .returning({ createdAt: events.createdAt }); + + if (!conflictValue) { + throw new WorkflowAPIError( + `Event ${eventId} could not be created`, + { status: 409 } + ); + } + + const conflictResult = { + eventType: 'hook_conflict' as const, + correlationId: data.correlationId, + eventData: conflictEventData, + ...conflictValue, + runId: effectiveRunId, + eventId, + }; + const parsedConflict = EventSchema.parse(conflictResult); + const resolveData = params?.resolveData ?? 'all'; + return { + event: filterEventData(parsedConflict, resolveData), + run, + step, + hook: undefined, + }; + } + + const [hookValue] = await drizzle + .insert(Schema.hooks) + .values({ + runId: effectiveRunId, + hookId: data.correlationId!, + token: eventData.token, + metadata: eventData.metadata as SerializedContent, + ownerId: '', // TODO: get from context + projectId: '', // TODO: get from context + environment: '', // TODO: get from context + // Propagate specVersion from the event to the hook entity + specVersion: effectiveSpecVersion, + }) + .onConflictDoNothing() + .returning(); + if (hookValue) { + hookValue.metadata ||= hookValue.metadataJson; + hook = HookSchema.parse(compact(hookValue)); + } + } + + // Handle hook_disposed event: delete hook entity + if (data.eventType === 'hook_disposed' && data.correlationId) { + await drizzle + .delete(Schema.hooks) + .where(eq(Schema.hooks.hookId, data.correlationId)); + } - return { - async create(runId, data, params) { - const eventId = `wevt_${ulid()}`; const [value] = await drizzle .insert(events) .values({ - runId, + runId: effectiveRunId, eventId, correlationId: data.correlationId, eventType: data.eventType, eventData: 'eventData' in data ? data.eventData : undefined, + specVersion: effectiveSpecVersion, }) .returning({ createdAt: events.createdAt }); if (!value) { @@ -320,10 +891,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { status: 409, }); } - const result = { ...data, ...value, runId, eventId }; + const result = { ...data, ...value, runId: effectiveRunId, eventId }; const parsed = EventSchema.parse(result); const resolveData = params?.resolveData ?? 'all'; - return filterEventData(parsed, resolveData); + return { event: filterEventData(parsed, resolveData), run, step, hook }; }, async list(params: ListEventsParams): Promise> { const limit = params?.pagination?.limit ?? 100; @@ -417,30 +988,6 @@ export function createHooksStorage(drizzle: Drizzle): Storage['hooks'] { const resolveData = params?.resolveData ?? 'all'; return filterHookData(parsed, resolveData); }, - async create(runId, data, params) { - const [value] = await drizzle - .insert(hooks) - .values({ - runId, - hookId: data.hookId, - token: data.token, - metadata: data.metadata as SerializedContent, - ownerId: '', // TODO: get from context - projectId: '', // TODO: get from context - environment: '', // TODO: get from context - }) - .onConflictDoNothing() - .returning(); - if (!value) { - throw new WorkflowAPIError(`Hook ${data.hookId} already exists`, { - status: 409, - }); - } - value.metadata ||= value.metadataJson; - const parsed = HookSchema.parse(compact(value)); - const resolveData = params?.resolveData ?? 'all'; - return filterHookData(parsed, resolveData); - }, async getByToken(token, params) { const [value] = await getByToken.execute({ token }); if (!value) { @@ -481,20 +1028,6 @@ export function createHooksStorage(drizzle: Drizzle): Storage['hooks'] { hasMore, }; }, - async dispose(hookId, params) { - const [value] = await drizzle - .delete(hooks) - .where(eq(hooks.hookId, hookId)) - .returning(); - if (!value) { - throw new WorkflowAPIError(`Hook not found: ${hookId}`, { - status: 404, - }); - } - const parsed = HookSchema.parse(compact(value)); - const resolveData = params?.resolveData ?? 'all'; - return filterHookData(parsed, resolveData); - }, }; } @@ -502,28 +1035,6 @@ export function createStepsStorage(drizzle: Drizzle): Storage['steps'] { const { steps } = Schema; return { - async create(runId, data) { - const [value] = await drizzle - .insert(steps) - .values({ - runId, - stepId: data.stepId, - stepName: data.stepName, - input: data.input as SerializedContent, - status: 'pending', - attempt: 0, - }) - .onConflictDoNothing() - .returning(); - - if (!value) { - throw new WorkflowAPIError(`Step ${data.stepId} already exists`, { - status: 409, - }); - } - return deserializeStepError(compact(value)); - }, - async get(runId, stepId, params) { // If runId is not provided, query only by stepId const whereClause = runId @@ -542,52 +1053,13 @@ export function createStepsStorage(drizzle: Drizzle): Storage['steps'] { }); } value.output ||= value.outputJson; + value.input ||= value.inputJson; + value.error ||= parseErrorJson(value.errorJson); const deserialized = deserializeStepError(compact(value)); const parsed = StepSchema.parse(deserialized); const resolveData = params?.resolveData ?? 'all'; return filterStepData(parsed, resolveData); }, - async update(runId, stepId, data) { - // Fetch current step to check if startedAt is already set - const [currentStep] = await drizzle - .select() - .from(steps) - .where(and(eq(steps.stepId, stepId), eq(steps.runId, runId))) - .limit(1); - - if (!currentStep) { - throw new WorkflowAPIError(`Step not found: ${stepId}`, { - status: 404, - }); - } - - // Serialize the error field if present - const serialized = serializeStepError(data); - - const updates: Partial = { - ...serialized, - output: data.output as SerializedContent, - }; - const now = new Date(); - // Only set startedAt the first time the step transitions to 'running' - if (data.status === 'running' && !currentStep.startedAt) { - updates.startedAt = now; - } - if (data.status === 'completed' || data.status === 'failed') { - updates.completedAt = now; - } - const [value] = await drizzle - .update(steps) - .set(updates) - .where(and(eq(steps.stepId, stepId), eq(steps.runId, runId))) - .returning(); - if (!value) { - throw new WorkflowAPIError(`Step not found: ${stepId}`, { - status: 404, - }); - } - return deserializeStepError(compact(value)); - }, async list(params) { const limit = params?.pagination?.limit ?? 20; const fromCursor = params?.pagination?.cursor; @@ -609,6 +1081,9 @@ export function createStepsStorage(drizzle: Drizzle): Storage['steps'] { const resolveData = params?.resolveData ?? 'all'; return { data: values.map((v) => { + v.output ||= v.outputJson; + v.input ||= v.inputJson; + v.error ||= parseErrorJson(v.errorJson); const deserialized = deserializeStepError(compact(v)); const parsed = StepSchema.parse(deserialized); return filterStepData(parsed, resolveData); diff --git a/packages/world-postgres/test/storage.test.ts b/packages/world-postgres/test/storage.test.ts index ea57eeded1..7a206a4b66 100644 --- a/packages/world-postgres/test/storage.test.ts +++ b/packages/world-postgres/test/storage.test.ts @@ -1,5 +1,6 @@ import { execSync } from 'node:child_process'; import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import type { Hook, Step, WorkflowRun } from '@workflow/world'; import postgres from 'postgres'; import { afterAll, @@ -17,6 +18,103 @@ import { createStepsStorage, } from '../src/storage.js'; +// Helper types for events storage +type EventsStorage = ReturnType; + +// Helper functions to create entities through events.create +async function createRun( + events: EventsStorage, + data: { + deploymentId: string; + workflowName: string; + input: unknown[]; + executionContext?: Record; + } +): Promise { + const result = await events.create(null, { + eventType: 'run_created', + eventData: data, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run; +} + +async function updateRun( + events: EventsStorage, + runId: string, + eventType: 'run_started' | 'run_completed' | 'run_failed', + eventData?: Record +): Promise { + const result = await events.create(runId, { + eventType, + eventData, + }); + if (!result.run) { + throw new Error('Expected run to be updated'); + } + return result.run; +} + +async function createStep( + events: EventsStorage, + runId: string, + data: { + stepId: string; + stepName: string; + input: unknown[]; + } +): Promise { + const result = await events.create(runId, { + eventType: 'step_created', + correlationId: data.stepId, + eventData: { stepName: data.stepName, input: data.input }, + }); + if (!result.step) { + throw new Error('Expected step to be created'); + } + return result.step; +} + +async function updateStep( + events: EventsStorage, + runId: string, + stepId: string, + eventType: 'step_started' | 'step_completed' | 'step_failed', + eventData?: Record +): Promise { + const result = await events.create(runId, { + eventType, + correlationId: stepId, + eventData, + }); + if (!result.step) { + throw new Error('Expected step to be updated'); + } + return result.step; +} + +async function createHook( + events: EventsStorage, + runId: string, + data: { + hookId: string; + token: string; + metadata?: unknown; + } +): Promise { + const result = await events.create(runId, { + eventType: 'hook_created', + correlationId: data.hookId, + eventData: { token: data.token, metadata: data.metadata }, + }); + if (!result.hook) { + throw new Error('Expected hook to be created'); + } + return result.hook; +} + describe('Storage (Postgres integration)', () => { if (process.platform === 'win32') { test.skip('skipped on Windows since it relies on a docker container', () => {}); @@ -75,7 +173,7 @@ describe('Storage (Postgres integration)', () => { input: ['arg1', 'arg2'], }; - const run = await runs.create(runData); + const run = await createRun(events, runData); expect(run.runId).toMatch(/^wrun_/); expect(run.deploymentId).toBe('deployment-123'); @@ -98,7 +196,7 @@ describe('Storage (Postgres integration)', () => { input: [], }; - const run = await runs.create(runData); + const run = await createRun(events, runData); expect(run.executionContext).toBeUndefined(); expect(run.input).toEqual([]); @@ -107,7 +205,7 @@ describe('Storage (Postgres integration)', () => { describe('get', () => { it('should retrieve an existing run', async () => { - const created = await runs.create({ + const created = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: ['arg'], @@ -126,72 +224,59 @@ describe('Storage (Postgres integration)', () => { }); }); - describe('update', () => { - it('should update run status to running', async () => { - const created = await runs.create({ + describe('update via events', () => { + it('should update run status to running via run_started event', async () => { + const created = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], }); - const updated = await runs.update(created.runId, { - status: 'running', - }); + const updated = await updateRun(events, created.runId, 'run_started'); expect(updated.status).toBe('running'); expect(updated.startedAt).toBeInstanceOf(Date); }); - it('should update run status to completed', async () => { - const created = await runs.create({ + it('should update run status to completed via run_completed event', async () => { + const created = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], }); - const updated = await runs.update(created.runId, { - status: 'completed', - output: [{ result: 42 }], - }); + const updated = await updateRun( + events, + created.runId, + 'run_completed', + { + output: [{ result: 42 }], + } + ); expect(updated.status).toBe('completed'); expect(updated.completedAt).toBeInstanceOf(Date); expect(updated.output).toEqual([{ result: 42 }]); }); - it('should update run status to failed', async () => { - const created = await runs.create({ + it('should update run status to failed via run_failed event', async () => { + const created = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], }); - const updated = await runs.update(created.runId, { - status: 'failed', - error: { - message: 'Something went wrong', - code: 'ERR_001', - }, + const updated = await updateRun(events, created.runId, 'run_failed', { + error: 'Something went wrong', }); expect(updated.status).toBe('failed'); - expect(updated.error).toEqual({ - message: 'Something went wrong', - code: 'ERR_001', - }); + expect(updated.error?.message).toBe('Something went wrong'); expect(updated.completedAt).toBeInstanceOf(Date); }); - - it('should throw error for non-existent run', async () => { - await expect( - runs.update('missing', { status: 'running' }) - ).rejects.toMatchObject({ - status: 404, - }); - }); }); describe('list', () => { it('should list all runs', async () => { - const run1 = await runs.create({ + const run1 = await createRun(events, { deploymentId: 'deployment-1', workflowName: 'workflow-1', input: [], @@ -200,7 +285,7 @@ describe('Storage (Postgres integration)', () => { // Small delay to ensure different timestamps in createdAt await new Promise((resolve) => setTimeout(resolve, 2)); - const run2 = await runs.create({ + const run2 = await createRun(events, { deploymentId: 'deployment-2', workflowName: 'workflow-2', input: [], @@ -218,12 +303,12 @@ describe('Storage (Postgres integration)', () => { }); it('should filter runs by workflowName', async () => { - await runs.create({ + await createRun(events, { deploymentId: 'deployment-1', workflowName: 'workflow-1', input: [], }); - const run2 = await runs.create({ + const run2 = await createRun(events, { deploymentId: 'deployment-2', workflowName: 'workflow-2', input: [], @@ -238,7 +323,7 @@ describe('Storage (Postgres integration)', () => { it('should support pagination', async () => { // Create multiple runs for (let i = 0; i < 5; i++) { - await runs.create({ + await createRun(events, { deploymentId: `deployment-${i}`, workflowName: `workflow-${i}`, input: [], @@ -260,28 +345,13 @@ describe('Storage (Postgres integration)', () => { expect(page2.data[0].runId).not.toBe(page1.data[0].runId); }); }); - - describe('cancel', () => { - it('should cancel a run', async () => { - const created = await runs.create({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - input: [], - }); - - const cancelled = await runs.cancel(created.runId); - - expect(cancelled.status).toBe('cancelled'); - expect(cancelled.completedAt).toBeInstanceOf(Date); - }); - }); }); describe('steps', () => { let testRunId: string; beforeEach(async () => { - const run = await runs.create({ + const run = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -297,9 +367,9 @@ describe('Storage (Postgres integration)', () => { input: ['input1', 'input2'], }; - const step = await steps.create(testRunId, stepData); + const step = await createStep(events, testRunId, stepData); - expect(step).toEqual({ + expect(step).toMatchObject({ runId: testRunId, stepId: 'step-123', stepName: 'test-step', @@ -312,13 +382,14 @@ describe('Storage (Postgres integration)', () => { completedAt: undefined, createdAt: expect.any(Date), updatedAt: expect.any(Date), + specVersion: 2, }); }); }); describe('get', () => { it('should retrieve a step with runId and stepId', async () => { - const created = await steps.create(testRunId, { + const created = await createStep(events, testRunId, { stepId: 'step-123', stepName: 'test-step', input: ['input1'], @@ -330,7 +401,7 @@ describe('Storage (Postgres integration)', () => { }); it('should retrieve a step with only stepId', async () => { - const created = await steps.create(testRunId, { + const created = await createStep(events, testRunId, { stepId: 'unique-step-123', stepName: 'test-step', input: ['input1'], @@ -348,83 +419,76 @@ describe('Storage (Postgres integration)', () => { }); }); - describe('update', () => { - it('should update step status to running', async () => { - await steps.create(testRunId, { + describe('update via events', () => { + it('should update step status to running via step_started event', async () => { + await createStep(events, testRunId, { stepId: 'step-123', stepName: 'test-step', input: ['input1'], }); - const updated = await steps.update(testRunId, 'step-123', { - status: 'running', - }); + const updated = await updateStep( + events, + testRunId, + 'step-123', + 'step_started', + {} // step_started no longer needs attempt in eventData - World increments it + ); expect(updated.status).toBe('running'); expect(updated.startedAt).toBeInstanceOf(Date); + expect(updated.attempt).toBe(1); // Incremented by step_started }); - it('should update step status to completed', async () => { - await steps.create(testRunId, { + it('should update step status to completed via step_completed event', async () => { + await createStep(events, testRunId, { stepId: 'step-123', stepName: 'test-step', input: ['input1'], }); - const updated = await steps.update(testRunId, 'step-123', { - status: 'completed', - output: ['ok'], - }); + const updated = await updateStep( + events, + testRunId, + 'step-123', + 'step_completed', + { result: ['ok'] } + ); expect(updated.status).toBe('completed'); expect(updated.completedAt).toBeInstanceOf(Date); expect(updated.output).toEqual(['ok']); }); - it('should update step status to failed', async () => { - await steps.create(testRunId, { + it('should update step status to failed via step_failed event', async () => { + await createStep(events, testRunId, { stepId: 'step-123', stepName: 'test-step', input: ['input1'], }); - const updated = await steps.update(testRunId, 'step-123', { - status: 'failed', - error: { - message: 'Step failed', - code: 'STEP_ERR', - }, - }); + const updated = await updateStep( + events, + testRunId, + 'step-123', + 'step_failed', + { error: 'Step failed' } + ); expect(updated.status).toBe('failed'); expect(updated.error?.message).toBe('Step failed'); - expect(updated.error?.code).toBe('STEP_ERR'); expect(updated.completedAt).toBeInstanceOf(Date); }); - - it('should update attempt count', async () => { - await steps.create(testRunId, { - stepId: 'step-123', - stepName: 'test-step', - input: ['input1'], - }); - - const updated = await steps.update(testRunId, 'step-123', { - attempt: 2, - }); - - expect(updated.attempt).toBe(2); - }); }); describe('list', () => { it('should list all steps for a run', async () => { - const step1 = await steps.create(testRunId, { + const step1 = await createStep(events, testRunId, { stepId: 'step-1', stepName: 'first-step', input: [], }); - const step2 = await steps.create(testRunId, { + const step2 = await createStep(events, testRunId, { stepId: 'step-2', stepName: 'second-step', input: [], @@ -446,7 +510,7 @@ describe('Storage (Postgres integration)', () => { it('should support pagination', async () => { // Create multiple steps for (let i = 0; i < 5; i++) { - await steps.create(testRunId, { + await createStep(events, testRunId, { stepId: `step-${i}`, stepName: `step-name-${i}`, input: [], @@ -476,7 +540,7 @@ describe('Storage (Postgres integration)', () => { let testRunId: string; beforeEach(async () => { - const run = await runs.create({ + const run = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -486,56 +550,82 @@ describe('Storage (Postgres integration)', () => { describe('create', () => { it('should create a new event', async () => { + // Create step before step_started event + await createStep(events, testRunId, { + stepId: 'corr_123', + stepName: 'test-step', + input: [], + }); + const eventData = { eventType: 'step_started' as const, correlationId: 'corr_123', }; - const event = await events.create(testRunId, eventData); + const result = await events.create(testRunId, eventData); - expect(event.runId).toBe(testRunId); - expect(event.eventId).toMatch(/^wevt_/); - expect(event.eventType).toBe('step_started'); - expect(event.correlationId).toBe('corr_123'); - expect(event.createdAt).toBeInstanceOf(Date); + expect(result.event.runId).toBe(testRunId); + expect(result.event.eventId).toMatch(/^wevt_/); + expect(result.event.eventType).toBe('step_started'); + expect(result.event.correlationId).toBe('corr_123'); + expect(result.event.createdAt).toBeInstanceOf(Date); }); it('should create a new event with null byte in payload', async () => { - const event = await events.create(testRunId, { + // Create step before step_failed event + await createStep(events, testRunId, { + stepId: 'corr_123_null', + stepName: 'test-step-null', + input: [], + }); + await events.create(testRunId, { + eventType: 'step_started', + correlationId: 'corr_123_null', + }); + + const result = await events.create(testRunId, { eventType: 'step_failed', - correlationId: 'corr_123', + correlationId: 'corr_123_null', eventData: { error: 'Error with null byte \u0000 in message' }, }); - expect(event.runId).toBe(testRunId); - expect(event.eventId).toMatch(/^wevt_/); - expect(event.eventType).toBe('step_failed'); - expect(event.correlationId).toBe('corr_123'); - expect(event.createdAt).toBeInstanceOf(Date); + expect(result.event.runId).toBe(testRunId); + expect(result.event.eventId).toMatch(/^wevt_/); + expect(result.event.eventType).toBe('step_failed'); + expect(result.event.correlationId).toBe('corr_123_null'); + expect(result.event.createdAt).toBeInstanceOf(Date); }); - it('should handle workflow completed events', async () => { + it('should handle run completed events', async () => { const eventData = { - eventType: 'workflow_completed' as const, + eventType: 'run_completed' as const, + eventData: { output: { result: 'done' } }, }; - const event = await events.create(testRunId, eventData); + const result = await events.create(testRunId, eventData); - expect(event.eventType).toBe('workflow_completed'); - expect(event.correlationId).toBeUndefined(); + expect(result.event.eventType).toBe('run_completed'); + expect(result.event.correlationId).toBeUndefined(); }); }); describe('list', () => { it('should list all events for a run', async () => { - const event1 = await events.create(testRunId, { - eventType: 'workflow_started' as const, + const result1 = await events.create(testRunId, { + eventType: 'run_started' as const, }); // Small delay to ensure different timestamps in event IDs await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(testRunId, { + // Create step before step_started event + await createStep(events, testRunId, { + stepId: 'corr-step-1', + stepName: 'test-step', + input: [], + }); + + const result2 = await events.create(testRunId, { eventType: 'step_started' as const, correlationId: 'corr-step-1', }); @@ -545,24 +635,33 @@ describe('Storage (Postgres integration)', () => { pagination: { sortOrder: 'asc' }, // Explicitly request ascending order }); - expect(result.data).toHaveLength(2); + // 4 events: run_created (from createRun), run_started, step_created, step_started + expect(result.data).toHaveLength(4); // Should be in chronological order (oldest first) - expect(result.data[0].eventId).toBe(event1.eventId); - expect(result.data[1].eventId).toBe(event2.eventId); - expect(result.data[1].createdAt.getTime()).toBeGreaterThanOrEqual( - result.data[0].createdAt.getTime() + expect(result.data[0].eventType).toBe('run_created'); + expect(result.data[1].eventId).toBe(result1.event.eventId); + expect(result.data[3].eventId).toBe(result2.event.eventId); + expect(result.data[3].createdAt.getTime()).toBeGreaterThanOrEqual( + result.data[1].createdAt.getTime() ); }); it('should list events in descending order when explicitly requested (newest first)', async () => { - const event1 = await events.create(testRunId, { - eventType: 'workflow_started' as const, + const result1 = await events.create(testRunId, { + eventType: 'run_started' as const, }); // Small delay to ensure different timestamps in event IDs await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(testRunId, { + // Create step before step_started event + await createStep(events, testRunId, { + stepId: 'corr-step-1', + stepName: 'test-step', + input: [], + }); + + const result2 = await events.create(testRunId, { eventType: 'step_started' as const, correlationId: 'corr-step-1', }); @@ -572,18 +671,31 @@ describe('Storage (Postgres integration)', () => { pagination: { sortOrder: 'desc' }, }); - expect(result.data).toHaveLength(2); + // 4 events: run_created (from createRun), run_started, step_created, step_started + expect(result.data).toHaveLength(4); // Should be in reverse chronological order (newest first) - expect(result.data[0].eventId).toBe(event2.eventId); - expect(result.data[1].eventId).toBe(event1.eventId); + expect(result.data[0].eventId).toBe(result2.event.eventId); + expect(result.data[1].eventType).toBe('step_created'); + expect(result.data[2].eventId).toBe(result1.event.eventId); + expect(result.data[3].eventType).toBe('run_created'); expect(result.data[0].createdAt.getTime()).toBeGreaterThanOrEqual( - result.data[1].createdAt.getTime() + result.data[2].createdAt.getTime() ); }); it('should support pagination', async () => { - // Create multiple events + // Create multiple events - must create steps first for (let i = 0; i < 5; i++) { + await createStep(events, testRunId, { + stepId: `corr_${i}`, + stepName: `test-step-${i}`, + input: [], + }); + // Start the step before completing + await events.create(testRunId, { + eventType: 'step_started', + correlationId: `corr_${i}`, + }); await events.create(testRunId, { eventType: 'step_completed', correlationId: `corr_${i}`, @@ -613,27 +725,40 @@ describe('Storage (Postgres integration)', () => { it('should list all events with a specific correlation ID', async () => { const correlationId = 'step-abc123'; + // Create step before step events + await createStep(events, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + // Create events with the target correlation ID - const event1 = await events.create(testRunId, { + const result1 = await events.create(testRunId, { eventType: 'step_started', correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(testRunId, { + const result2 = await events.create(testRunId, { eventType: 'step_completed', correlationId, eventData: { result: 'success' }, }); // Create events with different correlation IDs (should be filtered out) + await createStep(events, testRunId, { + stepId: 'different-step', + stepName: 'different-step', + input: [], + }); await events.create(testRunId, { eventType: 'step_started', correlationId: 'different-step', }); await events.create(testRunId, { - eventType: 'workflow_completed', + eventType: 'run_completed', + eventData: { output: { result: 'done' } }, }); const result = await events.listByCorrelationId({ @@ -641,32 +766,35 @@ describe('Storage (Postgres integration)', () => { pagination: {}, }); - expect(result.data).toHaveLength(2); - expect(result.data[0].eventId).toBe(event1.eventId); - expect(result.data[0].correlationId).toBe(correlationId); - expect(result.data[1].eventId).toBe(event2.eventId); + // 3 events: step_created, step_started, step_completed + expect(result.data).toHaveLength(3); + expect(result.data[0].eventType).toBe('step_created'); + expect(result.data[1].eventId).toBe(result1.event.eventId); expect(result.data[1].correlationId).toBe(correlationId); + expect(result.data[2].eventId).toBe(result2.event.eventId); + expect(result.data[2].correlationId).toBe(correlationId); }); it('should list events across multiple runs with same correlation ID', async () => { const correlationId = 'hook-xyz789'; // Create another run - const run2 = await runs.create({ + const run2 = await createRun(events, { deploymentId: 'deployment-456', workflowName: 'test-workflow-2', input: [], }); // Create events in both runs with same correlation ID - const event1 = await events.create(testRunId, { + const result1 = await events.create(testRunId, { eventType: 'hook_created', correlationId, + eventData: { token: 'test-token-1' }, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(run2.runId, { + const result2 = await events.create(run2.runId, { eventType: 'hook_received', correlationId, eventData: { payload: { data: 'test' } }, @@ -674,7 +802,7 @@ describe('Storage (Postgres integration)', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const event3 = await events.create(testRunId, { + const result3 = await events.create(testRunId, { eventType: 'hook_disposed', correlationId, }); @@ -685,15 +813,21 @@ describe('Storage (Postgres integration)', () => { }); expect(result.data).toHaveLength(3); - expect(result.data[0].eventId).toBe(event1.eventId); + expect(result.data[0].eventId).toBe(result1.event.eventId); expect(result.data[0].runId).toBe(testRunId); - expect(result.data[1].eventId).toBe(event2.eventId); + expect(result.data[1].eventId).toBe(result2.event.eventId); expect(result.data[1].runId).toBe(run2.runId); - expect(result.data[2].eventId).toBe(event3.eventId); + expect(result.data[2].eventId).toBe(result3.event.eventId); expect(result.data[2].runId).toBe(testRunId); }); it('should return empty list for non-existent correlation ID', async () => { + // Create a step and start it + await createStep(events, testRunId, { + stepId: 'existing-step', + stepName: 'existing-step', + input: [], + }); await events.create(testRunId, { eventType: 'step_started', correlationId: 'existing-step', @@ -712,6 +846,13 @@ describe('Storage (Postgres integration)', () => { it('should respect pagination parameters', async () => { const correlationId = 'step_paginated'; + // Create step first + await createStep(events, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + // Create multiple events await events.create(testRunId, { eventType: 'step_started', @@ -723,7 +864,15 @@ describe('Storage (Postgres integration)', () => { await events.create(testRunId, { eventType: 'step_retrying', correlationId, - eventData: { attempt: 1 }, + eventData: { error: 'retry error' }, + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + + // Start again after retry + await events.create(testRunId, { + eventType: 'step_started', + correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); @@ -734,27 +883,38 @@ describe('Storage (Postgres integration)', () => { eventData: { result: 'success' }, }); - // Get first page + // Get first page (step_created, step_started, step_retrying) const page1 = await events.listByCorrelationId({ correlationId, - pagination: { limit: 2 }, + pagination: { limit: 3 }, }); - expect(page1.data).toHaveLength(2); + expect(page1.data).toHaveLength(3); expect(page1.hasMore).toBe(true); expect(page1.cursor).toBeDefined(); - // Get second page + // Get second page (step_started, step_completed) const page2 = await events.listByCorrelationId({ correlationId, - pagination: { limit: 2, cursor: page1.cursor || undefined }, + pagination: { limit: 3, cursor: page1.cursor || undefined }, }); - expect(page2.data).toHaveLength(1); + expect(page2.data).toHaveLength(2); expect(page2.hasMore).toBe(false); }); it('should always return full event data', async () => { + // Create step first + await createStep(events, testRunId, { + stepId: 'step-with-data', + stepName: 'step-with-data', + input: [], + }); + // Start the step before completing + await events.create(testRunId, { + eventType: 'step_started', + correlationId: 'step-with-data', + }); await events.create(testRunId, { eventType: 'step_completed', correlationId: 'step-with-data', @@ -767,22 +927,30 @@ describe('Storage (Postgres integration)', () => { pagination: {}, }); - expect(result.data).toHaveLength(1); - expect(result.data[0].correlationId).toBe('step-with-data'); + // 3 events: step_created, step_started, step_completed + expect(result.data).toHaveLength(3); + expect(result.data[2].correlationId).toBe('step-with-data'); }); it('should return events in ascending order by default', async () => { const correlationId = 'step-ordering'; + // Create step first + await createStep(events, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + // Create events with slight delays to ensure different timestamps - const event1 = await events.create(testRunId, { + const result1 = await events.create(testRunId, { eventType: 'step_started', correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(testRunId, { + const result2 = await events.create(testRunId, { eventType: 'step_completed', correlationId, eventData: { result: 'success' }, @@ -793,25 +961,33 @@ describe('Storage (Postgres integration)', () => { pagination: {}, }); - expect(result.data).toHaveLength(2); - expect(result.data[0].eventId).toBe(event1.eventId); - expect(result.data[1].eventId).toBe(event2.eventId); - expect(result.data[0].createdAt.getTime()).toBeLessThanOrEqual( - result.data[1].createdAt.getTime() + // 3 events: step_created, step_started, step_completed + expect(result.data).toHaveLength(3); + expect(result.data[1].eventId).toBe(result1.event.eventId); + expect(result.data[2].eventId).toBe(result2.event.eventId); + expect(result.data[1].createdAt.getTime()).toBeLessThanOrEqual( + result.data[2].createdAt.getTime() ); }); it('should support descending order', async () => { const correlationId = 'step-desc-order'; - const event1 = await events.create(testRunId, { + // Create step first + await createStep(events, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + + const result1 = await events.create(testRunId, { eventType: 'step_started', correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(testRunId, { + const result2 = await events.create(testRunId, { eventType: 'step_completed', correlationId, eventData: { result: 'success' }, @@ -822,9 +998,10 @@ describe('Storage (Postgres integration)', () => { pagination: { sortOrder: 'desc' }, }); - expect(result.data).toHaveLength(2); - expect(result.data[0].eventId).toBe(event2.eventId); - expect(result.data[1].eventId).toBe(event1.eventId); + // 3 events in descending order: step_completed, step_started, step_created + expect(result.data).toHaveLength(3); + expect(result.data[0].eventId).toBe(result2.event.eventId); + expect(result.data[1].eventId).toBe(result1.event.eventId); expect(result.data[0].createdAt.getTime()).toBeGreaterThanOrEqual( result.data[1].createdAt.getTime() ); @@ -834,14 +1011,15 @@ describe('Storage (Postgres integration)', () => { const hookId = 'hook_test123'; // Create a typical hook lifecycle - const created = await events.create(testRunId, { + const createdResult = await events.create(testRunId, { eventType: 'hook_created' as const, correlationId: hookId, + eventData: { token: 'lifecycle-test-token' }, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const received1 = await events.create(testRunId, { + const received1Result = await events.create(testRunId, { eventType: 'hook_received' as const, correlationId: hookId, eventData: { payload: { request: 1 } }, @@ -849,7 +1027,7 @@ describe('Storage (Postgres integration)', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const received2 = await events.create(testRunId, { + const received2Result = await events.create(testRunId, { eventType: 'hook_received' as const, correlationId: hookId, eventData: { payload: { request: 2 } }, @@ -857,7 +1035,7 @@ describe('Storage (Postgres integration)', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const disposed = await events.create(testRunId, { + const disposedResult = await events.create(testRunId, { eventType: 'hook_disposed' as const, correlationId: hookId, }); @@ -868,15 +1046,1101 @@ describe('Storage (Postgres integration)', () => { }); expect(result.data).toHaveLength(4); - expect(result.data[0].eventId).toBe(created.eventId); + expect(result.data[0].eventId).toBe(createdResult.event.eventId); expect(result.data[0].eventType).toBe('hook_created'); - expect(result.data[1].eventId).toBe(received1.eventId); + expect(result.data[1].eventId).toBe(received1Result.event.eventId); expect(result.data[1].eventType).toBe('hook_received'); - expect(result.data[2].eventId).toBe(received2.eventId); + expect(result.data[2].eventId).toBe(received2Result.event.eventId); expect(result.data[2].eventType).toBe('hook_received'); - expect(result.data[3].eventId).toBe(disposed.eventId); + expect(result.data[3].eventId).toBe(disposedResult.event.eventId); expect(result.data[3].eventType).toBe('hook_disposed'); }); + + it('should enforce token uniqueness across different runs', async () => { + const token = 'unique-token-test'; + + // Create first hook with the token + await events.create(testRunId, { + eventType: 'hook_created' as const, + correlationId: 'hook_1', + eventData: { token }, + }); + + // Create another run + const run2 = await createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: [], + }); + + // Try to create another hook with the same token - should return hook_conflict event + const result = await events.create(run2.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_2', + eventData: { token }, + }); + + // Should return a hook_conflict event instead of throwing + expect(result.event.eventType).toBe('hook_conflict'); + expect(result.event.correlationId).toBe('hook_2'); + expect((result.event as any).eventData.token).toBe(token); + // No hook entity should be created + expect(result.hook).toBeUndefined(); + }); + + it('should allow token reuse after hook is disposed', async () => { + const token = 'reusable-token-test'; + + // Create first hook with the token + await events.create(testRunId, { + eventType: 'hook_created' as const, + correlationId: 'hook_reuse_1', + eventData: { token }, + }); + + // Dispose the first hook + await events.create(testRunId, { + eventType: 'hook_disposed' as const, + correlationId: 'hook_reuse_1', + }); + + // Create another run + const run2 = await createRun(events, { + deploymentId: 'deployment-789', + workflowName: 'test-workflow-3', + input: [], + }); + + // Now creating a hook with the same token should succeed + const result = await events.create(run2.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_reuse_2', + eventData: { token }, + }); + + expect(result.hook).toBeDefined(); + expect(result.hook!.token).toBe(token); + }); + }); + }); + + describe('step terminal state validation', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); + + describe('completed step', () => { + it('should reject step_started on completed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_terminal_1', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_terminal_1', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep(events, testRunId, 'step_terminal_1', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_completed on already completed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_terminal_2', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_terminal_2', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep(events, testRunId, 'step_terminal_2', 'step_completed', { + result: 'done again', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_failed on completed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_terminal_3', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_terminal_3', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep(events, testRunId, 'step_terminal_3', 'step_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('failed step', () => { + it('should reject step_started on failed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_failed_1', + stepName: 'test-step', + input: [], + }); + await updateStep(events, testRunId, 'step_failed_1', 'step_failed', { + error: 'Failed permanently', + }); + + await expect( + updateStep(events, testRunId, 'step_failed_1', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_completed on failed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_failed_2', + stepName: 'test-step', + input: [], + }); + await updateStep(events, testRunId, 'step_failed_2', 'step_failed', { + error: 'Failed permanently', + }); + + await expect( + updateStep(events, testRunId, 'step_failed_2', 'step_completed', { + result: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_failed on already failed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_failed_3', + stepName: 'test-step', + input: [], + }); + await updateStep(events, testRunId, 'step_failed_3', 'step_failed', { + error: 'Failed once', + }); + + await expect( + updateStep(events, testRunId, 'step_failed_3', 'step_failed', { + error: 'Failed again', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_retrying on failed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_failed_retry', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_failed_retry', + 'step_failed', + { + error: 'Failed permanently', + } + ); + + await expect( + updateStep(events, testRunId, 'step_failed_retry', 'step_retrying', { + error: 'Retry attempt', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('step_retrying validation', () => { + it('should reject step_retrying on completed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_completed_retry', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_completed_retry', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep( + events, + testRunId, + 'step_completed_retry', + 'step_retrying', + { + error: 'Retry attempt', + } + ) + ).rejects.toThrow(/terminal/i); + }); + }); + }); + + describe('run terminal state validation', () => { + describe('completed run', () => { + it('should reject run_started on completed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + await expect( + updateRun(events, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_failed on completed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + await expect( + updateRun(events, run.runId, 'run_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_cancelled on completed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + await expect( + events.create(run.runId, { eventType: 'run_cancelled' }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('failed run', () => { + it('should reject run_started on failed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + updateRun(events, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_completed on failed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + updateRun(events, run.runId, 'run_completed', { + output: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_cancelled on failed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + events.create(run.runId, { eventType: 'run_cancelled' }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('cancelled run', () => { + it('should reject run_started on cancelled run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(events, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_completed on cancelled run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(events, run.runId, 'run_completed', { + output: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_failed on cancelled run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(events, run.runId, 'run_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + }); + + describe('allowed operations on terminal runs', () => { + it('should allow step_completed on completed run for in-progress step', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step (making it in-progress) + await createStep(events, run.runId, { + stepId: 'step_in_progress', + stepName: 'test-step', + input: [], + }); + await updateStep(events, run.runId, 'step_in_progress', 'step_started'); + + // Complete the run while step is still running + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + // Should succeed - completing an in-progress step on a terminal run is allowed + const result = await updateStep( + events, + run.runId, + 'step_in_progress', + 'step_completed', + { result: 'step done' } + ); + expect(result.status).toBe('completed'); + }); + + it('should allow step_failed on completed run for in-progress step', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step + await createStep(events, run.runId, { + stepId: 'step_in_progress_fail', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + run.runId, + 'step_in_progress_fail', + 'step_started' + ); + + // Complete the run + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + // Should succeed - failing an in-progress step on a terminal run is allowed + const result = await updateStep( + events, + run.runId, + 'step_in_progress_fail', + 'step_failed', + { error: 'step failed' } + ); + expect(result.status).toBe('failed'); + }); + + it('should auto-delete hooks when run completes (postgres-specific behavior)', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a hook + await createHook(events, run.runId, { + hookId: 'hook_auto_deleted', + token: 'test-token-dispose', + }); + + // Complete the run - this auto-deletes the hook + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + // The hook should no longer exist because run completion auto-deletes hooks + // This is intentional behavior to allow token reuse across runs + await expect( + events.create(run.runId, { + eventType: 'hook_disposed', + correlationId: 'hook_auto_deleted', + }) + ).rejects.toThrow(/not found/i); + }); + }); + + describe('disallowed operations on terminal runs', () => { + it('should reject step_created on completed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + await expect( + createStep(events, run.runId, { + stepId: 'new_step', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_started on completed run for pending step', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a step but don't start it + await createStep(events, run.runId, { + stepId: 'pending_step', + stepName: 'test-step', + input: [], + }); + + // Complete the run + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + // Should reject - cannot start a pending step on a terminal run + await expect( + updateStep(events, run.runId, 'pending_step', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on completed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + await expect( + createHook(events, run.runId, { + hookId: 'new_hook', + token: 'new-token', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_created on failed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + createStep(events, run.runId, { + stepId: 'new_step_failed', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_created on cancelled run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createStep(events, run.runId, { + stepId: 'new_step_cancelled', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on failed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + createHook(events, run.runId, { + hookId: 'new_hook_failed', + token: 'new-token-failed', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on cancelled run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createHook(events, run.runId, { + hookId: 'new_hook_cancelled', + token: 'new-token-cancelled', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('idempotent operations', () => { + it('should allow run_cancelled on already cancelled run (idempotent)', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should succeed - idempotent operation + const result = await events.create(run.runId, { + eventType: 'run_cancelled', + }); + expect(result.run?.status).toBe('cancelled'); + }); + }); + + describe('step_retrying event handling', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); + + it('should set step status to pending and record error', async () => { + await createStep(events, testRunId, { + stepId: 'step_retry_1', + stepName: 'test-step', + input: [], + }); + await updateStep(events, testRunId, 'step_retry_1', 'step_started'); + + const result = await events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_1', + eventData: { + error: 'Temporary failure', + retryAfter: new Date(Date.now() + 5000), + }, + }); + + expect(result.step?.status).toBe('pending'); + expect(result.step?.error?.message).toBe('Temporary failure'); + expect(result.step?.retryAfter).toBeInstanceOf(Date); + }); + + it('should increment attempt when step_started is called after step_retrying', async () => { + await createStep(events, testRunId, { + stepId: 'step_retry_2', + stepName: 'test-step', + input: [], + }); + + // First attempt + const started1 = await updateStep( + events, + testRunId, + 'step_retry_2', + 'step_started' + ); + expect(started1.attempt).toBe(1); + + // Retry + await events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_2', + eventData: { error: 'Temporary failure' }, + }); + + // Second attempt + const started2 = await updateStep( + events, + testRunId, + 'step_retry_2', + 'step_started' + ); + expect(started2.attempt).toBe(2); + }); + + it('should reject step_retrying on completed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_retry_completed', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_retry_completed', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_completed', + eventData: { error: 'Should not work' }, + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_retrying on failed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_retry_failed', + stepName: 'test-step', + input: [], + }); + await updateStep(events, testRunId, 'step_retry_failed', 'step_failed', { + error: 'Permanent failure', + }); + + await expect( + events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_failed', + eventData: { error: 'Should not work' }, + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('run cancellation with in-flight entities', () => { + it('should allow in-progress step to complete after run cancelled', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step + await createStep(events, run.runId, { + stepId: 'step_in_flight', + stepName: 'test-step', + input: [], + }); + await updateStep(events, run.runId, 'step_in_flight', 'step_started'); + + // Cancel the run + await events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should succeed - completing an in-progress step is allowed + const result = await updateStep( + events, + run.runId, + 'step_in_flight', + 'step_completed', + { result: 'done' } + ); + expect(result.status).toBe('completed'); + }); + + it('should reject step_created after run cancelled', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createStep(events, run.runId, { + stepId: 'new_step_after_cancel', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_started for pending step after run cancelled', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a step but don't start it + await createStep(events, run.runId, { + stepId: 'pending_after_cancel', + stepName: 'test-step', + input: [], + }); + + // Cancel the run + await events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should reject - cannot start a pending step on a cancelled run + await expect( + updateStep(events, run.runId, 'pending_after_cancel', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('event ordering validation', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); + + it('should reject step_completed before step_created', async () => { + await expect( + events.create(testRunId, { + eventType: 'step_completed', + correlationId: 'nonexistent_step', + eventData: { result: 'done' }, + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject step_started before step_created', async () => { + await expect( + events.create(testRunId, { + eventType: 'step_started', + correlationId: 'nonexistent_step_started', + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject step_failed before step_created', async () => { + await expect( + events.create(testRunId, { + eventType: 'step_failed', + correlationId: 'nonexistent_step_failed', + eventData: { error: 'Failed' }, + }) + ).rejects.toThrow(/not found/i); + }); + + it('should allow step_completed without step_started (instant completion)', async () => { + await createStep(events, testRunId, { + stepId: 'instant_complete', + stepName: 'test-step', + input: [], + }); + + // Should succeed - instant completion without starting + const result = await updateStep( + events, + testRunId, + 'instant_complete', + 'step_completed', + { result: 'instant' } + ); + expect(result.status).toBe('completed'); + }); + + it('should reject hook_disposed before hook_created', async () => { + await expect( + events.create(testRunId, { + eventType: 'hook_disposed', + correlationId: 'nonexistent_hook', + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject hook_received before hook_created', async () => { + await expect( + events.create(testRunId, { + eventType: 'hook_received', + correlationId: 'nonexistent_hook_received', + eventData: { payload: {} }, + }) + ).rejects.toThrow(/not found/i); + }); + }); + + describe('legacy/backwards compatibility', () => { + // Helper to create a legacy run directly in the database (bypassing events.create) + // Column mapping: id (runId), deployment_id, name (workflowName), spec_version, status, input + async function createLegacyRun(runId: string, specVersion: number | null) { + await sql` + INSERT INTO workflow.workflow_runs (id, deployment_id, name, spec_version, status, input, created_at, updated_at) + VALUES (${runId}, 'legacy-deployment', 'legacy-workflow', ${specVersion}, 'running', '[]'::jsonb, NOW(), NOW()) + `; + } + + describe('legacy runs (specVersion < 2 or null)', () => { + it('should handle run_cancelled on legacy run with specVersion=1', async () => { + const runId = 'wrun_legacy_v1'; + await createLegacyRun(runId, 1); + + const result = await events.create(runId, { + eventType: 'run_cancelled', + }); + + // Legacy behavior: run is updated but event is not stored + expect(result.run?.status).toBe('cancelled'); + expect(result.event).toBeUndefined(); + }); + + it('should handle run_cancelled on legacy run with specVersion=null', async () => { + const runId = 'wrun_legacy_null'; + await createLegacyRun(runId, null); + + const result = await events.create(runId, { + eventType: 'run_cancelled', + }); + + // Legacy behavior: run is updated but event is not stored + expect(result.run?.status).toBe('cancelled'); + expect(result.event).toBeUndefined(); + }); + + it('should handle wait_completed on legacy run', async () => { + const runId = 'wrun_legacy_wait'; + await createLegacyRun(runId, 1); + + const result = await events.create(runId, { + eventType: 'wait_completed', + correlationId: 'wait_123', + eventData: { result: 'waited' }, + } as any); + + // Legacy behavior: event is stored but no entity mutation + expect(result.event).toBeDefined(); + expect(result.event?.eventType).toBe('wait_completed'); + expect(result.run).toBeUndefined(); + }); + + it('should handle hook_received on legacy run', async () => { + const runId = 'wrun_legacy_hook_received'; + await createLegacyRun(runId, 1); + + const result = await events.create(runId, { + eventType: 'hook_received', + correlationId: 'hook_123', + eventData: { payload: { data: 'test' } }, + } as any); + + // Legacy behavior: event is stored but no entity mutation + // (hooks exist via old system, not via events) + expect(result.event).toBeDefined(); + expect(result.event?.eventType).toBe('hook_received'); + expect(result.event?.correlationId).toBe('hook_123'); + expect(result.hook).toBeUndefined(); + }); + + it('should reject unsupported events on legacy runs', async () => { + const runId = 'wrun_legacy_unsupported'; + await createLegacyRun(runId, 1); + + // run_started is not supported for legacy runs + await expect( + events.create(runId, { eventType: 'run_started' }) + ).rejects.toThrow(/not supported for legacy runs/i); + + // run_completed is not supported for legacy runs + await expect( + events.create(runId, { + eventType: 'run_completed', + eventData: { output: 'done' }, + }) + ).rejects.toThrow(/not supported for legacy runs/i); + + // run_failed is not supported for legacy runs + await expect( + events.create(runId, { + eventType: 'run_failed', + eventData: { error: 'failed' }, + }) + ).rejects.toThrow(/not supported for legacy runs/i); + }); + + it('should delete hooks when legacy run is cancelled', async () => { + const runId = 'wrun_legacy_hooks'; + await createLegacyRun(runId, 1); + + // Create a hook directly in the database for this run + await sql` + INSERT INTO workflow.workflow_hooks (hook_id, run_id, token, owner_id, project_id, environment, created_at) + VALUES ('hook_legacy', ${runId}, 'legacy-token', 'owner', 'project', 'test', NOW()) + `; + + // Verify hook exists + const [hookBefore] = + await sql`SELECT hook_id FROM workflow.workflow_hooks WHERE hook_id = 'hook_legacy'`; + expect(hookBefore).toBeDefined(); + + // Cancel the legacy run + await events.create(runId, { eventType: 'run_cancelled' }); + + // Hook should be deleted + const [hookAfter] = + await sql`SELECT hook_id FROM workflow.workflow_hooks WHERE hook_id = 'hook_legacy'`; + expect(hookAfter).toBeUndefined(); + }); + }); + + describe('newer runs (specVersion > current)', () => { + it('should reject events on runs with newer specVersion', async () => { + const runId = 'wrun_future'; + // Create a run with a future spec version (higher than current) + await sql` + INSERT INTO workflow.workflow_runs (id, deployment_id, name, spec_version, status, input, created_at, updated_at) + VALUES (${runId}, 'future-deployment', 'future-workflow', 999, 'running', '[]'::jsonb, NOW(), NOW()) + `; + + await expect( + events.create(runId, { eventType: 'run_started' }) + ).rejects.toThrow(/requires spec version 999/i); + }); + }); + + describe('current version runs', () => { + it('should process events normally for current specVersion runs', async () => { + // Create run via events.create (gets current specVersion) + const run = await createRun(events, { + deploymentId: 'current-deployment', + workflowName: 'current-workflow', + input: [], + }); + + // Should work normally + const result = await events.create(run.runId, { + eventType: 'run_started', + }); + + expect(result.run?.status).toBe('running'); + expect(result.event?.eventType).toBe('run_started'); + }); + }); + + describe('legacy error parsing', () => { + it('should parse legacy errorJson field on runs', async () => { + const runId = 'wrun_legacy_error'; + // Create a run with legacy error format (error column is the text/JSON one) + // Failed runs need completed_at set + await sql` + INSERT INTO workflow.workflow_runs (id, deployment_id, name, spec_version, status, input, error, created_at, updated_at, completed_at) + VALUES (${runId}, 'deployment', 'workflow', 2, 'failed', '[]'::jsonb, '{"message":"Legacy error","stack":"at foo()"}', NOW(), NOW(), NOW()) + `; + + const run = await runs.get(runId); + expect(run.error?.message).toBe('Legacy error'); + expect(run.error?.stack).toBe('at foo()'); + }); + + it('should parse legacy errorJson as plain string', async () => { + const runId = 'wrun_legacy_string_error'; + // Create a run with plain string error + // Failed runs need completed_at set + await sql` + INSERT INTO workflow.workflow_runs (id, deployment_id, name, spec_version, status, input, error, created_at, updated_at, completed_at) + VALUES (${runId}, 'deployment', 'workflow', 2, 'failed', '[]'::jsonb, '"Simple error message"', NOW(), NOW(), NOW()) + `; + + const run = await runs.get(runId); + expect(run.error?.message).toBe('Simple error message'); + }); + + it('should parse legacy errorJson field on steps', async () => { + // First create a run and step + const run = await createRun(events, { + deploymentId: 'deployment', + workflowName: 'workflow', + input: [], + }); + + // Insert a step directly with legacy error format (error column is the text/JSON one) + // Failed steps need completed_at set + await sql` + INSERT INTO workflow.workflow_steps (run_id, step_id, step_name, status, input, error, attempt, created_at, updated_at, completed_at) + VALUES (${run.runId}, 'step_legacy_err', 'test-step', 'failed', '[]'::jsonb, '{"message":"Step error","stack":"at bar()"}', 1, NOW(), NOW(), NOW()) + `; + + const step = await steps.get(run.runId, 'step_legacy_err'); + expect(step.error?.message).toBe('Step error'); + expect(step.error?.stack).toBe('at bar()'); + }); }); }); }); diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 8b94d03bf4..0a291b6d15 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -1,15 +1,19 @@ import { + type AnyEventRequest, type CreateEventParams, - type CreateEventRequest, type Event, + type EventResult, EventSchema, EventTypeSchema, + HookSchema, type ListEventsByCorrelationIdParams, type ListEventsParams, type PaginatedResponse, PaginatedResponseSchema, + WorkflowRunSchema, } from '@workflow/world'; import z from 'zod'; +import { deserializeStep, StepWireSchema } from './steps.js'; import type { APIConfig } from './utils.js'; import { DEFAULT_RESOLVE_DATA_OPTION, @@ -26,8 +30,18 @@ function filterEventData(event: any, resolveData: 'none' | 'all'): Event { return event; } +// Schema for EventResult wire format returned by events.create +// Uses wire format schemas for step to handle field name mapping +const EventResultWireSchema = z.object({ + event: EventSchema, + run: WorkflowRunSchema.optional(), + step: StepWireSchema.optional(), + hook: HookSchema.optional(), +}); + // Would usually "EventSchema.omit({ eventData: true })" but that doesn't work // on zod unions. Re-creating the schema manually. +// specVersion defaults to 1 (legacy) when parsing responses from storage const EventWithRefsSchema = z.object({ eventId: z.string(), runId: z.string(), @@ -35,6 +49,7 @@ const EventWithRefsSchema = z.object({ correlationId: z.string().optional(), eventDataRef: z.any().optional(), createdAt: z.coerce.date(), + specVersion: z.number().default(1), }); // Functions @@ -68,8 +83,8 @@ export async function getWorkflowRunEvents( const queryString = searchParams.toString(); const query = queryString ? `?${queryString}` : ''; const endpoint = correlationId - ? `/v1/events${query}` - : `/v1/runs/${runId}/events${query}`; + ? `/v2/events${query}` + : `/v2/runs/${runId}/events${query}`; const response = (await makeRequest({ endpoint, @@ -89,22 +104,31 @@ export async function getWorkflowRunEvents( } export async function createWorkflowRunEvent( - id: string, - data: CreateEventRequest, + id: string | null, + data: AnyEventRequest, params?: CreateEventParams, config?: APIConfig -): Promise { +): Promise { const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - const event = await makeRequest({ - endpoint: `/v1/runs/${id}/events`, + // For run_created events, runId is null - use "null" string in the URL path + const runIdPath = id === null ? 'null' : id; + + const wireResult = await makeRequest({ + endpoint: `/v2/runs/${runIdPath}/events`, options: { method: 'POST', body: JSON.stringify(data, dateToStringReplacer), }, config, - schema: EventSchema, + schema: EventResultWireSchema, }); - return filterEventData(event, resolveData); + // Transform wire format to interface format + return { + event: filterEventData(wireResult.event, resolveData), + run: wireResult.run, + step: wireResult.step ? deserializeStep(wireResult.step) : undefined, + hook: wireResult.hook, + }; } diff --git a/packages/world-vercel/src/hooks.ts b/packages/world-vercel/src/hooks.ts index dd5b8190ae..87aa7281be 100644 --- a/packages/world-vercel/src/hooks.ts +++ b/packages/world-vercel/src/hooks.ts @@ -52,7 +52,7 @@ export async function listHooks( if (runId) searchParams.set('runId', runId); const queryString = searchParams.toString(); - const endpoint = `/v1/hooks${queryString ? `?${queryString}` : ''}`; + const endpoint = `/v2/hooks${queryString ? `?${queryString}` : ''}`; const response = (await makeRequest({ endpoint, @@ -75,7 +75,7 @@ export async function getHook( config?: APIConfig ): Promise { const resolveData = params?.resolveData || 'all'; - const endpoint = `/v1/hooks/${hookId}`; + const endpoint = `/v2/hooks/${hookId}`; const hook = await makeRequest({ endpoint, @@ -93,7 +93,7 @@ export async function createHook( config?: APIConfig ): Promise { return makeRequest({ - endpoint: `/v1/hooks/create`, + endpoint: `/v2/hooks/create`, options: { method: 'POST', body: JSON.stringify( @@ -114,7 +114,7 @@ export async function getHookByToken( config?: APIConfig ): Promise { return makeRequest({ - endpoint: `/v1/hooks/by-token?token=${encodeURIComponent(token)}`, + endpoint: `/v2/hooks/by-token?token=${encodeURIComponent(token)}`, options: { method: 'GET', }, @@ -128,7 +128,7 @@ export async function disposeHook( config?: APIConfig ): Promise { return makeRequest({ - endpoint: `/v1/hooks/${hookId}`, + endpoint: `/v2/hooks/${hookId}`, options: { method: 'DELETE' }, config, schema: HookSchema, diff --git a/packages/world-vercel/src/queue.test.ts b/packages/world-vercel/src/queue.test.ts index de8e6a8c41..fe168b63bc 100644 --- a/packages/world-vercel/src/queue.test.ts +++ b/packages/world-vercel/src/queue.test.ts @@ -118,6 +118,53 @@ describe('createQueue', () => { } } }); + + it('should silently handle idempotency key conflicts', async () => { + mockSend.mockRejectedValue( + new Error('Duplicate idempotency key detected') + ); + + const originalEnv = process.env.VERCEL_DEPLOYMENT_ID; + process.env.VERCEL_DEPLOYMENT_ID = 'dpl_test'; + + try { + const queue = createQueue(); + const result = await queue.queue( + '__wkf_workflow_test', + { runId: 'run-123' }, + { idempotencyKey: 'my-key' } + ); + + // Should not throw, and should return a placeholder messageId + expect(result.messageId).toBe('msg_duplicate_my-key'); + } finally { + if (originalEnv !== undefined) { + process.env.VERCEL_DEPLOYMENT_ID = originalEnv; + } else { + delete process.env.VERCEL_DEPLOYMENT_ID; + } + } + }); + + it('should rethrow non-idempotency errors', async () => { + mockSend.mockRejectedValue(new Error('Some other error')); + + const originalEnv = process.env.VERCEL_DEPLOYMENT_ID; + process.env.VERCEL_DEPLOYMENT_ID = 'dpl_test'; + + try { + const queue = createQueue(); + await expect( + queue.queue('__wkf_workflow_test', { runId: 'run-123' }) + ).rejects.toThrow('Some other error'); + } finally { + if (originalEnv !== undefined) { + process.env.VERCEL_DEPLOYMENT_ID = originalEnv; + } else { + delete process.env.VERCEL_DEPLOYMENT_ID; + } + } + }); }); describe('createQueueHandler()', () => { diff --git a/packages/world-vercel/src/queue.ts b/packages/world-vercel/src/queue.ts index 9dc8ec6512..9c5d85e918 100644 --- a/packages/world-vercel/src/queue.ts +++ b/packages/world-vercel/src/queue.ts @@ -93,12 +93,34 @@ export function createQueue(config?: APIConfig): Queue { deploymentId: opts?.deploymentId, }); const sanitizedQueueName = queueName.replace(/[^A-Za-z0-9-_]/g, '-'); - const { messageId } = await queueClient.send( - sanitizedQueueName, - encoded, - opts - ); - return { messageId: MessageId.parse(messageId) }; + try { + const { messageId } = await queueClient.send( + sanitizedQueueName, + encoded, + opts + ); + return { messageId: MessageId.parse(messageId) }; + } catch (error) { + // Silently handle idempotency key conflicts - the message was already queued + // This matches the behavior of world-local and world-postgres + if ( + error instanceof Error && + // TODO: checking the error message is flaky. VQS should throw a special duplicate + // error class + error.message === 'Duplicate idempotency key detected' + ) { + // Return a placeholder messageId since the original is not available from the error. + // Callers using idempotency keys shouldn't depend on the returned messageId. + // TODO : VQS should just return the message ID of the exisitng message, or we should + // stop expecting any world to include this + return { + messageId: MessageId.parse( + `msg_duplicate_${opts?.idempotencyKey ?? 'unknown'}` + ), + }; + } + throw error; + } }; const createQueueHandler: Queue['createQueueHandler'] = (prefix, handler) => { diff --git a/packages/world-vercel/src/runs.ts b/packages/world-vercel/src/runs.ts index 6b624d7397..485e54c524 100644 --- a/packages/world-vercel/src/runs.ts +++ b/packages/world-vercel/src/runs.ts @@ -6,6 +6,7 @@ import { type ListWorkflowRunsParams, type PaginatedResponse, PaginatedResponseSchema, + StructuredErrorSchema, type UpdateWorkflowRunRequest, type WorkflowRun, WorkflowRunBaseSchema, @@ -22,17 +23,18 @@ import { /** * Wire format schema for workflow runs coming from the backend. - * The backend returns error as a JSON string, not an object, so we need - * a schema that accepts the wire format before deserialization. + * The backend may return error either as: + * - A JSON string (legacy format) that needs deserialization + * - An already structured object (new format) with { message, stack?, code? } * * This is used for validation in makeRequest(), then deserializeError() - * transforms the string into the expected StructuredError object. + * normalizes both formats into the expected StructuredError object. */ const WorkflowRunWireBaseSchema = WorkflowRunBaseSchema.omit({ error: true, }).extend({ - // Backend returns error as a JSON string, not an object - error: z.string().optional(), + // Backend returns error as either a JSON string or structured object + error: z.union([z.string(), StructuredErrorSchema]).optional(), }); // Wire schema for resolved data (full input/output) @@ -97,7 +99,7 @@ export async function listWorkflowRuns( searchParams.set('remoteRefBehavior', remoteRefBehavior); const queryString = searchParams.toString(); - const endpoint = `/v1/runs${queryString ? `?${queryString}` : ''}`; + const endpoint = `/v2/runs${queryString ? `?${queryString}` : ''}`; const response = (await makeRequest({ endpoint, @@ -121,7 +123,7 @@ export async function createWorkflowRun( config?: APIConfig ): Promise { const run = await makeRequest({ - endpoint: '/v1/runs/create', + endpoint: '/v2/runs/create', options: { method: 'POST', body: JSON.stringify(data, dateToStringReplacer), @@ -144,7 +146,7 @@ export async function getWorkflowRun( searchParams.set('remoteRefBehavior', remoteRefBehavior); const queryString = searchParams.toString(); - const endpoint = `/v1/runs/${id}${queryString ? `?${queryString}` : ''}`; + const endpoint = `/v2/runs/${id}${queryString ? `?${queryString}` : ''}`; try { const run = await makeRequest({ @@ -173,7 +175,7 @@ export async function updateWorkflowRun( try { const serialized = serializeError(data); const run = await makeRequest({ - endpoint: `/v1/runs/${id}`, + endpoint: `/v2/runs/${id}`, options: { method: 'PUT', body: JSON.stringify(serialized, dateToStringReplacer), @@ -202,7 +204,7 @@ export async function cancelWorkflowRun( searchParams.set('remoteRefBehavior', remoteRefBehavior); const queryString = searchParams.toString(); - const endpoint = `/v1/runs/${id}/cancel${queryString ? `?${queryString}` : ''}`; + const endpoint = `/v2/runs/${id}/cancel${queryString ? `?${queryString}` : ''}`; try { const run = await makeRequest({ diff --git a/packages/world-vercel/src/steps.ts b/packages/world-vercel/src/steps.ts index 83233446d7..8a9b578702 100644 --- a/packages/world-vercel/src/steps.ts +++ b/packages/world-vercel/src/steps.ts @@ -13,24 +13,31 @@ import type { APIConfig } from './utils.js'; import { DEFAULT_RESOLVE_DATA_OPTION, dateToStringReplacer, - deserializeError, makeRequest, - serializeError, } from './utils.js'; /** * Wire format schema for steps coming from the backend. - * The backend returns error as a JSON string, not an object, so we need - * a schema that accepts the wire format before deserialization. - * - * This is used for validation in makeRequest(), then deserializeStepError() - * transforms the string into the expected StructuredError object. + * Handles error deserialization from wire format. */ -const StepWireSchema = StepSchema.omit({ +export const StepWireSchema = StepSchema.omit({ error: true, }).extend({ - // Backend returns error as a JSON string, not an object - error: z.string().optional(), + // Backend returns error either as: + // - A JSON string (legacy/lazy mode) + // - An object {message, stack} (when errorRef is resolved) + // This will be deserialized and mapped to error + error: z + .union([ + z.string(), + z.object({ + message: z.string(), + stack: z.string().optional(), + code: z.string().optional(), + }), + ]) + .optional(), + errorRef: z.any().optional(), }); // Wire schema for lazy mode with refs instead of data @@ -45,18 +52,66 @@ const StepWireWithRefsSchema = StepWireSchema.omit({ output: z.any().optional(), }); +/** + * Transform step from wire format to Step interface format. + * Maps: + * - error/errorRef → error (deserializing JSON string to StructuredError) + */ +export function deserializeStep(wireStep: any): Step { + const { error, errorRef, ...rest } = wireStep; + + const result: any = { + ...rest, + }; + + // Deserialize error to StructuredError + // The backend returns error as: + // - error: JSON string (legacy) or object (when resolved) + // - errorRef: resolved object {message, stack} when remoteRefBehavior=resolve + const errorSource = error ?? errorRef; + if (errorSource) { + if (typeof errorSource === 'string') { + try { + const parsed = JSON.parse(errorSource); + if (typeof parsed === 'object' && parsed.message !== undefined) { + result.error = { + message: parsed.message, + stack: parsed.stack, + code: parsed.code, + }; + } else { + // Parsed but not an object with message + result.error = { message: String(parsed) }; + } + } catch { + // Not JSON, treat as plain string + result.error = { message: errorSource }; + } + } else if (typeof errorSource === 'object' && errorSource !== null) { + // Already an object (from resolved ref) + result.error = { + message: errorSource.message ?? 'Unknown error', + stack: errorSource.stack, + code: errorSource.code, + }; + } + } + + return result as Step; +} + // Helper to filter step data based on resolveData setting function filterStepData(step: any, resolveData: 'none' | 'all'): Step { if (resolveData === 'none') { const { inputRef: _inputRef, outputRef: _outputRef, ...rest } = step; - const deserialized = deserializeError(rest); + const deserialized = deserializeStep(rest); return { ...deserialized, input: [], output: undefined, }; } - return deserializeError(step); + return deserializeStep(step); } // Functions @@ -82,7 +137,7 @@ export async function listWorkflowRunSteps( searchParams.set('remoteRefBehavior', remoteRefBehavior); const queryString = searchParams.toString(); - const endpoint = `/v1/runs/${runId}/steps${queryString ? `?${queryString}` : ''}`; + const endpoint = `/v2/runs/${runId}/steps${queryString ? `?${queryString}` : ''}`; const response = (await makeRequest({ endpoint, @@ -105,7 +160,7 @@ export async function createStep( config?: APIConfig ): Promise { const step = await makeRequest({ - endpoint: `/v1/runs/${runId}/steps`, + endpoint: `/v2/runs/${runId}/steps`, options: { method: 'POST', body: JSON.stringify(data, dateToStringReplacer), @@ -113,7 +168,7 @@ export async function createStep( config, schema: StepWireSchema, }); - return deserializeError(step); + return deserializeStep(step); } export async function updateStep( @@ -122,17 +177,22 @@ export async function updateStep( data: UpdateStepRequest, config?: APIConfig ): Promise { - const serialized = serializeError(data); + // Map interface field names to wire format field names + const { error: stepError, ...rest } = data; + const wireData: any = { ...rest }; + if (stepError) { + wireData.error = JSON.stringify(stepError); + } const step = await makeRequest({ - endpoint: `/v1/runs/${runId}/steps/${stepId}`, + endpoint: `/v2/runs/${runId}/steps/${stepId}`, options: { method: 'PUT', - body: JSON.stringify(serialized, dateToStringReplacer), + body: JSON.stringify(wireData, dateToStringReplacer), }, config, schema: StepWireSchema, }); - return deserializeError(step); + return deserializeStep(step); } export async function getStep( @@ -149,8 +209,8 @@ export async function getStep( const queryString = searchParams.toString(); const endpoint = runId - ? `/v1/runs/${runId}/steps/${stepId}${queryString ? `?${queryString}` : ''}` - : `/v1/steps/${stepId}${queryString ? `?${queryString}` : ''}`; + ? `/v2/runs/${runId}/steps/${stepId}${queryString ? `?${queryString}` : ''}` + : `/v2/steps/${stepId}${queryString ? `?${queryString}` : ''}`; const step = await makeRequest({ endpoint, diff --git a/packages/world-vercel/src/storage.ts b/packages/world-vercel/src/storage.ts index 585aa7dc92..da874bbf7d 100644 --- a/packages/world-vercel/src/storage.ts +++ b/packages/world-vercel/src/storage.ts @@ -1,41 +1,19 @@ import type { Storage } from '@workflow/world'; import { createWorkflowRunEvent, getWorkflowRunEvents } from './events.js'; -import { - createHook, - disposeHook, - getHook, - getHookByToken, - listHooks, -} from './hooks.js'; -import { - cancelWorkflowRun, - createWorkflowRun, - getWorkflowRun, - listWorkflowRuns, - updateWorkflowRun, -} from './runs.js'; -import { - createStep, - getStep, - listWorkflowRunSteps, - updateStep, -} from './steps.js'; +import { getHook, getHookByToken, listHooks } from './hooks.js'; +import { getWorkflowRun, listWorkflowRuns } from './runs.js'; +import { getStep, listWorkflowRunSteps } from './steps.js'; import type { APIConfig } from './utils.js'; export function createStorage(config?: APIConfig): Storage { return { // Storage interface with namespaced methods runs: { - create: (data) => createWorkflowRun(data, config), get: (id, params) => getWorkflowRun(id, params, config), - update: (id, data) => updateWorkflowRun(id, data, config), list: (params) => listWorkflowRuns(params, config), - cancel: (id, params) => cancelWorkflowRun(id, params, config), }, steps: { - create: (runId, data) => createStep(runId, data, config), get: (runId, stepId, params) => getStep(runId, stepId, params, config), - update: (runId, stepId, data) => updateStep(runId, stepId, data, config), list: (params) => listWorkflowRunSteps(params, config), }, events: { @@ -45,11 +23,9 @@ export function createStorage(config?: APIConfig): Storage { listByCorrelationId: (params) => getWorkflowRunEvents(params, config), }, hooks: { - create: (runId, data) => createHook(runId, data, config), get: (hookId, params) => getHook(hookId, params, config), getByToken: (token) => getHookByToken(token, config), list: (params) => listHooks(params, config), - dispose: (hookId) => disposeHook(hookId, config), }, }; } diff --git a/packages/world-vercel/src/streamer.ts b/packages/world-vercel/src/streamer.ts index 24e1b00ac4..017b3eaed1 100644 --- a/packages/world-vercel/src/streamer.ts +++ b/packages/world-vercel/src/streamer.ts @@ -8,10 +8,10 @@ function getStreamUrl( ) { if (runId) { return new URL( - `${httpConfig.baseUrl}/v1/runs/${runId}/stream/${encodeURIComponent(name)}` + `${httpConfig.baseUrl}/v2/runs/${runId}/stream/${encodeURIComponent(name)}` ); } - return new URL(`${httpConfig.baseUrl}/v1/stream/${encodeURIComponent(name)}`); + return new URL(`${httpConfig.baseUrl}/v2/stream/${encodeURIComponent(name)}`); } export function createStreamer(config?: APIConfig): Streamer { @@ -58,7 +58,7 @@ export function createStreamer(config?: APIConfig): Streamer { async listStreamsByRunId(runId: string) { const httpConfig = await getHttpConfig(config); - const url = new URL(`${httpConfig.baseUrl}/v1/runs/${runId}/streams`); + const url = new URL(`${httpConfig.baseUrl}/v2/runs/${runId}/streams`); const res = await fetch(url, { headers: httpConfig.headers }); if (!res.ok) throw new Error(`Failed to list streams: ${res.status}`); return (await res.json()) as string[]; diff --git a/packages/world-vercel/src/utils.ts b/packages/world-vercel/src/utils.ts index 0c1ea5b75c..baf9566fd6 100644 --- a/packages/world-vercel/src/utils.ts +++ b/packages/world-vercel/src/utils.ts @@ -54,19 +54,20 @@ export function serializeError( /** * Helper to deserialize error field from the backend into a StructuredError object. - * Handles backwards compatibility: + * Handles multiple formats from the backend: + * - If error is already a structured object → validate and use directly * - If error is a JSON string with {message, stack, code} → parse into StructuredError * - If error is a plain string → treat as error message with no stack * - If no error → undefined * - * This function transforms objects from wire format (where error is a JSON string) - * to domain format (where error is a StructuredError object). The generic type - * parameter should be the expected output type (WorkflowRun or Step). + * This function transforms objects from wire format (where error may be a JSON string + * or already structured) to domain format (where error is a StructuredError object). + * The generic type parameter should be the expected output type (WorkflowRun or Step). * * Note: The type assertion is necessary because the wire format types from Zod schemas - * have `error?: string` while the domain types have complex error types (e.g., discriminated - * unions with `error: void` or `error: StructuredError` depending on status), but the - * transformation preserves all other fields correctly. + * have `error?: string | StructuredError` while the domain types have complex error types + * (e.g., discriminated unions with `error: void` or `error: StructuredError` depending on + * status), but the transformation preserves all other fields correctly. */ export function deserializeError>(obj: any): T { const { error, ...rest } = obj; @@ -75,26 +76,47 @@ export function deserializeError>(obj: any): T { return obj as T; } - // Try to parse as structured error JSON - try { - const parsed = StructuredErrorSchema.parse(JSON.parse(error)); - return { - ...rest, - error: { - message: parsed.message, - stack: parsed.stack, - code: parsed.code, - }, - } as T; - } catch { - // Backwards compatibility: error is just a plain string - return { - ...rest, - error: { - message: error, - }, - } as T; + // If error is already an object (new format), validate and use directly + if (typeof error === 'object' && error !== null) { + const result = StructuredErrorSchema.safeParse(error); + if (result.success) { + return { + ...rest, + error: { + message: result.data.message, + stack: result.data.stack, + code: result.data.code, + }, + } as T; + } + // Fall through to treat as unknown format } + + // If error is a string, try to parse as structured error JSON + if (typeof error === 'string') { + try { + const parsed = StructuredErrorSchema.parse(JSON.parse(error)); + return { + ...rest, + error: { + message: parsed.message, + stack: parsed.stack, + code: parsed.code, + }, + } as T; + } catch { + // Backwards compatibility: error is just a plain string + return { + ...rest, + error: { + message: error, + }, + } as T; + } + } + + // Unknown format - return as-is and let downstream handle it + return obj as T; } const getUserAgent = () => { diff --git a/packages/world/package.json b/packages/world/package.json index ab54a218e9..f55c321861 100644 --- a/packages/world/package.json +++ b/packages/world/package.json @@ -1,6 +1,6 @@ { "name": "@workflow/world", - "version": "4.0.1-beta.13", + "version": "4.1.0-beta.0", "description": "The Workflows World interface", "type": "module", "main": "dist/index.js", diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 56a9082e3d..efdf5c7d2b 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -3,27 +3,38 @@ import type { PaginationOptions, ResolveData } from './shared.js'; // Event type enum export const EventTypeSchema = z.enum([ + // Run lifecycle events + 'run_created', + 'run_started', + 'run_completed', + 'run_failed', + 'run_cancelled', + // Step lifecycle events + 'step_created', 'step_completed', 'step_failed', 'step_retrying', 'step_started', + // Hook lifecycle events 'hook_created', 'hook_received', 'hook_disposed', + 'hook_conflict', // Created by world when hook token already exists + // Wait lifecycle events 'wait_created', 'wait_completed', - 'workflow_completed', - 'workflow_failed', - 'workflow_started', ]); // Base event schema with common properties // TODO: Event data on all specific event schemas can actually be undefined, // as the world may omit eventData when resolveData is set to 'none'. // Changing the type here will mainly improve type safety for o11y consumers. +// Note: specVersion is optional for backwards compatibility with legacy data in storage, +// but is always sent by the runtime on new events. export const BaseEventSchema = z.object({ eventType: EventTypeSchema, correlationId: z.string().optional(), + specVersion: z.number().optional(), }); // Event schemas (shared between creation requests and server responses) @@ -41,28 +52,58 @@ const StepFailedEventSchema = BaseEventSchema.extend({ eventData: z.object({ error: z.any(), stack: z.string().optional(), - fatal: z.boolean().optional(), }), }); -// TODO: this is not actually used anywhere yet, we could remove it -// on client and server if needed +/** + * Event created when a step fails and will be retried. + * Sets the step status back to 'pending' and records the error. + * The error is stored in step.error for debugging. + */ const StepRetryingEventSchema = BaseEventSchema.extend({ eventType: z.literal('step_retrying'), correlationId: z.string(), eventData: z.object({ - attempt: z.number().min(1), + error: z.any(), + stack: z.string().optional(), + retryAfter: z.coerce.date().optional(), }), }); const StepStartedEventSchema = BaseEventSchema.extend({ eventType: z.literal('step_started'), correlationId: z.string(), + eventData: z + .object({ + attempt: z.number().optional(), + }) + .optional(), +}); + +/** + * Event created when a step is first invoked. The World implementation + * atomically creates both the event and the step entity. + */ +const StepCreatedEventSchema = BaseEventSchema.extend({ + eventType: z.literal('step_created'), + correlationId: z.string(), + eventData: z.object({ + stepName: z.string(), + input: z.any(), // SerializedData + }), }); +/** + * Event created when a hook is first invoked. The World implementation + * atomically creates both the event and the hook entity. + */ const HookCreatedEventSchema = BaseEventSchema.extend({ eventType: z.literal('hook_created'), correlationId: z.string(), + eventData: z.object({ + token: z.string(), + metadata: z.any().optional(), // SerializedData + }), }); const HookReceivedEventSchema = BaseEventSchema.extend({ @@ -78,6 +119,22 @@ const HookDisposedEventSchema = BaseEventSchema.extend({ correlationId: z.string(), }); +/** + * Event created by World implementations when a hook_created request + * conflicts with an existing hook token. This event is NOT user-creatable - + * it is only returned by the World when a token conflict is detected. + * + * When the hook consumer sees this event, it should reject any awaited + * promises with a HookTokenConflictError. + */ +const HookConflictEventSchema = BaseEventSchema.extend({ + eventType: z.literal('hook_conflict'), + correlationId: z.string(), + eventData: z.object({ + token: z.string(), + }), +}); + const WaitCreatedEventSchema = BaseEventSchema.extend({ eventType: z.literal('wait_created'), correlationId: z.string(), @@ -91,58 +148,170 @@ const WaitCompletedEventSchema = BaseEventSchema.extend({ correlationId: z.string(), }); -// TODO: not used yet -const WorkflowCompletedEventSchema = BaseEventSchema.extend({ - eventType: z.literal('workflow_completed'), +// ============================================================================= +// Run lifecycle events +// ============================================================================= + +/** + * Event created when a workflow run is first created. The World implementation + * atomically creates both the event and the run entity with status 'pending'. + */ +const RunCreatedEventSchema = BaseEventSchema.extend({ + eventType: z.literal('run_created'), + eventData: z.object({ + deploymentId: z.string(), + workflowName: z.string(), + input: z.array(z.any()), // SerializedData[] + executionContext: z.record(z.string(), z.any()).optional(), + }), +}); + +/** + * Event created when a workflow run starts executing. + * Updates the run entity to status 'running'. + */ +const RunStartedEventSchema = BaseEventSchema.extend({ + eventType: z.literal('run_started'), }); -// TODO: not used yet -const WorkflowFailedEventSchema = BaseEventSchema.extend({ - eventType: z.literal('workflow_failed'), +/** + * Event created when a workflow run completes successfully. + * Updates the run entity to status 'completed' with output. + */ +const RunCompletedEventSchema = BaseEventSchema.extend({ + eventType: z.literal('run_completed'), + eventData: z.object({ + output: z.any().optional(), // SerializedData + }), +}); + +/** + * Event created when a workflow run fails. + * Updates the run entity to status 'failed' with error. + */ +const RunFailedEventSchema = BaseEventSchema.extend({ + eventType: z.literal('run_failed'), eventData: z.object({ error: z.any(), + errorCode: z.string().optional(), }), }); -// TODO: not used yet -const WorkflowStartedEventSchema = BaseEventSchema.extend({ - eventType: z.literal('workflow_started'), +/** + * Event created when a workflow run is cancelled. + * Updates the run entity to status 'cancelled'. + */ +const RunCancelledEventSchema = BaseEventSchema.extend({ + eventType: z.literal('run_cancelled'), }); -// Discriminated union (used for both creation requests and server responses) +// Discriminated union for user-creatable events (requests to world.events.create) +// Note: hook_conflict is NOT included here - it can only be created by World implementations export const CreateEventSchema = z.discriminatedUnion('eventType', [ + // Run lifecycle events + RunCreatedEventSchema, + RunStartedEventSchema, + RunCompletedEventSchema, + RunFailedEventSchema, + RunCancelledEventSchema, + // Step lifecycle events + StepCreatedEventSchema, StepCompletedEventSchema, StepFailedEventSchema, StepRetryingEventSchema, StepStartedEventSchema, + // Hook lifecycle events HookCreatedEventSchema, HookReceivedEventSchema, HookDisposedEventSchema, + // Wait lifecycle events WaitCreatedEventSchema, WaitCompletedEventSchema, - WorkflowCompletedEventSchema, - WorkflowFailedEventSchema, - WorkflowStartedEventSchema, ]); -// Server response include runId, eventId, and createdAt -export const EventSchema = CreateEventSchema.and( +// Discriminated union for ALL events (includes World-only events like hook_conflict) +// This is used for reading events from the event log +const AllEventsSchema = z.discriminatedUnion('eventType', [ + // Run lifecycle events + RunCreatedEventSchema, + RunStartedEventSchema, + RunCompletedEventSchema, + RunFailedEventSchema, + RunCancelledEventSchema, + // Step lifecycle events + StepCreatedEventSchema, + StepCompletedEventSchema, + StepFailedEventSchema, + StepRetryingEventSchema, + StepStartedEventSchema, + // Hook lifecycle events + HookCreatedEventSchema, + HookReceivedEventSchema, + HookDisposedEventSchema, + HookConflictEventSchema, // World-only: created when hook token conflicts + // Wait lifecycle events + WaitCreatedEventSchema, + WaitCompletedEventSchema, +]); + +// Server response includes runId, eventId, and createdAt +// specVersion is optional in database for backwards compatibility +export const EventSchema = AllEventsSchema.and( z.object({ runId: z.string(), eventId: z.string(), createdAt: z.coerce.date(), + specVersion: z.number().optional(), }) ); // Inferred types export type Event = z.infer; -export type CreateEventRequest = z.infer; export type HookReceivedEvent = z.infer; +export type HookConflictEvent = z.infer; + +/** + * Union of all possible event request types. + * @internal Use CreateEventRequest or RunCreatedEventRequest instead. + */ +export type AnyEventRequest = z.infer; + +/** + * Event request for creating a new workflow run. + * Must be used with runId: null since the server generates the runId. + */ +export type RunCreatedEventRequest = z.infer; + +/** + * Event request types that require an existing runId. + * This is the common case for all events except run_created. + */ +export type CreateEventRequest = Exclude< + AnyEventRequest, + RunCreatedEventRequest +>; export interface CreateEventParams { resolveData?: ResolveData; } +/** + * Result of creating an event. Includes the created event and optionally + * the entity that was created or updated as a result of the event, with any updates applied to it. + * + * Note: `event` is optional to support legacy runs where event storage is skipped. + */ +export interface EventResult { + /** The created event (optional for legacy compatibility) */ + event?: Event; + /** The workflow run entity (for run_* events) */ + run?: import('./runs.js').WorkflowRun; + /** The step entity (for step_* events) */ + step?: import('./steps.js').Step; + /** The hook entity (for hook_created events) */ + hook?: import('./hooks.js').Hook; +} + export interface ListEventsParams { runId: string; pagination?: PaginationOptions; diff --git a/packages/world/src/hooks.ts b/packages/world/src/hooks.ts index 8314acabbf..1519e3e944 100644 --- a/packages/world/src/hooks.ts +++ b/packages/world/src/hooks.ts @@ -16,6 +16,8 @@ export const HookSchema = z.object({ environment: z.string(), metadata: zodJsonSchema.optional(), createdAt: z.coerce.date(), + // Optional in database for backwards compatibility, defaults to 1 (legacy) when reading + specVersion: z.number().optional(), }); /** @@ -38,6 +40,8 @@ export type Hook = z.infer & { metadata?: unknown; /** The timestamp when this hook was created. */ createdAt: Date; + /** The spec version when this hook was created. */ + specVersion?: number; }; // Request types diff --git a/packages/world/src/index.ts b/packages/world/src/index.ts index 161031ea14..68c2a6f13f 100644 --- a/packages/world/src/index.ts +++ b/packages/world/src/index.ts @@ -31,3 +31,10 @@ export { } from './shared.js'; export type * from './steps.js'; export { StepSchema, StepStatusSchema } from './steps.js'; +export type { SpecVersion } from './spec-version.js'; +export { + SPEC_VERSION_LEGACY, + SPEC_VERSION_CURRENT, + isLegacySpecVersion, + requiresNewerWorld, +} from './spec-version.js'; diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index fdee64c69a..ff706eb433 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -2,31 +2,23 @@ import type { CreateEventParams, CreateEventRequest, Event, + EventResult, ListEventsByCorrelationIdParams, ListEventsParams, + RunCreatedEventRequest, } from './events.js'; -import type { - CreateHookRequest, - GetHookParams, - Hook, - ListHooksParams, -} from './hooks.js'; +import type { GetHookParams, Hook, ListHooksParams } from './hooks.js'; import type { Queue } from './queue.js'; import type { - CancelWorkflowRunParams, - CreateWorkflowRunRequest, GetWorkflowRunParams, ListWorkflowRunsParams, - UpdateWorkflowRunRequest, WorkflowRun, } from './runs.js'; import type { PaginatedResponse } from './shared.js'; import type { - CreateStepRequest, GetStepParams, ListWorkflowRunStepsParams, Step, - UpdateStepRequest, } from './steps.js'; export interface Streamer { @@ -43,38 +35,69 @@ export interface Streamer { listStreamsByRunId(runId: string): Promise; } +/** + * Storage interface for workflow data. + * + * Workflow storage models an append-only event log, so all state changes are handled through `events.create()`. + * Run/Step/Hook entities provide materialized views into the current state, but entities can't be modified directly. + * + * User-originated state changes are also handled via events: + * - run_cancelled event for run cancellation + * - hook_disposed event for explicit hook disposal (optional) + * + * Note: Hooks are automatically disposed by the World implementation when a workflow + * reaches a terminal state (run_completed, run_failed, run_cancelled). This releases + * hook tokens for reuse by future workflows. The hook_disposed event is only needed + * for explicit disposal before workflow completion. + */ export interface Storage { runs: { - create(data: CreateWorkflowRunRequest): Promise; get(id: string, params?: GetWorkflowRunParams): Promise; - update(id: string, data: UpdateWorkflowRunRequest): Promise; list( params?: ListWorkflowRunsParams ): Promise>; - cancel(id: string, params?: CancelWorkflowRunParams): Promise; }; steps: { - create(runId: string, data: CreateStepRequest): Promise; get( runId: string | undefined, stepId: string, params?: GetStepParams ): Promise; - update( - runId: string, - stepId: string, - data: UpdateStepRequest - ): Promise; list(params: ListWorkflowRunStepsParams): Promise>; }; events: { + /** + * Create a run_created event to start a new workflow run. + * The runId parameter must be null - the server generates and returns the runId. + * + * @param runId - Must be null for run_created events + * @param data - The run_created event data + * @param params - Optional parameters for event creation + * @returns Promise resolving to the created event and run entity + */ + create( + runId: null, + data: RunCreatedEventRequest, + params?: CreateEventParams + ): Promise; + + /** + * Create an event for an existing workflow run and atomically update the entity. + * Returns both the event and the affected entity (run/step/hook). + * + * @param runId - The workflow run ID (required for all events except run_created) + * @param data - The event to create + * @param params - Optional parameters for event creation + * @returns Promise resolving to the created event and affected entity + */ create( runId: string, data: CreateEventRequest, params?: CreateEventParams - ): Promise; + ): Promise; + list(params: ListEventsParams): Promise>; listByCorrelationId( params: ListEventsByCorrelationIdParams @@ -82,15 +105,9 @@ export interface Storage { }; hooks: { - create( - runId: string, - data: CreateHookRequest, - params?: GetHookParams - ): Promise; get(hookId: string, params?: GetHookParams): Promise; getByToken(token: string, params?: GetHookParams): Promise; list(params: ListHooksParams): Promise>; - dispose(hookId: string, params?: GetHookParams): Promise; }; } diff --git a/packages/world/src/runs.ts b/packages/world/src/runs.ts index 64451c6f1e..cb9dc412a0 100644 --- a/packages/world/src/runs.ts +++ b/packages/world/src/runs.ts @@ -25,6 +25,8 @@ export const WorkflowRunBaseSchema = z.object({ status: WorkflowRunStatusSchema, deploymentId: z.string(), workflowName: z.string(), + // Optional in database for backwards compatibility, defaults to 1 (legacy) when reading + specVersion: z.number().optional(), executionContext: z.record(z.string(), z.any()).optional(), input: z.array(z.any()), output: z.any().optional(), diff --git a/packages/world/src/spec-version.ts b/packages/world/src/spec-version.ts new file mode 100644 index 0000000000..50c5b28d72 --- /dev/null +++ b/packages/world/src/spec-version.ts @@ -0,0 +1,47 @@ +/** + * Spec version utilities for backwards compatibility. + * + * Uses a branded type to ensure packages import the version constants + * from @workflow/world rather than using arbitrary numbers. + */ + +declare const SpecVersionBrand: unique symbol; + +/** + * Branded type for spec versions. Must be created via SPEC_VERSION constants. + * This ensures all packages use the canonical version from @workflow/world. + */ +export type SpecVersion = number & { + readonly [SpecVersionBrand]: typeof SpecVersionBrand; +}; + +/** Legacy spec version (pre-event-sourcing). Also used for runs without specVersion. */ +export const SPEC_VERSION_LEGACY = 1 as SpecVersion; + +/** Current spec version (event-sourced architecture). */ +export const SPEC_VERSION_CURRENT = 2 as SpecVersion; + +/** + * Check if a spec version is legacy (< SPEC_VERSION_CURRENT or undefined). + * Legacy runs require different handling - they use direct entity mutation + * instead of the event-sourced model. + * + * @param v - The spec version number, or undefined/null for legacy runs + * @returns true if the run is a legacy run + */ +export function isLegacySpecVersion(v: number | undefined | null): boolean { + if (v === undefined || v === null) return true; + return v < SPEC_VERSION_CURRENT; +} + +/** + * Check if a spec version requires a newer world (> SPEC_VERSION_CURRENT). + * This happens when a run was created by a newer SDK version. + * + * @param v - The spec version number, or undefined/null for legacy runs + * @returns true if the run requires a newer world version + */ +export function requiresNewerWorld(v: number | undefined | null): boolean { + if (v === undefined || v === null) return false; + return v > SPEC_VERSION_CURRENT; +} diff --git a/packages/world/src/steps.ts b/packages/world/src/steps.ts index 8c973f6b9d..0d96084f70 100644 --- a/packages/world/src/steps.ts +++ b/packages/world/src/steps.ts @@ -24,13 +24,24 @@ export const StepSchema = z.object({ status: StepStatusSchema, input: z.array(z.any()), output: z.any().optional(), + /** + * The error from a step_retrying or step_failed event. + * This tracks the most recent error the step encountered, which may + * be from a retry attempt (step_retrying) or the final failure (step_failed). + */ error: StructuredErrorSchema.optional(), attempt: z.number(), + /** + * When the step first started executing. Set by the first step_started event + * and not updated on subsequent retries. + */ startedAt: z.coerce.date().optional(), completedAt: z.coerce.date().optional(), createdAt: z.coerce.date(), updatedAt: z.coerce.date(), retryAfter: z.coerce.date().optional(), + // Optional in database for backwards compatibility, defaults to 1 (legacy) when reading + specVersion: z.number().optional(), }); // Inferred types diff --git a/workbench/example/workflows/99_e2e.ts b/workbench/example/workflows/99_e2e.ts index e7afc97407..776ae35c4a 100644 --- a/workbench/example/workflows/99_e2e.ts +++ b/workbench/example/workflows/99_e2e.ts @@ -381,6 +381,131 @@ export async function promiseRaceStressTestWorkflow() { ////////////////////////////////////////////////////////// +async function stepThatRetriesAndSucceeds() { + 'use step'; + const { attempt } = getStepMetadata(); + console.log(`stepThatRetriesAndSucceeds - attempt: ${attempt}`); + + // Fail on attempts 1 and 2, succeed on attempt 3 + if (attempt < 3) { + console.log(`Attempt ${attempt} - throwing error to trigger retry`); + throw new Error(`Failed on attempt ${attempt}`); + } + + console.log(`Attempt ${attempt} - succeeding`); + return attempt; +} + +export async function retryAttemptCounterWorkflow() { + 'use workflow'; + console.log('Starting retry attempt counter workflow'); + + // This step should fail twice and succeed on the third attempt + const finalAttempt = await stepThatRetriesAndSucceeds(); + + console.log(`Workflow completed with final attempt: ${finalAttempt}`); + return { finalAttempt }; +} + +////////////////////////////////////////////////////////// + +async function stepThatThrowsRetryableError() { + 'use step'; + const { attempt, stepStartedAt } = getStepMetadata(); + if (attempt === 1) { + throw new RetryableError('Retryable error', { + retryAfter: '10s', + }); + } + return { + attempt, + stepStartedAt, + duration: Date.now() - stepStartedAt.getTime(), + }; +} + +export async function crossFileErrorWorkflow() { + 'use workflow'; + // This will throw an error from the imported helpers.ts file + callThrower(); + return 'never reached'; +} + +////////////////////////////////////////////////////////// + +export async function retryableAndFatalErrorWorkflow() { + 'use workflow'; + + const retryableResult = await stepThatThrowsRetryableError(); + + let gotFatalError = false; + try { + await stepThatFails(); + } catch (error: any) { + if (FatalError.is(error)) { + gotFatalError = true; + } + } + + return { retryableResult, gotFatalError }; +} + +////////////////////////////////////////////////////////// + +// Test that maxRetries = 0 means the step runs once but does not retry on failure +async function stepWithNoRetries() { + 'use step'; + const { attempt } = getStepMetadata(); + console.log(`stepWithNoRetries - attempt: ${attempt}`); + // Always fail - with maxRetries = 0, this should only run once + throw new Error(`Failed on attempt ${attempt}`); +} +stepWithNoRetries.maxRetries = 0; + +// Test that maxRetries = 0 works when the step succeeds +async function stepWithNoRetriesThatSucceeds() { + 'use step'; + const { attempt } = getStepMetadata(); + console.log(`stepWithNoRetriesThatSucceeds - attempt: ${attempt}`); + return { attempt }; +} +stepWithNoRetriesThatSucceeds.maxRetries = 0; + +export async function maxRetriesZeroWorkflow() { + 'use workflow'; + console.log('Starting maxRetries = 0 workflow'); + + // First, verify that a step with maxRetries = 0 can still succeed + const successResult = await stepWithNoRetriesThatSucceeds(); + + // Now test that a failing step with maxRetries = 0 does NOT retry + let failedAttempt: number | null = null; + let gotError = false; + try { + await stepWithNoRetries(); + } catch (error: any) { + gotError = true; + console.log('Received error', typeof error, error, error.message); + // Extract the attempt number from the error message + const match = error.message?.match(/attempt (\d+)/); + if (match) { + failedAttempt = parseInt(match[1], 10); + } + } + + console.log( + `Workflow completed: successResult=${JSON.stringify(successResult)}, gotError=${gotError}, failedAttempt=${failedAttempt}` + ); + + return { + successResult, + gotError, + failedAttempt, + }; +} + +////////////////////////////////////////////////////////// + export async function hookCleanupTestWorkflow( token: string, customData: string