-
Notifications
You must be signed in to change notification settings - Fork 0
feat(node): enqueue interceptors #367
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| /** | ||
| * Producer-side enqueue interception. Interceptors registered via | ||
| * `Queue.intercept` run at the very start of every enqueue — before per-task | ||
| * defaults, middleware, gates, and serialization — and decide what happens | ||
| * to the call: | ||
| * | ||
| * - `pass` — enqueue unchanged. | ||
| * - `convert` — replace the args (task name unchanged), e.g. redact a field. | ||
| * - `redirect` — enqueue a different task with new args instead. | ||
| * - `reject` — block the enqueue; `Queue.enqueue` throws `InterceptionError`. | ||
| * | ||
| * Interceptors chain in registration order, each seeing the previous one's | ||
| * output. Distinct from `Middleware.onEnqueue` (mutate-in-place hooks) and | ||
| * from the Python SDK's argument-type interception. | ||
| */ | ||
| export type Interception = | ||
| | { readonly type: "pass" } | ||
| | { readonly type: "convert"; readonly args: unknown[] } | ||
| | { readonly type: "redirect"; readonly taskName: string; readonly args: unknown[] } | ||
| | { readonly type: "reject"; readonly reason: string }; | ||
|
|
||
| /** Decides the outcome of an enqueue. Runs synchronously before serialization. */ | ||
| export type Interceptor = (taskName: string, args: unknown[]) => Interception; | ||
|
|
||
| /** Factories for the four {@link Interception} outcomes. */ | ||
| export const Interception = { | ||
| /** Enqueue the original args unchanged. */ | ||
| pass(): Interception { | ||
| return { type: "pass" }; | ||
| }, | ||
| /** Replace the args; the task name stays the same. */ | ||
| convert(args: unknown[]): Interception { | ||
| return { type: "convert", args }; | ||
| }, | ||
| /** Enqueue a different task with new args instead of the original. */ | ||
| redirect(taskName: string, args: unknown[]): Interception { | ||
| return { type: "redirect", taskName, args }; | ||
| }, | ||
| /** Block the enqueue; the reason is surfaced on the thrown error. */ | ||
| reject(reason: string): Interception { | ||
| return { type: "reject", reason }; | ||
| }, | ||
| }; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| import { mkdtempSync } from "node:fs"; | ||
| import { tmpdir } from "node:os"; | ||
| import { join } from "node:path"; | ||
| import { afterEach, describe, expect, it } from "vitest"; | ||
| import { GzipCodec, Interception, InterceptionError, Queue, type Worker } from "../../src/index"; | ||
|
|
||
| let worker: Worker | undefined; | ||
|
|
||
| afterEach(() => { | ||
| worker?.stop(); | ||
| worker = undefined; | ||
| }); | ||
|
|
||
| function tempDb(): string { | ||
| return join(mkdtempSync(join(tmpdir(), "taskito-intercept-")), "queue.db"); | ||
| } | ||
|
|
||
| async function waitFor<T>(read: () => T | undefined, timeoutMs = 10_000): Promise<T> { | ||
| const deadline = Date.now() + timeoutMs; | ||
| while (Date.now() < deadline) { | ||
| const value = read(); | ||
| if (value !== undefined) { | ||
| return value; | ||
| } | ||
| await new Promise((resolve) => setTimeout(resolve, 25)); | ||
| } | ||
| throw new Error("timed out waiting for job result"); | ||
| } | ||
|
|
||
| describe("enqueue interceptors", () => { | ||
| it("convert rewrites the args the worker sees", async () => { | ||
| const queue = new Queue({ dbPath: tempDb() }) | ||
| .task("ic.echo", (x: number) => x) | ||
| .intercept((_taskName, args) => Interception.convert([(args[0] as number) * 10])); | ||
|
|
||
| const id = queue.enqueue("ic.echo", [5]); | ||
| worker = queue.runWorker(); | ||
| await expect(waitFor(() => queue.getResult(id))).resolves.toBe(50); | ||
| }); | ||
|
|
||
| it("redirect enqueues the other task; the original handler never runs", async () => { | ||
| const ranA: unknown[] = []; | ||
| const queue = new Queue({ dbPath: tempDb() }) | ||
| .task("ic.a", (x: number) => { | ||
| ranA.push(x); | ||
| return x; | ||
| }) | ||
| .task("ic.b", (x: number) => x + 100) | ||
| .intercept((taskName, args) => | ||
| taskName === "ic.a" ? Interception.redirect("ic.b", args) : Interception.pass(), | ||
| ); | ||
|
|
||
| const id = queue.enqueue("ic.a", [1]); | ||
| expect(queue.getJob(id)?.taskName).toBe("ic.b"); | ||
| worker = queue.runWorker(); | ||
| await expect(waitFor(() => queue.getResult(id))).resolves.toBe(101); | ||
| expect(ranA).toEqual([]); | ||
| }); | ||
|
|
||
| it("reject throws InterceptionError and enqueues nothing", async () => { | ||
| const queue = new Queue({ dbPath: tempDb() }) | ||
| .task("ic.noop", () => null) | ||
| .intercept(() => Interception.reject("blocked by policy")); | ||
|
|
||
| expect(() => queue.enqueue("ic.noop", [])).toThrow(InterceptionError); | ||
| expect(() => queue.enqueue("ic.noop", [])).toThrow(/blocked by policy/); | ||
| expect((await queue.stats()).pending).toBe(0); | ||
| }); | ||
|
|
||
| it("a null-ish interceptor result throws InterceptionError", () => { | ||
| const queue = new Queue({ dbPath: tempDb() }) | ||
| .task("ic.noop", () => null) | ||
| // biome-ignore lint/suspicious/noExplicitAny: deliberately misbehaving interceptor | ||
| .intercept((() => undefined) as any); | ||
|
|
||
| expect(() => queue.enqueue("ic.noop", [])).toThrow(/returned null/); | ||
| }); | ||
|
|
||
| it("interceptors chain in registration order", async () => { | ||
| const queue = new Queue({ dbPath: tempDb() }) | ||
| .task("ic.echo", (x: number) => x) | ||
| .intercept((_name, args) => Interception.convert([(args[0] as number) + 1])) | ||
| .intercept((_name, args) => Interception.convert([(args[0] as number) * 2])); | ||
|
|
||
| const id = queue.enqueue("ic.echo", [3]); // (3 + 1) * 2 | ||
| worker = queue.runWorker(); | ||
| await expect(waitFor(() => queue.getResult(id))).resolves.toBe(8); | ||
| }); | ||
|
|
||
| it("batch enqueue converts per item", async () => { | ||
| const queue = new Queue({ dbPath: tempDb() }) | ||
| .task("ic.echo", (x: number) => x) | ||
| .intercept((_name, args) => Interception.convert([(args[0] as number) * 10])); | ||
|
|
||
| const ids = queue.enqueueMany("ic.echo", [{ args: [1] }, { args: [2] }]); | ||
| worker = queue.runWorker(); | ||
| await expect(waitFor(() => queue.getResult(ids[0] as string))).resolves.toBe(10); | ||
| await expect(waitFor(() => queue.getResult(ids[1] as string))).resolves.toBe(20); | ||
| }); | ||
|
|
||
| it("batch reject fails the whole batch — zero jobs stored", async () => { | ||
| const queue = new Queue({ dbPath: tempDb() }) | ||
| .task("ic.echo", (x: number) => x) | ||
| .intercept((_name, args) => | ||
| (args[0] as number) < 0 ? Interception.reject("negative") : Interception.pass(), | ||
| ); | ||
|
|
||
| expect(() => | ||
| queue.enqueueMany("ic.echo", [{ args: [1] }, { args: [-2] }, { args: [3] }]), | ||
| ).toThrow(InterceptionError); | ||
| expect((await queue.stats()).pending).toBe(0); | ||
| }); | ||
|
|
||
| it("batch redirect is unsupported", () => { | ||
| const queue = new Queue({ dbPath: tempDb() }) | ||
| .task("ic.a", (x: number) => x) | ||
| .task("ic.b", (x: number) => x) | ||
| .intercept((_name, args) => Interception.redirect("ic.b", args)); | ||
|
|
||
| expect(() => queue.enqueueMany("ic.a", [{ args: [1] }])).toThrow( | ||
| /Redirect is not supported for batch/, | ||
| ); | ||
| }); | ||
|
|
||
| it("redirect of a task with per-task codecs is rejected", () => { | ||
| const queue = new Queue({ dbPath: tempDb(), codecs: { gzip: new GzipCodec() } }) | ||
| .task("ic.codec", (x: number) => x, { codecs: ["gzip"] }) | ||
| .task("ic.b", (x: number) => x) | ||
| .intercept((_name, args) => Interception.redirect("ic.b", args)); | ||
|
|
||
| expect(() => queue.enqueue("ic.codec", [1])).toThrow( | ||
| /Redirect is not supported for a task with payload codecs/, | ||
| ); | ||
| }); | ||
|
|
||
| it("interceptors run before gates: a gate sees the converted args", async () => { | ||
| const queue = new Queue({ dbPath: tempDb() }) | ||
| .task("ic.echo", (x: number) => x) | ||
| .gate("ic.echo", ({ args }) => (args[0] as number) > 0) | ||
| .intercept((_name, args) => Interception.convert([Math.abs(args[0] as number)])); | ||
|
|
||
| // Raw -5 would be gated; the interceptor converts to 5 first. | ||
| const id = queue.enqueue("ic.echo", [-5]); | ||
| worker = queue.runWorker(); | ||
| await expect(waitFor(() => queue.getResult(id))).resolves.toBe(5); | ||
| }); | ||
| }); |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.