From e80818cf61822481ce6182944def5fca4f834cf8 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:11:49 +0530 Subject: [PATCH 01/57] docs: rework onboarding path and consolidate capabilities --- docs/content/docs/python/capabilities.mdx | 91 ----------------------- 1 file changed, 91 deletions(-) delete mode 100644 docs/content/docs/python/capabilities.mdx diff --git a/docs/content/docs/python/capabilities.mdx b/docs/content/docs/python/capabilities.mdx deleted file mode 100644 index 21ba0fd0..00000000 --- a/docs/content/docs/python/capabilities.mdx +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: Capabilities at a glance -description: Everything taskito already does, in one place — with a link to the deep-dive guide for each. ---- - -import { Cards, Card } from "fumadocs-ui/components/card"; -import { - ShieldCheck, - Workflow, - Cpu, - CalendarClock, - Activity, - Plug, -} from "lucide-react"; - -taskito ships a broad feature set out of the box. If you're evaluating it — or -suspect a capability is missing — scan this page first. Every card links to the -guide that documents it. - -## If you think it's missing, it isn't - -These are the capabilities most often assumed absent. They aren't. - -| You might assume… | Reality | -| --- | --- | -| No canvas / workflows — no chain, group, or chord | [`chain` / `group` / `chord`](/python/guides/workflows/canvas) are exported from the top-level package, backed by a full workflow DSL. | -| No Flower-like dashboard | [`taskito dashboard --app myapp:queue`](/python/guides/dashboard) serves a built-in monitoring UI. | -| No retry backoff or per-exception retries | [`max_retries`, `retry_backoff`, `retry_on`, `dont_retry_on`](/python/guides/reliability/retries) are arguments on every task. | -| No soft timeouts | [`current_job.check_timeout()`](/python/guides/reliability/error-handling) gives cooperative soft timeouts. | -| GIL bottleneck with no escape hatch | [`--pool prefork`](/python/guides/advanced-execution/prefork) runs child processes with independent GILs for true CPU parallelism. | - -## By capability - - - } - title="Reliability" - href="/python/guides/reliability" - description="Retries with exponential backoff, per-exception rules, soft timeouts, dead-letter replay, circuit breakers, and idempotent enqueue." - /> - } - title="Workflows" - href="/python/guides/workflows/canvas" - description="Compose with chain, fan out with group, fan in with chord — plus dependency graphs with cascade cancel." - /> - } - title="Concurrency" - href="/python/guides/advanced-execution/prefork" - description="A thread pool for I/O-bound work and a prefork pool for true CPU parallelism, with a native async API." - /> - } - title="Scheduling" - href="/python/guides/core/scheduling" - description="Priorities, rate limiting, periodic (cron) tasks, delayed execution, and job expiration." - /> - } - title="Observability" - href="/python/guides/dashboard" - description="A built-in dashboard, events, HMAC-signed webhooks, Prometheus, OpenTelemetry, and structured logging." - /> - } - title="Extensibility" - href="/python/guides/extensibility" - description="Pluggable serializers, per-task middleware, a fully async API, and Postgres or Redis backends." - /> - - -## One-liners - -```python -# Reliability — backoff + per-exception retry rules -@queue.task(max_retries=5, retry_backoff=2.0, retry_on=[TimeoutError]) -def fetch_url(url: str) -> str: ... - -# Workflows — sequential pipeline (group / chord for fan-out / fan-in) -chain(fetch.s(url), parse.s(), store.s()).apply() - -# Scheduling — priority + rate limit -@queue.task(priority=9, rate_limit="100/m") -def notify(user_id: int) -> None: ... -``` - -```bash -taskito worker --pool prefork --app tasks:queue # CPU-bound: true parallelism -taskito dashboard --app tasks:queue # Flower-style monitoring UI -``` From 490c8fda3c58db1466b92b2e4e28c77525cdf5ea Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:11:50 +0530 Subject: [PATCH 02/57] docs: fix core guides (expires, batch, glosses) --- .../docs/python/guides/core/batching.mdx | 8 ++++++ .../python/guides/core/execution-model.mdx | 2 +- .../content/docs/python/guides/core/index.mdx | 2 ++ .../docs/python/guides/core/predicates.mdx | 6 ++--- .../docs/python/guides/core/scheduling.mdx | 4 +++ .../content/docs/python/guides/core/tasks.mdx | 27 +++++++++---------- .../docs/python/guides/core/workers.mdx | 3 ++- .../docs/python/guides/core/workflows.mdx | 2 +- 8 files changed, 33 insertions(+), 21 deletions(-) diff --git a/docs/content/docs/python/guides/core/batching.mdx b/docs/content/docs/python/guides/core/batching.mdx index 94aae586..c8602b81 100644 --- a/docs/content/docs/python/guides/core/batching.mdx +++ b/docs/content/docs/python/guides/core/batching.mdx @@ -5,6 +5,14 @@ description: Collect many small calls into one larger job When a task does small per-call work — sending an email, writing a row, calling an API — running one job per call is wasteful. `@queue.task(batch=...)` collects calls in memory and dispatches them as a single job whose payload is the list of accumulated items. + + Task batching coalesces many `.delay()` calls into **one** job. + [Batch enqueue](/python/guides/advanced-execution/batch-enqueue) + (`task.map()` / `enqueue_many()`) does the reverse — it writes **many** jobs in + one transaction. Reach for task batching to cut per-call overhead; reach for + batch enqueue for bulk-insert throughput. + + ```python from taskito import Queue diff --git a/docs/content/docs/python/guides/core/execution-model.mdx b/docs/content/docs/python/guides/core/execution-model.mdx index 64040d3a..b5986b31 100644 --- a/docs/content/docs/python/guides/core/execution-model.mdx +++ b/docs/content/docs/python/guides/core/execution-model.mdx @@ -29,7 +29,7 @@ native async. ## Thread pool (default) The default. Runs sync task functions on Rust `std::thread` threads. Each -worker acquires the Python GIL only during task execution — the scheduler and +worker acquires the Python GIL (Global Interpreter Lock — only one thread runs Python bytecode at a time) only during task execution — the scheduler and dispatch logic never touch it. ```python diff --git a/docs/content/docs/python/guides/core/index.mdx b/docs/content/docs/python/guides/core/index.mdx index f3b4463d..cbd88918 100644 --- a/docs/content/docs/python/guides/core/index.mdx +++ b/docs/content/docs/python/guides/core/index.mdx @@ -12,4 +12,6 @@ The building blocks of every taskito application. | [Execution Models](/python/guides/core/execution-model) | How tasks move from enqueue to completion | | [Queues & Priority](/python/guides/core/queues) | Named queues, priority levels, and routing | | [Scheduling](/python/guides/core/scheduling) | Periodic tasks with cron expressions | +| [Predicates](/python/guides/core/predicates) | Gate execution with a serializable predicate DSL — time windows, feature flags | +| [Task Batching](/python/guides/core/batching) | Collect many small calls into a single job | | [Workflows](/python/guides/core/workflows) | Chains, groups, and chords for multi-step pipelines | diff --git a/docs/content/docs/python/guides/core/predicates.mdx b/docs/content/docs/python/guides/core/predicates.mdx index 136be630..30e6f03b 100644 --- a/docs/content/docs/python/guides/core/predicates.mdx +++ b/docs/content/docs/python/guides/core/predicates.mdx @@ -289,7 +289,7 @@ restored = Predicate.from_dict(blob) ## Metrics & events ```python -queue._predicate_metrics.snapshot() +queue._predicate_metrics.snapshot() # internal accessor — not yet part of the stable API # {"allowed": 412, "denied": 3, "deferred": 8, "cancelled": 1, "errors": 0} ``` @@ -406,7 +406,7 @@ through a dashboard textbox without code deploys). ```python from taskito.events import EventType -def on_event(event_type: EventType, payload: dict) -> None: +def on_predicate_event(event_type: EventType, payload: dict) -> None: print(event_type.value, payload) for event in ( @@ -414,5 +414,5 @@ for event in ( EventType.PREDICATE_CANCELLED, EventType.PREDICATE_REJECTED, ): - queue._event_bus.on(event, on_event) + queue.on_event(event, on_predicate_event) ``` diff --git a/docs/content/docs/python/guides/core/scheduling.mdx b/docs/content/docs/python/guides/core/scheduling.mdx index 05f13654..a0fcb1c5 100644 --- a/docs/content/docs/python/guides/core/scheduling.mdx +++ b/docs/content/docs/python/guides/core/scheduling.mdx @@ -62,6 +62,10 @@ taskito uses **6-field cron expressions** (with seconds): * * * * * * ``` +> **Coming from Celery:** Celery's `crontab()` only maps to the last 5 fields +> (minute, hour, day of month, month, day of week) — prepend a seconds field +> to get taskito's 6-field format. + | Expression | Schedule | |---|---| | `*/30 * * * * *` | Every 30 seconds | diff --git a/docs/content/docs/python/guides/core/tasks.mdx b/docs/content/docs/python/guides/core/tasks.mdx index 534861c4..8e22a7ac 100644 --- a/docs/content/docs/python/guides/core/tasks.mdx +++ b/docs/content/docs/python/guides/core/tasks.mdx @@ -34,7 +34,6 @@ def process_data(data: dict) -> str: | `queue` | `str` | `"default"` | Named queue to submit to. | | `circuit_breaker` | `dict \| None` | `None` | Circuit breaker config: `{"threshold": 5, "window": 60, "cooldown": 120}`. | | `middleware` | `list[TaskMiddleware] \| None` | `None` | Per-task middleware, applied in addition to queue-level middleware. | -| `expires` | `float \| None` | `None` | Seconds until the job expires if not started. | | `inject` | `list[str] \| None` | `None` | Worker resource names to inject as keyword arguments. See [Resource System](/python/guides/resources). | | `serializer` | `Serializer \| None` | `None` | Per-task serializer override. Falls back to the queue-level serializer. | | `max_concurrent` | `int \| None` | `None` | Max concurrent running instances of this task. `None` means no limit. | @@ -107,12 +106,16 @@ def important_task(): ### Job expiration -Skip jobs that weren't started within the deadline: +`expires` is an **enqueue-time** option (not a decorator option) — skip a job that +wasn't started within the deadline: ```python -@queue.task(expires=300) # skip if not started within 5 minutes +@queue.task() def time_sensitive(): ... + +# Skip if not started within 5 minutes +time_sensitive.apply_async(args=(), expires=300) ``` ### Max retry delay @@ -142,9 +145,9 @@ def expensive_render(): Override the queue-level serializer for a specific task: ```python -from taskito.serializers import JSONSerializer +from taskito.serializers import JsonSerializer -@queue.task(serializer=JSONSerializer()) +@queue.task(serializer=JsonSerializer()) def api_event(payload: dict) -> dict: ... ``` @@ -209,24 +212,18 @@ result = send_email("user@example.com", "Hello", "World") # Runs immediately ## Batch enqueue -Enqueue many jobs in a single SQLite transaction: +Enqueue many jobs in one transaction with `task.map()` or `queue.enqueue_many()`: ```python -# Via task.map() jobs = send_email.map([ ("alice@example.com", "Hi", "Body"), ("bob@example.com", "Hi", "Body"), - ("carol@example.com", "Hi", "Body"), ]) - -# Via queue.enqueue_many() -jobs = queue.enqueue_many( - task_name=send_email.name, - args_list=[("alice@example.com",), ("bob@example.com",)], - kwargs_list=[{"subject": "Hi", "body": "Body"}] * 2, -) ``` +See [Batch enqueue](/python/guides/advanced-execution/batch-enqueue) for +`enqueue_many()`, per-item overrides, and per-item result tracking. + ## Metadata Attach arbitrary JSON metadata to jobs: diff --git a/docs/content/docs/python/guides/core/workers.mdx b/docs/content/docs/python/guides/core/workers.mdx index 7d640980..afe21689 100644 --- a/docs/content/docs/python/guides/core/workers.mdx +++ b/docs/content/docs/python/guides/core/workers.mdx @@ -73,7 +73,8 @@ queue = Queue(db_path="myapp.db", workers=8) # Explicit count ## Prefork pool -The default worker pool uses OS threads, which share a single Python GIL. For +The default worker pool uses OS threads, which share a single Python GIL +(Global Interpreter Lock — only one thread runs Python bytecode at a time). For CPU-bound tasks, use the prefork pool — it spawns separate child processes, each with its own GIL: diff --git a/docs/content/docs/python/guides/core/workflows.mdx b/docs/content/docs/python/guides/core/workflows.mdx index 2b87f29e..004f0caf 100644 --- a/docs/content/docs/python/guides/core/workflows.mdx +++ b/docs/content/docs/python/guides/core/workflows.mdx @@ -9,5 +9,5 @@ taskito provides two workflow models. See the dedicated - **[DAG Workflows](/python/guides/workflows)** — Multi-step pipelines as directed acyclic graphs with fan-out, conditions, gates, sub-workflows, incremental caching, and visualization. -- **[Canvas Primitives](/python/api-reference/canvas)** — Lightweight chain, +- **[Canvas Primitives](/python/guides/workflows/canvas)** — Lightweight chain, group, and chord composition for simple pipelines. From 2ea2b5f3e14ce89a186f6b7ff3f561e01f926368 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:11:50 +0530 Subject: [PATCH 03/57] docs: fix reliability guides (dedup, retries, Celery) --- .../guides/reliability/circuit-breakers.mdx | 5 +++ .../guides/reliability/error-handling.mdx | 4 +-- .../python/guides/reliability/guarantees.mdx | 11 ++++--- .../python/guides/reliability/idempotency.mdx | 2 +- .../docs/python/guides/reliability/index.mdx | 1 + .../python/guides/reliability/locking.mdx | 26 ++++++++++----- .../guides/reliability/rate-limiting.mdx | 11 +++++-- .../python/guides/reliability/retries.mdx | 32 +++++++++++++------ 8 files changed, 64 insertions(+), 28 deletions(-) diff --git a/docs/content/docs/python/guides/reliability/circuit-breakers.mdx b/docs/content/docs/python/guides/reliability/circuit-breakers.mdx index 7991cef9..491d38c0 100644 --- a/docs/content/docs/python/guides/reliability/circuit-breakers.mdx +++ b/docs/content/docs/python/guides/reliability/circuit-breakers.mdx @@ -7,6 +7,9 @@ Circuit breakers prevent cascading failures by temporarily stopping task execution when a task fails repeatedly. This is especially useful for tasks that call external APIs or services. +> Celery has no built-in circuit breaker — this is taskito-native, configured per +> task with `@queue.task(circuit_breaker={...})`. + ## How it works A circuit breaker tracks failures within a time window and transitions @@ -68,6 +71,8 @@ for cb in breakers: ### Dashboard API +Requires a running [dashboard](/python/guides/dashboard) (default port `8080`): + ```bash curl http://localhost:8080/api/circuit-breakers ``` diff --git a/docs/content/docs/python/guides/reliability/error-handling.mdx b/docs/content/docs/python/guides/reliability/error-handling.mdx index 2b48ae57..250cef80 100644 --- a/docs/content/docs/python/guides/reliability/error-handling.mdx +++ b/docs/content/docs/python/guides/reliability/error-handling.mdx @@ -136,7 +136,7 @@ them in your app's `__init__.py`. OperationalError: database is locked ``` -SQLite allows concurrent reads (WAL mode), but only one writer at a time. +SQLite allows concurrent reads (WAL mode — write-ahead logging, lets many readers and one writer run concurrently), but only one writer at a time. taskito sets `busy_timeout=5000ms` to wait for locks, but heavy write loads can still cause contention. @@ -149,7 +149,7 @@ can still cause contention. **Fixes:** - Avoid opening the database file with other tools while the worker is running -- Use `enqueue_many()` / `task.map()` for batch inserts — they use a single transaction +- Use [`enqueue_many()` / `task.map()`](/python/guides/advanced-execution/batch-enqueue) for batch inserts — they use a single transaction - If running multiple processes, ensure only one worker process writes at a time ### Timeout reaping diff --git a/docs/content/docs/python/guides/reliability/guarantees.mdx b/docs/content/docs/python/guides/reliability/guarantees.mdx index f3523034..4018ff81 100644 --- a/docs/content/docs/python/guides/reliability/guarantees.mdx +++ b/docs/content/docs/python/guides/reliability/guarantees.mdx @@ -16,7 +16,7 @@ crashes mid-execution. ## Why not exactly-once? Exactly-once delivery is impossible in distributed systems without two-phase -commit. Taskito's approach matches Celery, SQS, and most production job +commit (a distributed-transaction protocol that coordinates commit across systems). Taskito's approach matches Celery, SQS, and most production job systems: deliver at least once, design tasks to handle duplicates. ## How recovery works @@ -83,8 +83,10 @@ job = send_report.apply_async( ``` If a job with the same `unique_key` is already pending or running, the -duplicate is silently dropped. See -[Unique Tasks](/python/guides/advanced-execution) for details. +duplicate enqueue does **not** create a second job — it returns the existing +job's ID, so the two calls coalesce onto one execution. See +[Unique Tasks](/python/guides/advanced-execution/unique-tasks) and +[Idempotency](/python/guides/reliability/idempotency) for details. ### Avoid side effects that can't be undone @@ -110,7 +112,8 @@ the same `unique_key` can be enqueued again. ```python job1 = task.apply_async(args=(1,), unique_key="order-123") # Enqueued -job2 = task.apply_async(args=(1,), unique_key="order-123") # Skipped (job1 pending) +job2 = task.apply_async(args=(1,), unique_key="order-123") # Coalesced onto job1 +assert job2.id == job1.id # same job — no duplicate created # ... job1 completes ... job3 = task.apply_async(args=(1,), unique_key="order-123") # Enqueued (new job) ``` diff --git a/docs/content/docs/python/guides/reliability/idempotency.mdx b/docs/content/docs/python/guides/reliability/idempotency.mdx index e85aa29c..c5d5dcc9 100644 --- a/docs/content/docs/python/guides/reliability/idempotency.mdx +++ b/docs/content/docs/python/guides/reliability/idempotency.mdx @@ -51,7 +51,7 @@ charge_customer.apply_async(args=(42, 1000), idempotent=False) | `idempotent=True` (decorator) | `bool` | Default for every `delay()` / `apply_async()` | | `idempotency_key="…"` (call) | `str` | Explicit key. Overrides auto-derivation. | | `idempotent=False` (call) | `bool` | Disable auto-derivation for this submission | -| `unique_key="…"` (call) | `str` | Backwards-compat alias of `idempotency_key` | +| `unique_key="…"` (call) | `str` | The underlying storage dedup key. Highest precedence — overrides `idempotency_key` and auto-derivation. | Precedence (high to low): `unique_key` → `idempotency_key` → auto-derivation. diff --git a/docs/content/docs/python/guides/reliability/index.mdx b/docs/content/docs/python/guides/reliability/index.mdx index 70848e97..785af148 100644 --- a/docs/content/docs/python/guides/reliability/index.mdx +++ b/docs/content/docs/python/guides/reliability/index.mdx @@ -10,6 +10,7 @@ Harden your task queue for production workloads. | [Retries & Dead Letters](/python/guides/reliability/retries) | Automatic retries with exponential backoff, dead letter queue | | [Error Handling](/python/guides/reliability/error-handling) | Task failure lifecycle, error inspection, debugging patterns | | [Delivery Guarantees](/python/guides/reliability/guarantees) | At-least-once delivery, idempotency, and exactly-once patterns | +| [Idempotency](/python/guides/reliability/idempotency) | Auto-derived dedup keys — identical enqueues coalesce to one execution | | [Rate Limiting](/python/guides/reliability/rate-limiting) | Throttle task execution with token bucket rate limits | | [Circuit Breakers](/python/guides/reliability/circuit-breakers) | Protect downstream services from cascading failures | | [Distributed Locking](/python/guides/reliability/locking) | Mutual exclusion across workers with database-backed locks | diff --git a/docs/content/docs/python/guides/reliability/locking.mdx b/docs/content/docs/python/guides/reliability/locking.mdx index a04777b3..23c8f336 100644 --- a/docs/content/docs/python/guides/reliability/locking.mdx +++ b/docs/content/docs/python/guides/reliability/locking.mdx @@ -6,8 +6,13 @@ description: "Database-backed mutual-exclusion across workers — sync, async, a import { Callout } from "fumadocs-ui/components/callout"; taskito provides a distributed lock primitive backed by the same database -used for the task queue. Locks work across multiple worker processes and -machines sharing the same database. +used for the task queue. Locks work across multiple worker processes on one +machine, and across multiple machines when the backend is a shared server +(Postgres or Redis) rather than a local SQLite file. + +> Celery has no built-in lock — teams reach for `celery-once` or a Redis lock. +> taskito's `queue.lock()` uses the queue database directly, so there's nothing +> extra to run. ## Overview @@ -52,7 +57,7 @@ queue.lock( | Parameter | Type | Default | Description | |---|---|---|---| | `name` | `str` | — | Lock name. All processes using the same name compete for the same lock. | -| `ttl` | `int` | `30` | Lock TTL in seconds. Auto-extended if `auto_extend=True`. | +| `ttl` | `int` | `30` | Lock time-to-live (TTL) in seconds. Auto-extended if `auto_extend=True`. | | `auto_extend` | `bool` | `True` | Automatically extend the lock before it expires (background thread). | | `owner_id` | `str \| None` | `None` | Custom owner identifier. Defaults to a random UUID per acquisition. | | `timeout` | `float \| None` | `None` | Max seconds to wait for the lock. `None` raises immediately if unavailable. | @@ -101,8 +106,10 @@ exceeded. ## Cross-process locking Because lock state is stored in the database, locks are effective across -multiple worker processes on the same machine or different machines sharing -the same database: +multiple worker processes on the same machine. Locking across **different** +machines works too, but only when they share a networked database — SQLite +is a local file, so cross-machine locking requires the Postgres or Redis +backend: ```python # Process A (machine 1) @@ -114,10 +121,13 @@ with queue.lock("billing-run"): run_billing() ``` - + On SQLite, cross-process locking works via WAL mode and exclusive - transactions. For multi-machine deployments, use the PostgreSQL backend - where `SELECT FOR UPDATE SKIP LOCKED` provides true distributed semantics. + transactions, but SQLite file locking is machine-local — it only works + across processes on the same machine. For multi-machine deployments, use + the PostgreSQL backend, where `SELECT FOR UPDATE SKIP LOCKED` provides true + distributed semantics, or the Redis backend, which also supports + `queue.lock()`. ## Error handling diff --git a/docs/content/docs/python/guides/reliability/rate-limiting.mdx b/docs/content/docs/python/guides/reliability/rate-limiting.mdx index efe2a580..88a85aed 100644 --- a/docs/content/docs/python/guides/reliability/rate-limiting.mdx +++ b/docs/content/docs/python/guides/reliability/rate-limiting.mdx @@ -5,8 +5,8 @@ description: "Token bucket rate limits per task — count/period syntax, persist import { Callout } from "fumadocs-ui/components/callout"; -taskito uses a **token bucket** algorithm to limit how fast tasks execute. -Rate limits are per-task and persisted in SQLite. +taskito uses a **token bucket** (a standard rate-limit algorithm that refills allowance over time) algorithm to limit how fast tasks execute. +Rate limits are per-task and persisted in the queue database (SQLite or Postgres). ## Usage @@ -45,7 +45,7 @@ The token bucket algorithm: Token bucket state (current tokens, last refill time) is stored in the - `rate_limits` SQLite table. This means rate limits survive worker restarts. + queue database (SQLite or Postgres). This means rate limits survive worker restarts. ## Per-task, not per-queue @@ -62,6 +62,11 @@ send_email.delay("alice@example.com", "Hi", "Body") send_email.apply_async(args=("bob@example.com", "Hi", "Body"), queue="urgent") ``` +> **Coming from Celery:** Celery's `rate_limit` is enforced **per worker** — the +> effective rate scales with worker count. taskito enforces it **globally per task +> name** at the scheduler, so `"100/m"` means 100/min across the whole deployment +> regardless of how many workers run. + ## Combining with retries Rate limiting and retries work together seamlessly. If a rate-limited task diff --git a/docs/content/docs/python/guides/reliability/retries.mdx b/docs/content/docs/python/guides/reliability/retries.mdx index 4e09bbb8..f8afcef4 100644 --- a/docs/content/docs/python/guides/reliability/retries.mdx +++ b/docs/content/docs/python/guides/reliability/retries.mdx @@ -33,7 +33,7 @@ delay = min(max_delay, base_delay * 2^retry_count) + jitter - `base_delay` = `retry_backoff` (in seconds) - `max_delay` = 300 seconds (5 minutes) -- `jitter` = random 0–500ms to prevent thundering herd +- `jitter` = random 0–500ms to prevent thundering herd (many retries firing at the same instant) **Example with `retry_backoff=2.0`:** @@ -47,18 +47,21 @@ delay = min(max_delay, base_delay * 2^retry_count) + jitter ## Exception filtering -Control which exceptions trigger retries with `retry_on` and `dont_retry_on`: +Control which exceptions trigger retries with `retry_on` (a whitelist) **or** +`dont_retry_on` (a blacklist). Use one or the other — not both (see the note below). ```python -@queue.task( - max_retries=5, - retry_on=[ConnectionError, TimeoutError], - dont_retry_on=[ValueError], -) +# Whitelist: retry only these exceptions; everything else skips straight to DLQ. +@queue.task(max_retries=5, retry_on=[ConnectionError, TimeoutError]) def fetch_data(url): response = requests.get(url) response.raise_for_status() return response.json() + +# Blacklist: retry everything except these exceptions. +@queue.task(max_retries=5, dont_retry_on=[ValueError]) +def parse_data(raw): + return json.loads(raw) # a ValueError here is permanent — don't retry it ``` | Parameter | Description | @@ -70,10 +73,14 @@ If neither is set, all exceptions trigger retries (default behavior). `retry_on` and `dont_retry_on` are mutually exclusive in practice — if - `retry_on` is set, only those exceptions are retried regardless of - `dont_retry_on`. + `retry_on` is set, only those exceptions are retried and `dont_retry_on` is + ignored. Set one, not both. +> **Coming from Celery:** `retry_backoff` here is a **float** — the base delay in +> seconds — not Celery's boolean flag. Celery's `autoretry_for=(SomeError,)` maps +> to taskito's `retry_on=[SomeError]`. + ## Retry flow Date: Wed, 8 Jul 2026 20:11:50 +0530 Subject: [PATCH 04/57] docs: fix advanced-execution guides --- .../guides/advanced-execution/async-tasks.mdx | 16 ++++++++++++---- .../guides/advanced-execution/batch-enqueue.mdx | 11 ++++++++++- .../guides/advanced-execution/dependencies.mdx | 11 +++++++---- .../python/guides/advanced-execution/prefork.mdx | 6 +++++- .../guides/advanced-execution/streaming.mdx | 5 +++++ .../guides/advanced-execution/unique-tasks.mdx | 8 ++++++-- 6 files changed, 45 insertions(+), 12 deletions(-) diff --git a/docs/content/docs/python/guides/advanced-execution/async-tasks.mdx b/docs/content/docs/python/guides/advanced-execution/async-tasks.mdx index e0fdc959..7ae48079 100644 --- a/docs/content/docs/python/guides/advanced-execution/async-tasks.mdx +++ b/docs/content/docs/python/guides/advanced-execution/async-tasks.mdx @@ -6,8 +6,11 @@ description: "async def tasks dispatched directly onto a dedicated event loop." import { Callout } from "fumadocs-ui/components/callout"; taskito runs async task functions natively — no wrapping in `asyncio.run()`, -no thread-pool bridging. Define a coroutine and the worker dispatches it -directly onto a dedicated event loop. +no thread-pool bridging. Define a coroutine (an `async def` function) and the worker dispatches it +directly onto a dedicated event loop (the asyncio scheduler that runs coroutines). + +Unlike Celery, this needs no `asgiref` or `sync_to_async` shims — the worker +dispatches `async def` tasks natively. ```python from taskito import Queue @@ -73,11 +76,12 @@ queue = Queue( ## Job context -`current_job` works inside async tasks — it reads from `contextvars` rather +`current_job` works inside async tasks — it reads from `contextvars` (Python's async-safe per-task storage) rather than `threading.local`, so it's safe across `await` boundaries: ```python -from taskito.context import current_job +import asyncio +from taskito import current_job @queue.task() async def process(item_id: str) -> str: @@ -150,6 +154,8 @@ dispatches each job to the correct pool based on the `_taskito_is_async` attribute set at registration time: ```python +import asyncio + @queue.task() def sync_task(x: int) -> int: return x * 2 @@ -198,6 +204,8 @@ result = await job.aresult(timeout=30) ### Async worker ```python +import asyncio + async def main(): await queue.arun_worker(queues=["default"]) diff --git a/docs/content/docs/python/guides/advanced-execution/batch-enqueue.mdx b/docs/content/docs/python/guides/advanced-execution/batch-enqueue.mdx index 6fd84c48..78bb7b39 100644 --- a/docs/content/docs/python/guides/advanced-execution/batch-enqueue.mdx +++ b/docs/content/docs/python/guides/advanced-execution/batch-enqueue.mdx @@ -5,7 +5,16 @@ description: "Insert many jobs in a single SQLite transaction with task.map() an import { Callout } from "fumadocs-ui/components/callout"; -Insert many jobs in a single SQLite transaction for high throughput. +Insert many jobs in a single database transaction (SQLite or Postgres) for high +throughput. + + + *Batch enqueue* (`task.map()` / `enqueue_many()`) writes **many** separate jobs + at once — one job per item. [*Task batching*](/python/guides/core/batching) + (`@queue.task(batch=…)`) does the opposite: it collects many `.delay()` calls + into **one** job. Use batch enqueue for bulk-insert throughput; use task + batching to coalesce small units of work. + ## `task.map()` diff --git a/docs/content/docs/python/guides/advanced-execution/dependencies.mdx b/docs/content/docs/python/guides/advanced-execution/dependencies.mdx index 952ad20e..dd52538e 100644 --- a/docs/content/docs/python/guides/advanced-execution/dependencies.mdx +++ b/docs/content/docs/python/guides/advanced-execution/dependencies.mdx @@ -13,7 +13,9 @@ have completed successfully. ## Basic usage Pass `depends_on` when enqueuing a job to declare that it should wait for one -or more other jobs to finish: +or more other jobs to finish. `.delay()` is shorthand for `.apply_async()` +using the task's default options; `.apply_async()` exposes the full set of +per-call options — including `depends_on` — so it's what you reach for here: @@ -244,9 +246,10 @@ job_success = on_valid.apply_async( ``` - `depends_on` is a lower-level primitive than chains, groups, and chords. - Use `depends_on` when you need fine-grained control over a custom DAG. Use - the workflow primitives when your pipeline fits a standard pattern. + `depends_on` is a lower-level primitive than the [chains, groups, and + chords](/python/guides/workflows) built on top of it. Use `depends_on` when + you need fine-grained control over a custom DAG. Use the workflow + primitives when your pipeline fits a standard pattern. ## Combining with other features diff --git a/docs/content/docs/python/guides/advanced-execution/prefork.mdx b/docs/content/docs/python/guides/advanced-execution/prefork.mdx index 4427edbf..a09067b1 100644 --- a/docs/content/docs/python/guides/advanced-execution/prefork.mdx +++ b/docs/content/docs/python/guides/advanced-execution/prefork.mdx @@ -4,7 +4,7 @@ description: "Spawn child processes for true CPU parallelism — each child has --- Spawn separate child processes for true CPU parallelism. Each child has its -own Python GIL, so CPU-bound tasks don't block each other. +own Python GIL (Global Interpreter Lock — only one thread runs Python bytecode at a time), so CPU-bound tasks don't block each other. ## When to use @@ -28,6 +28,10 @@ taskito worker --app myapp:queue --pool prefork The `app` parameter tells each child process how to import your Queue instance. It must be an importable path in `module:attribute` format. +Mapping from Celery: `--pool prefork` / `--concurrency N` ≈ taskito's +`--pool prefork` / `workers=N`. The default pool differs, though — taskito +defaults to `thread`, while Celery defaults to `prefork`. + ## How it works Celery has no native dedup — you'd add `celery-once` plus a Redis lock. taskito +> needs neither: deduplication is enforced in the queue database. + ```python job1 = process.apply_async(args=("report",), unique_key="daily-report") job2 = process.apply_async(args=("report",), unique_key="daily-report") assert job1.id == job2.id # Same job, not duplicated ``` -Once the original job completes (or fails to DLQ), the key is released and a +Once the original job completes (or fails to DLQ — dead letter queue, a holding area for jobs that exhausted their retries), the key is released and a new job can be created with the same key. Deduplication uses a partial unique index: - `CREATE UNIQUE INDEX ... ON jobs(unique_key) WHERE unique_key IS NOT NULL AND status IN (0, 1)`. + `CREATE UNIQUE INDEX ... ON jobs(unique_key) WHERE unique_key IS NOT NULL AND status IN (0, 1)` + (`0` = pending, `1` = running). Only pending and running jobs participate. The check-and-insert is atomic (transaction-protected), so concurrent calls with the same `unique_key` are handled gracefully without race conditions. From a70a2daa921652664475a6a74b5d86dcc9a0c61a Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:11:50 +0530 Subject: [PATCH 05/57] docs: fix workflow guides (worker note, canvas mapping) --- .../docs/python/guides/workflows/analysis.mdx | 3 +++ .../docs/python/guides/workflows/building.mdx | 12 ++++++++-- .../docs/python/guides/workflows/canvas.mdx | 19 ++++++++++++++++ .../python/guides/workflows/composition.mdx | 8 +++++-- .../python/guides/workflows/conditions.mdx | 6 ++++- .../docs/python/guides/workflows/fan-out.mdx | 11 ++++++---- .../docs/python/guides/workflows/gates.mdx | 12 +++++++--- .../docs/python/guides/workflows/index.mdx | 22 ++++++++++++++++--- .../docs/python/guides/workflows/sagas.mdx | 6 ++--- 9 files changed, 81 insertions(+), 18 deletions(-) diff --git a/docs/content/docs/python/guides/workflows/analysis.mdx b/docs/content/docs/python/guides/workflows/analysis.mdx index 64588fba..9ffbd6de 100644 --- a/docs/content/docs/python/guides/workflows/analysis.mdx +++ b/docs/content/docs/python/guides/workflows/analysis.mdx @@ -31,6 +31,9 @@ wf.stats() | `topological_levels()` | `list[list[str]]` | Nodes grouped by depth | | `stats()` | `dict` | Node count, edge count, depth, width, density | +`density` is edges ÷ possible edges — how tightly connected the DAG is, +from `0.0` (no edges) to `1.0` (fully connected). + ## Critical path Find the longest-weighted path through the DAG: diff --git a/docs/content/docs/python/guides/workflows/building.mdx b/docs/content/docs/python/guides/workflows/building.mdx index cd13a2fb..35e422d9 100644 --- a/docs/content/docs/python/guides/workflows/building.mdx +++ b/docs/content/docs/python/guides/workflows/building.mdx @@ -3,7 +3,7 @@ title: Building Workflows description: "Workflow.step(), @queue.workflow() decorator, step configuration, DAG structure, node statuses." --- -A workflow is a DAG of steps. Each step wraps a registered task. The engine +A workflow is a DAG (directed acyclic graph — steps wired by dependencies with no cycles) of steps. Each step wraps a registered task. The engine creates jobs in topological order with `depends_on` chains so the existing scheduler handles execution. @@ -34,7 +34,11 @@ wf.step("fetch", fetch_task, args=("https://api.example.com",)) wf.step("process", process_task, after="fetch", kwargs={"mode": "strict"}) ``` -Arguments are serialized at submission time using the queue's serializer. +Arguments are serialized at submission time using the queue's serializer. A step +runs with **only** the args and kwargs you declare here — it does *not* +automatically receive its predecessor's return value. To pass data downstream, +use [fan-out / fan-in](/python/guides/workflows/fan-out), or read upstream +results from `WorkflowContext.results` in a condition or gate. ## Step configuration @@ -84,6 +88,10 @@ This creates a `WorkflowRun` handle. Under the hood: 3. For each step in topological order, a job is enqueued with `depends_on` chains 4. The run transitions to `RUNNING` +A worker must be consuming the queue for those step jobs to execute — `run.wait()` +blocks until they finish. Start one first: `taskito worker --app tasks:queue`, or +`queue.run_worker()` in a background thread. + + The biggest gotcha: in Celery, `.apply()` executes the canvas **locally and + synchronously**. In taskito, `.apply(queue)` **enqueues** it for workers, and + you await results with `.result()`. There is no eager local execution. + + ## Signatures A `Signature` wraps a task call for deferred execution: diff --git a/docs/content/docs/python/guides/workflows/composition.mdx b/docs/content/docs/python/guides/workflows/composition.mdx index 11aea72a..25fe40c3 100644 --- a/docs/content/docs/python/guides/workflows/composition.mdx +++ b/docs/content/docs/python/guides/workflows/composition.mdx @@ -15,7 +15,7 @@ Use `WorkflowProxy.as_step()` to embed one workflow inside another: @queue.workflow("etl") def etl_pipeline(region): wf = Workflow() - wf.step("extract", extract, args=[region]) + wf.step("extract", extract, args=(region,)) wf.step("load", load, after="extract") return wf @@ -78,8 +78,12 @@ outer run hanging), and the parent run finalizes normally. Stack `@queue.periodic()` on top of `@queue.workflow()`: +taskito cron is **6-field, seconds first** (`sec min hour day month weekday`), so +`0 0 2 * * *` is 02:00:00 daily — not Celery's 5-field `crontab`. Prepend a +seconds field when porting a Celery schedule. + ```python -@queue.periodic(cron="0 0 2 * * *") # 2:00 AM daily +@queue.periodic(cron="0 0 2 * * *") # sec=0 min=0 hour=2 → 2:00 AM daily @queue.workflow("nightly_analytics") def nightly(): wf = Workflow() diff --git a/docs/content/docs/python/guides/workflows/conditions.mdx b/docs/content/docs/python/guides/workflows/conditions.mdx index 97d3982b..462b0b58 100644 --- a/docs/content/docs/python/guides/workflows/conditions.mdx +++ b/docs/content/docs/python/guides/workflows/conditions.mdx @@ -48,7 +48,11 @@ wf.step("deploy", deploy, after="validate", condition=high_score) ## Error strategies -Set the workflow-level error strategy: +Set the workflow-level error strategy. Don't confuse this with the +step-level condition from the previous section: `Workflow(on_failure=...)` +picks the engine's global strategy (`"fail_fast"` or `"continue"`), while +`condition="on_failure"` on a step is a per-step predicate meaning "run this +step if a predecessor failed" — same word, different axis. diff --git a/docs/content/docs/python/guides/workflows/fan-out.mdx b/docs/content/docs/python/guides/workflows/fan-out.mdx index 5c6160fb..a930a679 100644 --- a/docs/content/docs/python/guides/workflows/fan-out.mdx +++ b/docs/content/docs/python/guides/workflows/fan-out.mdx @@ -4,7 +4,10 @@ description: "Split a step's result into parallel children, collect results into --- Split a step's result into parallel child jobs, then collect all results -into a downstream step. +into a downstream step. This is the closest analog to Celery's `chord` +(a group of parallel tasks followed by a callback), but the fan-out size is +derived at runtime from the predecessor's return value instead of being +fixed when the group is built. + `submit_workflow()` enqueues the step jobs; they execute only while a worker is + consuming the queue. Start one first — `taskito worker --app tasks:queue`, or + `queue.run_worker()` in a background thread — otherwise `run.wait()` blocks + until it times out. Workflows ship in the standard `pip install taskito` wheel; + no extra install is needed. + + ## Section overview | Page | What it covers | diff --git a/docs/content/docs/python/guides/workflows/sagas.mdx b/docs/content/docs/python/guides/workflows/sagas.mdx index eec54dac..91167d77 100644 --- a/docs/content/docs/python/guides/workflows/sagas.mdx +++ b/docs/content/docs/python/guides/workflows/sagas.mdx @@ -13,7 +13,7 @@ from taskito.workflows import Workflow queue = Queue() -@queue.task +@queue.task() def refund(forward_args, forward_kwargs, forward_result): charge_id = forward_result # the id returned by charge() payment_api.refund(charge_id) @@ -22,7 +22,7 @@ def refund(forward_args, forward_kwargs, forward_result): def charge(amount: int, customer_id: str) -> str: return payment_api.charge(amount, customer_id) -@queue.task +@queue.task() def ship(order_id: int): shipping_api.ship(order_id) @@ -149,4 +149,4 @@ Saga lifecycle emits dedicated events on the queue's event bus: | `NODE_COMPENSATED` | A node's compensator finished successfully | | `NODE_COMPENSATION_FAILED` | A node's compensator failed after retries | -Subscribe via `queue.on(EventType.WORKFLOW_COMPENSATED, callback)` like any other workflow event. +Subscribe via `queue.on_event(EventType.WORKFLOW_COMPENSATED, callback)` like any other workflow event. From bc609144e43450b185798e1912ffa72d4c36f330 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:12:59 +0530 Subject: [PATCH 06/57] docs: add no-broker story and worker-first onboarding --- .../python/getting-started/capabilities.mdx | 129 +++++++++++------- .../docs/python/getting-started/concepts.mdx | 84 ++++++++---- .../python/getting-started/installation.mdx | 26 +++- .../docs/python/getting-started/meta.json | 2 +- .../python/getting-started/quickstart.mdx | 48 ++++--- docs/content/docs/python/guides/index.mdx | 2 +- 6 files changed, 188 insertions(+), 103 deletions(-) diff --git a/docs/content/docs/python/getting-started/capabilities.mdx b/docs/content/docs/python/getting-started/capabilities.mdx index 9bb12b8e..21ba0fd0 100644 --- a/docs/content/docs/python/getting-started/capabilities.mdx +++ b/docs/content/docs/python/getting-started/capabilities.mdx @@ -1,68 +1,91 @@ --- title: Capabilities at a glance -description: "Everything the Python SDK can do, with a link to each guide." +description: Everything taskito already does, in one place — with a link to the deep-dive guide for each. --- -A map of the Python SDK's features. Everything here runs over the shared Rust -core — pick a backend, register tasks, run workers. +import { Cards, Card } from "fumadocs-ui/components/card"; +import { + ShieldCheck, + Workflow, + Cpu, + CalendarClock, + Activity, + Plug, +} from "lucide-react"; -## Core +taskito ships a broad feature set out of the box. If you're evaluating it — or +suspect a capability is missing — scan this page first. Every card links to the +guide that documents it. -| Feature | Guide | -|---|---| -| Task registry, enqueue, await results | [Tasks](/python/guides/core/tasks) · [Queue API](/python/api-reference/overview) | -| SQLite / Postgres / Redis backends | [Postgres](/python/guides/operations/postgres) | -| Named queues, priority, pause/resume | [Queues](/python/guides/core/queues) | -| Cron-scheduled & delayed jobs | [Scheduling](/python/guides/core/scheduling) | -| Cooperative cancellation | [Job management](/python/guides/operations/job-management) | -| Streaming partial results | [Streaming](/python/guides/advanced-execution/streaming) | -| Sync & async tasks, concurrency | [Async tasks](/python/guides/advanced-execution/async-tasks) · [Execution model](/python/guides/core/execution-model) | -| Batch enqueue, dependencies, unique tasks | [Batch enqueue](/python/guides/advanced-execution/batch-enqueue) · [Dependencies](/python/guides/advanced-execution/dependencies) · [Unique tasks](/python/guides/advanced-execution/unique-tasks) | -| Enqueue predicates & prefork pools | [Predicates](/python/guides/core/predicates) · [Prefork](/python/guides/advanced-execution/prefork) | +## If you think it's missing, it isn't -## Reliability +These are the capabilities most often assumed absent. They aren't. -| Feature | Guide | -|---|---| -| Retries with backoff, dead-letter queue | [Retries](/python/guides/reliability/retries) · [Error handling](/python/guides/reliability/error-handling) | -| Concurrency caps & rate limits | [Rate limiting](/python/guides/reliability/rate-limiting) | -| Circuit breakers | [Circuit breakers](/python/guides/reliability/circuit-breakers) | -| Idempotent enqueue (`unique_key`) | [Idempotency](/python/guides/reliability/idempotency) | -| Distributed locks | [Locking](/python/guides/reliability/locking) | -| Delivery guarantees | [Guarantees](/python/guides/reliability/guarantees) | +| You might assume… | Reality | +| --- | --- | +| No canvas / workflows — no chain, group, or chord | [`chain` / `group` / `chord`](/python/guides/workflows/canvas) are exported from the top-level package, backed by a full workflow DSL. | +| No Flower-like dashboard | [`taskito dashboard --app myapp:queue`](/python/guides/dashboard) serves a built-in monitoring UI. | +| No retry backoff or per-exception retries | [`max_retries`, `retry_backoff`, `retry_on`, `dont_retry_on`](/python/guides/reliability/retries) are arguments on every task. | +| No soft timeouts | [`current_job.check_timeout()`](/python/guides/reliability/error-handling) gives cooperative soft timeouts. | +| GIL bottleneck with no escape hatch | [`--pool prefork`](/python/guides/advanced-execution/prefork) runs child processes with independent GILs for true CPU parallelism. | -## Orchestration +## By capability -| Feature | Guide | -|---|---| -| DAG workflows: fan-out/in, conditions, gates, sub-workflows, saga | [Workflows](/python/guides/workflows) · [Sagas](/python/guides/workflows/sagas) | -| Decentralized work-stealing mesh | [Mesh](/python/guides/operations/mesh) | + + } + title="Reliability" + href="/python/guides/reliability" + description="Retries with exponential backoff, per-exception rules, soft timeouts, dead-letter replay, circuit breakers, and idempotent enqueue." + /> + } + title="Workflows" + href="/python/guides/workflows/canvas" + description="Compose with chain, fan out with group, fan in with chord — plus dependency graphs with cascade cancel." + /> + } + title="Concurrency" + href="/python/guides/advanced-execution/prefork" + description="A thread pool for I/O-bound work and a prefork pool for true CPU parallelism, with a native async API." + /> + } + title="Scheduling" + href="/python/guides/core/scheduling" + description="Priorities, rate limiting, periodic (cron) tasks, delayed execution, and job expiration." + /> + } + title="Observability" + href="/python/guides/dashboard" + description="A built-in dashboard, events, HMAC-signed webhooks, Prometheus, OpenTelemetry, and structured logging." + /> + } + title="Extensibility" + href="/python/guides/extensibility" + description="Pluggable serializers, per-task middleware, a fully async API, and Postgres or Redis backends." + /> + -## Extensibility +## One-liners -| Feature | Guide | -|---|---| -| Lifecycle events & middleware | [Events & webhooks](/python/guides/extensibility/events-webhooks) · [Middleware](/python/guides/extensibility/middleware) | -| Signed webhook delivery | [Events & webhooks](/python/guides/extensibility/events-webhooks) | -| Resource injection, interception & proxies | [Dependency injection](/python/guides/resources/dependency-injection) · [Interception](/python/guides/resources/interception) · [Proxies](/python/guides/resources/proxies) | -| Pluggable serializers (JSON / msgpack / signed / encrypted) | [Serializers](/python/guides/extensibility/serializers) | +```python +# Reliability — backoff + per-exception retry rules +@queue.task(max_retries=5, retry_backoff=2.0, retry_on=[TimeoutError]) +def fetch_url(url: str) -> str: ... -## Observability & ops +# Workflows — sequential pipeline (group / chord for fan-out / fan-in) +chain(fetch.s(url), parse.s(), store.s()).apply() -| Feature | Guide | -|---|---| -| Stats, metrics, progress, worker heartbeats | [Monitoring](/python/guides/observability/monitoring) | -| Structured logging & notes | [Logging](/python/guides/observability/logging) · [Notes](/python/guides/observability/notes) | -| OpenTelemetry / Prometheus / Sentry | [OTel](/python/guides/integrations/otel) · [Prometheus](/python/guides/integrations/prometheus) · [Sentry](/python/guides/integrations/sentry) | -| Dashboard + REST API + auth / SSO | [Dashboard](/python/guides/dashboard) · [REST API](/python/guides/dashboard/rest-api) · [Authentication](/python/guides/dashboard/authentication) | -| KEDA autoscaling (queue-depth scaler) | [KEDA](/python/guides/operations/keda) · [Autoscaler](/python/guides/operations/autoscaler) | -| Job management & CLI | [Job management](/python/guides/operations/job-management) | +# Scheduling — priority + rate limit +@queue.task(priority=9, rate_limit="100/m") +def notify(user_id: int) -> None: ... +``` -## Framework integrations - -[Django](/python/guides/integrations/django) · [FastAPI](/python/guides/integrations/fastapi) · [Flask](/python/guides/integrations/flask) - - - Workflows and mesh are optional capabilities built into the native extension. - See [installation](/python/getting-started/installation) to get set up. - +```bash +taskito worker --pool prefork --app tasks:queue # CPU-bound: true parallelism +taskito dashboard --app tasks:queue # Flower-style monitoring UI +``` diff --git a/docs/content/docs/python/getting-started/concepts.mdx b/docs/content/docs/python/getting-started/concepts.mdx index 77444fcd..641fd6a9 100644 --- a/docs/content/docs/python/getting-started/concepts.mdx +++ b/docs/content/docs/python/getting-started/concepts.mdx @@ -1,42 +1,78 @@ --- title: Concepts -description: "How taskito's pieces fit together: queues, tasks, workers, and results." +description: "How taskito's pieces fit together: queues, tasks, workers, results — and why there's no broker." --- import { Cards, Card } from "fumadocs-ui/components/card"; +import { Callout } from "fumadocs-ui/components/callout"; -Get taskito installed and running in under 5 minutes. +A **task queue** lets your app hand slow work — sending an email, processing an +upload, calling a third-party API — to a background **worker**, so the request +that triggered it can return right away. taskito is a task queue for Python with +one distinguishing trait: **it needs no broker.** - - - - +## Why "no broker" matters + +Most Python task queues (Celery, RQ, Dramatiq, Huey) split the job into separate +services you have to run and operate: + +- a **broker** (Redis or RabbitMQ) — a standalone server that holds the list of + pending jobs and hands them out to workers; +- a **result backend** (often another Redis, or a database) — where return + values are stored; and +- the **workers** themselves. + +taskito collapses the broker and the result backend into a single embedded +database — SQLite by default, or Postgres. Enqueued jobs and their results live +in the same file (or the same Postgres instance). There is no message server to +install, secure, or monitor: `pip install taskito`, point a queue at a path, and +start a worker. + + + Your `broker=redis://…` and `backend=redis://…` URLs both disappear — a single + `Queue(db_path="tasks.db")` replaces them. See the + [migration guide](/python/guides/operations/migration). + ## The mental model -A **queue** is a SQLite (or Postgres) database that holds enqueued jobs. You instantiate -one in your application code: +A **queue** is a SQLite (or Postgres) database that holds enqueued jobs. You +instantiate one in your application code: ```python from taskito import Queue queue = Queue(db_path="tasks.db") ``` -A **task** is any function decorated with `@queue.task()`. Calling `.delay(...)` on it -returns a `JobResult` handle without running the function — the call is enqueued instead. +A **task** is any function decorated with `@queue.task()`. **Enqueuing** a task — +calling `.delay(...)` on it — does *not* run the function. It writes a **job** +row to the database and immediately returns a `JobResult` handle. -A **worker** is a process (or thread, or async task) that pulls jobs from the queue and -runs the corresponding task function. The Rust scheduler handles dispatch, retries, rate -limits, and cleanup; Python only runs during actual task execution. +A **worker** is a process (or thread, or async task) that pulls pending jobs from +the queue and runs the matching task function. The Rust scheduler handles +dispatch, retries, rate limits, and cleanup; your Python code runs only during +actual task execution. **If no worker is running, enqueued jobs simply wait.** -A **result** lives in the same database as the job. Calling `job.result(timeout=30)` -blocks (or `await job.aresult(...)` yields) until the task finishes, raises if it -failed, and deserializes the return value. +A **result** — the task's return value, or the exception it raised — is written +back to the same database. Calling `job.result(timeout=30)` blocks (or +`await job.aresult(...)` yields) until the job finishes, then returns the value +or re-raises the error. + +When a task fails, taskito **retries** it with backoff; once it exhausts +`max_retries` the job moves to the **dead letter queue (DLQ)** — a holding area +for permanently-failed jobs that you can inspect and replay. + +## Next + + + + + diff --git a/docs/content/docs/python/getting-started/installation.mdx b/docs/content/docs/python/getting-started/installation.mdx index 22fd0fb4..4452dacf 100644 --- a/docs/content/docs/python/getting-started/installation.mdx +++ b/docs/content/docs/python/getting-started/installation.mdx @@ -6,7 +6,7 @@ description: "Install taskito and prepare your environment." import { Callout } from "fumadocs-ui/components/callout"; - See [**Capabilities at a glance**](/python/capabilities) for everything taskito does — + See [**Capabilities at a glance**](/python/getting-started/capabilities) for everything taskito does — workflows, retries, the dashboard, prefork pools — each with a link to its guide. @@ -25,6 +25,26 @@ and results. It is installed automatically. **not** need a system SQLite installation. +## Framework & feature extras + +taskito exposes optional pip extras for backends and integrations — install +only what you need: + +```bash +pip install "taskito[postgres]" # Postgres storage backend +pip install "taskito[redis]" # Redis storage backend +pip install "taskito[flask]" # Flask integration +pip install "taskito[fastapi]" # FastAPI integration +pip install "taskito[django]" # Django integration +pip install "taskito[otel]" # OpenTelemetry tracing/metrics +pip install "taskito[prometheus]" # Prometheus metrics exporter +pip install "taskito[sentry]" # Sentry error reporting +pip install "taskito[encryption]" # EncryptedSerializer (cryptography) +pip install "taskito[msgpack]" # MsgPackSerializer +``` + +Extras can be combined, e.g. `pip install "taskito[postgres,otel,sentry]"`. + ## Postgres backend To use PostgreSQL as the storage backend instead of SQLite: @@ -33,7 +53,7 @@ To use PostgreSQL as the storage backend instead of SQLite: pip install taskito[postgres] ``` -See the [Postgres backend guide](/python/guides/operations) for configuration details. +See the [Postgres backend guide](/python/guides/operations/postgres) for configuration details. ## From source @@ -63,7 +83,7 @@ pip install -e ".[docs]" # Documentation ```python import taskito -print(taskito.__version__) # 0.12.0 +print(taskito.__version__) # 0.19.0 (your version may differ) ``` ## Requirements diff --git a/docs/content/docs/python/getting-started/meta.json b/docs/content/docs/python/getting-started/meta.json index 854471bd..11f93873 100644 --- a/docs/content/docs/python/getting-started/meta.json +++ b/docs/content/docs/python/getting-started/meta.json @@ -1,5 +1,5 @@ { "title": "Getting Started", "root": true, - "pages": ["installation", "quickstart", "concepts", "capabilities"] + "pages": ["installation", "concepts", "quickstart", "capabilities"] } diff --git a/docs/content/docs/python/getting-started/quickstart.mdx b/docs/content/docs/python/getting-started/quickstart.mdx index 31041fb1..6c9f99f4 100644 --- a/docs/content/docs/python/getting-started/quickstart.mdx +++ b/docs/content/docs/python/getting-started/quickstart.mdx @@ -28,23 +28,16 @@ def send_email(to: str, subject: str, body: str) -> str: return f"sent to {to}" ``` -## 2. Enqueue jobs +## 2. Start a worker -```python -from tasks import add, send_email - -# Enqueue returns a JobResult handle -job = add.delay(2, 3) -print(f"Job ID: {job.id}") # Job ID: 01936... -print(f"Status: {job.status}") # Status: pending -``` - -## 3. Start a worker +A **worker** pulls jobs from the queue and runs them. Start one first — the CLI +worker runs in the foreground and stays up, processing jobs as they arrive: ```bash +# Runs in its own terminal and blocks, processing jobs until you stop it (Ctrl+C) taskito worker --app tasks:queue ``` @@ -52,6 +45,7 @@ taskito worker --app tasks:queue ```python +# Runs the worker in a background thread inside your own process import threading from tasks import queue @@ -75,22 +69,34 @@ asyncio.run(main()) -## 4. Get results + + Nothing executes until a worker is consuming the queue. Enqueued jobs sit in + `pending` until a worker claims them — so leave this worker running while you + enqueue jobs in the next step. + + +## 3. Enqueue jobs and get results + +From another terminal (or anywhere in your app), enqueue a job and block for its +result. The worker from step 2 picks it up and runs it: ```python from tasks import add +# Enqueue — returns a JobResult handle immediately; the function does NOT run here job = add.delay(2, 3) +print(f"Job ID: {job.id}") # 01936... +print(f"Status: {job.status}") # "pending" — until a worker claims it -# Block until complete (with exponential backoff polling) +# Block until the worker finishes the job (exponential-backoff polling) result = job.result(timeout=30) -print(result) # 5 +print(result) # 5 -# Or use async +# Async variant result = await job.aresult(timeout=30) ``` -## 5. Monitor +## 4. Monitor ```python from tasks import queue @@ -122,12 +128,12 @@ taskito dashboard --app tasks:queue Open `http://localhost:8080` in your browser. The dashboard includes pages covering every aspect of your task queue — no extra dependencies needed. -[Dashboard guide →](/python/guides/observability) +[Dashboard guide →](/python/guides/dashboard) ## Next steps -- [Tasks](/python/guides/core) — decorator options, `.delay()` vs `.apply_async()` -- [Workers](/python/guides/core) — CLI flags, graceful shutdown, worker count -- [Retries](/python/guides/reliability) — exponential backoff, dead letter queue +- [Tasks](/python/guides/core/tasks) — decorator options, `.delay()` vs `.apply_async()` +- [Workers](/python/guides/core/workers) — CLI flags, graceful shutdown, worker count +- [Retries](/python/guides/reliability/retries) — exponential backoff, dead letter queue - [Workflows](/python/guides/workflows) — chain, group, chord -- [Testing](/python/guides/operations) — run tasks synchronously in tests with `queue.test_mode()` +- [Testing](/python/guides/operations/testing) — run tasks synchronously in tests with `queue.test_mode()` diff --git a/docs/content/docs/python/guides/index.mdx b/docs/content/docs/python/guides/index.mdx index d8a8377e..413886be 100644 --- a/docs/content/docs/python/guides/index.mdx +++ b/docs/content/docs/python/guides/index.mdx @@ -19,7 +19,7 @@ import { Pick the topic you're working through. Each guide is self-contained, with code samples, gotchas, and links to the relevant API reference. For a one-page overview of everything taskito does, see -[**Capabilities at a glance**](/python/capabilities). +[**Capabilities at a glance**](/python/getting-started/capabilities). Date: Wed, 8 Jul 2026 20:12:59 +0530 Subject: [PATCH 07/57] docs: fix resource guides (DI rationale, terms) --- .../python/guides/resources/configuration.mdx | 10 +++-- .../guides/resources/dependency-injection.mdx | 37 ++++++++++++++----- .../docs/python/guides/resources/index.mdx | 18 ++++++++- .../python/guides/resources/interception.mdx | 15 +++++++- .../python/guides/resources/observability.mdx | 12 +++--- .../docs/python/guides/resources/proxies.mdx | 18 ++++++--- .../docs/python/guides/resources/testing.mdx | 5 +++ 7 files changed, 88 insertions(+), 27 deletions(-) diff --git a/docs/content/docs/python/guides/resources/configuration.mdx b/docs/content/docs/python/guides/resources/configuration.mdx index 971d6d0e..5c42be7a 100644 --- a/docs/content/docs/python/guides/resources/configuration.mdx +++ b/docs/content/docs/python/guides/resources/configuration.mdx @@ -73,7 +73,7 @@ pip install tomli | `max_lifetime` | float | `3600.0` | Task scope: max seconds an instance lives. | | `idle_timeout` | float | `300.0` | Task scope: max idle seconds before eviction. | | `reloadable` | bool | `false` | Allow hot reload via SIGHUP or CLI. | -| `frozen` | bool | `false` | Wrap instance in a read-only proxy. | +| `frozen` | bool | `false` | Wrap instance in a read-only wrapper. | ## Pool configuration @@ -106,7 +106,9 @@ startup. This avoids the cold-start latency on the first burst of tasks. ## Frozen resources -Wrap a resource in a read-only proxy to prevent accidental mutation: +Wrap a resource in a read-only wrapper to prevent accidental mutation. (This +is unrelated to the [Resource Proxies](/python/guides/resources/proxies) +layer — "wrapper" here just means attribute writes raise.) ```python @queue.worker_resource("config", frozen=True) @@ -141,7 +143,9 @@ taskito reload --pid taskito reload --pid --resource feature_flags # reload one resource ``` -Or programmatically from application code: +Or programmatically from application code (internal accessor — the supported +triggers are `SIGHUP` and `taskito reload` above; this attribute is not yet part +of the stable API): ```python results = queue._resource_runtime.reload() diff --git a/docs/content/docs/python/guides/resources/dependency-injection.mdx b/docs/content/docs/python/guides/resources/dependency-injection.mdx index c9f19003..6aba8687 100644 --- a/docs/content/docs/python/guides/resources/dependency-injection.mdx +++ b/docs/content/docs/python/guides/resources/dependency-injection.mdx @@ -9,6 +9,14 @@ Worker resources are long-lived objects initialized once at worker startup and injected into tasks by name. No serialization is involved — they live entirely in the worker process and are never put in the queue. + + Celery has no first-class DI, so you typically open a connection at import time + (`db = create_engine(...)`) and share that global. Worker resources replace that + pattern: taskito creates one instance per worker, scopes it + (worker / task / thread / request), injects it by name, and tears it down on + shutdown — no import-time side effects, and clean per-scope lifecycles. + + ## Declaring resources ```python @@ -53,7 +61,7 @@ tasks. | `max_lifetime` | `3600.0` | Task scope: max seconds an instance can live. | | `idle_timeout` | `300.0` | Task scope: max idle seconds before eviction. | | `reloadable` | `False` | Allow hot reload via SIGHUP or CLI. | -| `frozen` | `False` | Wrap instance in a read-only proxy. | +| `frozen` | `False` | Wrap instance in a read-only wrapper (see [Configuration](/python/guides/resources/configuration#frozen-resources)). | ## Injecting resources into tasks @@ -111,6 +119,11 @@ def create_session(db): @queue.worker_resource("local_cache", scope="thread") def create_cache(): return {} + +# Request scope: a fresh instance for every task, torn down right after +@queue.worker_resource("audit_context", scope="request") +def create_audit_context(): + return AuditContext(started_at=time.time()) ``` Pool configuration parameters (`pool_size`, `pool_min`, `acquire_timeout`, @@ -131,7 +144,8 @@ def load_config(): @queue.worker_resource("db", depends_on=["config"]) def create_db(config): - return create_engine(config.db_url, pool_size=10) + # Return a sessionmaker so the injected `db` is callable: `session = db()`. + return sessionmaker(create_engine(config.db_url, pool_size=10)) @queue.worker_resource("cache", depends_on=["config"]) @@ -150,12 +164,14 @@ Cycles are detected eagerly at registration time and raise Supply a teardown callable to clean up the resource on graceful shutdown: ```python +engine = create_engine("postgresql://localhost/myapp") + @queue.worker_resource( "db", - teardown=lambda engine: engine.dispose(), + teardown=lambda _factory: engine.dispose(), # dispose the engine on shutdown ) def create_db(): - return create_engine("postgresql://localhost/myapp") + return sessionmaker(engine) ``` Or use `register_resource()` for the programmatic API: @@ -181,10 +197,13 @@ thread. If the check returns falsy, the worker attempts to recreate the resource: ```python -def check_db(engine): - with engine.connect() as conn: - conn.execute(text("SELECT 1")) - return True +def check_db(db): + session = db() # `db` is the sessionmaker returned by the factory + try: + session.execute(text("SELECT 1")) + return True + finally: + session.close() @queue.worker_resource( @@ -194,7 +213,7 @@ def check_db(engine): max_recreation_attempts=3, # mark permanently unhealthy after 3 failures ) def create_db(): - return create_engine("postgresql://localhost/myapp") + return sessionmaker(create_engine("postgresql://localhost/myapp")) ``` The health checker runs in a single daemon thread. Each resource with a diff --git a/docs/content/docs/python/guides/resources/index.mdx b/docs/content/docs/python/guides/resources/index.mdx index fc53e608..188c868d 100644 --- a/docs/content/docs/python/guides/resources/index.mdx +++ b/docs/content/docs/python/guides/resources/index.mdx @@ -7,7 +7,19 @@ The resource system gives tasks clean access to external dependencies — database connections, HTTP clients, cloud clients — without passing live objects through the queue. It operates in three layers that together solve a fundamental distributed systems problem: task arguments must be -serializable, but most real-world dependencies are not. +serializable (able to be turned into bytes so they can cross the process +boundary into the worker), but most real-world dependencies are not. + +Coming from Celery (which has no first-class DI), resources replace the +module-level globals you'd otherwise create — a single `db = create_engine(...)` +at import time — with per-worker instances that taskito initializes, scopes, and +tears down for you. + +Two terms come up throughout this section: a **recipe** is the small, +serializable dict a proxy handler extracts from a live object (e.g. a file +path and mode) so the worker can rebuild it later, and a **DI marker** is the +placeholder that replaces a redirected argument (like a database session) so +the worker knows which named resource to inject in its place. Coming from Celery, `test_mode()` replaces `task_always_eager` — but as a +> context manager it's scoped, so it's safe to use in parallel test runs. + ## Injecting mock resources ```python @@ -43,6 +46,8 @@ When `test_mode(resources=...)` is active: ```python from taskito import MockResource +real_session_factory = sessionmaker(create_engine("sqlite:///test.db")) + def test_with_spy(): spy_db = MockResource("db", wraps=real_session_factory, track_calls=True) From 8c8cfb4164033c4d0e94cd4729c32df097c14e1b Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:12:59 +0530 Subject: [PATCH 08/57] docs: fix operations guides (pool, mesh, migration) --- .../python/guides/operations/deployment.mdx | 40 ++++++++++++++++++- .../docs/python/guides/operations/index.mdx | 6 ++- .../guides/operations/job-management.mdx | 9 ++++- .../docs/python/guides/operations/keda.mdx | 13 ++++++ .../docs/python/guides/operations/mesh.mdx | 20 ++++++++-- .../python/guides/operations/migration.mdx | 10 ++++- .../python/guides/operations/postgres.mdx | 30 ++++++++++---- .../python/guides/operations/security.mdx | 15 +++++-- .../docs/python/guides/operations/testing.mdx | 36 ++++++++++++++--- .../guides/operations/troubleshooting.mdx | 24 ++++++----- .../guides/operations/upgrading-0.15.mdx | 31 +++++--------- 11 files changed, 175 insertions(+), 59 deletions(-) diff --git a/docs/content/docs/python/guides/operations/deployment.mdx b/docs/content/docs/python/guides/operations/deployment.mdx index 10e4e874..ef3507e0 100644 --- a/docs/content/docs/python/guides/operations/deployment.mdx +++ b/docs/content/docs/python/guides/operations/deployment.mdx @@ -7,6 +7,13 @@ import { Callout } from "fumadocs-ui/components/callout"; This guide covers running taskito in production environments. + + Unlike Celery (which needs Redis or RabbitMQ as a message broker), taskito's + scheduler lives inside the worker process and reads directly from SQLite or + Postgres. There's no separate broker process to run, monitor, or fail over — + one less moving part in production. + + ## SQLite file location Choose a persistent, backed-up location for your database: @@ -15,6 +22,11 @@ Choose a persistent, backed-up location for your database: queue = Queue(db_path="/var/lib/myapp/taskito.db") ``` +SQLite runs in **WAL** (write-ahead logging) mode by default — see +[WAL mode and backups](#wal-mode-and-backups) below — which appends writes to +a separate log file instead of the main database file, letting many readers +proceed concurrently with the one writer. + **Best practices:** - Use an absolute path — relative paths depend on the working directory @@ -81,11 +93,26 @@ COPY . . # Store the database in a volume VOLUME /data -ENV TASKITO_DB_PATH=/data/taskito.db CMD ["taskito", "worker", "--app", "myapp:queue"] ``` + + The `taskito` CLI and the `Queue` constructor do not read a `TASKITO_DB_PATH` + environment variable — that env var name is only recognized by the Flask and + Django integrations. If your `myapp:queue` module builds the `Queue` + directly, read the path yourself (and keep the `ENV TASKITO_DB_PATH=...` + line in the Dockerfile above), as shown below. + + +```python +# myapp.py +import os +from taskito import Queue + +queue = Queue(db_path=os.environ.get("TASKITO_DB_PATH", "/data/taskito.db")) +``` + ### docker-compose.yml ```yaml @@ -340,7 +367,11 @@ are serialized. This is the primary throughput ceiling. - **1,000–5,000 jobs/second** for enqueue + dequeue cycles (small payloads) - Throughput decreases with larger payloads, complex queries, or spinning disks -- The connection pool size (default: 8) controls read concurrency — tune it based on your read/write ratio +- The connection pool (a fixed set of reusable database connections, avoiding + the overhead of opening a new one per query) size is fixed at 8 for the + SQLite backend — it is not configurable from the Python `Queue` + constructor. The `pool_size` argument on `Queue()` only takes effect on the + [Postgres backend](/python/guides/operations/postgres#connection-pooling) **When to upgrade to Postgres:** @@ -363,6 +394,11 @@ of these limitations while keeping the same API. | 20K–50K jobs/s | Postgres | 32+ | prefork | Multiple worker processes, tune `pool_size` | | > 50K jobs/s | — | — | — | Consider Celery + RabbitMQ for this scale | +`Pool` is the worker execution pool: `thread` (default, one process with +multiple OS threads) or `prefork` (multiple worker subprocesses, true CPU +parallelism). See [Prefork Workers](/python/guides/advanced-execution/prefork) +for when and how to switch. + These are rough guidelines for noop tasks. Real throughput depends on task duration, payload size, and I/O patterns. diff --git a/docs/content/docs/python/guides/operations/index.mdx b/docs/content/docs/python/guides/operations/index.mdx index 0866b9e1..b618fe97 100644 --- a/docs/content/docs/python/guides/operations/index.mdx +++ b/docs/content/docs/python/guides/operations/index.mdx @@ -3,15 +3,19 @@ title: Operations description: "Run taskito reliably in production — testing, deployment, scaling, migration." --- -Run taskito reliably in production. +Run taskito reliably in production — with no Redis or RabbitMQ broker to operate, +just an embedded SQLite file or a Postgres instance. | Guide | Description | |---|---| | [Testing](/python/guides/operations/testing) | Test mode, fixtures, mocking resources, and workflow testing | | [Job Management](/python/guides/operations/job-management) | Cancel, pause, archive, revoke, replay, and clean up jobs | | [Troubleshooting](/python/guides/operations/troubleshooting) | Diagnose stuck jobs, lock contention, and worker issues | +| [Security](/python/guides/operations/security) | Serializer signing/encryption, dashboard auth, and SSRF guards | | [Deployment](/python/guides/operations/deployment) | systemd, Docker, WAL mode, Postgres, and production checklists | +| [Mesh Scheduling](/python/guides/operations/mesh) | Decentralized multi-worker work-stealing overlay | | [Bare-Metal Autoscaler](/python/guides/operations/autoscaler) | Automatically scale worker processes without Kubernetes | | [KEDA Autoscaling](/python/guides/operations/keda) | Kubernetes event-driven autoscaling for workers | | [Postgres Backend](/python/guides/operations/postgres) | Set up and run taskito with PostgreSQL | | [Migrating from Celery](/python/guides/operations/migration) | Side-by-side comparison and step-by-step migration guide | +| [Upgrading to 0.15](/python/guides/operations/upgrading-0.15) | Breaking changes and migration steps for the 0.15 release | diff --git a/docs/content/docs/python/guides/operations/job-management.mdx b/docs/content/docs/python/guides/operations/job-management.mdx index 7618939d..d27ee6e2 100644 --- a/docs/content/docs/python/guides/operations/job-management.mdx +++ b/docs/content/docs/python/guides/operations/job-management.mdx @@ -47,7 +47,7 @@ queue = Queue( The scheduler checks every ~60 seconds and purges: - Completed jobs older than `result_ttl` -- Dead letter entries older than `result_ttl` +- Dead letter (DLQ — a holding area for jobs that exhausted their retries) entries older than `result_ttl` - Error history records older than `result_ttl` Set to `None` (default) to disable auto-cleanup. @@ -141,6 +141,8 @@ Archived jobs are no longer returned by `queue.stats()` or ### Scheduled archival ```python +from taskito import current_job + @queue.periodic(cron="0 0 2 * * *") # Daily at 2 AM def nightly_archival(): archived = queue.archive(older_than=7 * 86400) # Archive jobs older than 7 days @@ -191,7 +193,10 @@ for entry in dead: ## SQLite configuration -taskito configures SQLite for optimal performance: +Left at their defaults, SQLite's own pragmas favor safety over throughput +and let concurrent readers block on writers — so taskito sets these at +connection time to give WAL-mode concurrency and predictable behavior under +lock contention without any configuration on your part: | Pragma | Value | Purpose | |---|---|---| diff --git a/docs/content/docs/python/guides/operations/keda.mdx b/docs/content/docs/python/guides/operations/keda.mdx index e9702b83..8aabe033 100644 --- a/docs/content/docs/python/guides/operations/keda.mdx +++ b/docs/content/docs/python/guides/operations/keda.mdx @@ -9,6 +9,19 @@ import { Callout } from "fumadocs-ui/components/callout"; taskito worker deployment up and down based on queue depth. taskito ships a dedicated scaler server that KEDA queries directly. + + Requires a Kubernetes cluster with the [KEDA operator](https://keda.sh/docs/latest/deploy/) + installed. For bare metal, Docker, or systemd without Kubernetes, use the + [bare-metal autoscaler](/python/guides/operations/autoscaler) instead. + + + + The scaler's `/api/scaler` endpoint is unauthenticated by design — see + [Security: Scaler bind address](/python/guides/operations/security#scaler-bind-address). + Keep it reachable only from inside the cluster (`ClusterIP`, never a public + LoadBalancer or Ingress). + + ## Scaler server Start the scaler alongside your worker: diff --git a/docs/content/docs/python/guides/operations/mesh.mdx b/docs/content/docs/python/guides/operations/mesh.mdx index 728b3208..9943830f 100644 --- a/docs/content/docs/python/guides/operations/mesh.mdx +++ b/docs/content/docs/python/guides/operations/mesh.mdx @@ -11,10 +11,12 @@ Mesh scheduling adds a decentralized overlay network on top of the gossip, route tasks through a consistent-hashing ring, and steal work from busy peers — all without a central coordinator. - - Mesh is **opt-in** and requires the `mesh` cargo feature at build time. - Without it, `MeshWorker` is still importable but passing it to `run_worker` - has no effect. Build with: `uv run maturin develop --features mesh,workflows` + + The `taskito` package published to PyPI does **not** include the `mesh` + cargo feature. `MeshWorker` is importable from it, but passing it to + `run_worker` has no effect. To use mesh scheduling you must build the + extension from source with the feature enabled: + `uv run maturin develop --features mesh,workflows` ## Quick start @@ -186,6 +188,16 @@ services: - worker-1 ``` + + This compose file is illustrative, not runnable as-is: the `taskito worker` + CLI has no mesh flags and never reads `MESH_PORT`/`MESH_SEEDS`/`MESH_KEY`. + Mesh is only enabled by calling `queue.run_worker(mesh=...)` in your own + Python code. Replace `command: taskito worker --app myapp:queue` with an + entrypoint that imports `get_mesh()` below and calls + `queue.run_worker(mesh=get_mesh())` directly (e.g. + `command: python -m myapp`). + + ```python # myapp.py — reads mesh config from environment import os diff --git a/docs/content/docs/python/guides/operations/migration.mdx b/docs/content/docs/python/guides/operations/migration.mdx index 35e86682..c1059b67 100644 --- a/docs/content/docs/python/guides/operations/migration.mdx +++ b/docs/content/docs/python/guides/operations/migration.mdx @@ -16,7 +16,7 @@ with less infrastructure and simpler configuration. |---|---|---| | `Celery()` app | `Queue()` | `Queue(db_path=...)` replaces `Celery(broker=..., backend=...)` — no broker URL, no result-backend URL, no `app.conf` setup. | | `@app.task` | `@queue.task()` | Same decorator shape; `max_retries`, `rate_limit`, `bind=True`, `name=` carry over with identical names. | -| `.apply_async()` | `.apply_async()` | Same kwargs (`countdown`, `eta`, `priority`, `queue`); taskito adds `delay`, `unique_key`, `expires`, `depends_on`. | +| `.apply_async()` | `.apply_async()` | `priority` and `queue` carry over. Celery's `countdown` becomes `delay`; there is no `eta` (use `delay` with relative seconds). taskito adds `unique_key`, `expires`, `depends_on`. | | `.delay()` | `.delay()` | Drop-in. Returns `JobResult` instead of `AsyncResult`. | | `AsyncResult` | `JobResult` | `.result(timeout=...)` replaces `.get(timeout=...)`; async variant `await job.aresult()`. | | Canvas (`chain`, `group`, `chord`) | `chain`, `group`, `chord` | API-compatible. taskito adds a DAG `Workflow` builder with conditions, gates, and fan-in. | @@ -212,6 +212,12 @@ chord( Almost identical. The only change: `.apply_async()` becomes `.apply()`. + + In Celery, `.apply()` runs the canvas **locally and synchronously**. In taskito, + `.apply(queue)` **submits** it to the queue for workers to run — await results + with `.result()`. taskito has no eager local execution mode. + + ### Periodic tasks @@ -339,7 +345,7 @@ Some Celery features don't have taskito equivalents: | Distributed workers (multi-server) | Use Postgres backend | | Message routing (exchanges, topics) | Use named queues instead | | `celery multi` (process management) | Use systemd, supervisor, or Docker | -| Custom serializers (JSON, msgpack) | `JsonSerializer`, `CloudpickleSerializer` (default), or custom `Serializer` protocol | +| Custom serializers (JSON, msgpack) | `SmartSerializer` (default), `JsonSerializer`, `CloudpickleSerializer`, or custom `Serializer` protocol | | Task cancellation (mid-execution) | Cancel pending or running jobs (`cancel_running_job()` + `check_cancelled()`) | | ETA (absolute datetime scheduling) | Use `delay` (relative seconds) | | `bind=True` (self argument) | Use `current_job` context instead | diff --git a/docs/content/docs/python/guides/operations/postgres.mdx b/docs/content/docs/python/guides/operations/postgres.mdx index 47a4401b..ba1099ff 100644 --- a/docs/content/docs/python/guides/operations/postgres.mdx +++ b/docs/content/docs/python/guides/operations/postgres.mdx @@ -97,12 +97,25 @@ your application tables. ## Connection pooling -The Postgres backend uses Diesel's `r2d2` connection pool with a default -size of **10 connections**. Each connection has the `search_path` set to -the configured schema on acquisition. +The Postgres backend keeps a pool of reusable connections (default size: +**10**) rather than opening a new connection per query. Each connection has +the `search_path` set to the configured schema on acquisition. -The pool size is configured at the Rust layer. For most workloads, the -default of 10 connections is sufficient. +Unlike the SQLite backend — where the pool size is fixed and not exposed to +Python (see [Deployment](/python/guides/operations/deployment#sqlite-scaling-limits)) +— the Postgres pool size is tunable from the `Queue` constructor: + +```python +queue = Queue( + backend="postgres", + db_url="postgresql://user:password@localhost:5432/myapp", + pool_size=20, # default: 10 +) +``` + +For most workloads, the default of 10 connections is sufficient. Lower it for +managed services with strict per-database connection limits (e.g. Supabase); +raise it if workers are blocking waiting for a free connection under load. ## Migrations @@ -242,5 +255,8 @@ pg_dump -h localhost -U taskito -d myapp -n taskito > backup.sql psql -h localhost -U taskito -d myapp < backup.sql ``` -For continuous backups, use PostgreSQL's built-in WAL archiving or a tool -like [pgBackRest](https://pgbackrest.org/). +For continuous backups, use PostgreSQL's built-in WAL (write-ahead logging — +see [WAL mode and backups](/python/guides/operations/deployment#wal-mode-and-backups) +for what WAL means in the SQLite backend; Postgres uses WAL primarily for +crash recovery and replication rather than reader/writer concurrency) +archiving, or a tool like [pgBackRest](https://pgbackrest.org/). diff --git a/docs/content/docs/python/guides/operations/security.mdx b/docs/content/docs/python/guides/operations/security.mdx index 6a5f88f1..2e7c8aff 100644 --- a/docs/content/docs/python/guides/operations/security.mdx +++ b/docs/content/docs/python/guides/operations/security.mdx @@ -46,8 +46,17 @@ key = os.urandom(32) # 32+ bytes, identical on producers and workers queue = Queue(serializer=SignedSerializer(SmartSerializer(), key)) ``` -For confidentiality as well, wrap `EncryptedSerializer`. See -[Pluggable Serializers](/docs/guides/extensibility/serializers) for the full +For confidentiality as well, wrap `EncryptedSerializer` (AES-GCM, so it +authenticates the payload too — no need to also wrap `SignedSerializer`): + +```python +from taskito.serializers import EncryptedSerializer, SmartSerializer + +enc_key = os.urandom(32) # 16, 24, or 32 bytes for AES-128/192/256 +queue = Queue(serializer=EncryptedSerializer(SmartSerializer(), enc_key)) +``` + +See [Pluggable Serializers](/python/guides/extensibility/serializers) for the full matrix. ### Payloads at rest @@ -97,7 +106,7 @@ with `TASKITO_DASHBOARD_ADMIN_USER` / `TASKITO_DASHBOARD_ADMIN_PASSWORD` Outbound webhook URLs are validated against private/loopback/link-local and cloud-metadata addresses **at delivery time** (not just registration), which -closes the DNS-rebinding SSRF window, and redirects are not followed. For +closes the DNS-rebinding SSRF (server-side request forgery) window, and redirects are not followed. For local development against `http://localhost`, set `TASKITO_WEBHOOKS_ALLOW_PRIVATE=1`. diff --git a/docs/content/docs/python/guides/operations/testing.mdx b/docs/content/docs/python/guides/operations/testing.mdx index 7528e746..89a8a2df 100644 --- a/docs/content/docs/python/guides/operations/testing.mdx +++ b/docs/content/docs/python/guides/operations/testing.mdx @@ -155,8 +155,25 @@ def test_failure_propagated(): ## Testing workflows -Chains, groups, and chords work in test mode because they call `enqueue()` -internally, which is intercepted by the test mode patch. +Chains and groups work in test mode because they call `enqueue()` +internally, which is intercepted by the test mode patch. Chords do too — the +callback receives the list of group results as its first argument: + +```python +from taskito import chord + +@queue.task() +def sum_results(values: list[int]) -> int: + return sum(values) + +def test_chord(): + with queue.test_mode() as results: + chord([double.s(1), double.s(2)], sum_results.s()).apply() + + # 2 group tasks + 1 callback = 3 results + assert len(results) == 3 + assert results[-1].return_value == 6 # sum([double(1), double(2)]) = sum([2, 4]) +``` ### Chains @@ -414,7 +431,14 @@ def test_e2e(): tests without running a real worker. -## Running tests locally +## Contributing: taskito's own test suite + + + This section is for people contributing to taskito itself — building the + Rust extension and running its internal test suite. If you're testing + tasks in your own application, the sections above (`test_mode()`, + `TestResult`, `MockResource`) are what you want instead. + ```bash # Rust tests @@ -426,9 +450,9 @@ uv run maturin develop # Python tests uv run python -m pytest tests/python/ -v -# Linting -uv run ruff check py_src/ tests/ -uv run mypy py_src/taskito/ --no-incremental +# Linting (run from sdks/python/) +uv run ruff check taskito/ tests/ +uv run mypy taskito/ --no-incremental ``` To build with native async support: diff --git a/docs/content/docs/python/guides/operations/troubleshooting.mdx b/docs/content/docs/python/guides/operations/troubleshooting.mdx index 6268dc58..0dfc61e2 100644 --- a/docs/content/docs/python/guides/operations/troubleshooting.mdx +++ b/docs/content/docs/python/guides/operations/troubleshooting.mdx @@ -30,14 +30,14 @@ for job in stuck: have exceeded their `timeout_ms` and retries them. If a job has no timeout set, it stays stuck forever. -To recover a stuck job manually: +`timeout_ms` is the internal, database-stored deadline in milliseconds; you +set it in seconds via `@queue.task(timeout=...)` (or `default_timeout` on +`Queue()`) and taskito converts it under the hood. -```python -import time - -# Mark the job as failed so it retries -queue._inner.retry(job_id, int(time.time() * 1000)) -``` +Recovery is automatic once a `timeout` is set: the stale reaper marks the job +failed and then retries it (or moves it to the dead letter queue). A job stuck +with **no** timeout stays `running` indefinitely — set a timeout going forward, +and re-enqueue the work with a fresh `.delay()` if you need it to run now. To prevent this in future, always set a timeout on production tasks: @@ -66,11 +66,14 @@ for w in workers: **Possible causes**: -1. **GIL-bound CPU task**: a long-running CPU task is holding the GIL, +1. **GIL-bound CPU task**: a long-running CPU task is holding the GIL + (Global Interpreter Lock — only one thread runs Python bytecode at a time), blocking the scheduler thread from dispatching new jobs. The scheduler runs in Rust, but it still needs the GIL to call Python functions. - Fix: switch to the prefork pool for CPU-bound tasks. + Fix: switch to the [prefork pool](/python/guides/advanced-execution/prefork) + for CPU-bound tasks — it runs tasks in separate subprocesses instead of + threads, so one task's GIL usage can't block the scheduler. ```bash taskito worker --app myapp:queue --pool prefork @@ -241,8 +244,7 @@ job = queue.get_job(job_id) print(job.to_dict()["task_name"]) # e.g. "myapp.tasks.process" # Check what the worker has registered -# (add this temporarily to your worker startup) -print(list(queue._task_registry.keys())) +print([t["name"] for t in queue.registered_tasks()]) ``` **Fix**: use consistent import paths. If the task is `myapp/tasks.py:process`, diff --git a/docs/content/docs/python/guides/operations/upgrading-0.15.mdx b/docs/content/docs/python/guides/operations/upgrading-0.15.mdx index aecbb5c9..8b3c43cf 100644 --- a/docs/content/docs/python/guides/operations/upgrading-0.15.mdx +++ b/docs/content/docs/python/guides/operations/upgrading-0.15.mdx @@ -59,8 +59,11 @@ Pre-0.15 payloads (untagged cloudpickle) are read transparently by 0.15 workers ## 2. Per-task / per-queue stats are computed server-side -**What changed.** Per-task and per-queue statistics are now computed on the -server side using `SINTERCARD`, replacing the previous O(N) full-scan approach. +**What changed.** On the **Redis backend**, per-task and per-queue statistics +are now computed on the server side using `SINTERCARD` (a Redis 7.0+ command +that counts set intersection size without transferring the members), +replacing the previous O(N) full-scan approach. SQLite and Postgres already +computed these with a server-side `SELECT COUNT(...)` and are unaffected. This is a pure performance improvement. **Action required:** none. No data migration, no API change, no configuration @@ -143,26 +146,12 @@ and may take a few seconds on large databases. back to a pre-0.15 binary after deploying. -## 6. Job payloads moved to a side table (`job_payloads`) +## 6. Job payloads side table (`job_payloads`) — superseded - - **Superseded.** A later release **removed** the `job_payloads` side table and - moved payloads back inline on `jobs`/`archived_jobs`. The narrow dequeue scan - already excludes the blobs by column selection, so the side table bought - nothing. The `jobs.payload`/`jobs.result` columns were never dropped, so the - removal moves no data and needs no upgrade window. This section is kept only as - history for databases that passed through 0.15. - - -**What 0.15 did.** Payload BLOBs were stored in a separate `job_payloads` table -rather than inline on the `jobs` row, dual-written alongside the kept -`jobs.payload`/`jobs.result` columns. This was transparent — no API change, no -change to enqueue or result behaviour. - - - Redis was never affected: the Redis backend stores job payloads inline in the - job JSON object and never used a side table. - +**Superseded:** 0.15 briefly dual-wrote payload BLOBs to a separate +`job_payloads` table; a later release removed it and moved payloads back +inline on `jobs`/`archived_jobs` (no data moved, no upgrade window, no action +needed — this is kept only as history). Redis was never affected. ## Upgrade checklist From ea20bb225c44f8dea0a103f01c6134e420bbe176 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:13:00 +0530 Subject: [PATCH 09/57] docs: fix dashboard guides (pause, routes, port) --- .../guides/dashboard/authentication.mdx | 2 +- .../docs/python/guides/dashboard/index.mdx | 13 ++-- .../docs/python/guides/dashboard/rest-api.mdx | 13 +++- .../docs/python/guides/dashboard/sso.mdx | 11 +++- .../guides/dashboard/task-overrides.mdx | 59 +++++++++++-------- 5 files changed, 62 insertions(+), 36 deletions(-) diff --git a/docs/content/docs/python/guides/dashboard/authentication.mdx b/docs/content/docs/python/guides/dashboard/authentication.mdx index 1e333a26..bd20dcee 100644 --- a/docs/content/docs/python/guides/dashboard/authentication.mdx +++ b/docs/content/docs/python/guides/dashboard/authentication.mdx @@ -135,7 +135,7 @@ stats = s.get("http://localhost:8080/api/stats").json() s.post("http://localhost:8080/api/queues/default/pause") ``` -## SSRF guard for outbound URLs +## SSRF (server-side request forgery) guard for outbound URLs Webhook URLs entered through the dashboard are vetted before any delivery happens. By default the server rejects: diff --git a/docs/content/docs/python/guides/dashboard/index.mdx b/docs/content/docs/python/guides/dashboard/index.mdx index 0bf4266f..516816f7 100644 --- a/docs/content/docs/python/guides/dashboard/index.mdx +++ b/docs/content/docs/python/guides/dashboard/index.mdx @@ -7,11 +7,14 @@ import { Callout } from "fumadocs-ui/components/callout"; import { Tab, Tabs } from "fumadocs-ui/components/tabs"; taskito ships with a built-in web dashboard for monitoring jobs, inspecting -dead letters, configuring webhooks, tuning per-task runtime limits, and +dead letters (the DLQ — a holding area for jobs that exhausted their retries), configuring webhooks, tuning per-task runtime limits, and managing your task queue in real time. The dashboard is a single-page application served directly from the Python package — **zero extra dependencies required**. +> Coming from Celery? This is taskito's built-in equivalent of Flower — no +> separate monitoring service to install or run. + ![Overview page with stats cards, throughput chart, and recent activity](/screenshots/dashboard/overview.png) ## Launching the dashboard @@ -32,7 +35,7 @@ The `--app` argument uses the same `module:attribute` format as the worker. from taskito.dashboard import serve_dashboard from myapp import queue -serve_dashboard(queue, host="0.0.0.0", port=8000) +serve_dashboard(queue, host="0.0.0.0", port=8080) ``` @@ -104,7 +107,7 @@ The full REST API surface is documented at The dashboard is a React 19 + Vite 8 + TypeScript SPA routed via TanStack Router, styled with Tailwind v4 and shadcn/ui, and shipped as -hash-busted multi-file assets under `py_src/taskito/static/dashboard/`. +hash-busted multi-file assets under `sdks/python/taskito/static/dashboard/`. - **Dark and light mode** — Toggle via the sun/moon button in the header. Preference is stored in `localStorage` and follows the system scheme by @@ -123,7 +126,7 @@ hash-busted multi-file assets under `py_src/taskito/static/dashboard/`. The built SPA ships inside the Python wheel under - `py_src/taskito/static/dashboard/` and is served by the Python + `sdks/python/taskito/static/dashboard/` and is served by the Python dashboard process. No Node.js, no pnpm, no CDN at runtime — just `pip install taskito`. Node.js and pnpm are only needed by contributors rebuilding the dashboard source. @@ -213,7 +216,7 @@ pnpm run build The build produces a static `index.html` plus hashed JS/CSS chunks -under `py_src/taskito/static/dashboard/`. The built assets aren't +under `sdks/python/taskito/static/dashboard/`. The built assets aren't committed — release tooling runs `pnpm -C dashboard build` before packaging so the wheel ships them. diff --git a/docs/content/docs/python/guides/dashboard/rest-api.mdx b/docs/content/docs/python/guides/dashboard/rest-api.mdx index 7c4fc687..57d8dbfa 100644 --- a/docs/content/docs/python/guides/dashboard/rest-api.mdx +++ b/docs/content/docs/python/guides/dashboard/rest-api.mdx @@ -10,9 +10,11 @@ All endpoints return `application/json` and live under the same origin as the dashboard itself. - Every `/api/*` endpoint except the public set (`/api/auth/status`, - `/api/auth/login`, `/api/auth/setup`) requires a valid session cookie - obtained from `POST /api/auth/login`. State-changing requests + Every route requires a valid session cookie obtained from `POST + /api/auth/login` **except** this public set: `/api/auth/status`, + `/api/auth/login`, `/api/auth/setup`, `/api/auth/providers`, + `/api/auth/oauth/start/{slot}`, `/api/auth/oauth/callback/{slot}`, + `/health`, `/readiness`, and `/metrics`. State-changing requests (POST/PUT/DELETE) additionally require a CSRF header. See [Dashboard Authentication](/python/guides/dashboard/authentication) for the login flow and headless usage examples. @@ -51,6 +53,11 @@ Returns the current user, CSRF token, and expiry. `401` when no session. Body: `{"old_password": "...", "new_password": "..."}`. +OAuth/OIDC discovery and callback routes (`GET /api/auth/providers`, +`GET /api/auth/oauth/start/{slot}`, `GET /api/auth/oauth/callback/{slot}`) +are also public — see [SSO](/python/guides/dashboard/sso#api-surface) for +the full flow. + ## Stats ### `GET /api/stats` diff --git a/docs/content/docs/python/guides/dashboard/sso.mdx b/docs/content/docs/python/guides/dashboard/sso.mdx index 7771ac6e..ca9db382 100644 --- a/docs/content/docs/python/guides/dashboard/sso.mdx +++ b/docs/content/docs/python/guides/dashboard/sso.mdx @@ -14,6 +14,15 @@ own button on the login screen. OAuth is **off by default**. Setting any provider's env vars turns it on; password login remains enabled unless you opt out explicitly. +Quick glossary for the acronyms used on this page: + +- **OIDC** — OpenID Connect +- **IdP** — identity provider +- **PKCE** — Proof Key for Code Exchange +- **JWKS** — the provider's JSON Web Key Set +- **nonce** — a one-time value that ties a login response to your request +- **aud** / **iss** — the audience and issuer claims in the ID token + OAuth requires the `authlib` extra: @@ -71,7 +80,7 @@ S256, OIDC nonce, ID-token signature (via the provider's JWKS), https://taskito.your-company.com/api/auth/oauth/callback/google ``` - (For local development, `http://localhost:8000/api/auth/oauth/callback/google` + (For local development, `http://localhost:8080/api/auth/oauth/callback/google` works without HTTPS.) 2. **Set env vars** before starting the dashboard: diff --git a/docs/content/docs/python/guides/dashboard/task-overrides.mdx b/docs/content/docs/python/guides/dashboard/task-overrides.mdx index 6fb7c0c3..2f464599 100644 --- a/docs/content/docs/python/guides/dashboard/task-overrides.mdx +++ b/docs/content/docs/python/guides/dashboard/task-overrides.mdx @@ -18,7 +18,8 @@ Two surfaces: - **Queue overrides** — per-queue knobs: rate limit, concurrency, paused. -Plus a separate but related toggle for **middleware** on a per-task +Plus a separate but related toggle for +[**middleware**](/python/guides/extensibility/middleware) on a per-task basis (see [§ Middleware toggles](#middleware-toggles) below). ## Tasks page @@ -55,24 +56,26 @@ This is the most important thing to internalize: | Change | Takes effect | |---|---| -| Pausing a task | Next worker restart for the rate-limit/concurrency side effects, **but** the paused flag is plumbed through the live `paused_queues` mechanism so the scheduler stops dequeuing immediately for queue-level pauses | -| Pausing a queue | **Immediately** on running workers (writes to `paused_queues`) | -| Rate limit / max concurrent / retries / timeout / priority on a task | **Next worker restart** — the values are baked into `PyTaskConfig` at `run_worker` time and passed to the Rust scheduler | -| Rate limit / max concurrent on a queue | **Next worker restart** — same mechanism, merged into `queue_configs` JSON sent to Rust | -| Middleware on/off per task | **Next job** — middleware lookup happens at every task invocation | - -This split is intentional. Pause is a fast-path safety valve; -retry/rate-limit changes need scheduler buy-in and are deliberately -"restart to apply" so operators have a clear mental model of when the -new values take over. - - - Pulling rate-limit / retries / timeout into the Rust scheduler's - per-poll lookup would let those changes hot-reload too. The - ``PyTaskConfig`` → scheduler path would gain a cache-invalidation - counter (incremented on every override write) the poller checks - before each admission cycle. Until then, restart the worker to apply - changes to those knobs. +| Pausing a queue — via `queue.pause(name)`, the dashboard's Queues page, or the REST API's queue-override endpoint | **Immediately** on every running worker — the scheduler checks the live pause state on every poll cycle, no restart needed | +| Setting `paused=True` on a **queue override** through the Python API (`queue.set_queue_override(...)`) directly, without also calling `queue.pause()` | Recorded, but only applied the next time a worker starts. Call `queue.pause(name)` (or use the dashboard/REST API, which does this for you) for an immediate effect | +| Setting `paused=True` on a **task override** (`queue.set_task_override(...)`) | Recorded and shown in the dashboard/API as paused, but **not currently enforced by the scheduler** — the task keeps being dequeued and run normally. Treat it as an operator-visible annotation today, not a functional kill switch; pause the task's queue if you need it to actually stop | +| Rate limit / max concurrent / retries / timeout / priority on a task | **Next worker restart** — these values are read once when a worker starts and used for the rest of that worker's lifetime | +| Rate limit / max concurrent on a queue | **Next worker restart** — same mechanism | +| Middleware on/off per task | **Next job** — the middleware lookup runs on every task invocation | + +This split is intentional. Queue-level pause is a fast-path safety +valve wired directly into the scheduler's poll loop; retry/rate-limit +changes need scheduler buy-in and are deliberately "restart to apply" +so operators have a clear mental model of when the new values take +over. + + + Two gaps are tracked for a future release: wiring the per-task + `paused` override into the scheduler (today it's metadata only), and + pulling rate-limit / retries / timeout into the scheduler's per-poll + lookup so those changes hot-reload instead of requiring a restart. + Until then, restart the worker to apply changes to those knobs, and + use a queue-level pause if you need a task to actually stop running. ## Programmatic API @@ -91,12 +94,13 @@ queue.set_task_override( rate_limit="200/m", max_retries=10, ) -queue.set_task_override("myapp.tasks.send_email", paused=True) # immediate-ish +queue.set_task_override("myapp.tasks.send_email", paused=True) # recorded only — not enforced by the scheduler yet queue.clear_task_override("myapp.tasks.send_email") # Queues queue.set_queue_override("email", max_concurrent=5) -queue.set_queue_override("email", paused=True) # immediate +queue.set_queue_override("email", paused=True) # recorded — applies on next worker start +queue.pause("email") # this is what actually stops dequeuing immediately queue.clear_queue_override("email") # Discovery — what's registered + what's overridden @@ -185,15 +189,18 @@ queue.get_disabled_middleware_for("myapp.tasks.send_email") # ["demo.metrics"] ### Pause one task without redeploying A flaky third-party API is rate-limiting your `send_email` task and -you want to stop new sends while you investigate: +you want to stop new sends while you investigate. Marking the task +itself as paused only records the state for operators to see — it +doesn't stop the scheduler today. To actually stop new sends, pause +the queue that task runs on: ```python -queue.set_task_override("myapp.tasks.send_email", paused=True) -# ... or from the dashboard: Tasks → send_email → Edit → check "Pause this task" +queue.pause("email") +# ... or from the dashboard: Infrastructure → Queues → email → Pause ``` -Existing in-flight jobs finish normally; nothing new dequeues until -you clear the override. +Existing in-flight jobs finish normally; nothing new dequeues from +that queue until you call `queue.resume("email")`. ### Lower a rate limit during an incident From 7c4c3c97e8544b65943ad1ffcb0b2e69bbb411c5 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:13:00 +0530 Subject: [PATCH 10/57] docs: fix extensibility (serializer default, signals) --- .../guides/extensibility/events-webhooks.mdx | 7 +- .../guides/extensibility/middleware.mdx | 8 +- .../guides/extensibility/serializers.mdx | 83 +++++++++++++------ 3 files changed, 69 insertions(+), 29 deletions(-) diff --git a/docs/content/docs/python/guides/extensibility/events-webhooks.mdx b/docs/content/docs/python/guides/extensibility/events-webhooks.mdx index 73fec418..88d81461 100644 --- a/docs/content/docs/python/guides/extensibility/events-webhooks.mdx +++ b/docs/content/docs/python/guides/extensibility/events-webhooks.mdx @@ -29,7 +29,10 @@ This guide covers both, starting with the events that drive them. ## Event types -The `EventType` enum defines all available lifecycle events: +The `EventType` enum defines all available lifecycle events. Each +member's `.value` is the dotted, lowercase wire string used in webhook +payloads, delivery logs, and the REST API — e.g. +`EventType.JOB_FAILED.value == "job.failed"`: | Event | Fires when | Payload fields | |---|---|---| @@ -135,7 +138,7 @@ queue.remove_webhook(sub.id) | Parameter | Type | Default | Description | |---|---|---|---| -| `url` | `str` | — | http/https URL. SSRF-guarded — see below | +| `url` | `str` | — | http/https URL. SSRF-guarded (server-side request forgery) — see below | | `events` | `list[EventType] \| None` | `None` | Event types to subscribe to. `None` means all events | | `task_filter` | `list[str] \| None` | `None` | Restrict to specific task names. `None` means all tasks | | `headers` | `dict[str, str] \| None` | `None` | Extra HTTP headers (e.g. `Authorization`) | diff --git a/docs/content/docs/python/guides/extensibility/middleware.mdx b/docs/content/docs/python/guides/extensibility/middleware.mdx index ada55f11..bfebc9a1 100644 --- a/docs/content/docs/python/guides/extensibility/middleware.mdx +++ b/docs/content/docs/python/guides/extensibility/middleware.mdx @@ -37,7 +37,7 @@ class LoggingMiddleware(TaskMiddleware): | `after(ctx, result, error)` | After task execution (success or failure) | | `on_retry(ctx, error, retry_count)` | A job fails and will be retried | | `on_enqueue(task_name, args, kwargs, options)` | A job is about to be enqueued | -| `on_dead_letter(ctx, error)` | A job exhausts all retries and moves to the DLQ | +| `on_dead_letter(ctx, error)` | A job exhausts all retries and moves to the DLQ (dead letter queue — a holding area for jobs that exhausted their retries) | | `on_timeout(ctx)` | A job hits its timeout limit | | `on_cancel(ctx)` | A job is cancelled during execution | @@ -190,6 +190,10 @@ Middleware exceptions never prevent task execution or result handling. ## Middleware vs hooks +Coming from Celery signals? Use these instead of connecting to `task_prerun` / +`task_postrun` / `task_failure`: queue-level **hooks** for global behavior, or +per-task **middleware** for scoped behavior. + taskito has two systems for running code around tasks: | | Hooks (`@queue.on_failure`, etc.) | Middleware (`TaskMiddleware`) | @@ -222,4 +226,4 @@ queue = Queue(middleware=[ ]) ``` -See the [OpenTelemetry guide](/python/guides/integrations) for setup details. +See the [OpenTelemetry guide](/python/guides/integrations/otel) for setup details. diff --git a/docs/content/docs/python/guides/extensibility/serializers.mdx b/docs/content/docs/python/guides/extensibility/serializers.mdx index 3e8fef7d..ce037d77 100644 --- a/docs/content/docs/python/guides/extensibility/serializers.mdx +++ b/docs/content/docs/python/guides/extensibility/serializers.mdx @@ -6,20 +6,37 @@ description: "CloudpickleSerializer, JsonSerializer, MsgPackSerializer, Encrypte import { Callout } from "fumadocs-ui/components/callout"; taskito uses a pluggable serializer for task arguments and results. By -default, it uses `CloudpickleSerializer`, but you can switch to -`JsonSerializer` or provide your own. +default, it uses `SmartSerializer`, but you can switch to +`JsonSerializer`, `CloudpickleSerializer` directly, or provide your own. ## Built-in serializers -### CloudpickleSerializer (default) +### SmartSerializer (default) -Handles lambdas, closures, and complex Python objects. This is the default -— no configuration needed. +Encodes plain data (dicts, lists, strings, numbers, booleans, `None`, +tuples) via MessagePack — fast and compact — and transparently falls +back to `CloudpickleSerializer` for anything MessagePack can't encode +(lambdas, closures, custom class instances). A one-byte tag on each +payload records which codec produced it, so `loads()` always picks the +right path. This is the default — no configuration needed. ```python from taskito import Queue -queue = Queue() # uses CloudpickleSerializer +queue = Queue() # uses SmartSerializer +``` + +### CloudpickleSerializer + +Handles lambdas, closures, and complex Python objects unconditionally — +every payload goes through cloudpickle, skipping `SmartSerializer`'s +MessagePack fast path. Use it directly if you want that predictability, +or as the inner serializer for `EncryptedSerializer` / `SignedSerializer`. + +```python +from taskito import Queue, CloudpickleSerializer + +queue = Queue(serializer=CloudpickleSerializer()) ``` ### JsonSerializer @@ -36,10 +53,13 @@ queue = Queue(serializer=JsonSerializer()) ### MsgPackSerializer MessagePack serialization: faster than cloudpickle, produces smaller -payloads, and is cross-language compatible. Requires the `msgpack` package. +payloads, and is cross-language compatible. `msgpack` ships as a core +taskito dependency, so no extra install is required — `taskito[msgpack]` +exists for discoverability if you want to spell it out in your own +requirements file: ```bash -pip install msgpack +pip install taskito[msgpack] ``` ```python @@ -59,10 +79,10 @@ queue = Queue(serializer=MsgPackSerializer()) AES-256-GCM encryption for task arguments and results. Payloads stored in the database are opaque ciphertext — only the key holder can read them. -Requires the `cryptography` package. +Requires the `encryption` extra: ```bash -pip install cryptography +pip install taskito[encryption] ``` ```python @@ -123,28 +143,41 @@ queue = Queue(serializer=SignedSerializer(inner, sign_key)) ## When to use each -| | CloudpickleSerializer | JsonSerializer | MsgPackSerializer | EncryptedSerializer | -|---|---|---|---|---| -| **Complex objects** | Yes | No | No | Depends on inner serializer | -| **Debugging** | Binary payloads (opaque) | Human-readable JSON | Binary (opaque) | Ciphertext (opaque) | -| **Cross-language** | Python only | Any language | Any language | Python only (by default) | -| **Performance** | Good | Good for simple types | Best | Adds encryption overhead | -| **Security** | None | None | None | AES-256-GCM | -| **Extra dependency** | No | No | `msgpack` | `cryptography` | -| **Default** | Yes | No | No | No | - -**Rule of thumb**: use `CloudpickleSerializer` (default) unless you have a -specific reason to switch. Use `EncryptedSerializer` when tasks carry +| | SmartSerializer | CloudpickleSerializer | JsonSerializer | MsgPackSerializer | EncryptedSerializer | +|---|---|---|---|---|---| +| **Complex objects** | Yes (cloudpickle fallback) | Yes | No | No | Depends on inner serializer | +| **Debugging** | Binary payloads (opaque) | Binary payloads (opaque) | Human-readable JSON | Binary (opaque) | Ciphertext (opaque) | +| **Cross-language** | Python only | Python only | Any language | Any language | Python only (by default) | +| **Performance** | Best for plain data | Good | Good for simple types | Best | Adds encryption overhead | +| **Security** | None | None | None | None | AES-256-GCM | +| **Extra dependency** | No | No | No | No | `cryptography` (`taskito[encryption]`) | +| **Default** | Yes | No | No | No | No | + +**Rule of thumb**: use the default `SmartSerializer` unless you have a +specific reason to switch — it gets you compact MessagePack payloads for +plain data with an automatic cloudpickle fallback for anything +MessagePack can't encode. Use `EncryptedSerializer` when tasks carry sensitive data that must not be readable in the database. +**Coming from Celery?** Celery defaults to JSON (`task_serializer="json"`); +taskito defaults to `SmartSerializer`, which is pickle-based (via +cloudpickle) for anything MessagePack can't represent. It handles closures +and arbitrary objects that JSON can't, but that fallback path is +Python-only and **executes code on load** — use it only with trusted +payloads. Switch to `JsonSerializer` for cross-language or untrusted data, +or wrap any serializer in `SignedSerializer` to authenticate payloads +(HMAC) before they are deserialized. + ## Custom serializers -Implement the `Serializer` protocol with two methods: +Implement the `Serializer` protocol with two methods. `MsgPackSerializer` +already ships built-in (see above) — this example is a stand-in for any +serializer not covered by the built-ins: ```python from taskito import Serializer -class MsgpackSerializer: +class MyMsgpackSerializer: def dumps(self, obj) -> bytes: import msgpack return msgpack.packb(obj) @@ -153,7 +186,7 @@ class MsgpackSerializer: import msgpack return msgpack.unpackb(data, raw=False) -queue = Queue(serializer=MsgpackSerializer()) +queue = Queue(serializer=MyMsgpackSerializer()) ``` The protocol requires: From 57fd274115e625e1d55be0560d16441a1b34d1a4 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:13:00 +0530 Subject: [PATCH 11/57] docs: fix integration guides (extras, metrics) --- .../python/guides/integrations/django.mdx | 23 +++++++++++-- .../python/guides/integrations/fastapi.mdx | 4 +-- .../docs/python/guides/integrations/flask.mdx | 4 +++ .../docs/python/guides/integrations/index.mdx | 9 +++++- .../docs/python/guides/integrations/otel.mdx | 7 ++-- .../python/guides/integrations/prometheus.mdx | 32 ++++++++++++------- .../python/guides/integrations/sentry.mdx | 1 - 7 files changed, 59 insertions(+), 21 deletions(-) diff --git a/docs/content/docs/python/guides/integrations/django.mdx b/docs/content/docs/python/guides/integrations/django.mdx index 1b31753b..c1c8c529 100644 --- a/docs/content/docs/python/guides/integrations/django.mdx +++ b/docs/content/docs/python/guides/integrations/django.mdx @@ -84,9 +84,9 @@ TASKITO_DASHBOARD_PORT = 9000 ## Queue configuration -Create a `taskito` queue instance in your Django project. The -`get_queue()` function in `taskito.contrib.django.settings` is used to -retrieve the queue instance. +Create a `taskito` queue instance in your Django project — this is the +object your tasks are decorated on and the one the worker CLI runs +against (`--app myproject.tasks:queue`). ```python # myproject/tasks.py @@ -101,6 +101,23 @@ def send_welcome_email(user_id: int): user.email_user("Welcome!", "Thanks for signing up.") ``` +Task modules are located via Django's standard app autodiscovery: +`taskito.contrib.django`'s `AppConfig.ready()` imports a `tasks` module +(the name is configurable via `TASKITO_AUTODISCOVER_MODULE`) from every +installed app, which registers any `@queue.task()`-decorated functions +it imports. + + + `taskito.contrib.django.settings.get_queue()` — used internally by the + admin views and the `manage.py taskito_*` commands — builds its own + `Queue` from the `TASKITO_*` Django settings. It is **not** the same + Python object as the `queue` you create in `myproject/tasks.py`. Point + both at the same `TASKITO_DB_PATH` / `TASKITO_BACKEND` / `TASKITO_DB_URL` + so they read the same underlying storage — otherwise the admin views + show an empty or different queue than the one your worker is + processing. + + Import Django models inside the task function body to avoid app registry issues during startup. diff --git a/docs/content/docs/python/guides/integrations/fastapi.mdx b/docs/content/docs/python/guides/integrations/fastapi.mdx index bc729363..2c0269a0 100644 --- a/docs/content/docs/python/guides/integrations/fastapi.mdx +++ b/docs/content/docs/python/guides/integrations/fastapi.mdx @@ -61,7 +61,7 @@ uvicorn myapp:app --reload how results are serialized, and page sizes: ```python -from fastapi import Depends, HTTPException +from fastapi import Depends, Header, HTTPException from taskito.contrib.fastapi import TaskitoRouter def require_api_key(x_api_key: str = Header(...)): @@ -77,7 +77,7 @@ app.include_router( result_timeout=5.0, default_page_size=25, max_page_size=200, - result_serializer=lambda v: v if isinstance(v, (str, int, float, bool, None)) else str(v), + result_serializer=lambda v: v if isinstance(v, (str, int, float, bool, type(None))) else str(v), ), prefix="/tasks", ) diff --git a/docs/content/docs/python/guides/integrations/flask.mdx b/docs/content/docs/python/guides/integrations/flask.mdx index 1f6afe16..af264131 100644 --- a/docs/content/docs/python/guides/integrations/flask.mdx +++ b/docs/content/docs/python/guides/integrations/flask.mdx @@ -59,6 +59,10 @@ All configuration is read from `app.config`: | `TASKITO_RESULT_TTL` | `None` | Auto-purge completed jobs after N seconds | | `TASKITO_DRAIN_TIMEOUT` | `30` | Seconds to wait for running tasks on shutdown | +Using `TASKITO_BACKEND = "postgres"` requires the `taskito[postgres]` +extra (`pip install taskito[postgres]`) — see the +[PostgreSQL guide](/python/guides/operations/postgres). + ## Extension options The `Taskito` constructor accepts a `cli_group` parameter to rename the CLI diff --git a/docs/content/docs/python/guides/integrations/index.mdx b/docs/content/docs/python/guides/integrations/index.mdx index f5d61d84..94ba9e38 100644 --- a/docs/content/docs/python/guides/integrations/index.mdx +++ b/docs/content/docs/python/guides/integrations/index.mdx @@ -17,10 +17,17 @@ tools. Install only what you need. | **Prometheus** | `pip install taskito[prometheus]` | `PrometheusMiddleware`, queue depth gauges, `/metrics` server | | **Sentry** | `pip install taskito[sentry]` | `SentryMiddleware` with auto error capture and task tags | | **Encryption** | `pip install taskito[encryption]` | `EncryptedSerializer` for at-rest payload encryption | -| **MsgPack** | `pip install taskito[msgpack]` | `MsgpackSerializer` for compact binary serialization | +| **MsgPack** | `pip install taskito[msgpack]` | `MsgPackSerializer` for compact binary serialization | | **Postgres** | `pip install taskito[postgres]` | Multi-machine workers via PostgreSQL backend | | **Redis** | `pip install taskito[redis]` | Redis storage backend | + + Postgres and Redis here are **shared storage backends** — the queue itself, + used to run workers across multiple machines. They are not message brokers + (a separate server such as Redis or RabbitMQ that holds pending jobs): + taskito never needs a separate broker regardless of backend. + + ## Framework integrations - **[Flask](/python/guides/integrations/flask)** — full Flask extension with app config, factory pattern, and CLI commands diff --git a/docs/content/docs/python/guides/integrations/otel.mdx b/docs/content/docs/python/guides/integrations/otel.mdx index e3c59509..08aa63ed 100644 --- a/docs/content/docs/python/guides/integrations/otel.mdx +++ b/docs/content/docs/python/guides/integrations/otel.mdx @@ -16,7 +16,10 @@ Install with the `otel` extra: pip install taskito[otel] ``` -This installs `opentelemetry-api` as a dependency. +This installs `opentelemetry-api` and `opentelemetry-sdk` as +dependencies. It does **not** install an exporter — for the OTLP example +below, also run `pip install opentelemetry-exporter-otlp` (or whichever +exporter package matches your backend). ## Setup @@ -31,7 +34,7 @@ queue = Queue(middleware=[OpenTelemetryMiddleware()]) ## What gets traced -Each task execution produces a span with: +Each task execution produces a span (a timed unit of work in a distributed trace) with: - **Span name**: `taskito.execute.` (customizable) - **Attributes**: diff --git a/docs/content/docs/python/guides/integrations/prometheus.mdx b/docs/content/docs/python/guides/integrations/prometheus.mdx index f84dccc0..62f89eb2 100644 --- a/docs/content/docs/python/guides/integrations/prometheus.mdx +++ b/docs/content/docs/python/guides/integrations/prometheus.mdx @@ -103,6 +103,23 @@ start_metrics_server(port=9090) This uses `prometheus_client.start_http_server` under the hood. +### This is a different `/metrics` than the dashboard's + +`start_metrics_server` opens its own dedicated HTTP server — it is not +the same endpoint as the dashboard's `GET /metrics` (see +[REST API](/python/guides/dashboard/rest-api)). Both ultimately call +`prometheus_client.generate_latest()`, but each only serves metrics +registered in *its own process*: + +- Run `start_metrics_server` in the same process as your + `PrometheusMiddleware` / `PrometheusStatsCollector` (typically the + worker process) — this is the endpoint you'll usually scrape. +- The dashboard's `GET /metrics` only shows something useful if the + dashboard process itself registered metrics (e.g. it also runs + `PrometheusMiddleware`). If your worker and dashboard run as separate + processes — the common deployment — scrape the worker's + `start_metrics_server` instead. + ## Full example ```python @@ -163,19 +180,10 @@ queue = Queue( See the [Middleware guide](/python/guides/extensibility/middleware) for more on combining middleware. -## Example: alert on high DLQ size - -```python -from taskito.contrib.prometheus import PrometheusMiddleware, PrometheusStatsCollector, start_metrics_server - -queue = Queue(db_path="myapp.db", middleware=[PrometheusMiddleware()]) - -# Start metrics and collector -start_metrics_server(port=9090) -PrometheusStatsCollector(queue, interval=10).start() -``` +## Alerting on high DLQ size -Prometheus alerting rule: +Using the same setup as the [full example](#full-example) above, a +couple of alerting rules worth adding: ```yaml groups: diff --git a/docs/content/docs/python/guides/integrations/sentry.mdx b/docs/content/docs/python/guides/integrations/sentry.mdx index 0a01a86c..d53262c6 100644 --- a/docs/content/docs/python/guides/integrations/sentry.mdx +++ b/docs/content/docs/python/guides/integrations/sentry.mdx @@ -133,4 +133,3 @@ When `process_payment` fails: 1. The error appears in Sentry with tags `taskito.task_name=myapp.tasks.process_payment`, `taskito.job_id=...`, `taskito.queue=default` 2. If the task retries, each retry is recorded as a breadcrumb 3. The final failure (after all retries) includes the full breadcrumb trail -``` From 690c793d52232cdbe996147fecf050e37c983d4e Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:13:00 +0530 Subject: [PATCH 12/57] docs: fix observability guides --- .../docs/python/guides/observability/logging.mdx | 5 +++++ .../docs/python/guides/observability/monitoring.mdx | 13 ++++++++----- .../docs/python/guides/observability/notes.mdx | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/content/docs/python/guides/observability/logging.mdx b/docs/content/docs/python/guides/observability/logging.mdx index 8faa4e9d..55a8bd3a 100644 --- a/docs/content/docs/python/guides/observability/logging.mdx +++ b/docs/content/docs/python/guides/observability/logging.mdx @@ -43,6 +43,11 @@ def process_order(order_id: int): | `level` | `LogLevel` | `LogLevel.INFO` | One of `LogLevel.DEBUG`, `LogLevel.INFO`, `LogLevel.WARNING`, `LogLevel.ERROR`, `LogLevel.CRITICAL`, `LogLevel.RESULT` | | `extra` | `dict \| None` | `None` | Structured data to attach as JSON | +`LogLevel.RESULT` is written internally by `current_job.publish()` to +stream partial results to `job.stream()` consumers — you generally won't +pass it to `log()` yourself, but it's useful for filtering streamed +output out of regular log queries. + ## Querying logs ### Per-job logs diff --git a/docs/content/docs/python/guides/observability/monitoring.mdx b/docs/content/docs/python/guides/observability/monitoring.mdx index 08b0d70c..b0f08ca5 100644 --- a/docs/content/docs/python/guides/observability/monitoring.mdx +++ b/docs/content/docs/python/guides/observability/monitoring.mdx @@ -41,7 +41,7 @@ taskito queue statistics total 468 ``` -### Live dashboard +### Live CLI stats ```bash taskito info --app myapp:queue --watch @@ -129,7 +129,7 @@ events (`JOB_ENQUEUED`, `JOB_COMPLETED`, `JOB_FAILED`, `JOB_RETRYING`, `JOB_DEAD`, `JOB_CANCELLED`). Events can also be delivered as signed HTTP webhooks to external systems. -[Events & Webhooks guide →](/python/guides/extensibility) +[Events & Webhooks guide →](/python/guides/extensibility/events-webhooks) ## Prometheus metrics @@ -140,11 +140,14 @@ counters, histograms, and gauges for task execution: pip install taskito[prometheus] ``` -[Prometheus integration →](/python/guides/integrations) +[Prometheus integration →](/python/guides/integrations/prometheus) ## Hooks -Run code before/after every task, or on success/failure. +Run code before/after every task, or on success/failure. For a +comparison against `TaskMiddleware` — which covers retry, dead-letter, +timeout, and enqueue hooks these decorators don't — see +[Middleware vs hooks](/python/guides/extensibility/middleware#middleware-vs-hooks). ### `@queue.before_task` @@ -243,7 +246,7 @@ taskito_queue_depth{queue="default"} **Job processing rate** (rate): ```text -rate(taskito_jobs_completed_total[5m]) +rate(taskito_jobs_total{status="completed"}[5m]) ``` **Job duration p99** (histogram): diff --git a/docs/content/docs/python/guides/observability/notes.mdx b/docs/content/docs/python/guides/observability/notes.mdx index 70fd35a2..26d075e9 100644 --- a/docs/content/docs/python/guides/observability/notes.mdx +++ b/docs/content/docs/python/guides/observability/notes.mdx @@ -66,7 +66,7 @@ attached). The raw stored JSON string is also available on the underlying Validation runs at the Python boundary; the Rust storage layer receives an already-encoded JSON string and stores it verbatim. The exact contract is -defined in [`taskito.notes`](https://github.com/your-org/taskito/blob/master/py_src/taskito/notes.py): +defined in [`taskito.notes`](https://github.com/ByteVeda/taskito/blob/master/sdks/python/taskito/notes.py): | Constraint | Default | Constant | |---|---|---| From b03357c3b7b5b0816dea3331f9c3d594a769f8b4 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:13:14 +0530 Subject: [PATCH 13/57] docs: fix API reference (CLI, task params, status) --- .../docs/python/api-reference/batching.mdx | 4 +- .../docs/python/api-reference/canvas.mdx | 14 ++++ .../content/docs/python/api-reference/cli.mdx | 83 ++++++++++++++++++- .../docs/python/api-reference/index.mdx | 2 +- .../docs/python/api-reference/overview.mdx | 2 +- .../docs/python/api-reference/queue/index.mdx | 19 ++++- .../docs/python/api-reference/queue/jobs.mdx | 2 +- .../docs/python/api-reference/result.mdx | 5 ++ .../docs/python/api-reference/saga.mdx | 8 +- .../docs/python/api-reference/task.mdx | 8 ++ 10 files changed, 135 insertions(+), 12 deletions(-) diff --git a/docs/content/docs/python/api-reference/batching.mdx b/docs/content/docs/python/api-reference/batching.mdx index 107af927..a6c81925 100644 --- a/docs/content/docs/python/api-reference/batching.mdx +++ b/docs/content/docs/python/api-reference/batching.mdx @@ -9,7 +9,7 @@ Producer-side task batching primitives. Import paths: from taskito.batching import BatchAccumulator, BatchConfig, BatchedJobResult ``` -For the conceptual overview and trade-offs, see [Task batching](/docs/guides/core/batching). +For the conceptual overview and trade-offs, see [Task batching](/python/guides/core/batching). --- @@ -89,7 +89,7 @@ assert isinstance(result, BatchedJobResult) assert result.id is None ``` -If you need per-item results, batching is the wrong primitive — use [`canvas.chunks()`](/docs/api-reference/canvas) instead, which creates one job per chunk and supports per-chunk retry. +If you need per-item results, batching is the wrong primitive — use [`canvas.chunks()`](/python/api-reference/canvas) instead, which creates one job per chunk and supports per-chunk retry. --- diff --git a/docs/content/docs/python/api-reference/canvas.mdx b/docs/content/docs/python/api-reference/canvas.mdx index 1297d88c..d77c9610 100644 --- a/docs/content/docs/python/api-reference/canvas.mdx +++ b/docs/content/docs/python/api-reference/canvas.mdx @@ -9,6 +9,20 @@ Canvas primitives for composing task workflows. Import directly from the package from taskito import chain, group, chord, chunks, starmap ``` + + On `Signature`, `chain`, `group`, and `chord`, `apply_async()` is an + **`async def` coroutine** — you must `await` it. This is the opposite of + [`TaskWrapper.apply_async()`](/python/api-reference/task) (the method on a + plain `@queue.task()`-decorated function), which is synchronous and returns + a [`JobResult`](/python/api-reference/result) directly. + + +Canvas primitives are **in-thread orchestrators**: `chain.apply()` / `chord.apply()` +enqueue each step and block on its result in the calling thread, driving the +whole pipeline client-side. Unlike Celery, there is no broker-linked callback +chain running server-side — the state lives only in the process that called +`apply()`. + --- ## Signature diff --git a/docs/content/docs/python/api-reference/cli.mdx b/docs/content/docs/python/api-reference/cli.mdx index 9b146700..039eb412 100644 --- a/docs/content/docs/python/api-reference/cli.mdx +++ b/docs/content/docs/python/api-reference/cli.mdx @@ -1,6 +1,6 @@ --- title: CLI Reference -description: "The taskito command-line interface — worker, info, scaler, autoscale." +description: "The taskito command-line interface — worker, dashboard, info, pause, resume, scaler, autoscale, resources, reload." --- taskito provides a command-line interface for running workers and inspecting @@ -51,6 +51,51 @@ taskito worker --app myapp.tasks:queue --pool prefork The worker blocks until interrupted with `Ctrl+C`. It performs a graceful shutdown — in-flight tasks are allowed to complete before the process exits. +### `taskito dashboard` + +Start the web dashboard (SPA + REST API) for the queue. + +```bash +taskito dashboard --app [--host ] [--port ] [--insecure-cookies] +``` + +| Flag | Default | Description | +|---|---|---| +| `--app` | — | Python path to the `Queue` instance | +| `--host` | `127.0.0.1` | Bind address | +| `--port` | `8080` | Bind port | +| `--insecure-cookies` | `False` | Drop the `Secure` flag on session cookies (local HTTP dev only) | + +**Example:** + +```bash +taskito dashboard --app myapp.tasks:queue --port 8080 +``` + +See the [Dashboard guide](/python/guides/dashboard) for authentication and deployment. + +### `taskito pause` / `taskito resume` + +Pause or resume a named queue. A paused queue stops dequeuing new jobs; +in-flight jobs still run to completion. + +```bash +taskito pause --app +taskito resume --app +``` + +| Flag / Arg | Required | Description | +|---|---|---| +| `--app` | Yes | Python path to the `Queue` instance | +| `queue_name` | Yes | Name of the queue to pause or resume | + +**Example:** + +```bash +taskito pause --app myapp.tasks:queue emails +taskito resume --app myapp.tasks:queue emails +``` + ### `taskito info` Display queue statistics. @@ -197,6 +242,42 @@ taskito autoscale --app myapp.tasks:queue --min-workers 2 --max-workers 20 See the [Autoscaler guide](/python/guides/operations/autoscaler) for deployment patterns and tuning advice. +### `taskito resources` + +Show the status of every registered resource — scope, health, init time, +recreation count, and dependencies. + +```bash +taskito resources --app +``` + +| Flag | Required | Description | +|---|---|---| +| `--app` | Yes | Python path to the `Queue` instance | + +See the [Resource System](/python/guides/resources) guide for background. + +### `taskito reload` + +Send `SIGHUP` to a running worker process to hot-reload resources without +restarting the worker. + +```bash +taskito reload --pid [--resource ] +``` + +| Flag | Required | Description | +|---|---|---| +| `--pid` | Yes | PID of the running worker process | +| `--resource` | No | Reload only this resource (default: all reloadable resources) | + +**Example:** + +```bash +taskito reload --pid 12345 +taskito reload --pid 12345 --resource db_pool +``` + ## Error messages | Error | Cause | diff --git a/docs/content/docs/python/api-reference/index.mdx b/docs/content/docs/python/api-reference/index.mdx index db9be548..5b006c2b 100644 --- a/docs/content/docs/python/api-reference/index.mdx +++ b/docs/content/docs/python/api-reference/index.mdx @@ -72,6 +72,6 @@ signature, parameters, return values, and a short example. icon={} title="CLI" href="/python/api-reference/cli" - description="taskito worker, dashboard, info — every flag and subcommand." + description="taskito worker, dashboard, info, pause, resume, scaler, autoscale, resources, reload." /> diff --git a/docs/content/docs/python/api-reference/overview.mdx b/docs/content/docs/python/api-reference/overview.mdx index a3283418..72ccc395 100644 --- a/docs/content/docs/python/api-reference/overview.mdx +++ b/docs/content/docs/python/api-reference/overview.mdx @@ -14,4 +14,4 @@ Complete Python API reference for all public classes and methods. | [Canvas](/python/api-reference/canvas) | Workflow primitives — `Signature`, `chain`, `group`, `chord` | | [Workflows](/python/api-reference/workflows) | DAG workflow builder, `WorkflowRun`, gates, sub-workflows | | [Testing](/python/api-reference/testing) | Test mode, `TestResult`, `MockResource` for unit testing tasks | -| [CLI](/python/api-reference/cli) | `taskito` command-line interface — `worker`, `info`, `scaler` | +| [CLI](/python/api-reference/cli) | `taskito` command-line interface — `worker`, `dashboard`, `info`, `pause`, `resume`, `scaler`, `autoscale`, `resources`, `reload` | diff --git a/docs/content/docs/python/api-reference/queue/index.mdx b/docs/content/docs/python/api-reference/queue/index.mdx index 01b0cda3..488c1a0f 100644 --- a/docs/content/docs/python/api-reference/queue/index.mdx +++ b/docs/content/docs/python/api-reference/queue/index.mdx @@ -85,10 +85,14 @@ Queue( queue: str = "default", circuit_breaker: dict | None = None, middleware: list[TaskMiddleware] | None = None, - expires: float | None = None, inject: list[str] | None = None, serializer: Serializer | None = None, max_concurrent: int | None = None, + idempotent: bool = False, + compensates: TaskWrapper | str | None = None, + batch: bool | dict | None = None, + predicate: Predicate | Callable | None = None, + on_false: str = "defer", ) -> TaskWrapper ``` @@ -108,10 +112,21 @@ Register a function as a background task. Returns a [`TaskWrapper`](/python/api- | `queue` | `str` | `"default"` | Named queue to submit to. | | `circuit_breaker` | `dict \| None` | `None` | Circuit breaker config: `{"threshold": 5, "window": 60, "cooldown": 120}`. | | `middleware` | `list[TaskMiddleware] \| None` | `None` | Per-task middleware, applied in addition to queue-level middleware. | -| `expires` | `float \| None` | `None` | Seconds until the job expires if not started. | | `inject` | `list[str] \| None` | `None` | Resource names to inject as keyword arguments. See [Resource System](/python/guides/resources). | | `serializer` | `Serializer \| None` | `None` | Per-task serializer override. Falls back to queue-level serializer. | | `max_concurrent` | `int \| None` | `None` | Max concurrent running instances. `None` = no limit. | +| `idempotent` | `bool` | `False` | Auto-derive a dedup key from `sha256(task_name\|payload)` so duplicate `.delay()`/`.apply_async()` calls while a job is pending or running return the same job ID. | +| `compensates` | `TaskWrapper \| str \| None` | `None` | Saga compensator for this task, enqueued in reverse order if a later workflow step fails. See [Sagas](/python/guides/workflows/sagas). | +| `batch` | `bool \| dict \| None` | `None` | Enable producer-side batching (`True` for defaults, or a dict with `max_size`/`max_wait_ms`). See [Batching](/python/guides/core/batching). | +| `predicate` | `Predicate \| Callable \| None` | `None` | Gate execution on a runtime condition, checked at enqueue time and worker-dispatch time. See [Predicates](/python/guides/core/predicates). | +| `on_false` | `str` | `"defer"` | What to do when `predicate` returns `False` — `"defer"` (re-schedule) or `"cancel"` (terminally skip). | + + + `predicate_extras` and `default_defer_seconds` fine-tune `predicate`/`on_false` — + see [Predicates](/python/guides/core/predicates). Per-call idempotency overrides + (`idempotency_key`, per-call `idempotent=`) are covered in + [Idempotency](/python/guides/reliability/idempotency). + ### `@queue.periodic()` diff --git a/docs/content/docs/python/api-reference/queue/jobs.mdx b/docs/content/docs/python/api-reference/queue/jobs.mdx index cbd22d5b..a4b560da 100644 --- a/docs/content/docs/python/api-reference/queue/jobs.mdx +++ b/docs/content/docs/python/api-reference/queue/jobs.mdx @@ -33,7 +33,7 @@ namespace — pass `namespace=None` to see all namespaces. | Parameter | Type | Default | Description | |---|---|---|---| -| `status` | `str \| None` | `None` | Filter by status: `pending`, `running`, `completed`, `failed`, `dead`, `cancelled` | +| `status` | `str \| None` | `None` | Filter by status: `pending`, `running`, `complete`/`completed` (both accepted), `failed`, `dead`, `cancelled` | | `queue` | `str \| None` | `None` | Filter by queue name | | `task_name` | `str \| None` | `None` | Filter by task name | | `limit` | `int` | `50` | Maximum results to return | diff --git a/docs/content/docs/python/api-reference/result.mdx b/docs/content/docs/python/api-reference/result.mdx index 83611fa9..51b288fa 100644 --- a/docs/content/docs/python/api-reference/result.mdx +++ b/docs/content/docs/python/api-reference/result.mdx @@ -38,6 +38,11 @@ print(job.status) # "pending" print(job.status) # "complete" ``` +> **A note on naming.** The per-job status is `"complete"` (singular). The +> aggregate count bucket from [`queue.stats()`](/python/api-reference/queue/queues) +> is `"completed"` (past tense). Status **filters** such as `list_jobs(status=…)` +> accept either spelling. + ### `job.progress` ```python diff --git a/docs/content/docs/python/api-reference/saga.mdx b/docs/content/docs/python/api-reference/saga.mdx index 1f01674c..1d8b16d9 100644 --- a/docs/content/docs/python/api-reference/saga.mdx +++ b/docs/content/docs/python/api-reference/saga.mdx @@ -14,7 +14,7 @@ from taskito.workflows.saga import ( ) ``` -For the conceptual overview, see [Sagas](/docs/guides/workflows/sagas). +For the conceptual overview, see [Sagas](/python/guides/workflows/sagas). --- @@ -104,7 +104,7 @@ Frozen dataclass passed in via the contextvar. Fields: | `Compensated` | ✓ | All compensators succeeded | | `CompensationFailed` | ✓ | At least one compensator failed; subsequent waves were not dispatched | -`WorkflowState.is_terminal()` returns `True` for `Compensated` and `CompensationFailed`. `WorkflowRun.wait()` unblocks on any of the four terminal states (`Completed`, `Failed`, `Cancelled`, `Compensated`, `CompensationFailed`). +`WorkflowState.is_terminal()` returns `True` for `Compensated` and `CompensationFailed`. `WorkflowRun.wait()` unblocks on any of the five terminal states (`Completed`, `Failed`, `Cancelled`, `Compensated`, `CompensationFailed`). --- @@ -122,7 +122,7 @@ Only `Completed` and `CacheHit` nodes are eligible for compensation (`is_compens ## Events -Subscribe to these on the queue's event bus via `queue.on(event_type, callback)`: +Subscribe to these on the queue's event bus via `queue.on_event(event_type, callback)`: | Event | Payload | |------------------------------------|---------------------------------------------------------------------------------| @@ -156,4 +156,4 @@ When a parent workflow's saga reaches a sub-workflow node whose child run carrie 3. Child saga starts (depth = parent depth + 1). 4. When the child saga finalises (`Compensated` or `CompensationFailed`), the parent's node is marked accordingly and the parent's wave advances. -If the parent step has its own explicit `compensates=…`, that wins — no propagation happens. See [Sagas](/docs/guides/workflows/sagas) for a worked example. +If the parent step has its own explicit `compensates=…`, that wins — no propagation happens. See [Sagas](/python/guides/workflows/sagas) for a worked example. diff --git a/docs/content/docs/python/api-reference/task.mdx b/docs/content/docs/python/api-reference/task.mdx index 724226f9..a3b2046a 100644 --- a/docs/content/docs/python/api-reference/task.mdx +++ b/docs/content/docs/python/api-reference/task.mdx @@ -57,6 +57,14 @@ task.apply_async( Enqueue with full control over submission options. Any parameter not provided falls back to the decorator's default. + + On a `TaskWrapper` (a `@queue.task()`-decorated function), `apply_async()` is + a **plain synchronous call** that returns a [`JobResult`](/python/api-reference/result) + directly — do not `await` it. This is different from `Signature.apply_async()` / + `chain.apply_async()` / `group.apply_async()` / `chord.apply_async()` in + [Canvas](/python/api-reference/canvas), which **are** coroutines you must `await`. + + | Parameter | Type | Default | Description | |---|---|---|---| | `args` | `tuple` | `()` | Positional arguments for the task | From ebaf4ee9a06c0491159aad39ad3734c241aa8c71 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:13:14 +0530 Subject: [PATCH 14/57] docs: fix examples (worker startup, real APIs) --- .../python/more/examples/batch-emails.mdx | 2 +- .../python/more/examples/data-pipeline.mdx | 10 ++++++++- .../python/more/examples/fastapi-service.mdx | 3 ++- .../docs/python/more/examples/index.mdx | 2 ++ .../python/more/examples/notifications.mdx | 2 +- .../more/examples/predicate-gated-jobs.mdx | 22 ++++++++++--------- .../python/more/examples/saga-checkout.mdx | 6 ++--- .../docs/python/more/examples/web-scraper.mdx | 4 +++- .../docs/python/more/examples/workflows.mdx | 12 ++++++++++ 9 files changed, 45 insertions(+), 18 deletions(-) diff --git a/docs/content/docs/python/more/examples/batch-emails.mdx b/docs/content/docs/python/more/examples/batch-emails.mdx index 622cf6a8..ce8b8601 100644 --- a/docs/content/docs/python/more/examples/batch-emails.mdx +++ b/docs/content/docs/python/more/examples/batch-emails.mdx @@ -95,7 +95,7 @@ def send_emails(items: list[dict]) -> None: If you want **N separate jobs** each holding a sub-list (so each chunk retries independently), use `canvas.chunks()`: ```python -from taskito.canvas import chunks +from taskito import chunks # Five jobs, each handling 100 emails, each with independent retry. chunks(send_emails, all_recipients, chunk_size=100).apply(queue) diff --git a/docs/content/docs/python/more/examples/data-pipeline.mdx b/docs/content/docs/python/more/examples/data-pipeline.mdx index 8c97a870..db6216db 100644 --- a/docs/content/docs/python/more/examples/data-pipeline.mdx +++ b/docs/content/docs/python/more/examples/data-pipeline.mdx @@ -108,7 +108,7 @@ def build_pipeline(api_endpoint: str, csv_path: str, target_table: str): # Stage 2: Transform (each depends on its extract) job_norm_a = normalize.apply_async( - args=[[], "schema_v2"], # actual data passed via result + args=[[], "schema_v2"], # depends_on ORDERS only — see note below depends_on=job_api.id, metadata=json.dumps({"stage": "transform", "schema": "v2"}), ) @@ -155,6 +155,14 @@ if __name__ == "__main__": print(f" {ext_job.id} dependents: {fetched.dependents}") ``` +> **`depends_on` orders jobs; it does not pass data.** It only guarantees a job +> starts after its dependencies reach a terminal state — it does **not** inject +> their return values. That is why each `normalize`/`load` call above is enqueued +> with a placeholder `[]`: the task loads its own input (from the database, an +> object store, or a key the upstream task wrote). To thread one task's return +> value straight into the next, use a chain or the +> [canvas API](/python/guides/workflows/canvas) instead. + ## worker.py ```python diff --git a/docs/content/docs/python/more/examples/fastapi-service.mdx b/docs/content/docs/python/more/examples/fastapi-service.mdx index 776e684d..23b6e173 100644 --- a/docs/content/docs/python/more/examples/fastapi-service.mdx +++ b/docs/content/docs/python/more/examples/fastapi-service.mdx @@ -63,7 +63,8 @@ def apply_filters(image_url: str, filters: list[str]) -> dict: app = FastAPI(title="Image Processing Service") -# Mount the taskito router — adds /tasks/stats, /tasks/jobs/{id}, etc. +# Mount the taskito router — adds /tasks/stats, /tasks/jobs/{id}, +# /tasks/jobs/{id}/progress (SSE), and more. app.include_router(TaskitoRouter(queue), prefix="/tasks") @app.post("/process") diff --git a/docs/content/docs/python/more/examples/index.mdx b/docs/content/docs/python/more/examples/index.mdx index 6d1f43e6..280fdf3d 100644 --- a/docs/content/docs/python/more/examples/index.mdx +++ b/docs/content/docs/python/more/examples/index.mdx @@ -12,5 +12,7 @@ End-to-end examples demonstrating common taskito patterns. | [Web Scraper Pipeline](/python/more/examples/web-scraper) | Distributed scraping with chains and error handling | | [Data Pipeline](/python/more/examples/data-pipeline) | ETL pipeline with dependencies, groups, and chords | | [DAG Workflows](/python/more/examples/workflows) | Fan-out, conditions, gates, sub-workflows, incremental runs | +| [Saga Checkout](/python/more/examples/saga-checkout) | Distributed transaction with per-step compensation and rollback | +| [Batch Emails](/python/more/examples/batch-emails) | High-throughput fan-out with in-memory task batching | | [Predicate-Gated Jobs](/python/more/examples/predicate-gated-jobs) | Business-hours windows, load-shedding, feature flags, per-tenant quotas | | [Benchmark](/python/more/examples/benchmark) | Performance benchmarks comparing taskito to alternatives | diff --git a/docs/content/docs/python/more/examples/notifications.mdx b/docs/content/docs/python/more/examples/notifications.mdx index aaed82b0..05f8b31a 100644 --- a/docs/content/docs/python/more/examples/notifications.mdx +++ b/docs/content/docs/python/more/examples/notifications.mdx @@ -83,7 +83,7 @@ def send_sms(phone: str, message: str) -> dict: # ── Periodic Digest ────────────────────────────────────── -@queue.periodic(cron="0 9 * * * *") +@queue.periodic(cron="0 0 9 * * *") def daily_digest(): """Send daily digest email every day at 9 AM.""" print("[DIGEST] Sending daily digest to all subscribers...") diff --git a/docs/content/docs/python/more/examples/predicate-gated-jobs.mdx b/docs/content/docs/python/more/examples/predicate-gated-jobs.mdx index 37ac9944..19feca11 100644 --- a/docs/content/docs/python/more/examples/predicate-gated-jobs.mdx +++ b/docs/content/docs/python/more/examples/predicate-gated-jobs.mdx @@ -153,22 +153,24 @@ def main() -> None: # 1. Allowed (acme is in the allowlist). send_daily_report.delay(tenant="acme") - # 2. Cancel at enqueue: tenant not in allowlist + on_false='cancel' - # is implied by the Cancel return path of TenantIn... actually, - # TenantIn returns False which combines with on_false='defer' - # here, so the call is deferred. Use TenantIn(...).evaluate() - # behavior plus on_false='cancel' for hard rejection: + # 2. Deferred: initech is not in the allowlist. send_daily_report uses + # on_false="defer", so the enqueue succeeds and the job is held back to + # be re-evaluated later — no exception is raised. + send_daily_report.delay(tenant="initech") + + # 3. Rejected at enqueue: charge_card uses on_false="cancel", and the + # new_billing flag is unset, so the predicate denies and enqueue raises. try: - send_daily_report.delay(tenant="initech") + charge_card.delay(tenant="acme", cents=4200) except PredicateRejectedError: - print("not allowed") + print("billing flag off — charge rejected") - # 3. Bypass the billing flag by setting it in env. + # 4. Bypass the billing flag by setting it in env; now it enqueues. import os os.environ["FF_NEW_BILLING"] = "true" charge_card.delay(tenant="acme", cents=4200) - # 4. Async predicate + async task — both awaited transparently. + # 5. Async predicate + async task — both awaited transparently. index_document.delay(tenant="acme", doc_id="doc-1") # ── Inspection: dashboard reads the same shape ───────────────── @@ -253,5 +255,5 @@ for event in ( `circuit_breaker` — they are enforced in the Rust scheduler. Predicates are for the gates the scheduler doesn't already provide: time, payload, feature flags, custom logic. -* See the [Predicates guide](/docs/guides/core/predicates) for the +* See the [Predicates guide](/python/guides/core/predicates) for the full API surface. diff --git a/docs/content/docs/python/more/examples/saga-checkout.mdx b/docs/content/docs/python/more/examples/saga-checkout.mdx index ab6474eb..5ffdd702 100644 --- a/docs/content/docs/python/more/examples/saga-checkout.mdx +++ b/docs/content/docs/python/more/examples/saga-checkout.mdx @@ -11,7 +11,7 @@ Each forward step has a compensator declared via `@queue.task(compensates=...)`: ```python from taskito import Queue -from taskito.workflows import Workflow +from taskito.workflows import Workflow, WorkflowState queue = Queue() @@ -98,14 +98,14 @@ run = queue.submit_workflow(build_checkout("ABC", 2, 5000, "cust-42")) final = run.wait(timeout=60) if final.state == WorkflowState.COMPLETED: - print("Checkout succeeded:", final.nodes["ship"].result_hash) + print("Checkout succeeded; ship job:", final.nodes["ship"].job_id) elif final.state == WorkflowState.COMPENSATED: print("Checkout failed but rolled back cleanly") elif final.state == WorkflowState.COMPENSATION_FAILED: print("Manual cleanup needed — some compensators failed") for name, node in final.nodes.items(): if node.status.value == "compensation_failed": - print(f" - {name}: {node.compensation_error}") + print(f" - {name}: {node.error}") ``` ## What happens on failure diff --git a/docs/content/docs/python/more/examples/web-scraper.mdx b/docs/content/docs/python/more/examples/web-scraper.mdx index 67ffaba8..adb8208c 100644 --- a/docs/content/docs/python/more/examples/web-scraper.mdx +++ b/docs/content/docs/python/more/examples/web-scraper.mdx @@ -168,7 +168,9 @@ from tasks import queue if __name__ == "__main__": print("Starting scraper worker...") - queue.run_worker(queues=["scraping", "processing", "storage"]) + # "default" is required: scrape_page (run.py Option 1) runs on the default + # queue. Omit the queues= argument to process every registered queue. + queue.run_worker(queues=["default", "scraping", "processing", "storage"]) ``` ## Running it diff --git a/docs/content/docs/python/more/examples/workflows.mdx b/docs/content/docs/python/more/examples/workflows.mdx index ba01cda1..833bc08d 100644 --- a/docs/content/docs/python/more/examples/workflows.mdx +++ b/docs/content/docs/python/more/examples/workflows.mdx @@ -6,6 +6,18 @@ description: "ML training, map-reduce, resilient pipelines, sub-workflows, incre Real-world workflow patterns demonstrating fan-out, conditions, approval gates, sub-workflows, and incremental runs. + + Each example builds and submits a workflow; the step jobs run only while a + worker is consuming the queue. Start one — `taskito worker --app app:queue`, or + `queue.run_worker()` in a background thread — before calling `run.wait()`. + (`Queue(workers=4)` sets the worker *count*; it does not start a worker.) + + A step runs with the args you give it in `wf.step(...)`; it does not + auto-receive its predecessor's return value. Data flows between steps via + `fan_out`/`fan_in` and `WorkflowContext.results` — as in the map-reduce and + ML-gate examples below. + + ## ML training pipeline A training pipeline that evaluates a model, gates deployment on accuracy, From e21b25cd35817fed0c52d83fef49d42680f495f5 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:13:14 +0530 Subject: [PATCH 15/57] docs: fix architecture (no-broker, delivery, serializer) --- docs/content/docs/architecture/failure-model.mdx | 6 +++++- docs/content/docs/architecture/job-lifecycle.mdx | 16 ++++++++++++++++ docs/content/docs/architecture/mesh.mdx | 8 +++++--- docs/content/docs/architecture/overview.mdx | 15 +++++++++++++++ docs/content/docs/architecture/serialization.mdx | 16 +++++++++++++--- 5 files changed, 54 insertions(+), 7 deletions(-) diff --git a/docs/content/docs/architecture/failure-model.mdx b/docs/content/docs/architecture/failure-model.mdx index 411eefef..964e14bf 100644 --- a/docs/content/docs/architecture/failure-model.mdx +++ b/docs/content/docs/architecture/failure-model.mdx @@ -5,7 +5,11 @@ description: "At-least-once delivery semantics and how taskito recovers from cra import { Callout } from "fumadocs-ui/components/callout"; -Taskito provides **at-least-once delivery**. Here's what happens when things go wrong. +Taskito provides **exactly-once dispatch** — `claim_execution` ensures two workers +never run the same job at once — but **at-least-once execution**: a crash mid-task +is retried, so a task body can run more than once. (This is why the +[mesh](/architecture/mesh) can promise no double-*dispatch* while a task may still +execute twice across a retry.) Here's what happens when things go wrong. ## Worker crash mid-task diff --git a/docs/content/docs/architecture/job-lifecycle.mdx b/docs/content/docs/architecture/job-lifecycle.mdx index 9ee4ef10..d465eb99 100644 --- a/docs/content/docs/architecture/job-lifecycle.mdx +++ b/docs/content/docs/architecture/job-lifecycle.mdx @@ -7,6 +7,15 @@ Every job moves through a state machine from creation to completion (or death). +## Key transitions + +A job starts **Pending** and moves to **Running** once a worker claims it. From +there it either finishes as **Complete**, or the task raises and it goes back +to **Pending** to wait out the retry backoff (if attempts remain) or to **Dead** +once `max_retries` is exhausted — landing in the dead-letter queue. A job can +also move directly from **Pending** to **Cancelled** if it's cancelled before a +worker picks it up. + ## Status codes | Status | Integer | Description | @@ -17,3 +26,10 @@ Every job moves through a state machine from creation to completion (or death). | Failed | 3 | Last attempt failed (may retry) | | Dead | 4 | All retries exhausted, in DLQ | | Cancelled | 5 | Cancelled before execution | + + + A single job's `status` field is `"complete"` (matches the `Complete` row + above). Aggregate views — `queue.stats()` and the `taskito info` CLI — + bucket finished jobs under the key `"completed"` instead. `list_jobs(status=)` + accepts either spelling. + diff --git a/docs/content/docs/architecture/mesh.mdx b/docs/content/docs/architecture/mesh.mdx index b105d0df..16e594fb 100644 --- a/docs/content/docs/architecture/mesh.mdx +++ b/docs/content/docs/architecture/mesh.mdx @@ -297,9 +297,11 @@ values cloned atomically). | Deque full | New prefetch jobs are dropped, worker falls back to direct DB dispatch | In every failure case, the database remains the source of truth. The worst -outcome is reduced performance (more DB polls), never data loss or -double-execution — those guarantees come from the -[storage layer's](/architecture/storage) atomic claim mechanism. +outcome is reduced performance (more DB polls), never data loss or two workers +running the same job at once — the storage layer's atomic claim mechanism gives +**exactly-once dispatch**. Execution itself is **at-least-once**: a crash +mid-task is retried, so the same task body can run more than once. Design tasks +to be idempotent — see the [failure model](/architecture/failure-model). ## See also diff --git a/docs/content/docs/architecture/overview.mdx b/docs/content/docs/architecture/overview.mdx index 733d0ccd..647eb692 100644 --- a/docs/content/docs/architecture/overview.mdx +++ b/docs/content/docs/architecture/overview.mdx @@ -9,6 +9,21 @@ dispatch, rate limiting, and worker management. +## Why there's no broker + +Most task queues are three services glued together: a **message broker** (Redis +or RabbitMQ) that holds pending jobs, a **result backend** that stores return +values, and the workers. You install, secure, and monitor all three. + +taskito removes the first two. The queue **is** a database — SQLite in +[WAL mode](/architecture/storage) by default, or Postgres — and it serves as +both the broker (the durable, ordered job store the scheduler polls) and the +result backend (return values are rows in the same store). The +[scheduler](/architecture/scheduler) claims each job straight from that database +with an atomic `claim_execution`, so there is no separate queue server to hop +through. That is why setup is a single install, and why the database is the one +source of truth for every dispatch, failure, and recovery path. + ## Section overview | Page | What it covers | diff --git a/docs/content/docs/architecture/serialization.mdx b/docs/content/docs/architecture/serialization.mdx index bbea27e8..ad815df2 100644 --- a/docs/content/docs/architecture/serialization.mdx +++ b/docs/content/docs/architecture/serialization.mdx @@ -4,8 +4,17 @@ description: "Pluggable serializers for task arguments and results, with a per-S --- taskito uses a pluggable serializer for task arguments and results. The default -is CloudpickleSerializer} node={JsonSerializer} java={JsonSerializer} />, -which . +is SmartSerializer} node={JsonSerializer} java={JsonSerializer} />, +which . + + + +> Coming from Celery? Celery defaults to JSON; taskito's Python default is +> `SmartSerializer` (MessagePack with a cloudpickle fallback). The fallback +> **executes code on load**, and it's Python-only — use `JsonSerializer` for +> cross-language or untrusted payloads. + + @@ -50,7 +59,8 @@ Taskito queue = Taskito.builder() | Serializer | Format | Best for | |---|---|---| -| `CloudpickleSerializer` (default) | Binary (pickle) | Complex Python objects, lambdas, closures | +| `SmartSerializer` (default) | MessagePack + cloudpickle fallback | Mixed payloads; falls back to pickle for complex objects | +| `CloudpickleSerializer` | Binary (pickle) | Complex Python objects, lambdas, closures | | `JsonSerializer` | JSON | Simple types, cross-language interop, debugging | From 3f0242eb7ccd0ac5d1ff36d171df27baf682c9ed Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:13:15 +0530 Subject: [PATCH 16/57] docs: fix comparison and faq (distributed, serializer) --- docs/content/docs/resources/comparison.mdx | 8 ++++---- docs/content/docs/resources/faq.mdx | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/content/docs/resources/comparison.mdx b/docs/content/docs/resources/comparison.mdx index 64b4c8b1..d974de5d 100644 --- a/docs/content/docs/resources/comparison.mdx +++ b/docs/content/docs/resources/comparison.mdx @@ -4,7 +4,7 @@ description: "Taskito vs Celery, RQ, Dramatiq, Huey, TaskIQ — feature matrix a --- **TL;DR**: Taskito is without the -broker. Rust scheduler, no Redis/RabbitMQ, lower latency, better concurrency. +broker (a separate server such as Redis or RabbitMQ that holds pending jobs). Rust scheduler, no Redis/RabbitMQ, lower latency, better concurrency. Start with SQLite, scale to Postgres when needed. ## Feature matrix @@ -36,7 +36,7 @@ Start with SQLite, scale to Postgres when needed. | Async canvas | **Yes** | No | No | No | No | No | | OpenTelemetry | **Yes** (optional) | Yes (contrib) | No | No | No | Yes (built-in) | | CLI | **Yes** | Yes | Yes | Yes | Yes | Yes | -| Result backend | **Built-in** (SQLite) | Redis / DB / custom | Redis | Redis / custom | Redis / SQLite | Redis / custom | +| Result backend (where task return values are stored) | **Built-in** (SQLite) | Redis / DB / custom | Redis | Redis / custom | Redis / SQLite | Redis / custom | | Setup complexity | **`pip install`** | Broker + backend | Redis server | Broker | Redis server | Broker + backend | ## When to use taskito @@ -71,7 +71,7 @@ widely adopted. | **Dependencies** | 1 (cloudpickle) | 10+ (kombu, billiard, vine, etc.) | | **Configuration** | Constructor params | Settings module or app config | | **Worker model** | Rust OS threads | prefork/eventlet/gevent pools | -| **Distributed** | No (single process) | Yes (multi-server) | +| **Distributed** | Single machine by default; multi-server via Postgres/Redis backend | Yes (multi-server) | | **Canvas** | chain, group, chord, starmap, chunks | chain, group, chord, starmap, chunks, and more | **Choose taskito** if you want zero-infrastructure simplicity on a single machine. @@ -139,7 +139,7 @@ async and already have a broker. | **Broker** | None (DB-backed) | Redis / RabbitMQ / Nats | | **Async** | Native + sync | Async-first | | **Scheduler** | Rust (Tokio) | Python | -| **GIL** | Rust scheduler bypasses GIL | Python scheduler competes for GIL | +| **GIL** (Global Interpreter Lock — only one thread runs Python bytecode at a time) | Rust scheduler bypasses GIL | Python scheduler competes for GIL | | **Setup** | `pip install taskito` | Install broker + taskiq + broker plugin | Choose taskito if you want zero infrastructure. Choose TaskIQ if you're fully diff --git a/docs/content/docs/resources/faq.mdx b/docs/content/docs/resources/faq.mdx index 3dbddb28..47c098b6 100644 --- a/docs/content/docs/resources/faq.mdx +++ b/docs/content/docs/resources/faq.mdx @@ -209,7 +209,7 @@ context in async tasks. ## What serialization format does taskito use? -By default, CloudpickleSerializer} node={JsonSerializer} java={JsonSerializer} /> — — or provide a custom serializer: +By default, SmartSerializer} node={JsonSerializer} java={JsonSerializer} /> — — or provide a custom serializer: From d2b481dc52fe32d397082c5025d752237773f3d8 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Wed, 8 Jul 2026 22:26:39 +0530 Subject: [PATCH 17/57] docs: fix ML workflow example step arg/result flow --- .../docs/python/more/examples/workflows.mdx | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/content/docs/python/more/examples/workflows.mdx b/docs/content/docs/python/more/examples/workflows.mdx index 833bc08d..12940a0a 100644 --- a/docs/content/docs/python/more/examples/workflows.mdx +++ b/docs/content/docs/python/more/examples/workflows.mdx @@ -32,23 +32,27 @@ queue = Queue(db_path="ml.db", workers=4) @queue.task() def fetch_dataset() -> dict: + # Fetch the dataset and stage it in a store. The return value is recorded + # in the run's ctx.results so gates/conditions can read it. return {"rows": 50_000, "path": "/data/train.parquet"} @queue.task() -def train_model(dataset: dict) -> dict: - # ... training logic ... +def train_model() -> dict: + # Read the staged dataset from the store, train, and register the model. return {"model_id": "v3.2", "accuracy": 0.97, "loss": 0.08} @queue.task() -def evaluate(model: dict) -> dict: - return {"accuracy": model["accuracy"], "passed": model["accuracy"] > 0.90} +def evaluate() -> dict: + # Read the trained model's metrics from the registry. + return {"accuracy": 0.97, "passed": True} @queue.task() -def deploy(model_id: str) -> str: - return f"deployed {model_id}" +def deploy() -> str: + # Deploy the latest registered model. + return "deployed v3.2" @queue.task() @@ -57,6 +61,8 @@ def notify_failure() -> str: def accuracy_gate(ctx: WorkflowContext) -> bool: + # Steps don't receive each other's results as args — they share the model + # through the registry. The gate reads evaluate's RETURN VALUE via ctx.results. return ctx.results.get("evaluate", {}).get("passed", False) From 0a6a7fb9f254282b69d0f8d5c9274081f622e897 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:34:09 +0530 Subject: [PATCH 18/57] docs: rework landing hero copy and CTAs --- docs/app/components/landing/hero.tsx | 34 ++++++++++++---------------- docs/app/routes/home.tsx | 5 ++-- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/docs/app/components/landing/hero.tsx b/docs/app/components/landing/hero.tsx index 98a559ee..795be61d 100644 --- a/docs/app/components/landing/hero.tsx +++ b/docs/app/components/landing/hero.tsx @@ -32,7 +32,6 @@ export function Hero() { // The selected snippet IS the global SDK — clicking a tab sets it, so the hero // copy, the install/quickstart links, and the docs sidebar switch all follow. const active = HERO_PANES.find((p) => p.sdk === sdk) ?? HERO_PANES[0]; - const label = sdkProfile(active.sdk).label; const codeHtml = active.lang === "ts" ? highlightTs(active.code) @@ -44,30 +43,33 @@ export function Hero() {

- One queue. - Built for {label}. + Keep your app fast —{" "} + run slow work in the background.

- A Rust-powered task queue with a first-class {label} SDK over - one core and one store — no broker. Start on SQLite, - scale to Postgres. + A task queue with no message broker. Your app hands slow work — + sending email, processing uploads, running pipelines — to a background + worker and gets the result later; the queue, results, and schedules + all live in{" "} + + one SQLite file + {" "} + (scale to Postgres). First-class{" "} + Python, Node, and Java over one Rust core.

Quickstart → - - Install - GitHub ↗
- Brokerless + No broker Rust core - {label} SDK - DAG workflows + MIT licensed + Python · Node · Java
@@ -124,14 +126,6 @@ export function Hero() { ))} - -
- {HERO_PANES.map((p) => ( - - {p.docLabel} → - - ))} -
); diff --git a/docs/app/routes/home.tsx b/docs/app/routes/home.tsx index ce8bcc96..a0f3835d 100644 --- a/docs/app/routes/home.tsx +++ b/docs/app/routes/home.tsx @@ -18,10 +18,11 @@ import type { Route } from "./+types/home"; export function meta(_: Route.MetaArgs) { return [ - { title: "Taskito — one queue, built for Python" }, + { title: "Taskito — background jobs without a broker" }, { name: "description", - content: "Rust-powered task queue for Python. No broker required.", + content: + "Keep your app fast — run slow work in the background. A Rust-powered task queue with no message broker, for Python, Node, and Java.", }, ]; } From 79544b4754d90c8e83d9ded4838ee7636128107c Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:34:14 +0530 Subject: [PATCH 19/57] docs: fix hero code panel to a consistent size --- docs/app/lib/landing-content.ts | 4 ++-- docs/app/styles/landing.css | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/app/lib/landing-content.ts b/docs/app/lib/landing-content.ts index 529c2e72..508458d8 100644 --- a/docs/app/lib/landing-content.ts +++ b/docs/app/lib/landing-content.ts @@ -102,14 +102,14 @@ import org.byteveda.taskito.task.Task; import org.byteveda.taskito.worker.Worker; Task add = Task.of("add", int[].class).retries(3); - try (Taskito queue = Taskito.builder().sqlite("tasks.db").open(); Worker worker = queue.worker() .handle(add, p -> p[0] + p[1]) .start()) { String id = queue.enqueue(add, new int[] {2, 3}); queue.awaitJob(id, java.time.Duration.ofSeconds(10)); - System.out.println(queue.getResult(id, Integer.class).orElseThrow()); // → 5 + var sum = queue.getResult(id, Integer.class).orElseThrow(); + System.out.println(sum); // → 5 }`, output: [ { glyph: "$", glyphKind: "p", text: "java -cp app.jar Tasks" }, diff --git a/docs/app/styles/landing.css b/docs/app/styles/landing.css index 114a9942..52512fde 100644 --- a/docs/app/styles/landing.css +++ b/docs/app/styles/landing.css @@ -274,10 +274,12 @@ main { padding: 20px 22px; color: var(--txt); white-space: pre; - min-height: 296px; + /* Fixed height so every SDK tab renders the same box — no resize/layout shift + when switching Python/Node/Java. Snippets are sized to fit without scroll. */ + height: 384px; + box-sizing: border-box; tab-size: 4; - /* Long lines scroll within the terminal instead of being clipped on mobile. */ - overflow-x: auto; + overflow: auto; } .code .cur { display: inline-block; From 75e79f5eea2936e36c48617cdf61f367315b4cb9 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:34:36 +0530 Subject: [PATCH 20/57] docs(node): fix getting-started and core guides --- .../docs/node/getting-started/concepts.mdx | 35 ++++++++++++++++--- .../node/getting-started/installation.mdx | 2 +- .../docs/node/getting-started/quickstart.mdx | 8 ++--- .../docs/node/guides/core/enqueue-options.mdx | 15 ++++++-- docs/content/docs/node/guides/core/queues.mdx | 9 ++++- .../docs/node/guides/core/scheduling.mdx | 7 ++++ .../docs/node/guides/core/streaming.mdx | 8 +++++ docs/content/docs/node/guides/core/tasks.mdx | 11 ++++++ .../content/docs/node/guides/core/workers.mdx | 10 ++++++ 9 files changed, 91 insertions(+), 14 deletions(-) diff --git a/docs/content/docs/node/getting-started/concepts.mdx b/docs/content/docs/node/getting-started/concepts.mdx index cff9e94b..4ae5987a 100644 --- a/docs/content/docs/node/getting-started/concepts.mdx +++ b/docs/content/docs/node/getting-started/concepts.mdx @@ -1,11 +1,36 @@ --- title: Concepts -description: "Queue, task, worker, result — the Node.js mental model over the Rust core." +description: "Queue, task, worker, result — the Node.js mental model over the Rust core, and why there's no broker." --- -The Node SDK is a typed shell; the scheduler, dispatcher, worker pool, and -storage are all the same Rust engine the Python SDK uses. Four concepts cover -the surface. +A **task queue** hands off slow work — sending an email, resizing an upload, +calling a third-party API — to a background **worker** (a process that pulls +jobs off the queue and runs them), so the request that triggered it can return +right away. The Node SDK is a typed shell; the scheduler, dispatcher, worker +pool, and storage are all the same Rust engine the Python SDK uses. Four +concepts cover the surface. + +## Why no broker + +Most Node.js task queues — BullMQ, Bee-Queue, Agenda — need a standalone +**broker** (a separate server that holds and dispatches jobs): you provision a +Redis instance, secure it, and keep it running alongside your app. taskito has +no broker. The queue, job results, and cron schedules all live in one embedded +SQLite file — nothing extra to install or operate. + +```ts +import { Queue } from "@byteveda/taskito"; + +const queue = new Queue({ dbPath: "taskito.db" }); // that's the whole setup +``` + + + BullMQ's `new Queue(name, { connection: redis })` plus a separately-run + Redis server becomes a single `new Queue({ dbPath })` — no connection to + provision. Point every process at the same file (or DSN) and they share one + logical queue. See [installation](/node/getting-started/installation) if you + want Postgres or Redis as a shared backend instead of a file. + = 18. Ships as dual **ESM + CommonJS** with bundled type +Requires Node.js >= 20. Ships as dual **ESM + CommonJS** with bundled type definitions. ## Prebuilt native binaries diff --git a/docs/content/docs/node/getting-started/quickstart.mdx b/docs/content/docs/node/getting-started/quickstart.mdx index 13eb03ff..135392fa 100644 --- a/docs/content/docs/node/getting-started/quickstart.mdx +++ b/docs/content/docs/node/getting-started/quickstart.mdx @@ -3,7 +3,8 @@ title: Quickstart description: "Define a task, enqueue a job, run a worker, await the result — in TypeScript." --- -Four steps: construct a queue, register a task, enqueue a job, run a worker. +Four steps: construct a queue, register a task, enqueue a job (add it to +storage), run a worker (a process that claims and executes queued jobs). ```ts import { Queue } from "@byteveda/taskito"; @@ -33,9 +34,8 @@ The producer and the worker only need to share storage — `enqueue` from your w app, `runWorker` from a separate process pointed at the same `dbPath` / DSN. - Tasks can be `async`. The dispatcher awaits the returned promise; an - `async def`-style task that does I/O runs on the native async pool without a - thread per job. + Tasks can be `async`. The dispatcher awaits the returned promise; an `async` + task that does I/O runs on the native async pool without a thread per job. ## Where to next diff --git a/docs/content/docs/node/guides/core/enqueue-options.mdx b/docs/content/docs/node/guides/core/enqueue-options.mdx index 7f88ee50..7a281524 100644 --- a/docs/content/docs/node/guides/core/enqueue-options.mdx +++ b/docs/content/docs/node/guides/core/enqueue-options.mdx @@ -1,23 +1,24 @@ --- title: Enqueue options -description: "Priority, delay, idempotency, metadata, and namespace at enqueue time." +description: "Priority, delay, idempotency, metadata, and routing at enqueue time." --- The third argument to `enqueue` sets per-job options: ```ts queue.enqueue("send_email", [user], { + queue: "emails", priority: 5, delayMs: 30_000, uniqueKey: `welcome:${user.id}`, metadata: "source=signup", notes: { campaign: "welcome", attempt: 1 }, - namespace: "emails", }); ``` | Option | Description | |---|---| +| `queue` | The named queue to route to (default `"default"`) — see [Queues & Priority](/node/guides/core/queues). | | `priority` | Higher dequeues first within a queue. | | `maxRetries` | Override the task's retry budget for this job. | | `timeoutMs` | Override the per-attempt timeout for this job. | @@ -25,7 +26,15 @@ queue.enqueue("send_email", [user], { | `uniqueKey` | Idempotency key — a duplicate enqueue is a no-op while the first job is pending/running. | | `metadata` | Free-form string stored with the job (surfaces on the dashboard / inspection). | | `notes` | Structured annotations — a JSON object, ≤15 fields and ≤4 KiB encoded. | -| `namespace` | The queue name to route to. | +| `namespace` | Tenant/environment isolation — a worker only claims jobs whose `namespace` exactly matches its own (unset by default). Orthogonal to `queue`: two jobs can share a `queue` name but sit in different namespaces, invisible to each other's workers. | + +> **Coming from BullMQ?** BullMQ's `add(name, data, opts)` third argument maps +> field-for-field: `opts.delay` → `delayMs`, `opts.priority` → `priority` +> (inverted direction — see [queues](/node/guides/core/queues)), and an +> explicit custom `opts.jobId` (BullMQ's usual dedup trick) → `uniqueKey`. +> taskito's `uniqueKey` dedup is automatic and scoped to "while the first job +> is pending/running" — it frees up once that job finishes, rather than +> permanently reserving the key. ## Idempotency diff --git a/docs/content/docs/node/guides/core/queues.mdx b/docs/content/docs/node/guides/core/queues.mdx index c467a25d..7667a5e2 100644 --- a/docs/content/docs/node/guides/core/queues.mdx +++ b/docs/content/docs/node/guides/core/queues.mdx @@ -7,7 +7,7 @@ Every job belongs to a queue (default `"default"`). Queues are just routing labels in shared storage — a worker chooses which queues it serves. ```ts -const id = queue.enqueue("send_email", [user], { namespace: "emails" }); +const id = queue.enqueue("send_email", [user], { queue: "emails" }); queue.runWorker({ queues: ["emails", "default"] }); // serve two queues ``` @@ -20,6 +20,13 @@ Higher `priority` dequeues first within a queue: queue.enqueue("report", [id], { priority: 10 }); // ahead of priority 0 jobs ``` + + Priority direction is **inverted**. BullMQ treats a *lower* `priority` + number as higher priority (`1` runs before `10`). taskito is the other way + round — a *higher* `priority` number dequeues first. Porting a BullMQ + priority scheme means inverting the numbers, not copying them. + + ## Pause and resume Pausing stops dispatch for a queue without stopping the worker — in-flight jobs diff --git a/docs/content/docs/node/guides/core/scheduling.mdx b/docs/content/docs/node/guides/core/scheduling.mdx index 3231227f..646405b4 100644 --- a/docs/content/docs/node/guides/core/scheduling.mdx +++ b/docs/content/docs/node/guides/core/scheduling.mdx @@ -33,3 +33,10 @@ queue.registerPeriodic("daily-digest", "digest", "0 0 9 * * *", { For a one-off future run, use [`delayMs`](/node/guides/core/enqueue-options#delayed-jobs) instead of a periodic schedule. + +> **Coming from BullMQ?** BullMQ attaches a repeat schedule to a job via +> `add(name, data, { repeat: { pattern: cron } })` — the schedule lives inside +> an enqueue call. taskito separates the two: `registerPeriodic(name, +> taskName, cron, options?)` is its own call, independent of any single +> `enqueue()`, and `deletePeriodic`/`pausePeriodic`/`resumePeriodic` manage it +> by name afterward. diff --git a/docs/content/docs/node/guides/core/streaming.mdx b/docs/content/docs/node/guides/core/streaming.mdx index b5725019..960208e8 100644 --- a/docs/content/docs/node/guides/core/streaming.mdx +++ b/docs/content/docs/node/guides/core/streaming.mdx @@ -32,6 +32,7 @@ order, then ends when the job reaches a terminal state. ```ts const id = queue.enqueue("train", [10]); +queue.runWorker(); // a worker must be consuming the queue for `train` to run for await (const update of queue.stream(id)) { console.log(`epoch ${update.epoch}: loss ${update.loss}`); @@ -44,6 +45,13 @@ const result = await queue.result(id); // "trained" interval (default 200 ms). It works from any process sharing the storage, and a late consumer still drains every value already published. + + `stream()` and `result()` only resolve once a worker actually runs the job. + If nothing is consuming the queue, `stream()` yields nothing until its + timeout and `result()` rejects with a timeout error — start a worker + (`queue.runWorker()`) before or right after enqueuing. + + ## Raw logs `queue.taskLogs(jobId)` returns the raw log entries (published partials have diff --git a/docs/content/docs/node/guides/core/tasks.mdx b/docs/content/docs/node/guides/core/tasks.mdx index 64b75b04..17d83b58 100644 --- a/docs/content/docs/node/guides/core/tasks.mdx +++ b/docs/content/docs/node/guides/core/tasks.mdx @@ -49,3 +49,14 @@ Beyond per-task config, you can set defaults for an entire named queue: ```ts queue.configureQueue("emails", { rateLimit: "50/s" }); ``` + + + BullMQ splits a job into `queue.add(name, data)` on one side and a + `new Worker(queueName, processor)` callback on the other — the processor + lives wherever you construct the `Worker`. taskito ties the handler to the + task name once, via `queue.task(name, handler, options)`; `enqueue()` just + references that name, and `runWorker()` takes no processor argument. Also + note: `enqueue()` returns the job id (a `string`), not a rich `Job` object — + fetch a snapshot with `queue.getJob(id)` or block on + [`queue.result(id)`](/node/api-reference/result). + diff --git a/docs/content/docs/node/guides/core/workers.mdx b/docs/content/docs/node/guides/core/workers.mdx index d33fabf0..8c5eb20b 100644 --- a/docs/content/docs/node/guides/core/workers.mdx +++ b/docs/content/docs/node/guides/core/workers.mdx @@ -39,3 +39,13 @@ queue.task("fetch", async (url: string) => (await fetch(url)).text()); Producers and workers only share storage. Enqueue from a web process and run workers from a separate process pointed at the same `dbPath` / DSN. + + + BullMQ's `new Worker(queueName, processor, { concurrency: N })` runs `N` + jobs at once **per worker process** — concurrency scales with how many + workers you start. taskito's `runWorker()` takes no processor or + concurrency argument (handlers come from `queue.task()`); simultaneous + execution is bounded by per-task/per-queue + [`maxConcurrent`](/node/guides/reliability/concurrency), enforced by the + scheduler across *all* workers on shared storage, not per process. + From e02238a1819a48960e2b2193ffb526d817e7a568 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:34:37 +0530 Subject: [PATCH 21/57] docs(node): fix reliability and workflow guides --- .../guides/reliability/circuit-breakers.mdx | 11 +++++- .../node/guides/reliability/concurrency.mdx | 5 +++ .../node/guides/reliability/dead-letter.mdx | 6 +++ .../guides/reliability/error-handling.mdx | 6 ++- .../node/guides/reliability/guarantees.mdx | 7 ++++ .../node/guides/reliability/idempotency.mdx | 6 +++ .../docs/node/guides/reliability/locks.mdx | 6 +++ .../node/guides/reliability/rate-limiting.mdx | 8 ++++ .../docs/node/guides/reliability/retries.mdx | 13 ++++++- .../docs/node/guides/reliability/timeouts.mdx | 13 +++++-- .../docs/node/guides/workflows/conditions.mdx | 5 +++ .../docs/node/guides/workflows/fan-out.mdx | 5 +++ .../docs/node/guides/workflows/gates.mdx | 6 +++ .../docs/node/guides/workflows/index.mdx | 39 +++++++++++++++++-- .../docs/node/guides/workflows/saga.mdx | 4 ++ .../node/guides/workflows/sub-workflows.mdx | 6 +++ 16 files changed, 134 insertions(+), 12 deletions(-) diff --git a/docs/content/docs/node/guides/reliability/circuit-breakers.mdx b/docs/content/docs/node/guides/reliability/circuit-breakers.mdx index 3af22f6a..7177104a 100644 --- a/docs/content/docs/node/guides/reliability/circuit-breakers.mdx +++ b/docs/content/docs/node/guides/reliability/circuit-breakers.mdx @@ -17,13 +17,20 @@ queue.task("call_flaky_api", callApi, { | `threshold` | Failures within `windowMs` that trip the breaker. | | `windowMs` | Rolling window for counting failures. | | `cooldownMs` | How long the breaker stays open before a trial dispatch. | +| `halfOpenMaxProbes` | Probe jobs let through while half-open (default 5). | +| `halfOpenSuccessRate` | Success rate (0.0–1.0) among probes required to close (default 0.8). | While **open**, jobs of that task are held rather than dispatched. After -`cooldownMs` the breaker half-opens and lets a job through; success closes it, -failure re-opens it. +`cooldownMs` the breaker half-opens and lets a limited number of probe jobs +through (`halfOpenMaxProbes`); if enough of them succeed +(`halfOpenSuccessRate`) the breaker closes, otherwise it re-opens. Pair with [retries](/node/guides/reliability/retries): retries handle transient blips on a healthy dependency; the breaker handles a dependency that is down, so you stop spending the retry budget on it. + +> **Coming from BullMQ?** BullMQ has no built-in circuit breaker — it's a +> taskito-specific addition. You'd otherwise reach for a separate library +> (e.g. `opossum`) alongside a BullMQ `Worker`. diff --git a/docs/content/docs/node/guides/reliability/concurrency.mdx b/docs/content/docs/node/guides/reliability/concurrency.mdx index 89d6da7a..0f058c5f 100644 --- a/docs/content/docs/node/guides/reliability/concurrency.mdx +++ b/docs/content/docs/node/guides/reliability/concurrency.mdx @@ -25,3 +25,8 @@ queue.configureQueue("video", { maxConcurrent: 4 }); limiting](/node/guides/reliability/rate-limiting) bounds *throughput over time*. They compose — a task can be both `maxConcurrent: 2` and `100/min`. + +> **Coming from BullMQ?** See the note on [workers](/node/guides/core/workers) +> — BullMQ's `concurrency` option scales with worker-process count; taskito's +> `maxConcurrent` is a single global cap the scheduler enforces no matter how +> many workers are running. diff --git a/docs/content/docs/node/guides/reliability/dead-letter.mdx b/docs/content/docs/node/guides/reliability/dead-letter.mdx index 20da7899..7df42785 100644 --- a/docs/content/docs/node/guides/reliability/dead-letter.mdx +++ b/docs/content/docs/node/guides/reliability/dead-letter.mdx @@ -32,3 +32,9 @@ taskito --db taskito.db dlq retry metadata survive the DLQ round-trip, so context attached at enqueue is preserved through replay. + +> **Coming from BullMQ?** BullMQ has no separate dead-letter store — exhausted +> jobs stay in the same queue's `failed` set (`queue.getFailed()`, +> `job.retry()`). taskito moves them into a distinct DLQ with its own +> `deadLetters()` / `retryDead()` / `purgeDead()` API, so failed jobs don't +> clutter the same listings as active work. diff --git a/docs/content/docs/node/guides/reliability/error-handling.mdx b/docs/content/docs/node/guides/reliability/error-handling.mdx index 38ce8d3c..0365264a 100644 --- a/docs/content/docs/node/guides/reliability/error-handling.mdx +++ b/docs/content/docs/node/guides/reliability/error-handling.mdx @@ -7,9 +7,11 @@ When a task throws (or its promise rejects), Taskito decides the job's fate from its retry budget. Each failing attempt runs the `onError` middleware hook, then: 1. **Retry** — if attempts remain, the job is rescheduled with - [backoff](/node/guides/reliability/retries). + [backoff](/node/guides/reliability/retries) (a growing delay between + attempts, so retries don't hammer a struggling dependency). 2. **Dead-letter** — once the budget is exhausted, the job moves to the - [dead-letter queue](/node/guides/reliability/dead-letter). + [dead-letter queue](/node/guides/reliability/dead-letter) (DLQ — a holding + area for permanently-failed jobs you can inspect and replay). ```ts queue.task("fetch", fetchUrl, { diff --git a/docs/content/docs/node/guides/reliability/guarantees.mdx b/docs/content/docs/node/guides/reliability/guarantees.mdx index f5e5771a..723e9235 100644 --- a/docs/content/docs/node/guides/reliability/guarantees.mdx +++ b/docs/content/docs/node/guides/reliability/guarantees.mdx @@ -45,3 +45,10 @@ queue.task("charge", async (orderId: string) => { A completed job's result is stored and can be awaited by id from any process sharing the storage — see [result](/node/api-reference/result). + +> **Coming from BullMQ?** BullMQ's `QueueEvents` is a dedicated Redis pub/sub +> subscriber, so any process can listen for job outcomes. taskito's +> `queue.on(event, handler)` fires inside the **worker process** handling the +> job — it isn't a cross-process subscription. For visibility from another +> process, poll `queue.getJob(id)` / `queue.stats()`, or use the +> [dashboard](/node/guides/operations/dashboard) / REST API. diff --git a/docs/content/docs/node/guides/reliability/idempotency.mdx b/docs/content/docs/node/guides/reliability/idempotency.mdx index ffd7d74a..91e978a9 100644 --- a/docs/content/docs/node/guides/reliability/idempotency.mdx +++ b/docs/content/docs/node/guides/reliability/idempotency.mdx @@ -16,6 +16,12 @@ queue.enqueue("welcome", [user.id], { uniqueKey: `welcome:${user.id}` }); // ign Backed by a partial unique index in the store, so the dedup is atomic across producers and processes. Once the first job finishes, the key frees up. +> **Coming from BullMQ?** BullMQ's usual dedup trick is an explicit custom +> `jobId` passed to `add()`. taskito's `uniqueKey` is the same idea, but +> scoped to "while the first job is pending/running" rather than a permanent +> reservation — enqueue the same key again later, after the job completes, +> and it creates a fresh job. + ## Idempotent tasks Separately from enqueue dedup, **task functions should be safe to run more than diff --git a/docs/content/docs/node/guides/reliability/locks.mdx b/docs/content/docs/node/guides/reliability/locks.mdx index f76bae65..7760de61 100644 --- a/docs/content/docs/node/guides/reliability/locks.mdx +++ b/docs/content/docs/node/guides/reliability/locks.mdx @@ -46,3 +46,9 @@ if (lock.acquire()) { never holds a lock past its TTL. Choose a `ttlMs` comfortably longer than the protected work's worst case. + +> **Coming from BullMQ?** BullMQ's locking is internal — a `lockDuration` +> per job that stops two workers double-processing it, not something you can +> call from application code. taskito's `queue.lock()` / `withLock()` is a +> general-purpose distributed lock you can use for *any* critical section, job +> processing or not. diff --git a/docs/content/docs/node/guides/reliability/rate-limiting.mdx b/docs/content/docs/node/guides/reliability/rate-limiting.mdx index 06f22f15..426cb6ab 100644 --- a/docs/content/docs/node/guides/reliability/rate-limiting.mdx +++ b/docs/content/docs/node/guides/reliability/rate-limiting.mdx @@ -25,6 +25,14 @@ queue.configureQueue("emails", { rateLimit: "50/s" }); Queue-level limits are evaluated before per-task limits. Rate-limited jobs stay `pending` and retry on the next tick — they do not consume the retry budget. +> **Coming from BullMQ?** BullMQ configures rate limiting on the `Worker` +> (`limiter: { max, duration }`), but it's enforced storage-side across every +> worker consuming that queue — not literally per process despite living on +> the `Worker` constructor. taskito's `rateLimit` just moves that config to +> where the limit conceptually applies: the task (`queue.task(name, fn, { +> rateLimit })`) or the queue (`configureQueue(name, { rateLimit })`), +> enforced by the scheduler the same way regardless of worker count. + Rate limiting throttles *throughput*; to bound *simultaneous* executions use [concurrency](/node/guides/reliability/concurrency). diff --git a/docs/content/docs/node/guides/reliability/retries.mdx b/docs/content/docs/node/guides/reliability/retries.mdx index a6dab9d3..52b9036e 100644 --- a/docs/content/docs/node/guides/reliability/retries.mdx +++ b/docs/content/docs/node/guides/reliability/retries.mdx @@ -3,8 +3,9 @@ title: Retries description: "Automatic retries with exponential backoff before dead-lettering." --- -A task that throws is retried up to `maxRetries` times before it dead-letters. -Retries are scheduled with exponential backoff. +A task that throws is retried up to `maxRetries` times before it +[dead-letters](/node/guides/reliability/dead-letter). Retries are scheduled +with exponential backoff — the delay roughly doubles each attempt. ```ts queue.task("charge", chargeCard, { @@ -34,3 +35,11 @@ retry; once exhausted, the job moves to the [idempotent](/node/guides/reliability/idempotency) — a crash after a side effect but before the result write will re-execute. + +> **Coming from BullMQ?** Watch the off-by-one: BullMQ's `attempts` counts the +> *total* tries including the first (`attempts: 3` = 1 try + 2 retries). +> taskito's `maxRetries` counts only the retries *after* the first +> (`maxRetries: 3` = 1 try + 3 retries = 4 total). BullMQ's +> `backoff: { type: "exponential", delay }` maps to `retryBackoff.baseMs`; +> taskito only offers exponential backoff, not BullMQ's pluggable custom +> backoff strategies. diff --git a/docs/content/docs/node/guides/reliability/timeouts.mdx b/docs/content/docs/node/guides/reliability/timeouts.mdx index a3b19baa..7b9f222b 100644 --- a/docs/content/docs/node/guides/reliability/timeouts.mdx +++ b/docs/content/docs/node/guides/reliability/timeouts.mdx @@ -14,9 +14,11 @@ queue.enqueue("scrape", [url], { timeoutMs: 10_000 }); // per-job override ``` A timeout counts as a failure: it consumes a retry and, if retries remain, the -job is re-enqueued. When the budget is exhausted the job dead-letters with its -`timedOut` flag set — surfaced to [middleware](/node/guides/extensibility/middleware) -`onTimeout` and the Sentry/observability integrations. +job is re-enqueued. When the budget is exhausted the job dead-letters with +`timedOut: true` on the outcome — passed to the `onRetry` / `onDeadLetter` +[middleware](/node/guides/extensibility/middleware) hooks and the `job.retrying` +/ `job.dead` [events](/node/guides/extensibility/events), so handlers can tell +a timeout apart from any other error. For `async` tasks, the timeout aborts the task's @@ -26,3 +28,8 @@ job is re-enqueued. When the budget is exhausted the job dead-letters with its Always set a timeout on production tasks — without one, a wedged task can hold a worker slot indefinitely. + +> **Coming from BullMQ?** BullMQ has no first-class per-job timeout option — +> processors typically race their own work against a manual `Promise.race` (or +> rely on `lockDuration` as an indirect signal). taskito's `timeoutMs` is +> enforced by the dispatcher directly, no manual race needed. diff --git a/docs/content/docs/node/guides/workflows/conditions.mdx b/docs/content/docs/node/guides/workflows/conditions.mdx index 76ba2a35..37c64b94 100644 --- a/docs/content/docs/node/guides/workflows/conditions.mdx +++ b/docs/content/docs/node/guides/workflows/conditions.mdx @@ -110,3 +110,8 @@ queue.workflows | `timeoutMs` | `number` | Per-attempt timeout | | `priority` | `number` | Queue priority | | `queue` | `string` | Queue name | + +> **Coming from BullMQ?** `FlowProducer` runs every child unconditionally — +> there's no per-child `on_success` / `on_failure` branching. Conditional +> steps are taskito-only; in BullMQ you'd inspect the parent job's outcome +> yourself and decide whether to `add()` the next job at all. diff --git a/docs/content/docs/node/guides/workflows/fan-out.mdx b/docs/content/docs/node/guides/workflows/fan-out.mdx index f56a23da..827fc97c 100644 --- a/docs/content/docs/node/guides/workflows/fan-out.mdx +++ b/docs/content/docs/node/guides/workflows/fan-out.mdx @@ -41,6 +41,11 @@ const run = await handle.wait(); console.log(run.state); // "completed" ``` +> **Coming from BullMQ?** `FlowProducer` builds a **static** parent/child tree +> — `children` is a fixed array at definition time. taskito's fan-out is +> **dynamic**: the child count comes from `list`'s return value at run time, so +> a batch of 3 files and a batch of 3,000 use the same workflow definition. + `itemsFrom` names the predecessor whose result is the item array. It defaults to the sole predecessor when omitted. Each item is passed to the child task as its single positional argument. diff --git a/docs/content/docs/node/guides/workflows/gates.mdx b/docs/content/docs/node/guides/workflows/gates.mdx index 385a14f7..72c7a130 100644 --- a/docs/content/docs/node/guides/workflows/gates.mdx +++ b/docs/content/docs/node/guides/workflows/gates.mdx @@ -114,3 +114,9 @@ app.post("/hooks/deploy-approval", async (req, res) => { res.sendStatus(204); }); ``` + +> **Coming from BullMQ?** `FlowProducer` has no pause/resume primitive — a +> human-in-the-loop approval step is something you'd build yourself (e.g. a +> job that polls a database flag). taskito's `.gate()` is a first-class step +> kind: the run genuinely pauses, and `approveGate`/`rejectGate` resolve it +> from any process. diff --git a/docs/content/docs/node/guides/workflows/index.mdx b/docs/content/docs/node/guides/workflows/index.mdx index 0cd8b7bb..b5898a6a 100644 --- a/docs/content/docs/node/guides/workflows/index.mdx +++ b/docs/content/docs/node/guides/workflows/index.mdx @@ -6,9 +6,10 @@ description: "Orchestrate multi-step DAGs in Node.js with the Taskito workflow b import { Cards, Card } from "fumadocs-ui/components/card"; import { Callout } from "fumadocs-ui/components/callout"; -Orchestrate multi-step DAGs where each step is a registered task and `after` -declares its dependencies. The Rust core schedules steps in topological order; -a worker advances the run as each step settles. +Orchestrate multi-step DAGs (directed acyclic graphs — steps with dependencies +and no cycles) where each step is a registered task and `after` declares its +dependencies. The Rust core schedules steps in topological order; a worker +advances the run as each step settles. ```ts const handle = queue.workflows @@ -28,6 +29,23 @@ handle.nodes(); // per-step status snapshot Requires the addon built with the `workflows` cargo feature (enabled by `pnpm build:native`). + + `.submit()` enqueues the step jobs; they only execute while a worker (a + process that claims and executes queued jobs) is consuming the queue. Start + one first — `queue.runWorker()`, as above — otherwise `handle.wait()` blocks + until it times out. + + + + BullMQ's closest equivalent is `FlowProducer` — `.add({ name, queueName, + data, children: [...] })` builds a parent/child job **tree**: one parent, + many children, no joins. taskito's workflow builder is a general **DAG**: a + step's `after` can list multiple predecessors, so two branches can merge + back into one step (a join `FlowProducer` can't express). Conditions, gates, + sagas, and caching are also taskito-only — there's no BullMQ equivalent for + those. + + + + + diff --git a/docs/content/docs/node/guides/workflows/saga.mdx b/docs/content/docs/node/guides/workflows/saga.mdx index e8951e14..22bfed9c 100644 --- a/docs/content/docs/node/guides/workflows/saga.mdx +++ b/docs/content/docs/node/guides/workflows/saga.mdx @@ -81,6 +81,10 @@ rollback. partially consistent. +> **Coming from BullMQ?** There's no built-in saga/compensation concept in +> BullMQ — you'd hand-roll rollback logic in your own failure handler. taskito's +> `compensate` option and automatic reverse-order rollback are taskito-only. + ## Compensate task signature The compensate task receives the forward step's return value as its single diff --git a/docs/content/docs/node/guides/workflows/sub-workflows.mdx b/docs/content/docs/node/guides/workflows/sub-workflows.mdx index a8ada562..00027cc1 100644 --- a/docs/content/docs/node/guides/workflows/sub-workflows.mdx +++ b/docs/content/docs/node/guides/workflows/sub-workflows.mdx @@ -125,3 +125,9 @@ the DAG — downstream steps are skipped. To handle child failures without failing the parent, wrap the sub-workflow node in a conditional branch: place an `on_failure` step after it and an `on_success` step for the happy path. See [Conditions](/node/guides/workflows/conditions). + +> **Coming from BullMQ?** `FlowProducer`'s nested `children` arrays are the +> closest BullMQ concept, but they're structural, not a linked *run*: taskito's +> child is a fully independent workflow run with its own id, state, and +> `queue.workflows.children()` listing — closer to calling `.submit()` from +> inside a step than to a static tree. From 709fc2432ddcd3f9026bc32316108a5eb279d713 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:34:37 +0530 Subject: [PATCH 22/57] docs(node): fix operations and integration guides --- .../node/guides/extensibility/middleware.mdx | 7 ++-- .../docs/node/guides/integrations/express.mdx | 4 +- .../docs/node/guides/integrations/otel.mdx | 3 +- .../node/guides/observability/monitoring.mdx | 15 ++++--- .../docs/node/guides/operations/backends.mdx | 8 +++- .../docs/node/guides/operations/cli.mdx | 3 +- .../node/guides/operations/dashboard-api.mdx | 2 +- .../docs/node/guides/operations/migration.mdx | 6 +++ .../docs/node/guides/operations/security.mdx | 6 +-- .../node/guides/resources/interception.mdx | 40 +++++++++++++++++++ 10 files changed, 77 insertions(+), 17 deletions(-) diff --git a/docs/content/docs/node/guides/extensibility/middleware.mdx b/docs/content/docs/node/guides/extensibility/middleware.mdx index bad54c6f..7b9202ce 100644 --- a/docs/content/docs/node/guides/extensibility/middleware.mdx +++ b/docs/content/docs/node/guides/extensibility/middleware.mdx @@ -34,9 +34,10 @@ The execution hooks (`before`/`after`/`onError`) receive a context with `taskName`, `jobId`, and `args`. The outcome hooks receive an event with `taskName`, `jobId`, `queue`, `retryCount`, and `timedOut` (which separates a timeout from other failures). `onEnqueue` receives a *mutable* enqueue context — -see [Enqueue interception](/node/guides/resources/interception). Multiple -middlewares run in registration order; `before` outermost-first, `after` in -reverse. +see [Enqueue interception](/node/guides/resources/interception). Every hook — +`before`, `after`, `onError`, and the outcome hooks — runs in the same +registration order for every middleware; the Node SDK doesn't reverse `after` +the way an "onion" middleware model would. Middleware wraps execution and can do async work in the hot path — keep it diff --git a/docs/content/docs/node/guides/integrations/express.mdx b/docs/content/docs/node/guides/integrations/express.mdx index fec646a0..694952b0 100644 --- a/docs/content/docs/node/guides/integrations/express.mdx +++ b/docs/content/docs/node/guides/integrations/express.mdx @@ -59,6 +59,6 @@ app.use("/admin", taskitoDashboard(queue)); The SPA assets must be built first: `pnpm build:dashboard`. The built files - land in `node_modules/taskito/static/dashboard` by default; pass `staticDir` - to point elsewhere. + land in `node_modules/@byteveda/taskito/static/dashboard` by default; pass + `staticDir` to point elsewhere. diff --git a/docs/content/docs/node/guides/integrations/otel.mdx b/docs/content/docs/node/guides/integrations/otel.mdx index 6715cd09..c7bd95a5 100644 --- a/docs/content/docs/node/guides/integrations/otel.mdx +++ b/docs/content/docs/node/guides/integrations/otel.mdx @@ -12,7 +12,8 @@ import { otelMiddleware } from "@byteveda/taskito/contrib/otel"; queue.use(otelMiddleware()); ``` -One span is created per execution attempt, named `taskito.execute.`. +One span (a single traced operation, OpenTelemetry's unit of work) is created +per execution attempt, named `taskito.execute.`. The span ends with status `OK` on success and `ERROR` (with the exception recorded) on throw. A retry is a new attempt, so it produces a new span. diff --git a/docs/content/docs/node/guides/observability/monitoring.mdx b/docs/content/docs/node/guides/observability/monitoring.mdx index 95b0459a..52ff5624 100644 --- a/docs/content/docs/node/guides/observability/monitoring.mdx +++ b/docs/content/docs/node/guides/observability/monitoring.mdx @@ -20,15 +20,20 @@ await queue.statsAllQueues(); // Record ## Metrics -`getMetrics(windowMs, task?)` aggregates finished jobs over a trailing window -into per-task counts, a success/failure split, and latency percentiles -(p50/p95/p99): +`getMetrics(sinceMs, task?)` returns the raw per-execution metric rows +(`taskName`, `jobId`, `wallTimeNs`, `memoryBytes`, `succeeded`, `recordedAt`) +recorded at or after `sinceMs` (a Unix-ms timestamp, not a duration) — one +entry per finished job, not a pre-aggregated summary: ```ts -await queue.getMetrics(3_600_000); // last hour, all tasks -await queue.getMetrics(3_600_000, "add"); // one task +await queue.getMetrics(Date.now() - 3_600_000); // last hour, all tasks +await queue.getMetrics(Date.now() - 3_600_000, "add"); // one task ``` +Aggregate them yourself (counts, success/failure split, p50/p95/p99 latency +percentiles) — the [dashboard](/node/guides/operations/dashboard) does exactly +this over `/api/metrics` and `/api/metrics/timeseries`. + ## Progress A running task reports progress through its job context; inspection and the diff --git a/docs/content/docs/node/guides/operations/backends.mdx b/docs/content/docs/node/guides/operations/backends.mdx index fed98238..61834298 100644 --- a/docs/content/docs/node/guides/operations/backends.mdx +++ b/docs/content/docs/node/guides/operations/backends.mdx @@ -7,6 +7,12 @@ The queue stores everything — jobs, results, schedules, locks, rate-limit stat — in one backend. The API is identical across all three; only the constructor differs. +Unlike BullMQ, there's no separate broker (message-routing middleman process) +to run — SQLite and Postgres need nothing beyond a database, and even the Redis +backend is just a data store here, not a required dependency of the queue +itself. Start on SQLite with zero infrastructure and move to Postgres or Redis +later without changing task code. + ```ts new Queue({ dbPath: "taskito.db" }); // SQLite (default) new Queue({ backend: "postgres", dsn: process.env.PG_URL, schema: "taskito" }); @@ -15,7 +21,7 @@ new Queue({ backend: "redis", dsn: "redis://localhost", prefix: "taskito" }); | Backend | Use when | Notes | |---|---|---| -| **SQLite** | Single host, simplest deploy | One file, WAL mode, no server. Bundled. | +| **SQLite** | Single host, simplest deploy | One file, WAL (write-ahead log) mode, no server. Bundled. | | **PostgreSQL** | Multiple hosts, durability | `SELECT ... FOR UPDATE SKIP LOCKED` dispatch; isolated `schema`. | | **Redis** | High throughput, ephemeral OK | JSON values + sorted sets; atomic Lua claims. | diff --git a/docs/content/docs/node/guides/operations/cli.mdx b/docs/content/docs/node/guides/operations/cli.mdx index ba4e11e9..bc124d41 100644 --- a/docs/content/docs/node/guides/operations/cli.mdx +++ b/docs/content/docs/node/guides/operations/cli.mdx @@ -12,8 +12,9 @@ taskito --db taskito.db stats taskito --db taskito.db jobs --status failed taskito --db taskito.db dlq list taskito --db taskito.db dlq retry -taskito --db taskito.db queues taskito --db taskito.db pause default +taskito --db taskito.db resume default +taskito --db taskito.db paused taskito --db taskito.db cancel ``` diff --git a/docs/content/docs/node/guides/operations/dashboard-api.mdx b/docs/content/docs/node/guides/operations/dashboard-api.mdx index d3afdc81..0e71ec6c 100644 --- a/docs/content/docs/node/guides/operations/dashboard-api.mdx +++ b/docs/content/docs/node/guides/operations/dashboard-api.mdx @@ -28,7 +28,7 @@ directly too. All timestamps are Unix milliseconds. ### Webhook deliveries -`GET /api/webhooks/:id/deliveries` supports `?status=&limit=&offset=` and +`GET /api/webhooks/:id/deliveries` supports `?status=&event=&limit=&offset=` and returns a page, fields in snake_case: ```json diff --git a/docs/content/docs/node/guides/operations/migration.mdx b/docs/content/docs/node/guides/operations/migration.mdx index bc415f66..2f47dd3e 100644 --- a/docs/content/docs/node/guides/operations/migration.mdx +++ b/docs/content/docs/node/guides/operations/migration.mdx @@ -17,9 +17,11 @@ handlers are **registered by name** on the queue rather than wired to a separate | `queue.add("job", data, opts)` | `queue.task("job", fn)` once, then `queue.enqueue("job", args, opts)` | | `job.data` | the positional `args` passed to the handler | | `job.id`, progress | `currentJob()` → `jobId`, `setProgress()` | +| `await job.waitUntilFinished(queueEvents)` | `await queue.result(id)` | | `attempts` + `backoff` | `maxRetries` + `retryBackoff` | | `rateLimiter` | per-task / per-queue `rateLimit: "100/m"` | | `QueueEvents` | `queue.on(event, …)` + [middleware](/node/guides/extensibility/middleware) | +| `QueueScheduler` | not needed — a running `queue.runWorker()` handles delayed and repeatable jobs itself | | Repeatable jobs (cron) | `queue.registerPeriodic(name, task, cron)` | | Flows (parent/child) | [Workflows](/node/guides/workflows) (DAG, fan-out, gates, saga) | | `queue.getJobCounts()` | `await queue.stats()` / `statsAllQueues()` | @@ -52,6 +54,10 @@ queue.runWorker({ queues: ["emails"] }); - **Built-in orchestration** — DAG [workflows](/node/guides/workflows) and a [mesh](/node/guides/operations/mesh) replace hand-rolled flows and external schedulers. +- **No separate scheduler process** — delayed and `registerPeriodic` (cron) jobs + fire from the same `queue.runWorker()` poll loop that runs your tasks. If no + worker process is running, cron jobs simply don't fire — there's no + `QueueScheduler`-equivalent to run independently. Migrating incrementally? Point Taskito at the same Redis you already run, move diff --git a/docs/content/docs/node/guides/operations/security.mdx b/docs/content/docs/node/guides/operations/security.mdx index 44b2d692..b0e616ab 100644 --- a/docs/content/docs/node/guides/operations/security.mdx +++ b/docs/content/docs/node/guides/operations/security.mdx @@ -41,9 +41,9 @@ const expected = "sha256=" + createHmac("sha256", secret).update(rawBody).digest // timing-safe compare against the x-taskito-signature header ``` -The deliverer fetches the configured URL directly — there is **no SSRF allowlist**. -Only register webhook URLs you control, and validate them before storing if they -come from users. +The deliverer fetches the configured URL directly — there is **no SSRF (server-side +request forgery) allowlist**. Only register webhook URLs you control, and +validate them before storing if they come from users. ## Dashboard diff --git a/docs/content/docs/node/guides/resources/interception.mdx b/docs/content/docs/node/guides/resources/interception.mdx index 2d8781ca..b1c201c0 100644 --- a/docs/content/docs/node/guides/resources/interception.mdx +++ b/docs/content/docs/node/guides/resources/interception.mdx @@ -63,6 +63,46 @@ queue.enqueue("charge", [-5]); // throws — nothing is enqueued in [Middleware](/node/guides/extensibility/middleware). +## Enqueue interceptors + +`queue.intercept(interceptor)` is a separate, more structural mechanism. It +runs *before* `onEnqueue`, per-task defaults, middleware, and gates, and +decides the fate of the whole enqueue rather than mutating a shared context +in place: + +```ts +import { Interception } from "@byteveda/taskito"; + +queue.intercept((taskName, args) => { + if (taskName === "chargeCard" && (args[0] as number) < 0) { + return Interception.reject("charge amount must be non-negative"); + } + return Interception.pass(); +}); +``` + +An interceptor returns one of four outcomes: + +| Outcome | Effect | +|---|---| +| `Interception.pass()` | Enqueue unchanged. | +| `Interception.convert(args)` | Replace the args; the task name stays the same. | +| `Interception.redirect(taskName, args)` | Enqueue a different task with new args instead. | +| `Interception.reject(reason)` | Block the enqueue — `enqueue`/`enqueueMany` throws `InterceptionError`. | + +Multiple interceptors chain in registration order, each seeing the previous +one's (possibly redirected) task name and args. `redirect` is rejected for +`enqueueMany` (a batch is stored under one task name) and for tasks +registered with per-task [codecs](/node/api-reference/serializers) — the +redirect target's codec chain can't be resolved from a bare name. + + + `intercept` and `onEnqueue` solve overlapping problems differently: + `intercept` runs first and can redirect to a different task or reject + outright; `onEnqueue` runs after and mutates the (possibly redirected) + context in place. Most apps need only one. + + ## Why no proxies? Python's SDK also ships object *proxies* — a system for shipping non-serializable From 56eacc2f757686468b3ac4dccf8d393fa44209d2 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:34:37 +0530 Subject: [PATCH 23/57] docs(node): fix API reference and examples --- docs/content/docs/node/api-reference/cli.mdx | 9 ++-- .../docs/node/api-reference/errors.mdx | 5 +++ .../content/docs/node/api-reference/queue.mdx | 15 +++++-- .../docs/node/api-reference/result.mdx | 8 ++-- .../docs/node/api-reference/serializers.mdx | 41 +++++++++++++++++++ docs/content/docs/node/api-reference/task.mdx | 11 +++-- .../docs/node/api-reference/workflows.mdx | 15 ++++++- .../docs/node/more/examples/benchmark.mdx | 4 +- .../docs/node/more/examples/bulk-emails.mdx | 2 +- .../docs/node/more/examples/data-pipeline.mdx | 9 ++-- .../docs/node/more/examples/web-scraper.mdx | 4 +- .../docs/node/more/examples/workflows.mdx | 2 +- 12 files changed, 99 insertions(+), 26 deletions(-) diff --git a/docs/content/docs/node/api-reference/cli.mdx b/docs/content/docs/node/api-reference/cli.mdx index d14ae9f0..e1dd0935 100644 --- a/docs/content/docs/node/api-reference/cli.mdx +++ b/docs/content/docs/node/api-reference/cli.mdx @@ -15,14 +15,15 @@ on read commands prints machine-readable output. | Command | Description | |---|---| | `enqueue ''` | Enqueue a job. | -| `stats` | Overall queue stats. | +| `stats [-q ]` | Overall stats, or a single queue's with `-q`. | | `jobs [--status ] [--limit ]` | List jobs. | -| `queues` | Per-queue stats. | -| `cancel ` | Request cancellation. | +| `cancel ` | Cancel a pending job, or request cancellation of a running one. | | `pause ` / `resume ` | Queue control. | -| `dlq list` / `dlq retry ` / `dlq delete ` | Dead-letter ops. | +| `paused` | List paused queues. | +| `dlq list` / `dlq retry ` / `dlq delete ` / `dlq purge [--older-than-ms ]` | Dead-letter (DLQ) ops. | | `run [--queues a,b]` | Load a module exporting a `Queue` and run its worker. | | `dashboard [--port ]` | Serve the [dashboard](/node/guides/operations/dashboard). | +| `scaler [--port ] [--target-queue-depth ]` | Serve the [KEDA](/node/guides/operations/keda) scaler metric endpoint. | ```bash taskito --db taskito.db enqueue add '[2,3]' diff --git a/docs/content/docs/node/api-reference/errors.mdx b/docs/content/docs/node/api-reference/errors.mdx index ef4c1e77..7f19664a 100644 --- a/docs/content/docs/node/api-reference/errors.mdx +++ b/docs/content/docs/node/api-reference/errors.mdx @@ -29,13 +29,18 @@ try { | `ResultTimeoutError` | `TaskitoError` | `result()` times out before the job settles. | | `QueueError` | `TaskitoError` | Queue construction / operational error (e.g. no `dbPath`/`dsn`). | | `LockNotAcquiredError` | `TaskitoError` | `withLock` can't acquire a held lock. | +| `LockLostError` | `TaskitoError` | A held lock's lease expired before `withLock`'s guarded section finished. | | `SerializationError` | `TaskitoError` | (De)serialization or payload-integrity failure (e.g. a bad `SignedSerializer` signature). | +| `CryptoError` | `SerializationError` | A [payload codec](/node/api-reference/serializers) (`HmacCodec`, `AesGcmCodec`) fails to decrypt or verify. | | `NotesValidationError` | `TaskitoError` | A `notes` object breaks the contract (>15 fields or >4 KiB). | | `WorkflowError` | `TaskitoError` | Workflow definition, submission, or query error. | | `PredicateRejectedError` | `TaskitoError` | An enqueue-time [gate](/node/guides/core/predicates) rejected the submission. | +| `InterceptionError` | `TaskitoError` | An [enqueue interceptor](/node/guides/resources/interception) rejects, misbehaves, or redirects illegally. | +| `ProxyError` | `TaskitoError` | A [proxy handler](/node/guides/resources/interception) signature, expiry, purpose, or allowlist failure. | | `ResourceError` | `TaskitoError` | Base for resource dependency-injection errors. | | `ResourceNotFoundError` | `ResourceError` | Resolving a resource name that was never registered. | | `ResourceScopeError` | `ResourceError` | Resolving a task-scoped resource at worker scope. | +| `ResourceUnavailableError` | `ResourceError` | A pooled resource couldn't be checked out before its acquire timeout. | Selected errors carry context: `TaskNotRegisteredError.taskName`, `JobFailedError.jobId`, `ResultTimeoutError.timeoutMs`, diff --git a/docs/content/docs/node/api-reference/queue.mdx b/docs/content/docs/node/api-reference/queue.mdx index 80f29143..0b008c25 100644 --- a/docs/content/docs/node/api-reference/queue.mdx +++ b/docs/content/docs/node/api-reference/queue.mdx @@ -21,6 +21,8 @@ const queue = new Queue(options); | `prefix` | `string` | Redis key prefix. | | `namespace` | `string` | Namespace applied to enqueued jobs + the worker scheduler. | | `serializer` | `Serializer` | Arg/result serializer (default `JsonSerializer`). | +| `codec` | `PayloadCodec \| PayloadCodec[]` | Global codec chain wrapping the serializer — covers every payload and result. See [Serializers](/node/api-reference/serializers). | +| `codecs` | `Record` | Named codec registry for per-task `codecs` (payload only). | ## Tasks & enqueue @@ -35,10 +37,13 @@ const queue = new Queue(options); | Method | Description | |---|---| -| `result(id, { timeoutMs? })` | Await a job's result. | +| `getJob(id)` → `Job \| null` | Fetch a job's current state. | +| `getResult(id)` | Deserialized result of a completed job, or `undefined` if not yet ready. | +| `result(id, { timeoutMs?, pollMs? })` | Await a job's terminal result; rejects on failure/cancellation/timeout. | | `stream(id, { timeoutMs?, pollMs? })` | Async-iterate a job's [published partials](/node/guides/core/streaming). | | `taskLogs(id)` | Raw task-log entries for a job. | -| `requestCancel(id)` | Abort a running job's signal. | +| `requestCancel(id)` | Request cooperative cancellation of a running job. | +| `isCancelRequested(id)` | Whether cancellation has been requested for a job. | | `cancelJob(id)` | Cancel a pending job. | ## Inspection & management @@ -51,8 +56,8 @@ background thread pool. | `stats()` → `Promise` / `statsByQueue(q)` → `Promise` / `statsAllQueues()` → `Promise>` | Counts by status. | | `listJobs(filter)` → `Promise` | List jobs. | | `getJobErrors(id)` → `Promise` | Per-attempt error history. | -| `getMetrics(windowMs, task?)` → `Promise` | Aggregated metrics. | -| `deadLetters()` → `Promise` / `retryDead(id)` / `deleteDead(id)` / `purgeDead(ms)` → `Promise` | DLQ ops. | +| `getMetrics(sinceMs, task?)` → `Promise` | Raw per-execution metric rows recorded at or after the Unix-ms `sinceMs`. | +| `deadLetters()` → `Promise` / `retryDead(id)` / `deleteDead(id)` / `purgeDead(ms)` → `Promise` | DLQ (dead-letter queue) ops. | | `purgeCompleted(ms)` → `Promise` | Drop old completed jobs. | | `pauseQueue(q)` / `resumeQueue(q)` / `listPausedQueues()` | Queue control. | | `listWorkers()` → `Promise` | Registered workers. | @@ -64,6 +69,8 @@ background thread pool. | `runWorker(options?)` | Start a [worker](/node/api-reference/worker). | | `on(event, handler)` | [Events](/node/guides/extensibility/events). | | `use(middleware)` | [Middleware](/node/guides/extensibility/middleware) (incl. the `onEnqueue` [interception](/node/guides/resources/interception) hook). | +| `intercept(interceptor)` | Register an [enqueue interceptor](/node/guides/resources/interception) — runs before defaults, middleware, and gates. | +| `gate(name, predicate)` | Reject an enqueue when the [predicate](/node/guides/core/predicates) returns `false`. | | `resource(name, factory, opts?)` | Register an injectable [resource](/node/guides/resources/dependency-injection) (`scope`, `dispose`). | | `webhooks` | [Webhooks](/node/guides/extensibility/webhooks) manager. | | `workflows` | [Workflow](/node/api-reference/workflows) builder + queries. | diff --git a/docs/content/docs/node/api-reference/result.mdx b/docs/content/docs/node/api-reference/result.mdx index dfd939a1..696c57ee 100644 --- a/docs/content/docs/node/api-reference/result.mdx +++ b/docs/content/docs/node/api-reference/result.mdx @@ -10,9 +10,11 @@ const value = await queue.result( ): Promise; ``` -Resolves with the task's return value once the job finishes. Rejects if the job -dead-letters or the optional `timeoutMs` (default 30000) elapses before -completion. `pollMs` (default 50) sets the store poll interval. +Resolves with the task's return value once the job finishes. Rejects with +`JobFailedError` if the job fails or dead-letters, `JobCancelledError` if it +was cancelled, or `ResultTimeoutError` once the optional `timeoutMs` (default +30000) elapses before a terminal state is reached. `pollMs` (default 50) sets +the store poll interval. ```ts const id = queue.enqueue("add", [2, 3]); diff --git a/docs/content/docs/node/api-reference/serializers.mdx b/docs/content/docs/node/api-reference/serializers.mdx index b0951ab9..a03cf2f8 100644 --- a/docs/content/docs/node/api-reference/serializers.mdx +++ b/docs/content/docs/node/api-reference/serializers.mdx @@ -34,6 +34,47 @@ new Queue({ }); ``` +## Payload codecs + +A `PayloadCodec` is a reversible byte transform layered *around* a serializer +— compression, encryption, signing — rather than a replacement for it. Codecs +compose: a chain encodes in list order on the producer and decodes in reverse +on the worker. Wire formats are part of the cross-SDK contract, so a +codec-framed payload decodes from any Taskito SDK. + +```ts +interface PayloadCodec { + encode(data: Uint8Array): Uint8Array; + decode(data: Uint8Array): Uint8Array; +} +``` + +| Class | Wire format | Description | +|---|---|---| +| `GzipCodec(maxDecompressedBytes?)` | standard gzip stream | Compresses; decompression capped at 64 MiB by default (zip-bomb guard). | +| `HmacCodec(key)` | `[32-byte mac][body]` | HMAC-SHA256 signs; rejects tampered or wrong-key payloads. | +| `AesGcmCodec(key)` | `[12-byte IV][ciphertext \|\| 16-byte tag]` | AES-128/192/256-GCM (by key length); confidentiality + integrity. | + +Apply a chain queue-wide with `QueueOptions.codec` (wraps the serializer, +covers every payload and result), or register named codecs via +`QueueOptions.codecs` and opt individual tasks in with `TaskOptions.codecs` +(payload only — results still use the plain serializer): + +```ts +import { Queue, GzipCodec, HmacCodec } from "@byteveda/taskito"; + +const queue = new Queue({ + dbPath: "taskito.db", + codec: [new GzipCodec(), new HmacCodec(hmacKey)], // encodes gzip, then hmac +}); +``` + + + `CodecSerializer` is the internal `Serializer` that layers a codec chain + around a delegate serializer — `QueueOptions.codec` builds one for you, so + you rarely construct it directly. + + ## `Serializer` interface ```ts diff --git a/docs/content/docs/node/api-reference/task.mdx b/docs/content/docs/node/api-reference/task.mdx index 582351b0..0c381726 100644 --- a/docs/content/docs/node/api-reference/task.mdx +++ b/docs/content/docs/node/api-reference/task.mdx @@ -4,13 +4,15 @@ description: "queue.task registration and per-task configuration." --- ```ts -queue.task(name: string, fn: (...args) => R | Promise, config?: TaskConfig): void +queue.task(name: string, fn: (...args) => R | Promise, options?: TaskOptions): Queue ``` -Registers a task. `fn` may be sync or `async`; its return value becomes the job -result. Re-registering a name replaces the function and config. +Registers a task and returns the queue (chainable) so `enqueue` can infer each +task's argument types from successive `.task()` calls. `fn` may be sync or +`async`; its return value becomes the job result. Re-registering a name +replaces the function and options. -## `TaskConfig` +## `TaskOptions` | Field | Type | Description | |---|---|---| @@ -21,6 +23,7 @@ result. Re-registering a name replaces the function and config. | `rateLimit` | `` `"100/m"` `` | Rate limit as `count/unit`, unit `s` \| `m` \| `h`. | | `circuitBreaker` | `{ threshold, windowMs, cooldownMs }` | Trip after repeated failures. | | `inject` | `string[]` | Resource names injected as a trailing `deps` argument. See [Dependency injection](/node/guides/resources/dependency-injection). | +| `codecs` | `string[]` | Names of registered [payload codecs](/node/api-reference/serializers) applied to this task's payload only. | ```ts queue.task("add", (a: number, b: number) => a + b, { diff --git a/docs/content/docs/node/api-reference/workflows.mdx b/docs/content/docs/node/api-reference/workflows.mdx index d4f2d123..87ef121e 100644 --- a/docs/content/docs/node/api-reference/workflows.mdx +++ b/docs/content/docs/node/api-reference/workflows.mdx @@ -17,7 +17,7 @@ queue.workflows | `step(name, task, opts?)` | A task node. `opts`: `after`, `args`, `queue`, `maxRetries`, `timeoutMs`, `priority`, `condition`, `compensate`, `cache`. | | `fanOut(name, { after, task, itemsFrom })` | Expand into one child per item. | | `fanIn(name, { after, task })` | Collect children's results into an array. | -| `gate(name, { after, timeoutMs?, onTimeout? })` | Pause for [approval](/node/guides/workflows/gates). | +| `gate(name, { after, timeoutMs?, onTimeout?, message? })` | Pause for [approval](/node/guides/workflows/gates). | | `subWorkflow(name, { after, workflow })` | Run a child workflow as a node. | | `chain(steps[], { after? })` | Canvas: wire `steps` sequentially. | | `group(steps[], { after? })` | Canvas: run `steps` in parallel. | @@ -29,6 +29,16 @@ Canvas `steps` are `{ name, task, ...stepOptions }` (`after` is managed by the h `condition`: `"on_success"` (default) · `"on_failure"` · `"always"`. +The chained `submit()` takes no arguments. To set submit-time +`queueDefault`/`params`, stop before the final `.submit()` and pass the +builder to `queue.workflows.submit(builder, { queueDefault?, params? })` +instead: + +```ts +const workflow = queue.workflows.define("etl").step("extract", "extract"); +const run = queue.workflows.submit(workflow, { queueDefault: "io", params: { runId } }); +``` + ## `WorkflowHandle` | Member | Description | @@ -44,7 +54,7 @@ Canvas `steps` are `{ name, task, ...stepOptions }` (`after` is managed by the h queue.workflows.list({ state: "running" }); queue.workflows.run(runId); // a single run snapshot queue.workflows.nodes(runId); // per-step status -queue.workflows.dag(runId); // the DAG graph (nodes + edges) +queue.workflows.dag(runId); // the DAG graph as a JSON string (nodes + edges); JSON.parse it, or use analyze() queue.workflows.children(runId); // spawned sub-workflow runs queue.workflows.approveGate(runId, "review"); queue.workflows.rejectGate(runId, "review", reason); @@ -59,6 +69,7 @@ queue.workflows.clearCache(); // drop all cached step results ```ts const a = queue.workflows.analyze(runId); +a?.node("load"); // one node's status record, or undefined a?.roots(); // entry nodes a?.leaves(); // exit nodes a?.ancestors("load"); // transitive upstream deps diff --git a/docs/content/docs/node/more/examples/benchmark.mdx b/docs/content/docs/node/more/examples/benchmark.mdx index e4367cd6..da46a7fb 100644 --- a/docs/content/docs/node/more/examples/benchmark.mdx +++ b/docs/content/docs/node/more/examples/benchmark.mdx @@ -27,7 +27,7 @@ const enqueueMs = Date.now() - t0; // 2. Processing throughput — drain the backlog, polling stats until empty. const t1 = Date.now(); const worker = queue.runWorker({ queues: ["default"], batchSize: 64, channelCapacity: 512 }); -while ((await queue.stats()).complete < N) { +while ((await queue.stats()).completed < N) { await new Promise((r) => setTimeout(r, 20)); } const processMs = Date.now() - t1; @@ -82,6 +82,6 @@ latency: p50 8ms p99 42ms | Pattern | Where | | --- | --- | | Batched staging | `queue.enqueueMany` | -| Drain to a target | poll `await queue.stats()` for `.complete` | +| Drain to a target | poll `await queue.stats()` for `.completed` | | Worker throughput knobs | `runWorker({ batchSize, channelCapacity })` | | Latency percentiles | `listJobs` + `completedAt - createdAt` | diff --git a/docs/content/docs/node/more/examples/bulk-emails.mdx b/docs/content/docs/node/more/examples/bulk-emails.mdx index 293af852..2771e73c 100644 --- a/docs/content/docs/node/more/examples/bulk-emails.mdx +++ b/docs/content/docs/node/more/examples/bulk-emails.mdx @@ -70,7 +70,7 @@ export function sendCampaign(recipients: string[], subject: string, body: string Watch the backlog drain and catch addresses that exhausted their retries. ```ts -console.log(await queue.statsByQueue("default")); // { pending, running, complete, failed, dead, cancelled } +console.log(await queue.statsByQueue("default")); // { pending, running, completed, failed, dead, cancelled } for (const dead of await queue.deadLetters(50)) { console.warn(`gave up on ${dead.taskName}: ${dead.error}`); diff --git a/docs/content/docs/node/more/examples/data-pipeline.mdx b/docs/content/docs/node/more/examples/data-pipeline.mdx index f530b6f9..4bd6f953 100644 --- a/docs/content/docs/node/more/examples/data-pipeline.mdx +++ b/docs/content/docs/node/more/examples/data-pipeline.mdx @@ -65,13 +65,16 @@ queue.task("load", async (runId: string) => { }); export function runPipeline(runId: string) { - return queue.workflows + // `queueDefault`/`params` are submit-time options, so build the workflow + // without the chained `.submit()` and pass the builder to + // `queue.workflows.submit(builder, options)` instead. + const workflow = queue.workflows .define("etl", 1) .step("extract", "extract", { args: [runId] }) .step("clean", "clean", { args: [runId], after: "extract" }) .step("enrich", "enrich", { args: [runId], after: "extract", queue: "io" }) - .step("load", "load", { args: [runId], after: ["clean", "enrich"], priority: 10 }) - .submit({ queueDefault: "default", params: { runId } }); + .step("load", "load", { args: [runId], after: ["clean", "enrich"], priority: 10 }); + return queue.workflows.submit(workflow, { queueDefault: "default", params: { runId } }); } ``` diff --git a/docs/content/docs/node/more/examples/web-scraper.mdx b/docs/content/docs/node/more/examples/web-scraper.mdx index 1575e538..42affa85 100644 --- a/docs/content/docs/node/more/examples/web-scraper.mdx +++ b/docs/content/docs/node/more/examples/web-scraper.mdx @@ -89,7 +89,7 @@ const run = queue.workflows .step("list", "listUrls", { args: [urls] }) .fanOut("fetch", { after: "list", task: "fetchPage", itemsFrom: "list", queue: "network" }) .fanIn("aggregate", { after: "fetch", task: "aggregate" }) - .submit({ queueDefault: "default" }); + .submit(); const final = await run.wait({ timeoutMs: 120_000 }); console.log(run.runId, final.state); @@ -143,5 +143,5 @@ process.on("SIGINT", () => { | Transient-failure retries | `retryBackoff` | | Workload isolation | `network` queue + `runWorker({ queues })` | | Cross-cutting logging | `queue.use({ before, after, onError })` | -| Parallel-then-join | workflow `chord` | +| Parallel-then-join over results | workflow `fanOut` / `fanIn` | | Scheduled maintenance | `registerPeriodic` + `purgeCompleted` | diff --git a/docs/content/docs/node/more/examples/workflows.mdx b/docs/content/docs/node/more/examples/workflows.mdx index 0eda762c..0e817323 100644 --- a/docs/content/docs/node/more/examples/workflows.mdx +++ b/docs/content/docs/node/more/examples/workflows.mdx @@ -117,7 +117,7 @@ if (a) { console.log("roots:", a.roots()); console.log("levels:", a.topologicalLevels()); console.log("critical path:", a.criticalPath()); - console.log("stats:", a.stats()); // { total, completed, failed, running, pending } + console.log("stats:", a.stats()); // { total, byStatus, completed, failed, running, pending } } ``` From 32c618a5892ceab3da8d6ea5d89afe264e5592ad Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:34:49 +0530 Subject: [PATCH 24/57] docs(java): fix getting-started and core guides --- .../java/getting-started/capabilities.mdx | 2 +- .../docs/java/getting-started/concepts.mdx | 27 +++++++++++++++++++ docs/content/docs/java/guides/core/index.mdx | 1 + .../docs/java/guides/core/streaming.mdx | 7 +++++ .../content/docs/java/guides/core/workers.mdx | 9 +++++++ 5 files changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/content/docs/java/getting-started/capabilities.mdx b/docs/content/docs/java/getting-started/capabilities.mdx index 78849af6..092567a9 100644 --- a/docs/content/docs/java/getting-started/capabilities.mdx +++ b/docs/content/docs/java/getting-started/capabilities.mdx @@ -45,7 +45,7 @@ core — pick a backend, describe tasks, run workers. | Resource injection, enqueue predicates & interception | [Resources](/java/guides/resources) · [Predicates](/java/guides/core/predicates) | | Signed cross-process references (proxies) | [Resources](/java/guides/resources) | | Pluggable serializers (JSON / MessagePack / signed / encrypted) & payload codecs | [Serializers](/java/guides/extensibility/serializers) | -| Producer-side batching (`Batcher`) | [Extensibility](/java/guides/extensibility) | +| Producer-side batching (`Batcher`) | [Batching](/java/guides/core/batching) | ## Observability & ops diff --git a/docs/content/docs/java/getting-started/concepts.mdx b/docs/content/docs/java/getting-started/concepts.mdx index cb945029..3300da8c 100644 --- a/docs/content/docs/java/getting-started/concepts.mdx +++ b/docs/content/docs/java/getting-started/concepts.mdx @@ -6,6 +6,28 @@ description: "Client, task, job, worker, result — the Java mental model over t The Java SDK is a typed shell; the scheduler, dispatcher, worker pool, and storage are all in the shared Rust engine. Five concepts cover the surface. +## Why no broker + +A typical JVM background-job setup runs several separate services: a message +**broker** (a standalone server that holds pending jobs — RabbitMQ, JMS, or a +Redis-backed queue) that hands work to consumers, somewhere to store results, +and the worker processes themselves. Spring's `@Async` skips the broker but +skips durability too — queued calls live in an in-process executor and vanish +on restart. + +taskito collapses all of that into the `Taskito` client: one SQLite file (or a +Postgres schema, or Redis) holds the job queue, the results, and any cron +schedules together. There's no broker to install, secure, or monitor — +`Taskito.builder().sqlite(...).open()` is the whole setup. + + + Your `ConnectionFactory` / `spring.rabbitmq.*` / JMS listener config + disappears — a single `Taskito.builder().sqlite("taskito.db").open()` + replaces the broker, the result store, and the connection pool. Unlike + `@Async`, jobs survive an app restart: they're rows in storage, not + in-memory tasks. + + + This loop assumes a worker is already running the `train` handler + elsewhere — e.g. `taskito.worker().handle(train, p -> runTraining(p)) + .start()` in another process or thread. Without one, the job stays + `PENDING` and the loop polls forever. + + ```java String id = taskito.enqueue(train, config); diff --git a/docs/content/docs/java/guides/core/workers.mdx b/docs/content/docs/java/guides/core/workers.mdx index 40d16217..16474905 100644 --- a/docs/content/docs/java/guides/core/workers.mdx +++ b/docs/content/docs/java/guides/core/workers.mdx @@ -21,6 +21,15 @@ core, and handlers execute on a Java thread pool. The worker registers itself and heartbeats, so it appears in `listWorkers()` and on the dashboard; `close()` unregisters it. + + `@Async` runs the call inline on a Spring-managed thread pool the moment + it's invoked — there's no separate producer/consumer step, and queued calls + are lost on restart. A `Worker` is a separate, explicitly started process + (or thread) that polls durable storage; nothing runs until one is up, and + work already enqueued survives a restart because it lives in the store, not + in a JVM executor. + + ## Builder options | Option | Description | From 38d43fb5140b43c9898492e921bbd4923bc6a092 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:34:49 +0530 Subject: [PATCH 25/57] docs(java): fix reliability and workflow guides --- .../java/guides/reliability/dead-letter.mdx | 6 ++++ .../java/guides/reliability/guarantees.mdx | 6 ++++ .../docs/java/guides/workflows/caching.mdx | 9 ++++++ .../docs/java/guides/workflows/conditions.mdx | 7 +++++ .../docs/java/guides/workflows/index.mdx | 30 ++++++++++++++++++- 5 files changed, 57 insertions(+), 1 deletion(-) diff --git a/docs/content/docs/java/guides/reliability/dead-letter.mdx b/docs/content/docs/java/guides/reliability/dead-letter.mdx index adecb92a..490a1f34 100644 --- a/docs/content/docs/java/guides/reliability/dead-letter.mdx +++ b/docs/content/docs/java/guides/reliability/dead-letter.mdx @@ -7,6 +7,12 @@ When a job exhausts its retries it moves to the **dead-letter queue** (DLQ) and the `DEAD` event fires. The DLQ keeps the payload and error history so you can investigate and replay. + + There's no separate DLQ/DLX to declare, bind, or point a routing key at — + taskito's dead-letter store is built into the same client and queried + through the same API (`listDead`, `retryDead`) as everything else. + + ```java List dead = queue.listDead(50, 0); // newest first, paged List charges = queue.listDeadByTask("charge", 50, 0); diff --git a/docs/content/docs/java/guides/reliability/guarantees.mdx b/docs/content/docs/java/guides/reliability/guarantees.mdx index 33582282..d597487b 100644 --- a/docs/content/docs/java/guides/reliability/guarantees.mdx +++ b/docs/content/docs/java/guides/reliability/guarantees.mdx @@ -15,6 +15,12 @@ guarded update — before executing, so two workers never run the same job concurrently. If that worker dies, the claim lapses and the scheduler dispatches the job again. + + There's no manual ack/nack: a claimed job's redelivery is driven by the + claim lapsing, not by a consumer forgetting to acknowledge. You still get + the same at-least-once contract you'd design a JMS listener around. + + ## Staying correct: idempotency Because a job may run more than once, make side effects idempotent: diff --git a/docs/content/docs/java/guides/workflows/caching.mdx b/docs/content/docs/java/guides/workflows/caching.mdx index c7dfcf7b..b6c0e55f 100644 --- a/docs/content/docs/java/guides/workflows/caching.mdx +++ b/docs/content/docs/java/guides/workflows/caching.mdx @@ -17,6 +17,15 @@ Workflow report = Workflow.named("report") .step("render", renderTask, 1, "crunch"); ``` + + These snippets assume a worker is already running with `.trackWorkflows()` + and handlers for `seedTask`, `crunchTask`, and `renderTask` — e.g. + `queue.worker().handle(seedTask, ...).handle(crunchTask, + ...).handle(renderTask, ...).trackWorkflows().start()`. `submitWorkflow` + only creates the run; a tracking worker is what advances it and applies the + cache. + + On the first run `crunch` executes normally; on a second run within five minutes its node status is `CACHE_HIT` and the task is not re-executed — `render` still runs. diff --git a/docs/content/docs/java/guides/workflows/conditions.mdx b/docs/content/docs/java/guides/workflows/conditions.mdx index f4bc2d7a..26fdc8b2 100644 --- a/docs/content/docs/java/guides/workflows/conditions.mdx +++ b/docs/content/docs/java/guides/workflows/conditions.mdx @@ -25,6 +25,13 @@ WorkflowStatus status = run.await(Duration.ofSeconds(30)); status.state; // FAILED — even if recover ran (see below) ``` + + `run.await(...)` only returns once a worker is running and tracking this + workflow — e.g. `queue.worker().handle(riskyTask, ...).handle(celebrateTask, + ...).handle(recoverTask, ...).trackWorkflows(pipeline).start()`. Without a + tracking worker, the run stays `PENDING` forever. + + ## Condition values | Builder method | Wire value | When the step runs | diff --git a/docs/content/docs/java/guides/workflows/index.mdx b/docs/content/docs/java/guides/workflows/index.mdx index b6a90cfe..4313a035 100644 --- a/docs/content/docs/java/guides/workflows/index.mdx +++ b/docs/content/docs/java/guides/workflows/index.mdx @@ -3,7 +3,8 @@ title: Workflows description: "Orchestrate multi-step DAGs in Java with the Taskito workflow builder." --- -Orchestrate multi-step DAGs where each step is a registered task and its +Orchestrate multi-step DAGs (directed acyclic graphs — steps as nodes, `after` +edges between them, no cycles) where each step is a registered task and its `after` list declares its dependencies. The Rust core schedules steps in topological order; a worker that calls `trackWorkflows()` advances the run as each step settles. @@ -49,7 +50,19 @@ supply payloads at submit time via payloads and predicates. + + Wiring a multi-step pipeline by hand usually means one listener per step and + custom state (a status column, a Redis key) to track what's done. A + `Workflow` declares the whole DAG — steps, `after` dependencies, and + retries — once, and the tracker walks the graph instead of your code. + + + + + + From 5d26e404ef85953477eaa068b1bd5e2137d1fa08 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:34:49 +0530 Subject: [PATCH 26/57] docs(java): fix operations and integration guides --- docs/content/docs/java/guides/extensibility/webhooks.mdx | 4 +++- docs/content/docs/java/guides/integrations/spring.mdx | 2 +- docs/content/docs/java/guides/operations/backends.mdx | 8 +++++++- docs/content/docs/java/guides/operations/deployment.mdx | 7 +++++++ docs/content/docs/java/guides/operations/graalvm.mdx | 7 ++++--- docs/content/docs/java/guides/operations/index.mdx | 1 + docs/content/docs/java/guides/resources/index.mdx | 3 +++ 7 files changed, 26 insertions(+), 6 deletions(-) diff --git a/docs/content/docs/java/guides/extensibility/webhooks.mdx b/docs/content/docs/java/guides/extensibility/webhooks.mdx index 2910dcb9..7ec27d9d 100644 --- a/docs/content/docs/java/guides/extensibility/webhooks.mdx +++ b/docs/content/docs/java/guides/extensibility/webhooks.mdx @@ -76,7 +76,9 @@ with the shared secret's raw UTF-8 bytes and comparing against the ## SSRF guard -Webhook URLs are vetted by `WebhookUrlValidator` before dashboard-submitted +SSRF (tricking the server into fetching an internal URL) is the main risk of +letting users configure outbound webhook targets. Webhook URLs are vetted by +`WebhookUrlValidator` before dashboard-submitted subscriptions are stored, and **every delivery re-validates the URL right before sending** — regardless of how the subscription was created — so a hostname that starts safe and is later rebound to an internal address (DNS diff --git a/docs/content/docs/java/guides/integrations/spring.mdx b/docs/content/docs/java/guides/integrations/spring.mdx index 8acd3b53..091626c6 100644 --- a/docs/content/docs/java/guides/integrations/spring.mdx +++ b/docs/content/docs/java/guides/integrations/spring.mdx @@ -91,7 +91,7 @@ taskito: | Property | Type | Default | Description | |---|---|---|---| | `taskito.dashboard.enabled` | `boolean` | `false` | Auto-start the dashboard server | -| `taskito.dashboard.port` | `int` | `8080` | Bind port (`0` for ephemeral) | +| `taskito.dashboard.port` | `int` | `8081` | Bind port (`0` for ephemeral); defaults off Spring Boot's own `8080` | | `taskito.dashboard.token` | `String` | none | Legacy shared token gating `/api/*`; unset enables [session auth](/java/guides/operations/dashboard#session-auth-default) | | `taskito.dashboard.static-dir` | `String` | auto-discovered | Directory of a prebuilt SPA, overriding the jar's extracted copy | | `taskito.dashboard.secure-cookies` | `boolean` | `true` | Drop the `Secure` cookie attribute for local HTTP development | diff --git a/docs/content/docs/java/guides/operations/backends.mdx b/docs/content/docs/java/guides/operations/backends.mdx index 6c66e146..3276c2a4 100644 --- a/docs/content/docs/java/guides/operations/backends.mdx +++ b/docs/content/docs/java/guides/operations/backends.mdx @@ -7,6 +7,12 @@ The queue stores everything — jobs, results, schedules, locks, rate-limit stat — in one backend. The API is identical across all three; only the builder call differs. +Unlike a Celery/RabbitMQ setup, there's no separate broker (message-routing +middleman process) to run: the scheduler lives inside the worker process and +reads/writes the backend directly. SQLite and Postgres need nothing beyond a +database, and even the Redis backend is just a data store here, not a required +dependency of the queue itself. + ```java Taskito.builder().open(); // SQLite at .taskito/taskito.db (default) Taskito.builder().sqlite("/var/lib/app/taskito.db").open(); // SQLite, explicit path @@ -16,7 +22,7 @@ Taskito.builder().redis("redis://localhost").prefix("taskito").open(); | Backend | Use when | Notes | |---|---|---| -| **SQLite** | Single host, simplest deploy | One file, WAL mode, no server. Bundled. | +| **SQLite** | Single host, simplest deploy | One file, WAL (write-ahead log) mode, no server. Bundled. | | **PostgreSQL** | Multiple hosts, durability | `SELECT ... FOR UPDATE SKIP LOCKED` dispatch; isolated `schema`. | | **Redis** | High throughput, ephemeral OK | JSON values + sorted sets; atomic Lua claims. | diff --git a/docs/content/docs/java/guides/operations/deployment.mdx b/docs/content/docs/java/guides/operations/deployment.mdx index 10bd9fdf..ed5891ff 100644 --- a/docs/content/docs/java/guides/operations/deployment.mdx +++ b/docs/content/docs/java/guides/operations/deployment.mdx @@ -5,6 +5,13 @@ description: "Process model, native library packaging, and graceful shutdown in ## Process model + + Unlike Celery or a JMS/RabbitMQ setup, taskito's scheduler lives inside the + worker process and reads directly from SQLite, Postgres, or Redis. There's + no separate broker process to run, monitor, or fail over — one less moving + part in production. + + Producers and workers are separate processes that share storage. A typical deploy: diff --git a/docs/content/docs/java/guides/operations/graalvm.mdx b/docs/content/docs/java/guides/operations/graalvm.mdx index eefdc996..5065c0ba 100644 --- a/docs/content/docs/java/guides/operations/graalvm.mdx +++ b/docs/content/docs/java/guides/operations/graalvm.mdx @@ -4,9 +4,10 @@ description: "Compile taskito applications to a native binary with GraalVM." --- The SDK is built to stay native-image friendly, and CI proves it: every change -compiles a smoke application into a GraalVM native image (with -`--no-fallback`, so missing metadata fails the build) and runs the full -enqueue → execute → result path in the compiled binary. +compiles a smoke application into a GraalVM (ahead-of-time compiler to a +native binary) native image (with `--no-fallback`, so missing metadata fails +the build) and runs the full enqueue → execute → result path in the compiled +binary. Two design choices do the heavy lifting: diff --git a/docs/content/docs/java/guides/operations/index.mdx b/docs/content/docs/java/guides/operations/index.mdx index ca36cbf2..84586987 100644 --- a/docs/content/docs/java/guides/operations/index.mdx +++ b/docs/content/docs/java/guides/operations/index.mdx @@ -10,6 +10,7 @@ Run taskito reliably in production. | [Backends](/java/guides/operations/backends) | SQLite, Postgres, and Redis — setup and trade-offs | | [Job Management](/java/guides/operations/inspection) | Inspect, cancel, replay, and clean up jobs | | [Dashboard](/java/guides/operations/dashboard) | Serve the bundled web dashboard and its REST API | +| [SSO (OAuth & OIDC)](/java/guides/operations/sso) | Google, GitHub, and OIDC sign-in for the dashboard | | [Mesh Scheduling](/java/guides/operations/mesh) | Decentralized work-stealing across worker nodes | | [Autoscaling](/java/guides/operations/autoscaling) | In-process thread autoscaling and the KEDA scaler endpoint | | [CLI](/java/guides/operations/cli) | Operate the queue from the terminal | diff --git a/docs/content/docs/java/guides/resources/index.mdx b/docs/content/docs/java/guides/resources/index.mdx index 17b6d89b..59bbe8cc 100644 --- a/docs/content/docs/java/guides/resources/index.mdx +++ b/docs/content/docs/java/guides/resources/index.mdx @@ -25,8 +25,11 @@ try (Taskito queue = Taskito.builder().sqlite("tasks.db").open()) { | Page | What it covers | |---|---| | [Dependency injection](/java/guides/resources/dependency-injection) | `Taskito.resource()`, the five scopes, `Resources.use`, annotation-driven injection, teardown, metrics | +| [Configuration](/java/guides/resources/configuration) | The `resource()` overloads, choosing a scope, teardown, and `PoolConfig` tuning | | [Enqueue interception](/java/guides/resources/interception) | `Interceptor` strategies and the `onEnqueue` hook — validate, rewrite, redirect, and reject jobs before serialization | | [Resource proxies](/java/guides/resources/proxies) | Signed `ProxyRef`s for non-serializable values — HMAC signing, TTL, purpose binding, sessions | +| [Observability](/java/guides/resources/observability) | Reading `resourceMetrics()` — created/disposed/active counters | +| [Testing](/java/guides/resources/testing) | Registering stub resources with `InMemoryTaskito` | ## The five scopes From e8877669d389f79cc38130a8d63ebd088c39eed4 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Thu, 9 Jul 2026 11:34:49 +0530 Subject: [PATCH 27/57] docs(java): fix API reference and examples --- .../content/docs/java/api-reference/index.mdx | 5 +++ .../docs/java/more/examples/saga-checkout.mdx | 38 ++++++++++++++----- .../docs/java/more/examples/web-scraper.mdx | 20 ++++++++++ .../docs/java/more/examples/workflows.mdx | 17 +++++++-- 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/docs/content/docs/java/api-reference/index.mdx b/docs/content/docs/java/api-reference/index.mdx index 3435f7e5..2782a800 100644 --- a/docs/content/docs/java/api-reference/index.mdx +++ b/docs/content/docs/java/api-reference/index.mdx @@ -10,6 +10,7 @@ The public API lives under `org.byteveda.taskito` (Maven coordinates | Page | Covers | |---|---| +| [Overview](/java/api-reference/overview) | The layered architecture — `Taskito` down to the native `QueueBackend`. | | [Taskito](/java/api-reference/queue) | `Taskito.builder()`, enqueue, inspect, admin, locks, periodic. | | [Task](/java/api-reference/task) | `Task.of` typed tasks, `EnqueueOptions`, `RetryPolicy`, the annotations. | | [Worker](/java/api-reference/worker) | The `Worker.Builder` and the running worker handle. | @@ -17,6 +18,10 @@ The public API lives under `org.byteveda.taskito` (Maven coordinates | [Context](/java/api-reference/context) | `Resources.use` and the middleware `TaskContext`. | | [Resources](/java/api-reference/resources) | Worker dependency injection — scopes, pools, disposal. | | [Serializers](/java/api-reference/serializers) | `JsonSerializer`, `MsgpackSerializer`, payload codecs. | +| [Batching](/java/api-reference/batching) | `Batcher` — buffer payloads and flush them as one `enqueueMany` call. | | [Workflows](/java/api-reference/workflows) | The `Workflow` builder, `WorkflowRun`, gates, sagas, analysis. | +| [Canvas](/java/api-reference/canvas) | `Canvas.link`, `chain`, `group`, and `chord` shortcuts. | +| [Saga](/java/api-reference/saga) | `Step.Builder.compensate`, saga run/node states, the rollback contract. | | [Errors](/java/api-reference/errors) | The `TaskitoException` hierarchy. | +| [Testing](/java/api-reference/testing) | The in-memory backend for fast, native-library-free tests. | | [CLI](/java/api-reference/cli) | The `taskito` command. | diff --git a/docs/content/docs/java/more/examples/saga-checkout.mdx b/docs/content/docs/java/more/examples/saga-checkout.mdx index fba9752b..dab1c50d 100644 --- a/docs/content/docs/java/more/examples/saga-checkout.mdx +++ b/docs/content/docs/java/more/examples/saga-checkout.mdx @@ -67,17 +67,37 @@ public final class Checkout { ## Running it The worker registers the forward *and* compensation handlers, and tracks -workflows so the saga orchestrator can drive the rollback. +workflows so the saga orchestrator can drive the rollback. Start it before +submitting — an unclaimed job never progresses, so `run.await(...)` would +otherwise time out. ```java -WorkflowRun run = Checkout.submit(taskito, order); -WorkflowStatus done = run.await(Duration.ofMinutes(1)); - -switch (done.state) { - case COMPLETED -> System.out.println("order placed"); - case COMPENSATED -> System.out.println("rolled back cleanly — customer charged nothing"); - case COMPENSATION_FAILED -> System.err.println("manual intervention needed"); - default -> System.out.println(done.state.wire()); +import java.time.Duration; +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.worker.Worker; +import org.byteveda.taskito.workflows.WorkflowRun; +import org.byteveda.taskito.workflows.WorkflowStatus; + +try (Taskito taskito = Taskito.builder().sqlite("checkout.db").open(); + Worker worker = taskito.worker() + .handle(Tasks.RESERVE, order -> inventory.reserve(order)) + .handle(Tasks.RELEASE, reservation -> inventory.release(reservation)) + .handle(Tasks.CHARGE, order -> payments.charge(order)) + .handle(Tasks.REFUND, charge -> payments.refund(charge)) + .handle(Tasks.SHIP, order -> shipping.create(order)) + .handle(Tasks.CANCEL_SHIPMENT, shipment -> shipping.cancel(shipment)) + .trackWorkflows() + .start()) { + + WorkflowRun run = Checkout.submit(taskito, order); + WorkflowStatus done = run.await(Duration.ofMinutes(1)); + + switch (done.state) { + case COMPLETED -> System.out.println("order placed"); + case COMPENSATED -> System.out.println("rolled back cleanly — customer charged nothing"); + case COMPENSATION_FAILED -> System.err.println("manual intervention needed"); + default -> System.out.println(done.state.wire()); + } } ``` diff --git a/docs/content/docs/java/more/examples/web-scraper.mdx b/docs/content/docs/java/more/examples/web-scraper.mdx index 60ded2aa..7b16c4fb 100644 --- a/docs/content/docs/java/more/examples/web-scraper.mdx +++ b/docs/content/docs/java/more/examples/web-scraper.mdx @@ -169,6 +169,26 @@ public final class WorkerMain { } ``` +## Running it + +`Run.java` only submits and awaits the workflow — it enqueues no worker of its +own, so `run.await(...)` blocks until something dequeues the jobs. Start +`WorkerMain` first (it keeps running and serves both queues), then run `Run` +against the same database: + + + + ```bash + java -cp app.jar WorkerMain + ``` + + + ```bash + java -cp app.jar Run + ``` + + + ## Key patterns | Pattern | Where | diff --git a/docs/content/docs/java/more/examples/workflows.mdx b/docs/content/docs/java/more/examples/workflows.mdx index f4c8be82..eb90db68 100644 --- a/docs/content/docs/java/more/examples/workflows.mdx +++ b/docs/content/docs/java/more/examples/workflows.mdx @@ -3,9 +3,20 @@ title: DAG Workflow Examples description: "Fan-out/fan-in map-reduce, approval gates, error-handling conditions, sub-workflows, caching, and canvas shortcuts." --- -These are focused recipes for the `Workflow` builder. Each is independent — -combine them as your orchestration needs grow. See the -[Workflows reference](/java/api-reference/workflows) for the underlying model. +These are focused recipes for the `Workflow` builder — snippets, not full +programs. Each is independent — combine them as your orchestration needs +grow. See the [Workflows reference](/java/api-reference/workflows) for the +underlying model. + + + Every recipe below assumes a `Worker` is already running against the same + store, with handlers registered for each step's task and `trackWorkflows()` + set where a recipe uses gates, conditions, or sub-workflows — otherwise + `run.await(...)` times out with nothing to dequeue the jobs. See + [ETL Data Pipeline](/java/more/examples/data-pipeline) or + [Web Scraper Pipeline](/java/more/examples/web-scraper) for a full + worker-plus-workflow program. + ## Map-reduce with fan-out / fan-in From 7fd5e2ba92eda22847ce1048c5a6aabe3d06060d Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:51:54 +0530 Subject: [PATCH 28/57] feat(docs): add turquoise color token --- docs/app/styles/tokens.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/app/styles/tokens.css b/docs/app/styles/tokens.css index 75f1660c..ec177fc8 100644 --- a/docs/app/styles/tokens.css +++ b/docs/app/styles/tokens.css @@ -26,6 +26,7 @@ --indigo-soft: rgba(31, 157, 84, 0.12); --indigo-line: rgba(31, 157, 84, 0.26); --cyan: #29b8a8; + --turquoise: #40e0d0; --grn: #4ec985; --amber: #ffb86b; --red: #ff6b6b; @@ -65,6 +66,8 @@ --indigo-soft: rgba(16, 161, 82, 0.1); --indigo-line: rgba(16, 161, 82, 0.26); --cyan: #0d9488; + /* deeper turquoise on warm paper — bright #40e0d0 fails contrast on light bg */ + --turquoise: #0d9e8f; --grn: #15a34a; --amber: #c2410c; --red: #dc2626; From 223e00a10df8ab3356ce1b8378c6522a964d433c Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:51:58 +0530 Subject: [PATCH 29/57] docs: make FAQ content SDK-specific --- docs/content/docs/resources/faq.mdx | 280 +++++++++++++++++++++++++++- 1 file changed, 272 insertions(+), 8 deletions(-) diff --git a/docs/content/docs/resources/faq.mdx b/docs/content/docs/resources/faq.mdx index 47c098b6..767af656 100644 --- a/docs/content/docs/resources/faq.mdx +++ b/docs/content/docs/resources/faq.mdx @@ -3,6 +3,8 @@ title: FAQ description: "Frequently asked questions about taskito." --- + + ## Can I use taskito with Django? Yes. Create a `Queue` instance in one of your Django apps and import it where needed: @@ -45,6 +47,88 @@ def generate_report(report_id: int): ... ``` + + + + +## Can I use taskito with Express, Fastify, or NestJS? + +Yes — all three. Define the queue and tasks once, then mount the REST API and +dashboard with the matching contrib helper. The worker runs the same way +regardless of framework; each helper's peer (`express` / `fastify` / +`@nestjs/common`) is installed separately. + +**Express** — `@byteveda/taskito/contrib/express`: + +```ts +import express from "express"; +import { taskitoRouter, taskitoDashboard } from "@byteveda/taskito/contrib/express"; + +const app = express(); +app.use("/tasks", taskitoRouter(queue)); // REST API +app.use("/admin", taskitoDashboard(queue)); // dashboard SPA +app.listen(3000); +``` + +**Fastify** — `@byteveda/taskito/contrib/fastify`: + +```ts +import Fastify from "fastify"; +import { taskitoFastify, taskitoDashboardPlugin } from "@byteveda/taskito/contrib/fastify"; + +const app = Fastify(); +await app.register(taskitoFastify, { queue, prefix: "/tasks" }); +await app.register(taskitoDashboardPlugin, { queue, prefix: "/admin" }); +await app.listen({ port: 3000 }); +``` + +**NestJS** — `@byteveda/taskito/contrib/nest`: + +```ts +import { Module } from "@nestjs/common"; +import { TaskitoModule } from "@byteveda/taskito/contrib/nest"; + +@Module({ imports: [TaskitoModule.forRoot(queue)] }) +export class AppModule {} +``` + +Inject `TaskitoService` into your providers to enqueue and read results. + + + + + +## Can I use taskito with Spring Boot? + +Yes, via the `org.byteveda:taskito-spring` starter (Spring Boot 3). Add it and +the auto-configuration builds a `Taskito` bean from your `taskito.*` properties +— inject it anywhere: + +```java +@Service +public class SignupService { + private final Taskito taskito; + public SignupService(Taskito taskito) { this.taskito = taskito; } + + public void register(User user) { + taskito.enqueue("send_welcome_email", user.email()); + } +} +``` + +```yaml +taskito: + url: postgres://localhost/taskito + pool-size: 8 + namespace: my-app +``` + +The worker is not auto-started — build one from the injected bean in an +`ApplicationRunner`, or set `taskito.dashboard.enabled=true` to auto-start the +dashboard. Non-Spring apps just build a `Taskito` and `Worker` programmatically. + + + ## Can multiple processes share the same SQLite file? Yes, with caveats. SQLite in WAL mode allows concurrent readers and one writer @@ -64,22 +148,70 @@ The job stays in `running` status in SQLite. On the next worker start, the If no timeout is set, stale jobs remain in `running` status indefinitely. **Always set a timeout on your tasks.** + + ```python @queue.task(timeout=300) # 5 minute timeout def process(data): ... ``` + + + + +```ts +queue.task("process", process, { timeoutMs: 300_000 }); // 5 minute timeout +``` + + + + + +```java +Task process = Task.of("process", Data.class) + .timeout(Duration.ofMinutes(5)); +``` + + + ## How big can the SQLite database get? SQLite can handle databases up to 281 TB (theoretical limit). In practice, -taskito databases stay small if you set `result_ttl` to auto-purge old jobs: +taskito databases stay small if you purge finished jobs. Without cleanup, +expect ~1 KB per job — a million completed jobs ≈ 1 GB. + + + +Set `result_ttl` to auto-purge old jobs: ```python queue = Queue(db_path="myapp.db", result_ttl=86400) # Purge after 24h ``` -Without cleanup, expect ~1 KB per job. A million completed jobs ≈ 1 GB. + + + + +There is no result-TTL knob; call `purgeCompleted` on a schedule (the argument +is milliseconds): + +```ts +setInterval(() => queue.purgeCompleted(86_400_000), 3_600_000); // keep 24h +``` + + + + + +There is no result-TTL knob; call `purgeCompleted` on a schedule (the argument +is milliseconds): + +```java +taskito.purgeCompleted(Duration.ofHours(24).toMillis()); // keep 24h +``` + + ## Can I use a remote or networked SQLite? @@ -89,14 +221,48 @@ SQLite depends on. Always place the database on local storage. ## When should I use Postgres instead of SQLite? -Use the **Postgres backend** (`pip install taskito[postgres]`) when you need: +Use the **Postgres backend** when you need: - **Multi-machine workers** — run workers on separate servers sharing the same queue - **Higher write throughput** — Postgres handles concurrent writers better than SQLite - **Existing Postgres infrastructure** — leverage your existing database and backups -For single-machine workloads, SQLite is simpler and requires zero setup. See -the Postgres backend guide. +For single-machine workloads, SQLite is simpler and requires zero setup. + + + +Install the extra, then point the queue at Postgres: + +```python +# pip install taskito[postgres] +queue = Queue(db_url="postgres://localhost/taskito") +``` + + + + + +Postgres is already compiled into the prebuilt native binary — no extra install, +just switch the backend: + +```ts +new Queue({ backend: "postgres", dsn: process.env.PG_URL, schema: "taskito" }); +``` + + + + + +Postgres ships inside the bundled native library — no extra dependency, just +switch the backend: + +```java +Taskito.builder().postgres(System.getenv("PG_URL")).open(); +``` + + + +See the Postgres backend guide. ## Is taskito production-ready? @@ -108,7 +274,10 @@ multi-server setups, use the Postgres backend + +taskito offers three observability integrations, each implemented as a +`TaskMiddleware` and combinable: | Integration | Best for | Install | |-------------|----------|---------| @@ -116,7 +285,45 @@ taskito offers three observability integrations, each suited to different needs: | **Prometheus** | Metrics dashboards, alerting on queue depth/error rates | `pip install taskito[prometheus]` | | **Sentry** | Error tracking with rich context and breadcrumbs | `pip install taskito[sentry]` | -All three are implemented as `TaskMiddleware` and can be combined together. + + + + +taskito ships three observability middlewares under `@byteveda/taskito/contrib/*`. +Install the peer you use, then register with `queue.use(...)`: + +| Integration | Best for | Peer | +|-------------|----------|------| +| **OpenTelemetry** | Distributed tracing, correlating tasks with HTTP requests | `@opentelemetry/api` | +| **Prometheus** | Metrics dashboards, alerting on queue depth/error rates | `prom-client` | +| **Sentry** | Error tracking with rich context and breadcrumbs | `@sentry/node` | + +```ts +import { otelMiddleware } from "@byteveda/taskito/contrib/otel"; +queue.use(otelMiddleware()); +``` + + + + + +taskito ships two contrib middlewares, registered with `taskito.use(...)`. Their +third-party dependency is `compileOnly` — add the runtime dep yourself: + +| Integration | Best for | Runtime dep | +|-------------|----------|-------------| +| **Micrometer** (`TaskitoObservation`) | One instrumentation → both a metrics timer and a trace span | `io.micrometer:micrometer-observation` | +| **Sentry** (`SentryMiddleware`) | Error tracking on failed attempts and dead-letters | `io.sentry:sentry` | + +```java +taskito.use(new TaskitoObservation(registry)); +``` + +OpenTelemetry and Prometheus are reached downstream of Micrometer — OTel as a +`micrometer-tracing` backend, Prometheus as the dashboard's `/metrics` endpoint +or a Micrometer `MeterRegistry`. There is no dedicated OTel/Prometheus class. + + ## How does taskito compare to running Celery with SQLite? @@ -256,7 +463,9 @@ handles, database connections, or thread locks. ## Can I run the dashboard and worker in the same process? -They're designed to run as separate processes sharing the same database: +They're designed to run as separate processes sharing the same database. + + ```bash # Terminal 1 @@ -269,8 +478,41 @@ taskito dashboard --app myapp:queue For embedding in a FastAPI app, use `TaskitoRouter` instead — it provides the same stats and job management as REST endpoints. + + + + +```bash +# Terminal 1 — run a worker over a module that exports a Queue +taskito run ./app.js --queues default + +# Terminal 2 — serve the dashboard +taskito --db taskito.db dashboard --port 8787 +``` + +To embed the dashboard in an existing HTTP server, call `serveDashboard(queue, { port })`. + + + + + +Workers are started programmatically (there is no `taskito worker` command); +the dashboard has both a programmatic API and a CLI command: + +```java +// In your app — start a worker +Worker worker = taskito.worker().handle(myTask, ...).concurrency(4).start(); + +// Anywhere — start the dashboard +taskito.dashboard(8081); // or DashboardServer.start(taskito, 8081) +``` + + + ## How do I reset / clear all jobs? + + ```python # Purge all completed jobs queue.purge_completed(older_than=0) @@ -279,6 +521,28 @@ queue.purge_completed(older_than=0) queue.purge_dead(older_than=0) ``` + + + + +```ts +// Purge all completed jobs and dead letters (argument is milliseconds; 0 = all) +await queue.purgeCompleted(0); +await queue.purgeDead(0); +``` + + + + + +```java +// Purge all completed jobs and dead letters (argument is milliseconds; 0 = all) +taskito.purgeCompleted(0L); +taskito.purgeDead(0L); +``` + + + Or delete the database file and restart: ```bash From b23a17df9f5b604b862cce0df9e02cda37e61a59 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:51:58 +0530 Subject: [PATCH 30/57] fix(docs): filter on-this-page TOC by active SDK --- docs/app/components/docs/toc.tsx | 36 ++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/docs/app/components/docs/toc.tsx b/docs/app/components/docs/toc.tsx index 5c9f0f77..65903880 100644 --- a/docs/app/components/docs/toc.tsx +++ b/docs/app/components/docs/toc.tsx @@ -1,5 +1,7 @@ import { useEffect, useState } from "react"; import { useLocation } from "react-router"; +import { useActiveSdk } from "@/hooks"; +import type { Sdk } from "@/lib"; interface Heading { id: string; @@ -7,19 +9,30 @@ interface Heading { level: number; } -function readHeadings(article: Element): Heading[] { - return [...article.querySelectorAll("h2[id], h3[id]")].map( - (el) => ({ - id: el.id, - text: el.textContent ?? "", - level: el.tagName === "H3" ? 3 : 2, - }), +/** Headings inside an inactive `SdkOnly` block ship in the HTML but are hidden + * via CSS, so skip any heading whose nearest `data-sdk-variant` ancestor isn't + * the active SDK — otherwise the TOC lists every SDK's questions at once. */ +function visibleHeadingEls(article: Element, sdk: Sdk): HTMLElement[] { + return [...article.querySelectorAll("h2[id], h3[id]")].filter( + (el) => { + const variant = el.closest("[data-sdk-variant]"); + return !variant || variant.getAttribute("data-sdk-variant") === sdk; + }, ); } +function toHeading(el: HTMLElement): Heading { + return { + id: el.id, + text: el.textContent ?? "", + level: el.tagName === "H3" ? 3 : 2, + }; +} + /** On-this-page TOC, built from the rendered article headings with scroll-spy. */ export function Toc() { const { pathname } = useLocation(); + const sdk = useActiveSdk(); const [headings, setHeadings] = useState([]); const [active, setActive] = useState(""); @@ -27,7 +40,7 @@ export function Toc() { // when this effect first runs after a client-side navigation. Re-scan on every // article mutation (via MutationObserver) so the TOC fills in once the page // content actually mounts, and re-arm scroll-spy whenever the heading set changes. - // biome-ignore lint/correctness/useExhaustiveDependencies: re-scan when the path changes + // biome-ignore lint/correctness/useExhaustiveDependencies: re-scan on path or SDK change useEffect(() => { const article = document.querySelector(".article"); if (!article) { @@ -38,7 +51,8 @@ export function Toc() { let lastKey = ""; const scan = () => { - const next = readHeadings(article); + const els = visibleHeadingEls(article, sdk); + const next = els.map(toHeading); const key = next.map((h) => h.id).join("|"); if (key === lastKey) { return; @@ -59,7 +73,7 @@ export function Toc() { }, { rootMargin: "-80px 0px -70% 0px" }, ); - for (const el of article.querySelectorAll("h2[id], h3[id]")) { + for (const el of els) { spy.observe(el); } }; @@ -71,7 +85,7 @@ export function Toc() { content.disconnect(); spy?.disconnect(); }; - }, [pathname]); + }, [pathname, sdk]); if (headings.length === 0) { return