From c0bdcc906a7dc0c333861becfcd896b78002f075 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Tue, 24 Feb 2026 01:34:37 +0530 Subject: [PATCH 1/3] feat: added enhancements --- .github/workflows/ci.yml | 2 +- .github/workflows/publish.yml | 72 ++ .gitignore | 5 +- Cargo.toml | 6 +- README.md | 72 ++ mkdocs.yml | 107 +++ pyproject.toml | 1 + python/quickq/__init__.py | 13 +- python/quickq/_quickq.pyi | 29 + python/quickq/app.py | 244 ++++++- python/quickq/canvas.py | 143 ++++ python/quickq/cli.py | 77 +- python/quickq/context.py | 108 +++ python/quickq/result.py | 45 +- python/quickq/task.py | 37 + rust/quickq-core/Cargo.toml | 6 +- rust/quickq-core/src/dlq.rs | 2 +- rust/quickq-core/src/error.rs | 7 +- rust/quickq-core/src/job.rs | 38 + rust/quickq-core/src/lib.rs | 1 + rust/quickq-core/src/periodic.rs | 44 ++ rust/quickq-core/src/rate_limiter.rs | 3 +- rust/quickq-core/src/scheduler.rs | 121 +++- rust/quickq-core/src/storage/mod.rs | 2 + rust/quickq-core/src/storage/models.rs | 138 ++++ rust/quickq-core/src/storage/schema.rs | 70 ++ rust/quickq-core/src/storage/sqlite.rs | 944 ++++++++++++++++--------- rust/quickq-python/src/py_job.rs | 9 + rust/quickq-python/src/py_queue.rs | 203 +++++- rust/quickq-python/src/py_worker.rs | 28 +- tests/python/test_batch.py | 63 ++ tests/python/test_cancel.py | 69 ++ tests/python/test_chain.py | 81 +++ tests/python/test_cli.py | 39 + tests/python/test_context.py | 63 ++ tests/python/test_hooks.py | 90 +++ tests/python/test_periodic.py | 60 ++ tests/python/test_progress.py | 49 ++ tests/python/test_retry_history.py | 57 ++ tests/python/test_shutdown.py | 60 ++ tests/python/test_unique.py | 73 ++ 41 files changed, 2900 insertions(+), 381 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100644 mkdocs.yml create mode 100644 python/quickq/canvas.py create mode 100644 python/quickq/context.py create mode 100644 rust/quickq-core/src/periodic.rs create mode 100644 rust/quickq-core/src/storage/models.rs create mode 100644 rust/quickq-core/src/storage/schema.rs create mode 100644 tests/python/test_batch.py create mode 100644 tests/python/test_cancel.py create mode 100644 tests/python/test_chain.py create mode 100644 tests/python/test_cli.py create mode 100644 tests/python/test_context.py create mode 100644 tests/python/test_hooks.py create mode 100644 tests/python/test_periodic.py create mode 100644 tests/python/test_progress.py create mode 100644 tests/python/test_retry_history.py create mode 100644 tests/python/test_shutdown.py create mode 100644 tests/python/test_unique.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73aacd29..b4df2de5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: build-wheels: needs: test - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + if: github.event_name == 'push' && github.ref == 'refs/heads/master' runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..46fd88b2 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,72 @@ +name: Publish to PyPI + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + build-wheels: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + args: --release --out dist + manylinux: auto + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: dist/*.whl + + build-sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + + publish: + needs: [build-wheels, build-sdist] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ diff --git a/.gitignore b/.gitignore index 03c53fd3..aecc3545 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,7 @@ Thumbs.db .pytest_cache/ .coverage htmlcov/ -junk/ \ No newline at end of file +junk/ + +# Docs +site/ \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index bc76d5d5..0bbe4244 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,9 +3,8 @@ members = ["rust/quickq-core", "rust/quickq-python"] resolver = "2" [workspace.dependencies] -rusqlite = { version = "0.31", features = ["bundled"] } -r2d2 = "0.8" -r2d2_sqlite = "0.24" +diesel = { version = "2.2", features = ["sqlite", "r2d2", "returning_clauses_for_sqlite_3_35"] } +libsqlite3-sys = { version = "0.30", features = ["bundled"] } uuid = { version = "1", features = ["v7"] } tokio = { version = "1", features = ["full"] } serde = { version = "1", features = ["derive"] } @@ -14,4 +13,5 @@ rand = "0.8" crossbeam-channel = "0.5" thiserror = "1" chrono = "0.4" +cron = "0.13" pyo3 = { version = "0.22", features = ["extension-module"] } diff --git a/README.md b/README.md index e69de29b..5dfd3fe0 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,72 @@ +# quickq + +A Rust-powered task queue for Python. No broker required — just SQLite. + +``` +pip install quickq +``` + +## Quickstart + +```python +import threading +from quickq import Queue + +queue = Queue(db_path="tasks.db") + +@queue.task() +def add(a: int, b: int) -> int: + return a + b + +job = add.delay(2, 3) + +t = threading.Thread(target=queue.run_worker, daemon=True) +t.start() + +print(job.result(timeout=10)) # 5 +``` + +## Why quickq? + +Most Python task queues require a separate broker (Redis, RabbitMQ) even for single-machine workloads. quickq embeds everything — storage, scheduling, and worker management — into a single `pip install` with no external dependencies beyond Python itself. + +The heavy lifting runs in Rust: a Tokio async scheduler, OS thread worker pool with crossbeam channels, and Diesel ORM over SQLite in WAL mode. Python's GIL is only held during task execution. + +## Features + +- **Priority queues** — higher priority jobs run first +- **Retry with exponential backoff** — automatic retries with jitter +- **Dead letter queue** — inspect and replay failed jobs +- **Rate limiting** — token bucket with `"100/m"` syntax +- **Task workflows** — `chain`, `group`, `chord` primitives +- **Periodic tasks** — cron scheduling with seconds granularity +- **Progress tracking** — report and read progress from inside tasks +- **Job cancellation** — cancel pending jobs before execution +- **Unique tasks** — deduplicate active jobs by key +- **Batch enqueue** — `task.map()` for high-throughput bulk inserts +- **Named queues** — route tasks to isolated queues +- **Hooks** — before/after/success/failure middleware +- **Async support** — `await job.aresult()`, `await queue.astats()` +- **CLI** — `quickq worker --app myapp:queue` and `quickq info --watch` + +## Documentation + +Full documentation with guides, API reference, architecture diagrams, and examples: + +**[Read the docs →](https://quickq.dev)** *(coming soon)* + +## Comparison + +| Feature | quickq | Celery | RQ | Dramatiq | Huey | +|---|---|---|---|---|---| +| Broker required | **No** | Yes | Yes | Yes | Yes | +| Core language | **Rust + Python** | Python | Python | Python | Python | +| Priority queues | **Yes** | Yes | No | No | Yes | +| Rate limiting | **Yes** | Yes | No | Yes | No | +| Dead letter queue | **Yes** | No | Yes | No | No | +| Task chaining | **Yes** | Yes | No | Yes | No | +| Setup | **`pip install`** | Broker + backend | Redis | Broker | Redis | + +## License + +MIT diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..0d366a77 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,107 @@ +site_name: quickq +site_description: "Rust-powered task queue for Python. No broker required." +site_url: https://pratyush618.github.io/quickq +repo_url: https://github.com/pratyush618/quickq +repo_name: pratyush618/quickq + +theme: + name: material + palette: + - scheme: default + primary: deep purple + accent: amber + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - scheme: slate + primary: deep purple + accent: amber + toggle: + icon: material/brightness-4 + name: Switch to light mode + font: + text: Inter + code: JetBrains Mono + icon: + repo: fontawesome/brands/github + features: + - navigation.instant + - navigation.tracking + - navigation.sections + - navigation.expand + - navigation.top + - content.code.copy + - content.code.annotate + - content.tabs.link + - search.highlight + - search.suggest + - toc.follow + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.mark + - pymdownx.keys + - pymdownx.emoji: + emoji_index: !!python/name:zensical.extensions.emoji.twemoji + emoji_generator: !!python/name:zensical.extensions.emoji.to_svg + - attr_list + - md_in_html + - tables + - toc: + permalink: true + - def_list + - pymdownx.tasklist: + custom_checkbox: true + +plugins: + - search + +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/pratyush618/quickq + - icon: fontawesome/brands/python + link: https://pypi.org/project/quickq/ + +nav: + - Home: index.md + - Getting Started: + - Installation: getting-started/installation.md + - Quickstart: getting-started/quickstart.md + - User Guide: + - Tasks: guide/tasks.md + - Queues & Priority: guide/queues.md + - Workers: guide/workers.md + - Retries & Dead Letters: guide/retries.md + - Rate Limiting: guide/rate-limiting.md + - Workflows: guide/workflows.md + - Scheduling: guide/scheduling.md + - Monitoring & Hooks: guide/monitoring.md + - Advanced: guide/advanced.md + - Architecture: architecture.md + - API Reference: + - Queue: api/queue.md + - TaskWrapper: api/task.md + - JobResult: api/result.md + - Canvas: api/canvas.md + - JobContext: api/context.md + - CLI: api/cli.md + - Examples: + - Web Scraper Pipeline: examples/web-scraper.md + - Benchmark: examples/benchmark.md + - Comparison: comparison.md + - Changelog: changelog.md diff --git a/pyproject.toml b/pyproject.toml index 3e8f3338..8fabf569 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = ["cloudpickle>=3.0"] [project.optional-dependencies] dev = ["pytest>=7.0", "pytest-asyncio>=0.21", "ruff>=0.8", "mypy>=1.13"] +docs = ["zensical"] [tool.maturin] manifest-path = "rust/quickq-python/Cargo.toml" diff --git a/python/quickq/__init__.py b/python/quickq/__init__.py index 5ceae994..46f1c36c 100644 --- a/python/quickq/__init__.py +++ b/python/quickq/__init__.py @@ -1,8 +1,19 @@ """quickq — Rust-powered task queue for Python. No broker required.""" from quickq.app import Queue +from quickq.canvas import Signature, chain, chord, group +from quickq.context import current_job from quickq.result import JobResult from quickq.task import TaskWrapper -__all__ = ["JobResult", "Queue", "TaskWrapper"] +__all__ = [ + "JobResult", + "Queue", + "Signature", + "TaskWrapper", + "chain", + "chord", + "current_job", + "group", +] __version__ = "0.1.0" diff --git a/python/quickq/_quickq.pyi b/python/quickq/_quickq.pyi index cc51ea61..e959a62f 100644 --- a/python/quickq/_quickq.pyi +++ b/python/quickq/_quickq.pyi @@ -37,6 +37,9 @@ class PyJob: completed_at: int | None error: str | None timeout_ms: int + unique_key: str | None + progress: int | None + metadata: str | None @property def status(self) -> str: ... @@ -51,7 +54,9 @@ class PyQueue: default_retry: int = 3, default_timeout: int = 300, default_priority: int = 0, + result_ttl: int | None = None, ) -> None: ... + def request_shutdown(self) -> None: ... def enqueue( self, task_name: str, @@ -61,12 +66,36 @@ class PyQueue: delay_seconds: float | None = None, max_retries: int | None = None, timeout: int | None = None, + unique_key: str | None = None, + metadata: str | None = None, ) -> PyJob: ... + def enqueue_batch( + self, + task_names: list[str], + payloads: list[bytes], + queues: list[str] | None = None, + priorities: list[int] | None = None, + max_retries_list: list[int] | None = None, + timeouts: list[int] | None = None, + ) -> list[PyJob]: ... def get_job(self, job_id: str) -> PyJob | None: ... + def get_job_errors(self, job_id: str) -> list[dict[str, Any]]: ... + def cancel_job(self, job_id: str) -> bool: ... + def update_progress(self, job_id: str, progress: int) -> None: ... + def purge_completed(self, older_than_seconds: int) -> int: ... def stats(self) -> dict[str, int]: ... def dead_letters(self, limit: int = 10, offset: int = 0) -> list[dict[str, Any]]: ... def retry_dead(self, dead_id: str) -> str: ... def purge_dead(self, older_than_seconds: int) -> int: ... + def register_periodic( + self, + name: str, + task_name: str, + cron_expr: str, + args: bytes | None = None, + kwargs: bytes | None = None, + queue: str = "default", + ) -> None: ... def run_worker( self, task_registry: dict[str, Any], diff --git a/python/quickq/app.py b/python/quickq/app.py index fde5f52c..adcc7a90 100644 --- a/python/quickq/app.py +++ b/python/quickq/app.py @@ -4,6 +4,7 @@ import asyncio import functools +import logging import signal from collections.abc import Sequence from concurrent.futures import ThreadPoolExecutor @@ -15,6 +16,8 @@ from quickq.result import JobResult from quickq.task import TaskWrapper +logger = logging.getLogger("quickq") + class Queue: """ @@ -39,6 +42,7 @@ def __init__( default_retry: int = 3, default_timeout: int = 300, default_priority: int = 0, + result_ttl: int | None = None, ): """Initialize a new task queue backed by SQLite. @@ -48,6 +52,8 @@ def __init__( default_retry: Default max retry attempts for tasks. default_timeout: Default task timeout in seconds. default_priority: Default task priority (higher = more urgent). + result_ttl: Auto-cleanup completed/dead jobs older than this many + seconds. None disables auto-cleanup. """ self._inner = PyQueue( db_path=db_path, @@ -55,10 +61,18 @@ def __init__( default_retry=default_retry, default_timeout=default_timeout, default_priority=default_priority, + result_ttl=result_ttl, ) self._task_registry: dict[str, Callable] = {} self._task_configs: list[PyTaskConfig] = [] self._executor = ThreadPoolExecutor(max_workers=2) + self._periodic_configs: list[dict[str, Any]] = [] + self._hooks: dict[str, list[Callable]] = { + "before_task": [], + "after_task": [], + "on_success": [], + "on_failure": [], + } def task( self, @@ -69,14 +83,33 @@ def task( priority: int = 0, rate_limit: str | None = None, queue: str = "default", - ) -> Callable: - """Decorator to register a function as a task.""" + ) -> Callable[[Callable[..., Any]], TaskWrapper]: + """Decorator to register a function as a background task. + + Args: + name: Explicit task name. Defaults to ``module.qualname``. + max_retries: Max retry attempts on failure before moving to DLQ. + retry_backoff: Base delay in seconds for exponential backoff between retries. + timeout: Max execution time in seconds before the task is killed. + priority: Priority level (higher = more urgent). + rate_limit: Rate limit string, e.g. ``"100/m"``, ``"10/s"``, ``"3600/h"``. + queue: Named queue to submit to. + + Example:: + + @queue.task(max_retries=5, rate_limit="100/m", queue="emails") + def send_email(to: str, subject: str, body: str): + ... + + send_email.delay("user@example.com", "Hello", "World") + """ def decorator(fn: Callable) -> TaskWrapper: task_name = name or f"{fn.__module__}.{fn.__qualname__}" - # Register in the task registry - self._task_registry[task_name] = fn + # Wrap the function with hooks and context + wrapped = self._wrap_task(fn, task_name) + self._task_registry[task_name] = wrapped # Store config for worker startup config = PyTaskConfig( @@ -108,6 +141,107 @@ def decorator(fn: Callable) -> TaskWrapper: return decorator + def periodic( + self, + cron: str, + name: str | None = None, + args: tuple = (), + kwargs: dict | None = None, + queue: str = "default", + ) -> Callable[[Callable[..., Any]], TaskWrapper]: + """Decorator to register a periodic (cron-scheduled) task. + + Args: + cron: Cron expression (6-field with seconds), e.g. ``"0 */5 * * * *"`` + for every 5 minutes. + name: Explicit task name. Defaults to ``module.qualname``. + args: Positional arguments to pass to the task on each run. + kwargs: Keyword arguments to pass to the task on each run. + queue: Named queue to submit to. + + Example:: + + @queue.periodic(cron="0 */5 * * * *") + def cleanup(): + ... + """ + + def decorator(fn: Callable) -> TaskWrapper: + # Register as a normal task first + wrapper = self.task(name=name, queue=queue)(fn) + + # Store periodic config for registration at worker startup + payload = cloudpickle.dumps((args, kwargs or {})) + self._periodic_configs.append( + { + "name": name or f"{fn.__module__}.{fn.__qualname__}", + "task_name": wrapper.name, + "cron_expr": cron, + "payload": payload, + "queue": queue, + } + ) + + return wrapper + + return decorator + + # -- Hooks / middleware -- + + def before_task(self, fn: Callable) -> Callable: + """Register a hook called before each task: ``fn(task_name, args, kwargs)``.""" + self._hooks["before_task"].append(fn) + return fn + + def after_task(self, fn: Callable) -> Callable: + """Register a hook called after each task. + + Signature: ``fn(task_name, args, kwargs, result, error)`` + """ + self._hooks["after_task"].append(fn) + return fn + + def on_success(self, fn: Callable) -> Callable: + """Register a hook called on task success: ``fn(task_name, args, kwargs, result)``.""" + self._hooks["on_success"].append(fn) + return fn + + def on_failure(self, fn: Callable) -> Callable: + """Register a hook called on task failure: ``fn(task_name, args, kwargs, error)``.""" + self._hooks["on_failure"].append(fn) + return fn + + def _wrap_task(self, fn: Callable, task_name: str) -> Callable: + """Wrap a task function with hooks and job context.""" + from quickq.context import _clear_context + + hooks = self._hooks + + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + for hook in hooks["before_task"]: + hook(task_name, args, kwargs) + + error = None + result = None + try: + result = fn(*args, **kwargs) + except Exception as exc: + error = exc + for hook in hooks["on_failure"]: + hook(task_name, args, kwargs, exc) + raise + else: + for hook in hooks["on_success"]: + hook(task_name, args, kwargs, result) + return result + finally: + for hook in hooks["after_task"]: + hook(task_name, args, kwargs, result, error) + _clear_context() + + return wrapper + def enqueue( self, task_name: str, @@ -118,6 +252,8 @@ def enqueue( queue: str | None = None, max_retries: int | None = None, timeout: int | None = None, + unique_key: str | None = None, + metadata: str | None = None, ) -> JobResult: """Enqueue a task for execution.""" payload = cloudpickle.dumps((args, kwargs or {})) @@ -130,10 +266,57 @@ def enqueue( delay_seconds=delay, max_retries=max_retries, timeout=timeout, + unique_key=unique_key, + metadata=metadata, ) return JobResult(py_job=py_job, queue=self) + def enqueue_many( + self, + task_name: str, + args_list: list[tuple], + kwargs_list: list[dict] | None = None, + priority: int | None = None, + queue: str | None = None, + max_retries: int | None = None, + timeout: int | None = None, + ) -> list[JobResult]: + """Enqueue multiple jobs for the same task in a single transaction. + + Args: + task_name: The registered task name. + args_list: List of positional argument tuples, one per job. + kwargs_list: Optional list of kwarg dicts, one per job. + priority: Priority for all jobs (uses default if None). + queue: Queue name for all jobs (uses "default" if None). + max_retries: Max retries for all jobs (uses default if None). + timeout: Timeout in seconds for all jobs (uses default if None). + + Returns: + List of JobResult handles, one per enqueued job. + """ + count = len(args_list) + kw_list = kwargs_list or [{}] * count + payloads = [cloudpickle.dumps((args, kw)) for args, kw in zip(args_list, kw_list)] + task_names = [task_name] * count + + queues_list = [queue or "default"] * count if queue else None + priorities_list = [priority] * count if priority is not None else None + retries_list = [max_retries] * count if max_retries is not None else None + timeouts_list = [timeout] * count if timeout is not None else None + + py_jobs = self._inner.enqueue_batch( + task_names=task_names, + payloads=payloads, + queues=queues_list, + priorities=priorities_list, + max_retries_list=retries_list, + timeouts=timeouts_list, + ) + + return [JobResult(py_job=pj, queue=self) for pj in py_jobs] + def get_job(self, job_id: str) -> JobResult | None: """Get a job by ID.""" py_job = self._inner.get_job(job_id) @@ -159,6 +342,22 @@ def purge_dead(self, older_than: int = 86400) -> int: """Purge dead letter entries older than N seconds ago.""" return self._inner.purge_dead(older_than) + def cancel_job(self, job_id: str) -> bool: + """Cancel a pending job. Returns True if cancelled, False if not pending.""" + return self._inner.cancel_job(job_id) + + def update_progress(self, job_id: str, progress: int) -> None: + """Update progress for a running job (0-100).""" + self._inner.update_progress(job_id, progress) + + def job_errors(self, job_id: str) -> list[dict]: + """Get error history for a job (one entry per failed attempt).""" + return self._inner.get_job_errors(job_id) + + def purge_completed(self, older_than: int = 86400) -> int: + """Purge completed jobs older than N seconds ago. Returns count deleted.""" + return self._inner.purge_completed(older_than) + # -- Async inspection methods -- async def astats(self) -> dict[str, int]: @@ -178,6 +377,11 @@ async def aretry_dead(self, dead_id: str) -> str: loop = asyncio.get_event_loop() return await loop.run_in_executor(self._executor, lambda: self.retry_dead(dead_id)) + async def acancel_job(self, job_id: str) -> bool: + """Async version of cancel_job().""" + loop = asyncio.get_event_loop() + return await loop.run_in_executor(self._executor, lambda: self.cancel_job(job_id)) + # -- Worker startup -- def run_worker(self, queues: Sequence[str] | None = None) -> None: @@ -187,12 +391,30 @@ def run_worker(self, queues: Sequence[str] | None = None) -> None: """ queue_list = list(queues) if queues else None + # Make queue accessible from job context (for current_job.update_progress()) + from quickq.context import _set_queue_ref + + _set_queue_ref(self) + + # Register periodic tasks with Rust scheduler + for pc in self._periodic_configs: + self._inner.register_periodic( + name=pc["name"], + task_name=pc["task_name"], + cron_expr=pc["cron_expr"], + args=pc["payload"], + queue=pc["queue"], + ) + # Print startup info num_tasks = len(self._task_registry) + num_periodic = len(self._periodic_configs) worker_queues = queue_list or ["default"] - print("[quickq] Starting worker...") - print(f"[quickq] Registered tasks: {num_tasks}") - print(f"[quickq] Queues: {', '.join(worker_queues)}") + logger.info("Starting worker...") + logger.info("Registered tasks: %d", num_tasks) + if num_periodic: + logger.info("Periodic tasks: %d", num_periodic) + logger.info("Queues: %s", ", ".join(worker_queues)) # Set up signal handler for graceful shutdown (only in main thread) import threading @@ -204,9 +426,10 @@ def run_worker(self, queues: Sequence[str] | None = None) -> None: original_sigint = signal.getsignal(signal.SIGINT) def shutdown_handler(signum: int, frame: Any) -> None: - print("\n[quickq] Shutting down gracefully...") + logger.info("Shutting down gracefully (waiting for in-flight jobs)...") + self._inner.request_shutdown() + # Restore original handler so a second Ctrl+C force-kills signal.signal(signal.SIGINT, original_sigint) - raise KeyboardInterrupt signal.signal(signal.SIGINT, shutdown_handler) @@ -217,8 +440,9 @@ def shutdown_handler(signum: int, frame: Any) -> None: queues=queue_list, ) except KeyboardInterrupt: - print("[quickq] Worker stopped.") + logger.info("Worker force-stopped.") finally: + logger.info("Worker stopped.") if is_main and original_sigint is not None: signal.signal(signal.SIGINT, original_sigint) diff --git a/python/quickq/canvas.py b/python/quickq/canvas.py new file mode 100644 index 00000000..40c7cc8d --- /dev/null +++ b/python/quickq/canvas.py @@ -0,0 +1,143 @@ +"""Task chaining primitives: Signature, chain, group, chord.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from quickq.app import Queue + from quickq.result import JobResult + from quickq.task import TaskWrapper + + +@dataclass(frozen=True) +class Signature: + """A frozen task call spec — what to call and with what arguments. + + Created via ``task.s()`` or ``task.si()``:: + + sig = add.s(2, 3) # mutable — receives previous result + sig = add.si(2, 3) # immutable — ignores previous result + """ + + task: TaskWrapper + args: tuple = () + kwargs: dict = field(default_factory=dict) + options: dict = field(default_factory=dict) + immutable: bool = False + + def apply(self, queue: Queue | None = None) -> JobResult: + """Enqueue this signature immediately.""" + q = queue or self.task._queue + return q.enqueue( + task_name=self.task.name, + args=self.args, + kwargs=self.kwargs if self.kwargs else None, + **self.options, + ) + + +class chain: + """Execute signatures sequentially, piping each result to the next. + + Usage:: + + result = chain(add.s(2, 3), multiply.s(10)).apply(queue) + print(result.result(timeout=30)) # (2 + 3) * 10 = 50 + """ + + def __init__(self, *signatures: Signature): + if len(signatures) < 1: + raise ValueError("chain requires at least one signature") + self.signatures = list(signatures) + + def apply(self, queue: Queue | None = None) -> JobResult: + """Execute the chain synchronously by polling for results.""" + q = queue or self.signatures[0].task._queue + + prev_result: Any = None + last_job: JobResult | None = None + + for sig in self.signatures: + args = sig.args + if prev_result is not None and not sig.immutable: + args = (prev_result, *sig.args) + + last_job = q.enqueue( + task_name=sig.task.name, + args=args, + kwargs=sig.kwargs if sig.kwargs else None, + **sig.options, + ) + prev_result = last_job.result(timeout=300) + + return last_job # type: ignore[return-value] + + +class group: + """Execute signatures in parallel and collect all results. + + Usage:: + + results = group(add.s(1, 2), add.s(3, 4)).apply(queue) + print([r.result(timeout=30) for r in results]) # [3, 7] + """ + + def __init__(self, *signatures: Signature): + if len(signatures) < 1: + raise ValueError("group requires at least one signature") + self.signatures = list(signatures) + + def apply(self, queue: Queue | None = None) -> list[JobResult]: + """Enqueue all signatures and return a list of JobResult handles.""" + q = queue or self.signatures[0].task._queue + + jobs: list[JobResult] = [] + for sig in self.signatures: + job = q.enqueue( + task_name=sig.task.name, + args=sig.args, + kwargs=sig.kwargs if sig.kwargs else None, + **sig.options, + ) + jobs.append(job) + + return jobs + + +class chord: + """Run a group in parallel, then a callback with all results. + + Usage:: + + result = chord( + group(add.s(1, 2), add.s(3, 4)), + total.s() + ).apply(queue) + print(result.result(timeout=30)) # sum([3, 7]) = 10 + """ + + def __init__(self, group_: group, callback: Signature): + self.group = group_ + self.callback = callback + + def apply(self, queue: Queue | None = None) -> JobResult: + """Execute the group, collect results, then run callback.""" + q = queue or self.callback.task._queue + + # Run group and wait for all results + jobs = self.group.apply(queue=q) + results = [job.result(timeout=300) for job in jobs] + + # Run callback with collected results + args = self.callback.args + if not self.callback.immutable: + args = (results, *self.callback.args) + + return q.enqueue( + task_name=self.callback.task.name, + args=args, + kwargs=self.callback.kwargs if self.callback.kwargs else None, + **self.callback.options, + ) diff --git a/python/quickq/cli.py b/python/quickq/cli.py index 87657117..8f161408 100644 --- a/python/quickq/cli.py +++ b/python/quickq/cli.py @@ -1,10 +1,15 @@ -"""CLI entry point for quickq worker.""" +"""CLI entry point for quickq worker and info commands.""" from __future__ import annotations import argparse import importlib import sys +import time +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from quickq.app import Queue def main() -> None: @@ -28,19 +33,33 @@ def main() -> None: help="Comma-separated list of queues to process (default: all registered)", ) + # info subcommand + info_parser = subparsers.add_parser("info", help="Show queue statistics") + info_parser.add_argument( + "--app", + required=True, + help="Python path to the Queue instance (e.g., 'myapp.tasks:queue')", + ) + info_parser.add_argument( + "--watch", + action="store_true", + default=False, + help="Continuously refresh stats every 2 seconds", + ) + args = parser.parse_args() if args.command == "worker": run_worker(args) + elif args.command == "info": + run_info(args) else: parser.print_help() sys.exit(1) -def run_worker(args: argparse.Namespace) -> None: - """Import the user's Queue instance and start the worker.""" - app_path = args.app - +def _load_queue(app_path: str) -> Queue: + """Import and return a Queue instance from a 'module:attribute' path.""" if ":" not in app_path: print( f"Error: --app must be in 'module:attribute' format, got '{app_path}'", @@ -74,9 +93,57 @@ def run_worker(args: argparse.Namespace) -> None: ) sys.exit(1) + return queue + + +def run_worker(args: argparse.Namespace) -> None: + """Import the user's Queue instance and start the worker.""" + queue = _load_queue(args.app) queues = args.queues.split(",") if args.queues else None queue.run_worker(queues=queues) +def run_info(args: argparse.Namespace) -> None: + """Print queue statistics.""" + queue = _load_queue(args.app) + + if args.watch: + _watch_stats(queue) + else: + _print_stats(queue) + + +def _print_stats(queue: Queue) -> None: + """Print stats once.""" + stats = queue.stats() + print("quickq queue statistics") + print("-" * 30) + for key in ("pending", "running", "completed", "failed", "dead", "cancelled"): + print(f" {key:<12} {stats.get(key, 0)}") + total = sum(stats.values()) + print("-" * 30) + print(f" {'total':<12} {total}") + + +def _watch_stats(queue: Queue) -> None: + """Refresh stats every 2 seconds with throughput.""" + prev_completed = 0 + try: + while True: + print("\033[2J\033[H", end="") + stats = queue.stats() + completed = stats.get("completed", 0) + throughput = (completed - prev_completed) / 2.0 + prev_completed = completed + + _print_stats(queue) + if throughput > 0: + print(f"\n throughput {throughput:.1f} jobs/s") + print("\nRefreshing every 2s... (Ctrl+C to stop)") + time.sleep(2) + except KeyboardInterrupt: + pass + + if __name__ == "__main__": main() diff --git a/python/quickq/context.py b/python/quickq/context.py new file mode 100644 index 00000000..67078e28 --- /dev/null +++ b/python/quickq/context.py @@ -0,0 +1,108 @@ +"""Thread-local context for the currently executing job.""" + +from __future__ import annotations + +import threading +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from quickq.app import Queue + +_local = threading.local() +_queue_ref: Queue | None = None + + +class JobContext: + """Provides access to the currently executing job's metadata and controls. + + Usage inside a task:: + + from quickq.context import current_job + + @queue.task() + def process(data): + current_job.update_progress(50) + ... + current_job.update_progress(100) + """ + + @property + def id(self) -> str: + """The ID of the currently executing job.""" + return self._require_context().job_id + + @property + def task_name(self) -> str: + """The name of the currently executing task.""" + return self._require_context().task_name + + @property + def retry_count(self) -> int: + """How many times this job has been retried.""" + return self._require_context().retry_count + + @property + def queue_name(self) -> str: + """The queue this job is running on.""" + return self._require_context().queue_name + + def update_progress(self, progress: int) -> None: + """Update the job's progress (0-100).""" + ctx = self._require_context() + if _queue_ref is None: + raise RuntimeError("Queue reference not set. Cannot update progress.") + _queue_ref._inner.update_progress(ctx.job_id, progress) + + @staticmethod + def _require_context() -> _ActiveContext: + ctx: _ActiveContext | None = getattr(_local, "context", None) + if ctx is None: + raise RuntimeError( + "No active job context. current_job can only be used inside a running task." + ) + return ctx + + +class _ActiveContext: + __slots__ = ("job_id", "queue_name", "retry_count", "task_name") + + def __init__( + self, + job_id: str, + task_name: str, + retry_count: int, + queue_name: str, + ): + self.job_id = job_id + self.task_name = task_name + self.retry_count = retry_count + self.queue_name = queue_name + + +def _set_queue_ref(queue: Any) -> None: + """Store the Queue instance at module level for use by job context.""" + global _queue_ref + _queue_ref = queue + + +def _set_context( + job_id: str, + task_name: str, + retry_count: int, + queue_name: str, +) -> None: + """Set the thread-local job context. Called from Rust worker before each task.""" + _local.context = _ActiveContext( + job_id=job_id, + task_name=task_name, + retry_count=retry_count, + queue_name=queue_name, + ) + + +def _clear_context() -> None: + """Clear the thread-local job context. Called from Rust worker after each task.""" + _local.context = None + + +current_job = JobContext() diff --git a/python/quickq/result.py b/python/quickq/result.py index 75872a40..180952e7 100644 --- a/python/quickq/result.py +++ b/python/quickq/result.py @@ -38,13 +38,21 @@ def id(self) -> str: def status(self) -> str: """ Current job status. Fetches fresh from the database. - Returns one of: "pending", "running", "complete", "failed", "dead". + Returns one of: "pending", "running", "complete", "failed", "dead", "cancelled". """ refreshed = self._queue._inner.get_job(self._py_job.id) if refreshed is not None: self._py_job = refreshed return self._py_job.status + @property + def progress(self) -> int | None: + """Current progress (0-100) if reported by the task.""" + refreshed = self._queue._inner.get_job(self._py_job.id) + if refreshed is not None: + self._py_job = refreshed + return self._py_job.progress + @property def error(self) -> str | None: """Error message if the job failed.""" @@ -53,13 +61,27 @@ def error(self) -> str | None: self._py_job = refreshed return self._py_job.error - def result(self, timeout: float = 30.0, poll_interval: float = 0.1) -> Any: + @property + def errors(self) -> list[dict]: + """Error history for this job (one entry per failed attempt).""" + return self._queue.job_errors(self.id) + + def result( + self, + timeout: float = 30.0, + poll_interval: float = 0.05, + max_poll_interval: float = 0.5, + ) -> Any: """ Block until the job completes and return the result. + Uses exponential backoff: starts polling at ``poll_interval`` and + gradually increases to ``max_poll_interval``. + Args: timeout: Maximum seconds to wait. - poll_interval: Seconds between status checks. + poll_interval: Initial seconds between status checks. + max_poll_interval: Maximum seconds between status checks. Returns: The deserialized return value of the task function. @@ -69,6 +91,7 @@ def result(self, timeout: float = 30.0, poll_interval: float = 0.1) -> Any: RuntimeError: If the job failed or was moved to DLQ. """ deadline = time.monotonic() + timeout + current_interval = poll_interval while time.monotonic() < deadline: refreshed = self._queue._inner.get_job(self._py_job.id) @@ -86,22 +109,31 @@ def result(self, timeout: float = 30.0, poll_interval: float = 0.1) -> Any: f"Job {self.id} {status}: {self._py_job.error or 'unknown error'}" ) - time.sleep(poll_interval) + time.sleep(current_interval) + current_interval = min(current_interval * 1.5, max_poll_interval) raise TimeoutError( f"Job {self.id} did not complete within {timeout}s " f"(current status: {self._py_job.status})" ) - async def aresult(self, timeout: float = 30.0, poll_interval: float = 0.1) -> Any: + async def aresult( + self, + timeout: float = 30.0, + poll_interval: float = 0.05, + max_poll_interval: float = 0.5, + ) -> Any: """ Async version of result(). Awaitable, non-blocking. + Uses exponential backoff like :meth:`result`. + Usage:: result = await job.aresult(timeout=30) """ deadline = time.monotonic() + timeout + current_interval = poll_interval while time.monotonic() < deadline: refreshed = self._queue._inner.get_job(self._py_job.id) @@ -119,7 +151,8 @@ async def aresult(self, timeout: float = 30.0, poll_interval: float = 0.1) -> An f"Job {self.id} {status}: {self._py_job.error or 'unknown error'}" ) - await asyncio.sleep(poll_interval) + await asyncio.sleep(current_interval) + current_interval = min(current_interval * 1.5, max_poll_interval) raise TimeoutError( f"Job {self.id} did not complete within {timeout}s " diff --git a/python/quickq/task.py b/python/quickq/task.py index df0631ce..1aece254 100644 --- a/python/quickq/task.py +++ b/python/quickq/task.py @@ -6,6 +6,7 @@ if TYPE_CHECKING: from quickq.app import Queue + from quickq.canvas import Signature from quickq.result import JobResult @@ -78,6 +79,8 @@ def apply_async( queue: str | None = None, max_retries: int | None = None, timeout: int | None = None, + unique_key: str | None = None, + metadata: str | None = None, ) -> JobResult: """ Enqueue with full control over submission options. @@ -90,6 +93,8 @@ def apply_async( queue: Override the default queue name. max_retries: Override the default max retry count. timeout: Override the default timeout in seconds. + unique_key: Deduplicate active jobs with the same key. + metadata: Arbitrary JSON metadata to attach to the job. """ return self._queue.enqueue( task_name=self._task_name, @@ -100,8 +105,40 @@ def apply_async( queue=queue or self._default_queue, max_retries=max_retries if max_retries is not None else self._default_max_retries, timeout=timeout if timeout is not None else self._default_timeout, + unique_key=unique_key, + metadata=metadata, ) + def map(self, iterable: list[tuple]) -> list[JobResult]: + """Enqueue one job per item in a single batch transaction. + + Args: + iterable: List of argument tuples, one per job. + + Returns: + List of JobResult handles. + """ + return self._queue.enqueue_many( + task_name=self._task_name, + args_list=iterable, + priority=self._default_priority, + queue=self._default_queue, + max_retries=self._default_max_retries, + timeout=self._default_timeout, + ) + + def s(self, *args: Any, **kwargs: Any) -> Signature: + """Create a mutable Signature (receives previous result in chains).""" + from quickq.canvas import Signature + + return Signature(task=self, args=args, kwargs=kwargs) + + def si(self, *args: Any, **kwargs: Any) -> Signature: + """Create an immutable Signature (ignores previous result in chains).""" + from quickq.canvas import Signature + + return Signature(task=self, args=args, kwargs=kwargs, immutable=True) + def __repr__(self) -> str: """Return a developer-friendly string representation.""" return f"" diff --git a/rust/quickq-core/Cargo.toml b/rust/quickq-core/Cargo.toml index a4ab8552..cd39a317 100644 --- a/rust/quickq-core/Cargo.toml +++ b/rust/quickq-core/Cargo.toml @@ -4,9 +4,8 @@ version = "0.1.0" edition = "2021" [dependencies] -rusqlite = { workspace = true } -r2d2 = { workspace = true } -r2d2_sqlite = { workspace = true } +diesel = { workspace = true } +libsqlite3-sys = { workspace = true } uuid = { workspace = true } tokio = { workspace = true } serde = { workspace = true } @@ -15,6 +14,7 @@ rand = { workspace = true } crossbeam-channel = { workspace = true } thiserror = { workspace = true } chrono = { workspace = true } +cron = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/rust/quickq-core/src/dlq.rs b/rust/quickq-core/src/dlq.rs index b64d555b..e559dfb2 100644 --- a/rust/quickq-core/src/dlq.rs +++ b/rust/quickq-core/src/dlq.rs @@ -1,6 +1,6 @@ use crate::error::Result; use crate::job::Job; -use crate::storage::sqlite::{DeadJob, SqliteStorage}; +use crate::storage::sqlite::{SqliteStorage, DeadJob}; /// Dead letter queue manager. pub struct DeadLetterQueue { diff --git a/rust/quickq-core/src/error.rs b/rust/quickq-core/src/error.rs index 87a4ddf6..f101c7ce 100644 --- a/rust/quickq-core/src/error.rs +++ b/rust/quickq-core/src/error.rs @@ -3,10 +3,10 @@ use thiserror::Error; #[derive(Error, Debug)] pub enum QueueError { #[error("storage error: {0}")] - Storage(#[from] rusqlite::Error), + Storage(#[from] diesel::result::Error), #[error("connection pool error: {0}")] - Pool(#[from] r2d2::Error), + Pool(#[from] diesel::r2d2::PoolError), #[error("job not found: {0}")] JobNotFound(String), @@ -29,6 +29,9 @@ pub enum QueueError { #[error("job timed out: {0}")] Timeout(String), + #[error("config error: {0}")] + Config(String), + #[error("{0}")] Other(String), } diff --git a/rust/quickq-core/src/job.rs b/rust/quickq-core/src/job.rs index f85f00ed..86142cf3 100644 --- a/rust/quickq-core/src/job.rs +++ b/rust/quickq-core/src/job.rs @@ -2,6 +2,8 @@ use serde::{Deserialize, Serialize}; use std::time::{SystemTime, UNIX_EPOCH}; use uuid::Uuid; +use crate::storage::models::JobRow; + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[repr(i32)] pub enum JobStatus { @@ -10,6 +12,7 @@ pub enum JobStatus { Complete = 2, Failed = 3, Dead = 4, + Cancelled = 5, } impl JobStatus { @@ -20,6 +23,7 @@ impl JobStatus { 2 => Some(Self::Complete), 3 => Some(Self::Failed), 4 => Some(Self::Dead), + 5 => Some(Self::Cancelled), _ => None, } } @@ -31,6 +35,7 @@ impl JobStatus { Self::Complete => "complete", Self::Failed => "failed", Self::Dead => "dead", + Self::Cancelled => "cancelled", } } } @@ -53,6 +58,34 @@ pub struct Job { pub result: Option>, pub error: Option, pub timeout_ms: i64, + pub unique_key: Option, + pub progress: Option, + pub metadata: Option, +} + +impl From for Job { + fn from(row: JobRow) -> Self { + Self { + id: row.id, + queue: row.queue, + task_name: row.task_name, + payload: row.payload, + status: JobStatus::from_i32(row.status).unwrap_or(JobStatus::Pending), + priority: row.priority, + created_at: row.created_at, + scheduled_at: row.scheduled_at, + started_at: row.started_at, + completed_at: row.completed_at, + retry_count: row.retry_count, + max_retries: row.max_retries, + result: row.result, + error: row.error, + timeout_ms: row.timeout_ms, + unique_key: row.unique_key, + progress: row.progress, + metadata: row.metadata, + } + } } /// Parameters for creating a new job. @@ -64,6 +97,8 @@ pub struct NewJob { pub scheduled_at: i64, pub max_retries: i32, pub timeout_ms: i64, + pub unique_key: Option, + pub metadata: Option, } impl NewJob { @@ -85,6 +120,9 @@ impl NewJob { result: None, error: None, timeout_ms: self.timeout_ms, + unique_key: self.unique_key, + progress: None, + metadata: self.metadata, } } } diff --git a/rust/quickq-core/src/lib.rs b/rust/quickq-core/src/lib.rs index 1266c173..3739fdf7 100644 --- a/rust/quickq-core/src/lib.rs +++ b/rust/quickq-core/src/lib.rs @@ -5,6 +5,7 @@ pub mod scheduler; pub mod retry; pub mod dlq; pub mod rate_limiter; +pub mod periodic; pub use job::{Job, JobStatus, NewJob}; pub use error::QueueError; diff --git a/rust/quickq-core/src/periodic.rs b/rust/quickq-core/src/periodic.rs new file mode 100644 index 00000000..e10b3a37 --- /dev/null +++ b/rust/quickq-core/src/periodic.rs @@ -0,0 +1,44 @@ +use std::str::FromStr; + +use chrono::Utc; +use cron::Schedule; + +use crate::error::{QueueError, Result}; + +/// Compute the next run time (in UNIX milliseconds) for a cron expression, +/// starting from `after_ms` (also UNIX milliseconds). +pub fn next_cron_time(cron_expr: &str, after_ms: i64) -> Result { + let schedule = Schedule::from_str(cron_expr).map_err(|e| { + QueueError::Config(format!("invalid cron expression '{cron_expr}': {e}")) + })?; + + let after_dt = chrono::DateTime::from_timestamp_millis(after_ms) + .unwrap_or_else(|| Utc::now()); + + let next = schedule + .after(&after_dt) + .next() + .ok_or_else(|| QueueError::Config(format!("no next run time for '{cron_expr}'")))?; + + Ok(next.timestamp_millis()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_next_cron_time_every_minute() { + let now = crate::job::now_millis(); + let next = next_cron_time("0 * * * * *", now).unwrap(); + // Next minute should be within 60 seconds + assert!(next > now); + assert!(next <= now + 60_000); + } + + #[test] + fn test_invalid_cron_expr() { + let result = next_cron_time("not a cron", 0); + assert!(result.is_err()); + } +} diff --git a/rust/quickq-core/src/rate_limiter.rs b/rust/quickq-core/src/rate_limiter.rs index 11ca20e9..45203cf3 100644 --- a/rust/quickq-core/src/rate_limiter.rs +++ b/rust/quickq-core/src/rate_limiter.rs @@ -1,6 +1,7 @@ use crate::error::Result; use crate::job::now_millis; -use crate::storage::sqlite::{RateLimitRow, SqliteStorage}; +use crate::storage::models::RateLimitRow; +use crate::storage::sqlite::SqliteStorage; /// Token bucket rate limiter backed by SQLite for persistence. pub struct RateLimiter { diff --git a/rust/quickq-core/src/scheduler.rs b/rust/quickq-core/src/scheduler.rs index d474ad94..9e043bc0 100644 --- a/rust/quickq-core/src/scheduler.rs +++ b/rust/quickq-core/src/scheduler.rs @@ -7,7 +7,8 @@ use tokio::sync::Notify; use crate::dlq::DeadLetterQueue; use crate::error::Result; -use crate::job::{Job, now_millis}; +use crate::job::{Job, NewJob, now_millis}; +use crate::periodic::next_cron_time; use crate::rate_limiter::{RateLimitConfig, RateLimiter}; use crate::retry::RetryPolicy; use crate::storage::sqlite::SqliteStorage; @@ -43,10 +44,11 @@ pub struct Scheduler { queues: Vec, poll_interval: Duration, shutdown: Arc, + result_ttl_ms: Option, } impl Scheduler { - pub fn new(storage: SqliteStorage, queues: Vec) -> Self { + pub fn new(storage: SqliteStorage, queues: Vec, result_ttl_ms: Option) -> Self { let rate_limiter = RateLimiter::new(storage.clone()); let dlq = DeadLetterQueue::new(storage.clone()); @@ -58,6 +60,7 @@ impl Scheduler { queues, poll_interval: Duration::from_millis(50), shutdown: Arc::new(Notify::new()), + result_ttl_ms, } } @@ -77,6 +80,8 @@ impl Scheduler { /// to the worker pool via the provided channel. pub async fn run(&self, job_tx: Sender) { let mut reap_counter = 0u32; + let mut periodic_counter = 0u32; + let mut cleanup_counter = 0u32; loop { // Check for shutdown @@ -100,13 +105,30 @@ impl Scheduler { } } - // Periodically reap stale jobs (every ~100 iterations = ~5s) reap_counter += 1; + periodic_counter += 1; + cleanup_counter += 1; + + // Periodically reap stale jobs (every ~100 iterations = ~5s) if reap_counter % 100 == 0 { if let Err(e) = self.reap_stale() { eprintln!("[quickq] reap error: {e}"); } } + + // Check periodic tasks (every ~60 iterations = ~3s) + if periodic_counter % 60 == 0 { + if let Err(e) = self.check_periodic() { + eprintln!("[quickq] periodic check error: {e}"); + } + } + + // Auto-cleanup old completed/dead jobs (every ~1200 iterations = ~60s) + if cleanup_counter % 1200 == 0 { + if let Err(e) = self.auto_cleanup() { + eprintln!("[quickq] auto-cleanup error: {e}"); + } + } } } @@ -150,6 +172,11 @@ impl Scheduler { max_retries, task_name, } => { + // Record the error for this attempt + if let Err(e) = self.storage.record_error(&job_id, retry_count, &error) { + eprintln!("[quickq] failed to record error for job {job_id}: {e}"); + } + let policy = self .task_configs .get(&task_name) @@ -196,4 +223,92 @@ impl Scheduler { Ok(()) } + + /// Purge old completed/dead jobs and their errors if result_ttl_ms is set. + fn auto_cleanup(&self) -> Result<()> { + let ttl = match self.result_ttl_ms { + Some(t) => t, + None => return Ok(()), + }; + + let cutoff = now_millis() - ttl; + let completed = self.storage.purge_completed(cutoff)?; + let dead = self.storage.purge_dead(cutoff)?; + let errors = self.storage.purge_job_errors(cutoff)?; + + if completed + dead + errors > 0 { + eprintln!( + "[quickq] auto-cleanup: purged {completed} completed, {dead} dead, {errors} error records" + ); + } + + Ok(()) + } + + /// Check for periodic tasks that are due and enqueue them. + fn check_periodic(&self) -> Result<()> { + let now = now_millis(); + let due_tasks = self.storage.get_due_periodic(now)?; + + for task in due_tasks { + // Enqueue a job for this periodic task + let new_job = NewJob { + queue: task.queue.clone(), + task_name: task.task_name.clone(), + payload: Self::build_periodic_payload(&task.args, &task.kwargs), + priority: 0, + scheduled_at: now, + max_retries: 3, + timeout_ms: 300_000, + unique_key: None, + metadata: None, + }; + + if let Err(e) = self.storage.enqueue(new_job) { + eprintln!("[quickq] failed to enqueue periodic task '{}': {e}", task.name); + continue; + } + + // Compute next run time + let next_run = match next_cron_time(&task.cron_expr, now) { + Ok(t) => t, + Err(e) => { + eprintln!( + "[quickq] failed to compute next run for '{}': {e}", + task.name + ); + continue; + } + }; + + if let Err(e) = self.storage.update_periodic_schedule(&task.name, now, next_run) { + eprintln!( + "[quickq] failed to update schedule for '{}': {e}", + task.name + ); + } + } + + Ok(()) + } + + /// Build a cloudpickle-compatible payload from stored args/kwargs. + /// If both are None, returns an empty tuple payload. + fn build_periodic_payload(args: &Option>, _kwargs: &Option>) -> Vec { + // The args and kwargs are stored as pre-pickled blobs from Python. + // We need to combine them into the (args, kwargs) tuple format. + // If they were stored as None, we use empty tuple/dict from cloudpickle. + // Since we can't easily construct Python pickle in Rust, the Python + // layer stores the full (args, kwargs) tuple as the args blob. + // So if args blob exists, use it directly as the payload. + match args { + Some(blob) => blob.clone(), + None => { + // Empty payload: cloudpickle.dumps(((), {})) + // This is a well-known pickle byte sequence for ((), {}) + // We'll store the full payload in the `args` column from Python. + Vec::new() + } + } + } } diff --git a/rust/quickq-core/src/storage/mod.rs b/rust/quickq-core/src/storage/mod.rs index 6b1c1083..3fc61b90 100644 --- a/rust/quickq-core/src/storage/mod.rs +++ b/rust/quickq-core/src/storage/mod.rs @@ -1 +1,3 @@ +pub mod models; +pub mod schema; pub mod sqlite; diff --git a/rust/quickq-core/src/storage/models.rs b/rust/quickq-core/src/storage/models.rs new file mode 100644 index 00000000..c2e827f5 --- /dev/null +++ b/rust/quickq-core/src/storage/models.rs @@ -0,0 +1,138 @@ +use diesel::prelude::*; + +use super::schema::{dead_letter, job_errors, jobs, periodic_tasks, rate_limits}; + +/// A row in the `jobs` table (for SELECT queries). +#[derive(Queryable, Selectable, Debug, Clone)] +#[diesel(table_name = jobs)] +pub struct JobRow { + pub id: String, + pub queue: String, + pub task_name: String, + pub payload: Vec, + pub status: i32, + pub priority: i32, + pub created_at: i64, + pub scheduled_at: i64, + pub started_at: Option, + pub completed_at: Option, + pub retry_count: i32, + pub max_retries: i32, + pub result: Option>, + pub error: Option, + pub timeout_ms: i64, + pub unique_key: Option, + pub progress: Option, + pub metadata: Option, +} + +/// Insertable struct for creating new jobs. +#[derive(Insertable, Debug)] +#[diesel(table_name = jobs)] +pub struct NewJobRow<'a> { + pub id: &'a str, + pub queue: &'a str, + pub task_name: &'a str, + pub payload: &'a [u8], + pub status: i32, + pub priority: i32, + pub created_at: i64, + pub scheduled_at: i64, + pub retry_count: i32, + pub max_retries: i32, + pub timeout_ms: i64, + pub unique_key: Option<&'a str>, + pub metadata: Option<&'a str>, +} + +/// A row in the `dead_letter` table. +#[derive(Queryable, Selectable, Debug, Clone)] +#[diesel(table_name = dead_letter)] +pub struct DeadLetterRow { + pub id: String, + pub original_job_id: String, + pub queue: String, + pub task_name: String, + pub payload: Vec, + pub error: Option, + pub retry_count: i32, + pub failed_at: i64, + pub metadata: Option, +} + +/// Insertable struct for dead letter entries. +#[derive(Insertable, Debug)] +#[diesel(table_name = dead_letter)] +pub struct NewDeadLetterRow<'a> { + pub id: &'a str, + pub original_job_id: &'a str, + pub queue: &'a str, + pub task_name: &'a str, + pub payload: &'a [u8], + pub error: Option<&'a str>, + pub retry_count: i32, + pub failed_at: i64, + pub metadata: Option<&'a str>, +} + +/// A row in the `rate_limits` table. +#[derive(Queryable, Selectable, Insertable, AsChangeset, Debug, Clone)] +#[diesel(table_name = rate_limits)] +pub struct RateLimitRow { + pub key: String, + pub tokens: f64, + pub max_tokens: f64, + pub refill_rate: f64, + pub last_refill: i64, +} + +/// A row in the `periodic_tasks` table. +#[derive(Queryable, Selectable, Debug, Clone)] +#[diesel(table_name = periodic_tasks)] +pub struct PeriodicTaskRow { + pub name: String, + pub task_name: String, + pub cron_expr: String, + pub args: Option>, + pub kwargs: Option>, + pub queue: String, + pub enabled: bool, + pub last_run: Option, + pub next_run: i64, +} + +/// Insertable struct for periodic tasks. +#[derive(Insertable, AsChangeset, Debug)] +#[diesel(table_name = periodic_tasks)] +pub struct NewPeriodicTaskRow<'a> { + pub name: &'a str, + pub task_name: &'a str, + pub cron_expr: &'a str, + pub args: Option<&'a [u8]>, + pub kwargs: Option<&'a [u8]>, + pub queue: &'a str, + pub enabled: bool, + pub next_run: i64, +} + +/// A row in the `job_errors` table (for SELECT queries). +#[derive(Queryable, Selectable, Debug, Clone)] +#[diesel(table_name = job_errors)] +pub struct JobErrorRow { + pub id: String, + pub job_id: String, + pub attempt: i32, + pub error: String, + pub failed_at: i64, +} + +/// Insertable struct for job error entries. +#[derive(Insertable, Debug)] +#[diesel(table_name = job_errors)] +pub struct NewJobErrorRow<'a> { + pub id: &'a str, + pub job_id: &'a str, + pub attempt: i32, + pub error: &'a str, + pub failed_at: i64, +} diff --git a/rust/quickq-core/src/storage/schema.rs b/rust/quickq-core/src/storage/schema.rs new file mode 100644 index 00000000..d55598d7 --- /dev/null +++ b/rust/quickq-core/src/storage/schema.rs @@ -0,0 +1,70 @@ +diesel::table! { + jobs (id) { + id -> Text, + queue -> Text, + task_name -> Text, + payload -> Binary, + status -> Integer, + priority -> Integer, + created_at -> BigInt, + scheduled_at -> BigInt, + started_at -> Nullable, + completed_at -> Nullable, + retry_count -> Integer, + max_retries -> Integer, + result -> Nullable, + error -> Nullable, + timeout_ms -> BigInt, + unique_key -> Nullable, + progress -> Nullable, + metadata -> Nullable, + } +} + +diesel::table! { + dead_letter (id) { + id -> Text, + original_job_id -> Text, + queue -> Text, + task_name -> Text, + payload -> Binary, + error -> Nullable, + retry_count -> Integer, + failed_at -> BigInt, + metadata -> Nullable, + } +} + +diesel::table! { + rate_limits (key) { + key -> Text, + tokens -> Double, + max_tokens -> Double, + refill_rate -> Double, + last_refill -> BigInt, + } +} + +diesel::table! { + periodic_tasks (name) { + name -> Text, + task_name -> Text, + cron_expr -> Text, + args -> Nullable, + kwargs -> Nullable, + queue -> Text, + enabled -> Bool, + last_run -> Nullable, + next_run -> BigInt, + } +} + +diesel::table! { + job_errors (id) { + id -> Text, + job_id -> Text, + attempt -> Integer, + error -> Text, + failed_at -> BigInt, + } +} diff --git a/rust/quickq-core/src/storage/sqlite.rs b/rust/quickq-core/src/storage/sqlite.rs index d781bb82..68ec40c2 100644 --- a/rust/quickq-core/src/storage/sqlite.rs +++ b/rust/quickq-core/src/storage/sqlite.rs @@ -1,54 +1,64 @@ -use r2d2::Pool; -use r2d2_sqlite::SqliteConnectionManager; -use rusqlite::params; +use diesel::prelude::*; +use diesel::r2d2::{ConnectionManager, Pool}; +use diesel::sqlite::SqliteConnection; use crate::error::{QueueError, Result}; use crate::job::{Job, JobStatus, NewJob, now_millis}; +use super::models::*; +use super::schema::{dead_letter, job_errors, jobs, periodic_tasks, rate_limits}; -/// SQLite-backed storage for the task queue. +type DbPool = Pool>; + +/// SQLite-backed storage for the task queue, using Diesel ORM. #[derive(Clone)] pub struct SqliteStorage { - pool: Pool, + pool: DbPool, } impl SqliteStorage { /// Open (or create) a SQLite database at the given path. pub fn new(db_path: &str) -> Result { - let manager = SqliteConnectionManager::file(db_path); + let manager = ConnectionManager::::new(db_path); let pool = Pool::builder() .max_size(8) - .build(manager) - .map_err(QueueError::Pool)?; + .build(manager)?; let storage = Self { pool }; - storage.init_tables()?; + storage.run_pragmas()?; + storage.run_migrations()?; Ok(storage) } /// Create an in-memory storage (useful for tests). pub fn in_memory() -> Result { - let manager = SqliteConnectionManager::memory(); + let manager = ConnectionManager::::new(":memory:"); let pool = Pool::builder() .max_size(1) - .build(manager) - .map_err(QueueError::Pool)?; + .build(manager)?; let storage = Self { pool }; - storage.init_tables()?; + storage.run_pragmas()?; + storage.run_migrations()?; Ok(storage) } - fn init_tables(&self) -> Result<()> { - let conn = self.pool.get()?; + fn conn(&self) -> Result>> { + Ok(self.pool.get()?) + } + + fn run_pragmas(&self) -> Result<()> { + let mut conn = self.conn()?; + diesel::sql_query("PRAGMA journal_mode = WAL").execute(&mut conn)?; + diesel::sql_query("PRAGMA busy_timeout = 5000").execute(&mut conn)?; + diesel::sql_query("PRAGMA journal_size_limit = 67108864").execute(&mut conn)?; + diesel::sql_query("PRAGMA synchronous = NORMAL").execute(&mut conn)?; + Ok(()) + } - conn.execute_batch( - "PRAGMA journal_mode = WAL; - PRAGMA busy_timeout = 5000; - PRAGMA journal_size_limit = 67108864; - PRAGMA synchronous = NORMAL;", - )?; + fn run_migrations(&self) -> Result<()> { + let mut conn = self.conn()?; - conn.execute_batch( + diesel::sql_query( "CREATE TABLE IF NOT EXISTS jobs ( id TEXT PRIMARY KEY, queue TEXT NOT NULL DEFAULT 'default', @@ -64,15 +74,29 @@ impl SqliteStorage { max_retries INTEGER NOT NULL DEFAULT 3, result BLOB, error TEXT, - timeout_ms INTEGER NOT NULL DEFAULT 300000 - ); - - CREATE INDEX IF NOT EXISTS idx_jobs_dequeue - ON jobs(queue, status, priority DESC, scheduled_at); - CREATE INDEX IF NOT EXISTS idx_jobs_status - ON jobs(status); - - CREATE TABLE IF NOT EXISTS dead_letter ( + timeout_ms INTEGER NOT NULL DEFAULT 300000, + unique_key TEXT, + progress INTEGER, + metadata TEXT + )" + ).execute(&mut conn)?; + + diesel::sql_query( + "CREATE INDEX IF NOT EXISTS idx_jobs_dequeue + ON jobs(queue, status, priority DESC, scheduled_at)" + ).execute(&mut conn)?; + + diesel::sql_query( + "CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)" + ).execute(&mut conn)?; + + diesel::sql_query( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_jobs_unique_key + ON jobs(unique_key) WHERE unique_key IS NOT NULL AND status IN (0, 1)" + ).execute(&mut conn)?; + + diesel::sql_query( + "CREATE TABLE IF NOT EXISTS dead_letter ( id TEXT PRIMARY KEY, original_job_id TEXT NOT NULL, queue TEXT NOT NULL, @@ -82,79 +106,200 @@ impl SqliteStorage { retry_count INTEGER NOT NULL, failed_at INTEGER NOT NULL, metadata TEXT - ); + )" + ).execute(&mut conn)?; - CREATE TABLE IF NOT EXISTS rate_limits ( + diesel::sql_query( + "CREATE TABLE IF NOT EXISTS rate_limits ( key TEXT PRIMARY KEY, tokens REAL NOT NULL, max_tokens REAL NOT NULL, refill_rate REAL NOT NULL, last_refill INTEGER NOT NULL - );", - )?; + )" + ).execute(&mut conn)?; + + diesel::sql_query( + "CREATE TABLE IF NOT EXISTS periodic_tasks ( + name TEXT PRIMARY KEY, + task_name TEXT NOT NULL, + cron_expr TEXT NOT NULL, + args BLOB, + kwargs BLOB, + queue TEXT NOT NULL DEFAULT 'default', + enabled INTEGER NOT NULL DEFAULT 1, + last_run INTEGER, + next_run INTEGER NOT NULL + )" + ).execute(&mut conn)?; + + diesel::sql_query( + "CREATE TABLE IF NOT EXISTS job_errors ( + id TEXT PRIMARY KEY, + job_id TEXT NOT NULL, + attempt INTEGER NOT NULL, + error TEXT NOT NULL, + failed_at INTEGER NOT NULL + )" + ).execute(&mut conn)?; + + diesel::sql_query( + "CREATE INDEX IF NOT EXISTS idx_job_errors_job_id ON job_errors(job_id)" + ).execute(&mut conn)?; Ok(()) } - /// Insert a new job into the queue. Returns the job ID. + // ── Job operations ───────────────────────────────────────────────── + + /// Insert a new job into the queue. Returns the job. pub fn enqueue(&self, new_job: NewJob) -> Result { let job = new_job.into_job(); - let conn = self.pool.get()?; - - conn.execute( - "INSERT INTO jobs (id, queue, task_name, payload, status, priority, - created_at, scheduled_at, retry_count, max_retries, timeout_ms) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", - params![ - job.id, - job.queue, - job.task_name, - job.payload, - job.status as i32, - job.priority, - job.created_at, - job.scheduled_at, - job.retry_count, - job.max_retries, - job.timeout_ms, - ], - )?; + let mut conn = self.conn()?; + + let row = NewJobRow { + id: &job.id, + queue: &job.queue, + task_name: &job.task_name, + payload: &job.payload, + status: job.status as i32, + priority: job.priority, + created_at: job.created_at, + scheduled_at: job.scheduled_at, + retry_count: job.retry_count, + max_retries: job.max_retries, + timeout_ms: job.timeout_ms, + unique_key: job.unique_key.as_deref(), + metadata: job.metadata.as_deref(), + }; + + diesel::insert_into(jobs::table) + .values(&row) + .execute(&mut conn)?; Ok(job) } - /// Atomically dequeue the highest-priority ready job from the given queue. - /// Returns `None` if no jobs are ready. - pub fn dequeue(&self, queue: &str, now: i64) -> Result> { - let conn = self.pool.get()?; - - let mut stmt = conn.prepare( - "UPDATE jobs SET status = ?1, started_at = ?2 - WHERE id = ( - SELECT id FROM jobs - WHERE queue = ?3 AND status = 0 AND scheduled_at <= ?4 - ORDER BY priority DESC, scheduled_at ASC - LIMIT 1 - ) - RETURNING id, queue, task_name, payload, status, priority, - created_at, scheduled_at, started_at, completed_at, - retry_count, max_retries, result, error, timeout_ms", - )?; - - let result = stmt - .query_row( - params![JobStatus::Running as i32, now, queue, now], - |row| row_to_job(row), - ) - .optional()?; + /// Enqueue multiple jobs in a single transaction. + pub fn enqueue_batch(&self, new_jobs: Vec) -> Result> { + let mut conn = self.conn()?; + let jobs: Vec = new_jobs.into_iter().map(|nj| nj.into_job()).collect(); + + conn.transaction(|conn| { + for job in &jobs { + let row = NewJobRow { + id: &job.id, + queue: &job.queue, + task_name: &job.task_name, + payload: &job.payload, + status: job.status as i32, + priority: job.priority, + created_at: job.created_at, + scheduled_at: job.scheduled_at, + retry_count: job.retry_count, + max_retries: job.max_retries, + timeout_ms: job.timeout_ms, + unique_key: job.unique_key.as_deref(), + metadata: job.metadata.as_deref(), + }; + + diesel::insert_into(jobs::table) + .values(&row) + .execute(conn)?; + } + Ok(jobs) + }) + } - Ok(result) + /// Enqueue with unique_key deduplication. Returns existing job if duplicate. + pub fn enqueue_unique(&self, new_job: NewJob) -> Result { + let job = new_job.into_job(); + let mut conn = self.conn()?; + + // Check for existing active job with same unique_key + if let Some(ref uk) = job.unique_key { + let existing: Option = jobs::table + .filter(jobs::unique_key.eq(uk)) + .filter(jobs::status.eq_any([ + JobStatus::Pending as i32, + JobStatus::Running as i32, + ])) + .select(JobRow::as_select()) + .first(&mut conn) + .optional()?; + + if let Some(row) = existing { + return Ok(Job::from(row)); + } + } + + let row = NewJobRow { + id: &job.id, + queue: &job.queue, + task_name: &job.task_name, + payload: &job.payload, + status: job.status as i32, + priority: job.priority, + created_at: job.created_at, + scheduled_at: job.scheduled_at, + retry_count: job.retry_count, + max_retries: job.max_retries, + timeout_ms: job.timeout_ms, + unique_key: job.unique_key.as_deref(), + metadata: job.metadata.as_deref(), + }; + + diesel::insert_into(jobs::table) + .values(&row) + .execute(&mut conn)?; + + Ok(job) + } + + /// Atomically dequeue the highest-priority ready job from the given queue. + pub fn dequeue(&self, queue_name: &str, now: i64) -> Result> { + let mut conn = self.conn()?; + + conn.transaction(|conn| { + // Find the next ready job + let candidate: Option = jobs::table + .filter(jobs::queue.eq(queue_name)) + .filter(jobs::status.eq(JobStatus::Pending as i32)) + .filter(jobs::scheduled_at.le(now)) + .order((jobs::priority.desc(), jobs::scheduled_at.asc())) + .select(JobRow::as_select()) + .first(conn) + .optional()?; + + let row = match candidate { + Some(r) => r, + None => return Ok(None), + }; + + // Atomically claim it + diesel::update(jobs::table) + .filter(jobs::id.eq(&row.id)) + .filter(jobs::status.eq(JobStatus::Pending as i32)) + .set(( + jobs::status.eq(JobStatus::Running as i32), + jobs::started_at.eq(now), + )) + .execute(conn)?; + + // Re-read the updated row + let updated: JobRow = jobs::table + .find(&row.id) + .select(JobRow::as_select()) + .first(conn)?; + + Ok(Some(Job::from(updated))) + }) } /// Dequeue from multiple queues, checking each in order. pub fn dequeue_from(&self, queues: &[String], now: i64) -> Result> { - for queue in queues { - if let Some(job) = self.dequeue(queue, now)? { + for queue_name in queues { + if let Some(job) = self.dequeue(queue_name, now)? { return Ok(Some(job)); } } @@ -162,21 +307,19 @@ impl SqliteStorage { } /// Mark a job as complete with the given result. - pub fn complete(&self, id: &str, result: Option>) -> Result<()> { - let conn = self.pool.get()?; + pub fn complete(&self, id: &str, result_bytes: Option>) -> Result<()> { let now = now_millis(); - - let affected = conn.execute( - "UPDATE jobs SET status = ?1, completed_at = ?2, result = ?3 - WHERE id = ?4 AND status = ?5", - params![ - JobStatus::Complete as i32, - now, - result, - id, - JobStatus::Running as i32, - ], - )?; + let mut conn = self.conn()?; + + let affected = diesel::update(jobs::table) + .filter(jobs::id.eq(id)) + .filter(jobs::status.eq(JobStatus::Running as i32)) + .set(( + jobs::status.eq(JobStatus::Complete as i32), + jobs::completed_at.eq(now), + jobs::result.eq(result_bytes), + )) + .execute(&mut conn)?; if affected == 0 { return Err(QueueError::JobNotFound(id.to_string())); @@ -186,20 +329,18 @@ impl SqliteStorage { /// Mark a job as failed with the given error message. pub fn fail(&self, id: &str, error: &str) -> Result<()> { - let conn = self.pool.get()?; let now = now_millis(); - - let affected = conn.execute( - "UPDATE jobs SET status = ?1, completed_at = ?2, error = ?3 - WHERE id = ?4 AND status = ?5", - params![ - JobStatus::Failed as i32, - now, - error, - id, - JobStatus::Running as i32, - ], - )?; + let mut conn = self.conn()?; + + let affected = diesel::update(jobs::table) + .filter(jobs::id.eq(id)) + .filter(jobs::status.eq(JobStatus::Running as i32)) + .set(( + jobs::status.eq(JobStatus::Failed as i32), + jobs::completed_at.eq(now), + jobs::error.eq(error), + )) + .execute(&mut conn)?; if affected == 0 { return Err(QueueError::JobNotFound(id.to_string())); @@ -207,18 +348,53 @@ impl SqliteStorage { Ok(()) } - /// Re-schedule a job for retry (set status back to pending with new scheduled_at). + /// Re-schedule a job for retry. pub fn retry(&self, id: &str, next_scheduled_at: i64) -> Result<()> { - let conn = self.pool.get()?; + let mut conn = self.conn()?; + + let affected = diesel::update(jobs::table) + .filter(jobs::id.eq(id)) + .set(( + jobs::status.eq(JobStatus::Pending as i32), + jobs::scheduled_at.eq(next_scheduled_at), + jobs::retry_count.eq(jobs::retry_count + 1), + jobs::started_at.eq(None::), + jobs::completed_at.eq(None::), + jobs::error.eq(None::), + )) + .execute(&mut conn)?; + + if affected == 0 { + return Err(QueueError::JobNotFound(id.to_string())); + } + Ok(()) + } + + /// Cancel a pending job. Returns true if cancelled, false if not pending. + pub fn cancel_job(&self, id: &str) -> Result { + let now = now_millis(); + let mut conn = self.conn()?; + + let affected = diesel::update(jobs::table) + .filter(jobs::id.eq(id)) + .filter(jobs::status.eq(JobStatus::Pending as i32)) + .set(( + jobs::status.eq(JobStatus::Cancelled as i32), + jobs::completed_at.eq(now), + )) + .execute(&mut conn)?; + + Ok(affected > 0) + } + + /// Update progress for a running job (0-100). + pub fn update_progress(&self, id: &str, progress: i32) -> Result<()> { + let mut conn = self.conn()?; - let affected = conn.execute( - "UPDATE jobs SET status = ?1, scheduled_at = ?2, - retry_count = retry_count + 1, - started_at = NULL, completed_at = NULL, - error = NULL - WHERE id = ?3", - params![JobStatus::Pending as i32, next_scheduled_at, id], - )?; + let affected = diesel::update(jobs::table) + .filter(jobs::id.eq(id)) + .set(jobs::progress.eq(progress)) + .execute(&mut conn)?; if affected == 0 { return Err(QueueError::JobNotFound(id.to_string())); @@ -228,72 +404,71 @@ impl SqliteStorage { /// Move a job to the dead letter queue. pub fn move_to_dlq(&self, job: &Job, error: &str, metadata: Option<&str>) -> Result<()> { - let conn = self.pool.get()?; let now = now_millis(); let dlq_id = uuid::Uuid::now_v7().to_string(); - - conn.execute( - "INSERT INTO dead_letter (id, original_job_id, queue, task_name, payload, - error, retry_count, failed_at, metadata) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", - params![ - dlq_id, - job.id, - job.queue, - job.task_name, - job.payload, - error, - job.retry_count, - now, + let mut conn = self.conn()?; + + conn.transaction(|conn| { + let dlq_row = NewDeadLetterRow { + id: &dlq_id, + original_job_id: &job.id, + queue: &job.queue, + task_name: &job.task_name, + payload: &job.payload, + error: Some(error), + retry_count: job.retry_count, + failed_at: now, metadata, - ], - )?; + }; - // Mark the original job as dead - conn.execute( - "UPDATE jobs SET status = ?1, error = ?2, completed_at = ?3 WHERE id = ?4", - params![JobStatus::Dead as i32, error, now, job.id], - )?; + diesel::insert_into(dead_letter::table) + .values(&dlq_row) + .execute(conn)?; - Ok(()) + diesel::update(jobs::table) + .filter(jobs::id.eq(&job.id)) + .set(( + jobs::status.eq(JobStatus::Dead as i32), + jobs::error.eq(error), + jobs::completed_at.eq(now), + )) + .execute(conn)?; + + Ok(()) + }) } /// Get a job by ID. pub fn get_job(&self, id: &str) -> Result> { - let conn = self.pool.get()?; + let mut conn = self.conn()?; - let mut stmt = conn.prepare( - "SELECT id, queue, task_name, payload, status, priority, - created_at, scheduled_at, started_at, completed_at, - retry_count, max_retries, result, error, timeout_ms - FROM jobs WHERE id = ?1", - )?; + let row: Option = jobs::table + .find(id) + .select(JobRow::as_select()) + .first(&mut conn) + .optional()?; - let result = stmt.query_row(params![id], |row| row_to_job(row)).optional()?; - Ok(result) + Ok(row.map(Job::from)) } /// Get queue statistics. pub fn stats(&self) -> Result { - let conn = self.pool.get()?; + let mut conn = self.conn()?; - let mut stmt = conn.prepare( - "SELECT status, COUNT(*) FROM jobs GROUP BY status", - )?; + let rows: Vec<(i32, i64)> = jobs::table + .group_by(jobs::status) + .select((jobs::status, diesel::dsl::count(jobs::id))) + .load(&mut conn)?; let mut stats = QueueStats::default(); - let rows = stmt.query_map([], |row| { - Ok((row.get::<_, i32>(0)?, row.get::<_, i64>(1)?)) - })?; - - for row in rows { - let (status, count) = row?; + for (status, count) in rows { match JobStatus::from_i32(status) { Some(JobStatus::Pending) => stats.pending = count, Some(JobStatus::Running) => stats.running = count, Some(JobStatus::Complete) => stats.completed = count, Some(JobStatus::Failed) => stats.failed = count, Some(JobStatus::Dead) => stats.dead = count, + Some(JobStatus::Cancelled) => stats.cancelled = count, None => {} } } @@ -301,199 +476,229 @@ impl SqliteStorage { Ok(stats) } + // ── Dead letter operations ───────────────────────────────────────── + /// List dead letter entries. pub fn list_dead(&self, limit: i64, offset: i64) -> Result> { - let conn = self.pool.get()?; - - let mut stmt = conn.prepare( - "SELECT id, original_job_id, queue, task_name, payload, - error, retry_count, failed_at, metadata - FROM dead_letter ORDER BY failed_at DESC LIMIT ?1 OFFSET ?2", - )?; - - let rows = stmt.query_map(params![limit, offset], |row| { - Ok(DeadJob { - id: row.get(0)?, - original_job_id: row.get(1)?, - queue: row.get(2)?, - task_name: row.get(3)?, - payload: row.get(4)?, - error: row.get(5)?, - retry_count: row.get(6)?, - failed_at: row.get(7)?, - metadata: row.get(8)?, - }) - })?; + let mut conn = self.conn()?; - let mut dead_jobs = Vec::new(); - for row in rows { - dead_jobs.push(row?); - } - Ok(dead_jobs) + let rows: Vec = dead_letter::table + .order(dead_letter::failed_at.desc()) + .limit(limit) + .offset(offset) + .select(DeadLetterRow::as_select()) + .load(&mut conn)?; + + Ok(rows.into_iter().map(DeadJob::from).collect()) } /// Re-enqueue a dead letter job. Returns the new job ID. pub fn retry_dead(&self, dead_id: &str) -> Result { - // Fetch dead letter entry (connection is dropped after this block) - let dead: DeadJob = { - let conn = self.pool.get()?; - let mut stmt = conn.prepare( - "SELECT id, original_job_id, queue, task_name, payload, - error, retry_count, failed_at, metadata - FROM dead_letter WHERE id = ?1", - )?; - let result = stmt - .query_row(params![dead_id], |row| { - Ok(DeadJob { - id: row.get(0)?, - original_job_id: row.get(1)?, - queue: row.get(2)?, - task_name: row.get(3)?, - payload: row.get(4)?, - error: row.get(5)?, - retry_count: row.get(6)?, - failed_at: row.get(7)?, - metadata: row.get(8)?, - }) - }) - .map_err(|_| QueueError::JobNotFound(dead_id.to_string()))?; - result - }; + let mut conn = self.conn()?; + + let dead_row: DeadLetterRow = dead_letter::table + .find(dead_id) + .select(DeadLetterRow::as_select()) + .first(&mut conn) + .map_err(|_| QueueError::JobNotFound(dead_id.to_string()))?; + + // Drop conn before calling enqueue (which gets its own conn) + drop(conn); - // Create new job from dead letter (gets its own connection) let new_job = NewJob { - queue: dead.queue, - task_name: dead.task_name, - payload: dead.payload, + queue: dead_row.queue, + task_name: dead_row.task_name, + payload: dead_row.payload, priority: 0, scheduled_at: now_millis(), max_retries: 3, timeout_ms: 300_000, + unique_key: None, + metadata: None, }; let job = self.enqueue(new_job)?; - // Remove from dead letter queue (gets its own connection) - let conn = self.pool.get()?; - conn.execute("DELETE FROM dead_letter WHERE id = ?1", params![dead_id])?; + let mut conn = self.conn()?; + diesel::delete(dead_letter::table.find(dead_id)) + .execute(&mut conn)?; Ok(job.id) } /// Purge dead letter entries older than the given timestamp. pub fn purge_dead(&self, older_than_ms: i64) -> Result { - let conn = self.pool.get()?; - let affected = conn.execute( - "DELETE FROM dead_letter WHERE failed_at < ?1", - params![older_than_ms], - )?; + let mut conn = self.conn()?; + + let affected = diesel::delete( + dead_letter::table.filter(dead_letter::failed_at.lt(older_than_ms)) + ).execute(&mut conn)?; + Ok(affected as u64) } /// Purge completed jobs older than the given timestamp. pub fn purge_completed(&self, older_than_ms: i64) -> Result { - let conn = self.pool.get()?; - let affected = conn.execute( - "DELETE FROM jobs WHERE status = ?1 AND completed_at < ?2", - params![JobStatus::Complete as i32, older_than_ms], - )?; + let mut conn = self.conn()?; + + let affected = diesel::delete( + jobs::table + .filter(jobs::status.eq(JobStatus::Complete as i32)) + .filter(jobs::completed_at.lt(older_than_ms)) + ).execute(&mut conn)?; + Ok(affected as u64) } - /// Find stale running jobs (started but exceeded timeout) and mark them as failed. + /// Find stale running jobs that exceeded their timeout. pub fn reap_stale_jobs(&self, now: i64) -> Result> { - let conn = self.pool.get()?; - - let mut stmt = conn.prepare( - "SELECT id, queue, task_name, payload, status, priority, - created_at, scheduled_at, started_at, completed_at, - retry_count, max_retries, result, error, timeout_ms - FROM jobs - WHERE status = ?1 AND started_at IS NOT NULL - AND (started_at + timeout_ms) < ?2", - )?; - - let rows = stmt.query_map(params![JobStatus::Running as i32, now], |row| { - row_to_job(row) - })?; - - let mut stale = Vec::new(); - for row in rows { - stale.push(row?); - } + let mut conn = self.conn()?; + + // SQLite doesn't support column arithmetic in Diesel DSL easily, + // so we use sql_query for the timeout comparison. + let rows: Vec = jobs::table + .filter(jobs::status.eq(JobStatus::Running as i32)) + .filter(jobs::started_at.is_not_null()) + .select(JobRow::as_select()) + .load(&mut conn)?; + + // Filter in Rust for the timeout condition + let stale: Vec = rows + .into_iter() + .filter(|r| { + if let Some(started) = r.started_at { + (started + r.timeout_ms) < now + } else { + false + } + }) + .map(Job::from) + .collect(); + Ok(stale) } - // -- Rate limit storage methods -- + // ── Rate limit operations ────────────────────────────────────────── pub fn get_rate_limit(&self, key: &str) -> Result> { - let conn = self.pool.get()?; - let mut stmt = conn.prepare( - "SELECT key, tokens, max_tokens, refill_rate, last_refill - FROM rate_limits WHERE key = ?1", - )?; - - let result = stmt - .query_row(params![key], |row| { - Ok(RateLimitRow { - key: row.get(0)?, - tokens: row.get(1)?, - max_tokens: row.get(2)?, - refill_rate: row.get(3)?, - last_refill: row.get(4)?, - }) - }) + let mut conn = self.conn()?; + + let row: Option = rate_limits::table + .find(key) + .select(RateLimitRow::as_select()) + .first(&mut conn) .optional()?; - Ok(result) + + Ok(row) } pub fn upsert_rate_limit(&self, row: &RateLimitRow) -> Result<()> { - let conn = self.pool.get()?; - conn.execute( - "INSERT INTO rate_limits (key, tokens, max_tokens, refill_rate, last_refill) - VALUES (?1, ?2, ?3, ?4, ?5) - ON CONFLICT(key) DO UPDATE SET tokens = ?2, last_refill = ?5", - params![row.key, row.tokens, row.max_tokens, row.refill_rate, row.last_refill], - )?; + let mut conn = self.conn()?; + + diesel::replace_into(rate_limits::table) + .values(row) + .execute(&mut conn)?; + Ok(()) } -} -/// Use rusqlite's optional extension for query_row returning Option. -trait OptionalExt { - fn optional(self) -> std::result::Result, rusqlite::Error>; -} + // ── Periodic task operations ─────────────────────────────────────── -impl OptionalExt for std::result::Result { - fn optional(self) -> std::result::Result, rusqlite::Error> { - match self { - Ok(v) => Ok(Some(v)), - Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), - Err(e) => Err(e), - } + /// Register or update a periodic task. + pub fn register_periodic(&self, task: &NewPeriodicTaskRow) -> Result<()> { + let mut conn = self.conn()?; + + diesel::replace_into(periodic_tasks::table) + .values(task) + .execute(&mut conn)?; + + Ok(()) + } + + /// Get all periodic tasks that are due to run. + pub fn get_due_periodic(&self, now: i64) -> Result> { + let mut conn = self.conn()?; + + let rows = periodic_tasks::table + .filter(periodic_tasks::enabled.eq(true)) + .filter(periodic_tasks::next_run.le(now)) + .select(PeriodicTaskRow::as_select()) + .load(&mut conn)?; + + Ok(rows) } -} -fn row_to_job(row: &rusqlite::Row) -> rusqlite::Result { - Ok(Job { - id: row.get(0)?, - queue: row.get(1)?, - task_name: row.get(2)?, - payload: row.get(3)?, - status: JobStatus::from_i32(row.get(4)?).unwrap_or(JobStatus::Pending), - priority: row.get(5)?, - created_at: row.get(6)?, - scheduled_at: row.get(7)?, - started_at: row.get(8)?, - completed_at: row.get(9)?, - retry_count: row.get(10)?, - max_retries: row.get(11)?, - result: row.get(12)?, - error: row.get(13)?, - timeout_ms: row.get(14)?, - }) + // ── Job error operations ────────────────────────────────────────── + + /// Record an error for a job attempt. + pub fn record_error(&self, job_id: &str, attempt: i32, error: &str) -> Result<()> { + let mut conn = self.conn()?; + let id = uuid::Uuid::now_v7().to_string(); + let now = now_millis(); + + let row = NewJobErrorRow { + id: &id, + job_id, + attempt, + error, + failed_at: now, + }; + + diesel::insert_into(job_errors::table) + .values(&row) + .execute(&mut conn)?; + + Ok(()) + } + + /// Get all errors for a job, ordered by attempt. + pub fn get_job_errors(&self, job_id: &str) -> Result> { + let mut conn = self.conn()?; + + let rows = job_errors::table + .filter(job_errors::job_id.eq(job_id)) + .order(job_errors::attempt.asc()) + .select(JobErrorRow::as_select()) + .load(&mut conn)?; + + Ok(rows) + } + + /// Purge job errors older than the given timestamp. + pub fn purge_job_errors(&self, older_than_ms: i64) -> Result { + let mut conn = self.conn()?; + + let affected = diesel::delete( + job_errors::table.filter(job_errors::failed_at.lt(older_than_ms)) + ).execute(&mut conn)?; + + Ok(affected as u64) + } + + // ── Periodic task operations ─────────────────────────────────────── + + /// Update a periodic task's schedule after execution. + pub fn update_periodic_schedule( + &self, + name: &str, + last_run: i64, + next_run: i64, + ) -> Result<()> { + let mut conn = self.conn()?; + + diesel::update(periodic_tasks::table.find(name)) + .set(( + periodic_tasks::last_run.eq(last_run), + periodic_tasks::next_run.eq(next_run), + )) + .execute(&mut conn)?; + + Ok(()) + } } +// ── Helper types ─────────────────────────────────────────────────────── + #[derive(Debug, Clone, Default)] pub struct QueueStats { pub pending: i64, @@ -501,6 +706,7 @@ pub struct QueueStats { pub completed: i64, pub failed: i64, pub dead: i64, + pub cancelled: i64, } #[derive(Debug, Clone)] @@ -516,13 +722,20 @@ pub struct DeadJob { pub metadata: Option, } -#[derive(Debug, Clone)] -pub struct RateLimitRow { - pub key: String, - pub tokens: f64, - pub max_tokens: f64, - pub refill_rate: f64, - pub last_refill: i64, +impl From for DeadJob { + fn from(row: DeadLetterRow) -> Self { + Self { + id: row.id, + original_job_id: row.original_job_id, + queue: row.queue, + task_name: row.task_name, + payload: row.payload, + error: row.error, + retry_count: row.retry_count, + failed_at: row.failed_at, + metadata: row.metadata, + } + } } #[cfg(test)] @@ -542,6 +755,8 @@ mod tests { scheduled_at: now_millis(), max_retries: 3, timeout_ms: 300_000, + unique_key: None, + metadata: None, } } @@ -577,11 +792,9 @@ mod tests { new_job.scheduled_at = future; storage.enqueue(new_job).unwrap(); - // Should not dequeue before scheduled_at let none = storage.dequeue("default", now_millis()).unwrap(); assert!(none.is_none()); - // Should dequeue after scheduled_at let some = storage.dequeue("default", future + 1).unwrap(); assert!(some.is_some()); } @@ -674,12 +887,10 @@ mod tests { let dead = storage.list_dead(10, 0).unwrap(); let new_id = storage.retry_dead(&dead[0].id).unwrap(); - // New job exists and is pending let new_job = storage.get_job(&new_id).unwrap().unwrap(); assert_eq!(new_job.status, JobStatus::Pending); assert_eq!(new_job.task_name, "retry_dead_task"); - // Dead letter entry removed let dead = storage.list_dead(10, 0).unwrap(); assert!(dead.is_empty()); } @@ -694,4 +905,103 @@ mod tests { assert_eq!(stats.pending, 2); assert_eq!(stats.running, 0); } + + #[test] + fn test_cancel_job() { + let storage = test_storage(); + let job = storage.enqueue(make_job("cancel_me")).unwrap(); + + assert!(storage.cancel_job(&job.id).unwrap()); + + let fetched = storage.get_job(&job.id).unwrap().unwrap(); + assert_eq!(fetched.status, JobStatus::Cancelled); + + // Cancelling again should return false + assert!(!storage.cancel_job(&job.id).unwrap()); + } + + #[test] + fn test_unique_key_dedup() { + let storage = test_storage(); + + let mut job1 = make_job("unique_task"); + job1.unique_key = Some("my-key".to_string()); + let j1 = storage.enqueue_unique(job1).unwrap(); + + let mut job2 = make_job("unique_task"); + job2.unique_key = Some("my-key".to_string()); + let j2 = storage.enqueue_unique(job2).unwrap(); + + // Should return the same job + assert_eq!(j1.id, j2.id); + } + + #[test] + fn test_enqueue_batch() { + let storage = test_storage(); + let jobs: Vec = (0..5).map(|i| { + let mut j = make_job(&format!("batch_task_{i}")); + j.priority = i; + j + }).collect(); + + let result = storage.enqueue_batch(jobs).unwrap(); + assert_eq!(result.len(), 5); + + let stats = storage.stats().unwrap(); + assert_eq!(stats.pending, 5); + } + + #[test] + fn test_record_and_get_job_errors() { + let storage = test_storage(); + let job = storage.enqueue(make_job("error_task")).unwrap(); + + storage.record_error(&job.id, 0, "first failure").unwrap(); + storage.record_error(&job.id, 1, "second failure").unwrap(); + + let errors = storage.get_job_errors(&job.id).unwrap(); + assert_eq!(errors.len(), 2); + assert_eq!(errors[0].attempt, 0); + assert_eq!(errors[0].error, "first failure"); + assert_eq!(errors[1].attempt, 1); + assert_eq!(errors[1].error, "second failure"); + } + + #[test] + fn test_job_errors_empty_for_success() { + let storage = test_storage(); + let job = storage.enqueue(make_job("ok_task")).unwrap(); + + let errors = storage.get_job_errors(&job.id).unwrap(); + assert!(errors.is_empty()); + } + + #[test] + fn test_purge_job_errors() { + let storage = test_storage(); + let job = storage.enqueue(make_job("purge_err_task")).unwrap(); + + storage.record_error(&job.id, 0, "old error").unwrap(); + // All errors are recorded at now_millis(), so purging with a future cutoff should remove them + let purged = storage.purge_job_errors(now_millis() + 10_000).unwrap(); + assert_eq!(purged, 1); + + let errors = storage.get_job_errors(&job.id).unwrap(); + assert!(errors.is_empty()); + } + + #[test] + fn test_progress_tracking() { + let storage = test_storage(); + let job = storage.enqueue(make_job("progress_task")).unwrap(); + + storage.update_progress(&job.id, 50).unwrap(); + let fetched = storage.get_job(&job.id).unwrap().unwrap(); + assert_eq!(fetched.progress, Some(50)); + + storage.update_progress(&job.id, 100).unwrap(); + let fetched = storage.get_job(&job.id).unwrap().unwrap(); + assert_eq!(fetched.progress, Some(100)); + } } diff --git a/rust/quickq-python/src/py_job.rs b/rust/quickq-python/src/py_job.rs index bf388a8d..2498f28c 100644 --- a/rust/quickq-python/src/py_job.rs +++ b/rust/quickq-python/src/py_job.rs @@ -30,6 +30,12 @@ pub struct PyJob { pub error: Option, #[pyo3(get)] pub timeout_ms: i64, + #[pyo3(get)] + pub unique_key: Option, + #[pyo3(get)] + pub progress: Option, + #[pyo3(get)] + pub metadata: Option, status_val: i32, result_bytes: Option>, @@ -75,6 +81,9 @@ impl From for PyJob { completed_at: job.completed_at, error: job.error, timeout_ms: job.timeout_ms, + unique_key: job.unique_key, + progress: job.progress, + metadata: job.metadata, status_val: job.status as i32, result_bytes: job.result, } diff --git a/rust/quickq-python/src/py_queue.rs b/rust/quickq-python/src/py_queue.rs index 168e6a0f..c3b1dec7 100644 --- a/rust/quickq-python/src/py_queue.rs +++ b/rust/quickq-python/src/py_queue.rs @@ -1,13 +1,16 @@ use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use crossbeam_channel; use pyo3::prelude::*; use pyo3::types::PyDict; use quickq_core::job::{NewJob, now_millis}; +use quickq_core::periodic::next_cron_time; use quickq_core::rate_limiter::RateLimitConfig; use quickq_core::retry::RetryPolicy; use quickq_core::scheduler::{Scheduler, TaskConfig}; +use quickq_core::storage::models::NewPeriodicTaskRow; use quickq_core::storage::sqlite::SqliteStorage; use crate::py_config::PyTaskConfig; @@ -23,18 +26,21 @@ pub struct PyQueue { default_retry: i32, default_timeout: i64, default_priority: i32, + shutdown_flag: Arc, + result_ttl_ms: Option, } #[pymethods] impl PyQueue { #[new] - #[pyo3(signature = (db_path="quickq.db", workers=0, default_retry=3, default_timeout=300, default_priority=0))] + #[pyo3(signature = (db_path="quickq.db", workers=0, default_retry=3, default_timeout=300, default_priority=0, result_ttl=None))] pub fn new( db_path: &str, workers: usize, default_retry: i32, default_timeout: i64, default_priority: i32, + result_ttl: Option, ) -> PyResult { let storage = SqliteStorage::new(db_path) .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; @@ -54,11 +60,18 @@ impl PyQueue { default_retry, default_timeout, default_priority, + shutdown_flag: Arc::new(AtomicBool::new(false)), + result_ttl_ms: result_ttl.map(|s| s * 1000), }) } - /// Enqueue a job. Called from Python with the task name and pickled (args, kwargs). - #[pyo3(signature = (task_name, payload, queue="default", priority=None, delay_seconds=None, max_retries=None, timeout=None))] + /// Signal the worker to shut down gracefully. + pub fn request_shutdown(&self) { + self.shutdown_flag.store(true, Ordering::SeqCst); + } + + /// Enqueue a job. + #[pyo3(signature = (task_name, payload, queue="default", priority=None, delay_seconds=None, max_retries=None, timeout=None, unique_key=None, metadata=None))] pub fn enqueue( &self, task_name: &str, @@ -68,6 +81,8 @@ impl PyQueue { delay_seconds: Option, max_retries: Option, timeout: Option, + unique_key: Option, + metadata: Option, ) -> PyResult { let now = now_millis(); let scheduled_at = match delay_seconds { @@ -83,14 +98,69 @@ impl PyQueue { scheduled_at, max_retries: max_retries.unwrap_or(self.default_retry), timeout_ms: timeout.unwrap_or(self.default_timeout) * 1000, + unique_key: unique_key.clone(), + metadata, }; - let job = self + let job = if unique_key.is_some() { + self.storage.enqueue_unique(new_job) + } else { + self.storage.enqueue(new_job) + }; + + let job = job.map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + Ok(PyJob::from(job)) + } + + /// Enqueue multiple jobs in a single transaction. + #[pyo3(signature = (task_names, payloads, queues=None, priorities=None, max_retries_list=None, timeouts=None))] + pub fn enqueue_batch( + &self, + task_names: Vec, + payloads: Vec>, + queues: Option>, + priorities: Option>, + max_retries_list: Option>, + timeouts: Option>, + ) -> PyResult> { + let count = task_names.len(); + if payloads.len() != count { + return Err(pyo3::exceptions::PyValueError::new_err( + "task_names and payloads must have the same length", + )); + } + + let now = now_millis(); + let mut new_jobs = Vec::with_capacity(count); + + for i in 0..count { + new_jobs.push(NewJob { + queue: queues.as_ref().map_or("default".to_string(), |q| { + q.get(i).cloned().unwrap_or_else(|| "default".to_string()) + }), + task_name: task_names[i].clone(), + payload: payloads[i].clone(), + priority: priorities.as_ref().map_or(self.default_priority, |p| { + p.get(i).copied().unwrap_or(self.default_priority) + }), + scheduled_at: now, + max_retries: max_retries_list.as_ref().map_or(self.default_retry, |r| { + r.get(i).copied().unwrap_or(self.default_retry) + }), + timeout_ms: timeouts.as_ref().map_or(self.default_timeout * 1000, |t| { + t.get(i).copied().unwrap_or(self.default_timeout) * 1000 + }), + unique_key: None, + metadata: None, + }); + } + + let jobs = self .storage - .enqueue(new_job) + .enqueue_batch(new_jobs) .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; - Ok(PyJob::from(job)) + Ok(jobs.into_iter().map(PyJob::from).collect()) } /// Get a job by ID. @@ -102,6 +172,28 @@ impl PyQueue { Ok(job.map(PyJob::from)) } + /// Cancel a pending job. Returns True if cancelled, False if not pending. + pub fn cancel_job(&self, job_id: &str) -> PyResult { + self.storage + .cancel_job(job_id) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + + /// Update progress for a running job (0-100). + pub fn update_progress(&self, job_id: &str, progress: i32) -> PyResult<()> { + self.storage + .update_progress(job_id, progress) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + + /// Purge completed jobs older than given seconds ago. Returns count deleted. + pub fn purge_completed(&self, older_than_seconds: i64) -> PyResult { + let cutoff = now_millis() - (older_than_seconds * 1000); + self.storage + .purge_completed(cutoff) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + /// Get queue statistics. pub fn stats(&self) -> PyResult { let stats = self @@ -116,6 +208,7 @@ impl PyQueue { dict.set_item("completed", stats.completed)?; dict.set_item("failed", stats.failed)?; dict.set_item("dead", stats.dead)?; + dict.set_item("cancelled", stats.cancelled)?; Ok(dict.into()) }) } @@ -161,9 +254,60 @@ impl PyQueue { .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) } + /// Get error history for a job. + pub fn get_job_errors(&self, job_id: &str) -> PyResult> { + let errors = self + .storage + .get_job_errors(job_id) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + + Python::with_gil(|py| { + let mut result = Vec::with_capacity(errors.len()); + for err in errors { + let dict = PyDict::new_bound(py); + dict.set_item("id", err.id)?; + dict.set_item("job_id", err.job_id)?; + dict.set_item("attempt", err.attempt)?; + dict.set_item("error", err.error)?; + dict.set_item("failed_at", err.failed_at)?; + result.push(dict.into()); + } + Ok(result) + }) + } + + /// Register a periodic task schedule. + #[pyo3(signature = (name, task_name, cron_expr, args=None, kwargs=None, queue="default"))] + pub fn register_periodic( + &self, + name: &str, + task_name: &str, + cron_expr: &str, + args: Option>, + kwargs: Option>, + queue: &str, + ) -> PyResult<()> { + let now = now_millis(); + let next_run = next_cron_time(cron_expr, now) + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e.to_string()))?; + + let row = NewPeriodicTaskRow { + name, + task_name, + cron_expr, + args: args.as_deref(), + kwargs: kwargs.as_deref(), + queue, + enabled: true, + next_run, + }; + + self.storage + .register_periodic(&row) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + /// Run the worker loop. This blocks until interrupted. - /// `task_registry` is a Python dict mapping task_name -> callable. - /// `task_configs` is a list of PyTaskConfig for retry/rate-limit settings. #[pyo3(signature = (task_registry, task_configs, queues=None))] pub fn run_worker( &self, @@ -172,12 +316,13 @@ impl PyQueue { task_configs: Vec, queues: Option>, ) -> PyResult<()> { + // Reset shutdown flag for this run + self.shutdown_flag.store(false, Ordering::SeqCst); + let queues = queues.unwrap_or_else(|| vec!["default".to_string()]); - // Build scheduler - let mut scheduler = Scheduler::new(self.storage.clone(), queues); + let mut scheduler = Scheduler::new(self.storage.clone(), queues, self.result_ttl_ms); - // Register task configs for tc in &task_configs { let retry_policy = RetryPolicy { max_retries: tc.max_retries, @@ -196,16 +341,13 @@ impl PyQueue { let shutdown = scheduler.shutdown_handle(); - // Create channels let (job_tx, job_rx) = crossbeam_channel::bounded(self.num_workers * 2); let (result_tx, result_rx) = crossbeam_channel::bounded(self.num_workers * 2); - // Start worker threads let registry_arc = Arc::new(task_registry); let worker_pool = WorkerPool::start(self.num_workers, job_rx, result_tx, registry_arc); - // Run scheduler in a Tokio runtime on a separate thread let scheduler_arc = Arc::new(scheduler); let scheduler_for_dispatch = scheduler_arc.clone(); @@ -217,12 +359,39 @@ impl PyQueue { rt.block_on(scheduler_for_dispatch.run(job_tx)); }); - // Process results on the main thread (with GIL release) let scheduler_for_results = scheduler_arc.clone(); + let flag = self.shutdown_flag.clone(); - // Release the GIL and process results py.allow_threads(move || { + let drain_timeout = std::time::Duration::from_secs(30); + loop { + // Check if graceful shutdown was requested + if flag.load(Ordering::SeqCst) { + // Stop the scheduler from dispatching new jobs + shutdown.notify_one(); + + // Drain remaining results with a timeout + let drain_start = std::time::Instant::now(); + while drain_start.elapsed() < drain_timeout { + match result_rx.recv_timeout(std::time::Duration::from_millis(100)) { + Ok(result) => { + if let Err(e) = scheduler_for_results.handle_result(result) { + eprintln!("[quickq] result handling error: {e}"); + } + } + Err(crossbeam_channel::RecvTimeoutError::Timeout) => { + // Check if all workers are done (channel disconnected) + continue; + } + Err(crossbeam_channel::RecvTimeoutError::Disconnected) => { + break; + } + } + } + break; + } + match result_rx.recv_timeout(std::time::Duration::from_millis(100)) { Ok(result) => { if let Err(e) = scheduler_for_results.handle_result(result) { @@ -239,8 +408,6 @@ impl PyQueue { } }); - // Cleanup - shutdown.notify_one(); let _ = scheduler_handle.join(); worker_pool.join(); diff --git a/rust/quickq-python/src/py_worker.rs b/rust/quickq-python/src/py_worker.rs index bb2e13c5..d32a41db 100644 --- a/rust/quickq-python/src/py_worker.rs +++ b/rust/quickq-python/src/py_worker.rs @@ -60,10 +60,9 @@ fn worker_loop( let task_name = job.task_name.clone(); let retry_count = job.retry_count; let max_retries = job.max_retries; - let payload = job.payload.clone(); let result = Python::with_gil(|py| -> PyResult>> { - execute_task(py, &task_registry, &task_name, &payload) + execute_task(py, &task_registry, &job) }); let job_result = match result { @@ -92,8 +91,7 @@ fn worker_loop( fn execute_task( py: Python<'_>, task_registry: &PyObject, - task_name: &str, - payload: &[u8], + job: &Job, ) -> PyResult>> { let cloudpickle = py.import_bound("cloudpickle")?; let registry = task_registry.bind(py); @@ -101,16 +99,23 @@ fn execute_task( // Look up the task function let registry_dict: &Bound<'_, PyDict> = registry.downcast()?; let task_fn = registry_dict - .get_item(task_name)? + .get_item(&job.task_name)? .ok_or_else(|| { pyo3::exceptions::PyKeyError::new_err(format!( "task '{}' not registered", - task_name + job.task_name )) })?; + // Set job context before execution + let context_mod = py.import_bound("quickq.context")?; + context_mod.call_method1( + "_set_context", + (&job.id, &job.task_name, job.retry_count, &job.queue), + )?; + // Deserialize arguments: (args, kwargs) - let payload_bytes = PyBytes::new_bound(py, payload); + let payload_bytes = PyBytes::new_bound(py, &job.payload); let unpickled = cloudpickle.call_method1("loads", (payload_bytes,))?; let args_tuple: Bound<'_, PyTuple> = unpickled.downcast_into()?; @@ -120,13 +125,18 @@ fn execute_task( // Call the task function let result = if kwargs.is_none() { let args_tuple_inner: Bound<'_, PyTuple> = args.downcast_into()?; - task_fn.call(args_tuple_inner, None)? + task_fn.call(args_tuple_inner, None) } else { let kwargs_dict: Bound<'_, PyDict> = kwargs.downcast_into()?; let args_tuple_inner: Bound<'_, PyTuple> = args.downcast_into()?; - task_fn.call(args_tuple_inner, Some(&kwargs_dict))? + task_fn.call(args_tuple_inner, Some(&kwargs_dict)) }; + // Clear context after execution (whether success or failure) + let _ = context_mod.call_method0("_clear_context"); + + let result = result?; + // Serialize result if result.is_none() { Ok(None) diff --git a/tests/python/test_batch.py b/tests/python/test_batch.py new file mode 100644 index 00000000..57e8425f --- /dev/null +++ b/tests/python/test_batch.py @@ -0,0 +1,63 @@ +"""Tests for batch enqueue (enqueue_many / task.map).""" + +import threading + +import pytest + +from quickq import Queue + + +@pytest.fixture +def queue(tmp_path): + db_path = str(tmp_path / "test_batch.db") + return Queue(db_path=db_path, workers=2) + + +def test_enqueue_many(queue): + """enqueue_many enqueues all items in a single batch.""" + + @queue.task() + def double(x): + return x * 2 + + jobs = queue.enqueue_many( + task_name=double.name, + args_list=[(i,) for i in range(10)], + ) + assert len(jobs) == 10 + + stats = queue.stats() + assert stats["pending"] == 10 + + +def test_task_map(queue): + """TaskWrapper.map() enqueues and returns results.""" + + @queue.task() + def add(a, b): + return a + b + + jobs = add.map([(1, 2), (3, 4), (5, 6)]) + assert len(jobs) == 3 + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + results = [j.result(timeout=10) for j in jobs] + assert sorted(results) == [3, 7, 11] + + +def test_batch_stats(queue): + """Batch enqueue of 50 items shows correct pending count.""" + + @queue.task() + def noop(): + pass + + queue.enqueue_many( + task_name=noop.name, + args_list=[() for _ in range(50)], + ) + + stats = queue.stats() + assert stats["pending"] == 50 diff --git a/tests/python/test_cancel.py b/tests/python/test_cancel.py new file mode 100644 index 00000000..4e01b615 --- /dev/null +++ b/tests/python/test_cancel.py @@ -0,0 +1,69 @@ +"""Tests for job cancellation.""" + +from __future__ import annotations + +import threading + +import pytest + +from quickq import Queue + + +@pytest.fixture +def queue(tmp_path): + db_path = str(tmp_path / "test_cancel.db") + return Queue(db_path=db_path, workers=2) + + +def test_cancel_pending_job(queue): + """A pending job can be cancelled.""" + + @queue.task() + def slow_task(): + return "done" + + job = slow_task.delay() + assert queue.cancel_job(job.id) is True + + refreshed = queue.get_job(job.id) + assert refreshed.status == "cancelled" + + +def test_cancel_nonexistent_job(queue): + """Cancelling a nonexistent job returns False.""" + + @queue.task() + def dummy(): + pass + + assert queue.cancel_job("nonexistent-id") is False + + +def test_cancel_completed_job(queue): + """Cancelling a completed job returns False (only pending can be cancelled).""" + + @queue.task() + def quick_task(): + return 42 + + job = quick_task.delay() + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + job.result(timeout=10) + assert queue.cancel_job(job.id) is False + + +def test_cancelled_in_stats(queue): + """Cancelled jobs show up in stats.""" + + @queue.task() + def task_a(): + pass + + job = task_a.delay() + queue.cancel_job(job.id) + + stats = queue.stats() + assert stats["cancelled"] == 1 diff --git a/tests/python/test_chain.py b/tests/python/test_chain.py new file mode 100644 index 00000000..60b6ec59 --- /dev/null +++ b/tests/python/test_chain.py @@ -0,0 +1,81 @@ +"""Tests for task chaining (chain, group, chord).""" + +from __future__ import annotations + +import threading + +import pytest + +from quickq import Queue, chain, chord, group + + +@pytest.fixture +def queue(tmp_path): + db_path = str(tmp_path / "test_chain.db") + q = Queue(db_path=db_path, workers=4) + + # Start worker in background + t = threading.Thread(target=q.run_worker, daemon=True) + t.start() + + return q + + +def test_chain_executes_in_order(queue): + """chain pipes results through signatures in order.""" + + @queue.task() + def add(a, b): + return a + b + + @queue.task() + def double(x): + return x * 2 + + result = chain(add.s(2, 3), double.s()) + last_job = result.apply(queue) + assert last_job.result(timeout=30) == 10 # (2+3) * 2 = 10 + + +def test_chain_with_immutable(queue): + """si() signatures ignore previous results.""" + + @queue.task() + def add(a, b): + return a + b + + @queue.task() + def constant(): + return 99 + + result = chain(add.s(1, 2), constant.si()) + last_job = result.apply(queue) + assert last_job.result(timeout=30) == 99 + + +def test_group_parallel(queue): + """group enqueues tasks in parallel.""" + + @queue.task() + def square(x): + return x * x + + jobs = group(square.s(2), square.s(3), square.s(4)).apply(queue) + results = [j.result(timeout=30) for j in jobs] + assert sorted(results) == [4, 9, 16] + + +def test_chord_callback(queue): + """chord runs group, then callback with collected results.""" + + @queue.task() + def add(a, b): + return a + b + + @queue.task() + def total(results): + return sum(results) + + grp = group(add.s(1, 2), add.s(3, 4), add.s(5, 6)) + result_job = chord(grp, total.s()).apply(queue) + assert result_job.result(timeout=30) == 21 # 3 + 7 + 11 = 21 diff --git a/tests/python/test_cli.py b/tests/python/test_cli.py new file mode 100644 index 00000000..ff138d93 --- /dev/null +++ b/tests/python/test_cli.py @@ -0,0 +1,39 @@ +"""Tests for CLI info command.""" + +import pytest + +from quickq.cli import _load_queue, _print_stats + + +def test_load_queue_invalid_format(): + """_load_queue rejects paths without a colon.""" + with pytest.raises(SystemExit): + _load_queue("no_colon_here") + + +def test_load_queue_missing_module(): + """_load_queue exits on missing module.""" + with pytest.raises(SystemExit): + _load_queue("nonexistent.module:queue") + + +def test_print_stats_format(capsys, tmp_path): + """_print_stats prints a formatted stats table.""" + from quickq import Queue + + db_path = str(tmp_path / "test_cli_stats.db") + queue = Queue(db_path=db_path) + + @queue.task() + def noop(): + pass + + noop.delay() + noop.delay() + + _print_stats(queue) + output = capsys.readouterr().out + + assert "quickq queue statistics" in output + assert "pending" in output + assert "total" in output diff --git a/tests/python/test_context.py b/tests/python/test_context.py new file mode 100644 index 00000000..4f39ab29 --- /dev/null +++ b/tests/python/test_context.py @@ -0,0 +1,63 @@ +"""Tests for job context — current_job inside running tasks.""" + +import threading + +import pytest + +from quickq import Queue +from quickq.context import current_job + + +@pytest.fixture +def queue(tmp_path): + db_path = str(tmp_path / "test_context.db") + return Queue(db_path=db_path, workers=1) + + +def test_current_job_raises_outside_task(): + """current_job properties raise RuntimeError outside a task.""" + with pytest.raises(RuntimeError, match="No active job context"): + _ = current_job.id + + +def test_current_job_id_available_in_task(queue): + """current_job.id is accessible inside a running task.""" + captured = {} + + @queue.task() + def capture_context(): + captured["id"] = current_job.id + captured["task_name"] = current_job.task_name + captured["retry_count"] = current_job.retry_count + captured["queue_name"] = current_job.queue_name + return "ok" + + job = capture_context.delay() + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + result = job.result(timeout=10) + assert result == "ok" + assert captured["id"] == job.id + assert captured["task_name"].endswith("capture_context") + assert captured["retry_count"] == 0 + assert captured["queue_name"] == "default" + + +def test_current_job_update_progress(queue): + """current_job.update_progress() works inside a running task.""" + + @queue.task() + def task_with_progress(): + current_job.update_progress(50) + current_job.update_progress(100) + return "done" + + job = task_with_progress.delay() + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + result = job.result(timeout=10) + assert result == "done" diff --git a/tests/python/test_hooks.py b/tests/python/test_hooks.py new file mode 100644 index 00000000..8c39e8b6 --- /dev/null +++ b/tests/python/test_hooks.py @@ -0,0 +1,90 @@ +"""Tests for the hooks/middleware system.""" + +from __future__ import annotations + +import threading + +import pytest + +from quickq import Queue + + +@pytest.fixture +def queue(tmp_path): + db_path = str(tmp_path / "test_hooks.db") + return Queue(db_path=db_path, workers=2) + + +def test_before_and_after_hooks(queue): + """before_task and after_task hooks fire around task execution.""" + events = [] + + @queue.before_task + def on_before(task_name, args, kwargs): + events.append(("before", task_name)) + + @queue.after_task + def on_after(task_name, args, kwargs, result, error): + events.append(("after", task_name, result, error)) + + @queue.task() + def add(a, b): + return a + b + + job = add.delay(1, 2) + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + result = job.result(timeout=10) + assert result == 3 + + # Verify hooks fired + assert any(e[0] == "before" for e in events) + assert any(e[0] == "after" and e[2] == 3 and e[3] is None for e in events) + + +def test_on_success_hook(queue): + """on_success hook fires when task succeeds.""" + success_results = [] + + @queue.on_success + def on_success(task_name, args, kwargs, result): + success_results.append(result) + + @queue.task() + def multiply(a, b): + return a * b + + job = multiply.delay(3, 4) + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + result = job.result(timeout=10) + assert result == 12 + assert 12 in success_results + + +def test_on_failure_hook(queue): + """on_failure hook fires when task raises.""" + failure_errors = [] + + @queue.on_failure + def on_failure(task_name, args, kwargs, error): + failure_errors.append(str(error)) + + @queue.task(max_retries=1, retry_backoff=0.1) + def always_fails(): + raise ValueError("boom") + + always_fails.delay() + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + import time + + time.sleep(3) + + assert any("boom" in e for e in failure_errors) diff --git a/tests/python/test_periodic.py b/tests/python/test_periodic.py new file mode 100644 index 00000000..a08c355f --- /dev/null +++ b/tests/python/test_periodic.py @@ -0,0 +1,60 @@ +"""Tests for periodic (cron-scheduled) tasks.""" + +import threading +import time + +import pytest + +from quickq import Queue + + +@pytest.fixture +def queue(tmp_path): + db_path = str(tmp_path / "test_periodic.db") + return Queue(db_path=db_path, workers=1) + + +def test_periodic_task_registration(queue): + """Periodic tasks are registered as both regular tasks and periodic configs.""" + + @queue.periodic(cron="0 * * * * *") + def every_minute(): + return "tick" + + assert every_minute.name.endswith("every_minute") + assert every_minute.name in queue._task_registry + assert len(queue._periodic_configs) == 1 + assert queue._periodic_configs[0]["cron_expr"] == "0 * * * * *" + + +def test_periodic_task_direct_call(queue): + """Periodic tasks can still be called directly like regular tasks.""" + + @queue.periodic(cron="0 * * * * *") + def add(a, b): + return a + b + + assert add(3, 4) == 7 + + +def test_periodic_task_triggers(queue): + """Periodic task gets enqueued by the scheduler when due.""" + results = [] + + @queue.periodic(cron="* * * * * *") # every second + def frequent_task(): + results.append(1) + return "done" + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + # Wait for the periodic task to trigger at least once + deadline = time.time() + 15 + while time.time() < deadline: + stats = queue.stats() + if stats["completed"] >= 1: + break + time.sleep(0.5) + + assert queue.stats()["completed"] >= 1 diff --git a/tests/python/test_progress.py b/tests/python/test_progress.py new file mode 100644 index 00000000..25e65ed1 --- /dev/null +++ b/tests/python/test_progress.py @@ -0,0 +1,49 @@ +"""Tests for progress tracking.""" + +from __future__ import annotations + +import time + +import pytest + +from quickq import Queue + + +@pytest.fixture +def queue(tmp_path): + db_path = str(tmp_path / "test_progress.db") + return Queue(db_path=db_path, workers=2) + + +def test_update_progress(queue): + """Progress can be updated and read back.""" + + @queue.task() + def slow_task(): + + time.sleep(0.5) + return "done" + + job = slow_task.delay() + + # Update progress directly via the queue API + queue.update_progress(job.id, 50) + + refreshed = queue.get_job(job.id) + assert refreshed.progress == 50 + + queue.update_progress(job.id, 100) + refreshed = queue.get_job(job.id) + assert refreshed.progress == 100 + + +def test_progress_starts_none(queue): + """Progress is None by default.""" + + @queue.task() + def task_a(): + return 1 + + job = task_a.delay() + refreshed = queue.get_job(job.id) + assert refreshed.progress is None diff --git a/tests/python/test_retry_history.py b/tests/python/test_retry_history.py new file mode 100644 index 00000000..a505dafd --- /dev/null +++ b/tests/python/test_retry_history.py @@ -0,0 +1,57 @@ +"""Tests for retry history (job_errors tracking).""" + +import threading + +import pytest + +from quickq import Queue + + +@pytest.fixture +def queue(tmp_path): + db_path = str(tmp_path / "test_retry_history.db") + return Queue(db_path=db_path, workers=1) + + +def test_retry_errors_recorded(queue): + """Failed attempts are recorded in job.errors.""" + call_count = {"n": 0} + + @queue.task(max_retries=3, retry_backoff=0.01) + def flaky(): + call_count["n"] += 1 + if call_count["n"] <= 3: + raise ValueError(f"attempt {call_count['n']}") + return "ok" + + job = flaky.delay() + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + result = job.result(timeout=15) + assert result == "ok" + + errors = job.errors + assert len(errors) == 3 + assert errors[0]["attempt"] == 0 + assert "attempt 1" in errors[0]["error"] + assert errors[1]["attempt"] == 1 + assert errors[2]["attempt"] == 2 + + +def test_errors_empty_on_success(queue): + """Successful jobs have an empty errors list.""" + + @queue.task() + def ok_task(): + return 42 + + job = ok_task.delay() + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + result = job.result(timeout=10) + assert result == 42 + assert job.errors == [] diff --git a/tests/python/test_shutdown.py b/tests/python/test_shutdown.py new file mode 100644 index 00000000..90a65424 --- /dev/null +++ b/tests/python/test_shutdown.py @@ -0,0 +1,60 @@ +"""Tests for graceful shutdown.""" + +import threading +import time + +import pytest + +from quickq import Queue + + +@pytest.fixture +def queue(tmp_path): + db_path = str(tmp_path / "test_shutdown.db") + return Queue(db_path=db_path, workers=2) + + +def test_graceful_shutdown_completes_inflight(queue): + """Graceful shutdown waits for in-flight tasks to complete.""" + completed = threading.Event() + + @queue.task() + def slow_task(): + time.sleep(1) + completed.set() + return "done" + + job = slow_task.delay() + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + # Wait a bit for the task to start + time.sleep(0.3) + + # Request graceful shutdown + queue._inner.request_shutdown() + + # Worker should finish the in-flight task + worker_thread.join(timeout=10) + + assert completed.is_set() + fetched = queue.get_job(job.id) + assert fetched.status == "complete" + + +def test_shutdown_stops_worker(queue): + """request_shutdown causes run_worker to return.""" + + @queue.task() + def noop(): + pass + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + time.sleep(0.3) + queue._inner.request_shutdown() + + worker_thread.join(timeout=10) + assert not worker_thread.is_alive() diff --git a/tests/python/test_unique.py b/tests/python/test_unique.py new file mode 100644 index 00000000..b5e9bec0 --- /dev/null +++ b/tests/python/test_unique.py @@ -0,0 +1,73 @@ +"""Tests for unique task deduplication.""" + +from __future__ import annotations + +import threading + +import pytest + +from quickq import Queue + + +@pytest.fixture +def queue(tmp_path): + db_path = str(tmp_path / "test_unique.db") + return Queue(db_path=db_path, workers=2) + + +def test_unique_key_dedup(queue): + """Two jobs with the same unique_key should return the same job ID.""" + + @queue.task() + def process(data): + return data + + job1 = process.apply_async(args=("a",), unique_key="dedup-1") + job2 = process.apply_async(args=("b",), unique_key="dedup-1") + + assert job1.id == job2.id + + +def test_different_unique_keys(queue): + """Different unique keys should create separate jobs.""" + + @queue.task() + def process(data): + return data + + job1 = process.apply_async(args=("a",), unique_key="key-a") + job2 = process.apply_async(args=("b",), unique_key="key-b") + + assert job1.id != job2.id + + +def test_unique_key_allows_after_complete(queue): + """After a unique job completes, a new one with the same key can be created.""" + + @queue.task() + def fast_task(): + return "done" + + job1 = fast_task.apply_async(unique_key="once") + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + job1.result(timeout=10) + + # Now enqueue again with the same key — should create a new job + job2 = fast_task.apply_async(unique_key="once") + assert job2.id != job1.id + + +def test_no_unique_key_allows_duplicates(queue): + """Without unique_key, duplicate jobs are allowed.""" + + @queue.task() + def process(data): + return data + + job1 = process.delay("a") + job2 = process.delay("a") + + assert job1.id != job2.id From f99223183e7a8c0b8e7aae7f8b113cf00343d5fe Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Tue, 24 Feb 2026 01:34:52 +0530 Subject: [PATCH 2/3] feat: added docs --- docs/api/canvas.md | 238 +++++++++++++++++++++++++++ docs/api/cli.md | 132 +++++++++++++++ docs/api/context.md | 105 ++++++++++++ docs/api/queue.md | 226 +++++++++++++++++++++++++ docs/api/result.md | 134 +++++++++++++++ docs/api/task.md | 128 ++++++++++++++ docs/architecture.md | 208 +++++++++++++++++++++++ docs/changelog.md | 37 +++++ docs/comparison.md | 107 ++++++++++++ docs/examples/benchmark.md | 186 +++++++++++++++++++++ docs/examples/web-scraper.md | 200 ++++++++++++++++++++++ docs/getting-started/installation.md | 49 ++++++ docs/getting-started/quickstart.md | 107 ++++++++++++ docs/guide/advanced.md | 129 +++++++++++++++ docs/guide/monitoring.md | 153 +++++++++++++++++ docs/guide/queues.md | 100 +++++++++++ docs/guide/rate-limiting.md | 67 ++++++++ docs/guide/retries.md | 127 ++++++++++++++ docs/guide/scheduling.md | 90 ++++++++++ docs/guide/tasks.md | 129 +++++++++++++++ docs/guide/workers.md | 109 ++++++++++++ docs/guide/workflows.md | 146 ++++++++++++++++ docs/index.md | 152 +++++++++++++++++ 23 files changed, 3059 insertions(+) create mode 100644 docs/api/canvas.md create mode 100644 docs/api/cli.md create mode 100644 docs/api/context.md create mode 100644 docs/api/queue.md create mode 100644 docs/api/result.md create mode 100644 docs/api/task.md create mode 100644 docs/architecture.md create mode 100644 docs/changelog.md create mode 100644 docs/comparison.md create mode 100644 docs/examples/benchmark.md create mode 100644 docs/examples/web-scraper.md create mode 100644 docs/getting-started/installation.md create mode 100644 docs/getting-started/quickstart.md create mode 100644 docs/guide/advanced.md create mode 100644 docs/guide/monitoring.md create mode 100644 docs/guide/queues.md create mode 100644 docs/guide/rate-limiting.md create mode 100644 docs/guide/retries.md create mode 100644 docs/guide/scheduling.md create mode 100644 docs/guide/tasks.md create mode 100644 docs/guide/workers.md create mode 100644 docs/guide/workflows.md create mode 100644 docs/index.md diff --git a/docs/api/canvas.md b/docs/api/canvas.md new file mode 100644 index 00000000..25c030a0 --- /dev/null +++ b/docs/api/canvas.md @@ -0,0 +1,238 @@ +# Canvas (Workflows) + +::: quickq.canvas + +Canvas primitives for composing task workflows. Import directly from the package: + +```python +from quickq import chain, group, chord +``` + +--- + +## Signature + +A frozen task call spec — describes *what* to call and *with what arguments*, without executing it. + +### Creating Signatures + +```python +# Mutable signature — receives previous result in chains +sig = add.s(2, 3) + +# Immutable signature — ignores previous result in chains +sig = add.si(2, 3) +``` + +### Fields + +| Field | Type | Description | +|---|---|---| +| `task` | `TaskWrapper` | The task to call | +| `args` | `tuple` | Positional arguments | +| `kwargs` | `dict` | Keyword arguments | +| `options` | `dict` | Enqueue options (priority, queue, etc.) | +| `immutable` | `bool` | If `True`, ignores previous result in chains | + +### `sig.apply()` + +```python +sig.apply(queue: Queue | None = None) -> JobResult +``` + +Enqueue this signature immediately. If `queue` is `None`, uses the task's parent queue. + +```python +sig = add.s(2, 3) +job = sig.apply() +print(job.result(timeout=10)) # 5 +``` + +### Mutable vs Immutable + +In a [`chain`](#chain), the previous task's return value is **prepended** to a mutable signature's args: + +```python +# add.s(10) in a chain where previous step returned 5: +# → add(5, 10) = 15 + +# add.si(2, 3) in a chain: +# → add(2, 3) = 5 (always, regardless of previous result) +``` + +--- + +## chain + +Execute signatures sequentially, piping each result to the next. + +### Constructor + +```python +chain(*signatures: Signature) +``` + +Requires at least one signature. + +### `chain.apply()` + +```python +chain.apply(queue: Queue | None = None) -> JobResult +``` + +Execute the chain by enqueuing and waiting for each step sequentially. Returns the [`JobResult`](result.md) of the **last** step. + +Each step's return value is prepended to the next mutable signature's args. Immutable signatures (`task.si()`) receive their args as-is. + +```python +@queue.task() +def double(x): + return x * 2 + +@queue.task() +def add_ten(x): + return x + 10 + +# double(3) → 6, then add_ten(6) → 16 +result = chain(double.s(3), add_ten.s()).apply() +print(result.result(timeout=10)) # 16 +``` + +```mermaid +graph LR + A["double(3)"] -->|"6"| B["add_ten(6)"] + B -->|"16"| C["Result: 16"] +``` + +--- + +## group + +Execute signatures in parallel and collect all results. + +### Constructor + +```python +group(*signatures: Signature) +``` + +Requires at least one signature. + +### `group.apply()` + +```python +group.apply(queue: Queue | None = None) -> list[JobResult] +``` + +Enqueue all signatures and return a list of [`JobResult`](result.md) handles. Jobs run concurrently across available workers. + +```python +jobs = group( + add.s(1, 2), + add.s(3, 4), + add.s(5, 6), +).apply() + +results = [j.result(timeout=10) for j in jobs] +print(results) # [3, 7, 11] +``` + +```mermaid +graph LR + A["add(1,2)"] --> D["Results: [3, 7, 11]"] + B["add(3,4)"] --> D + C["add(5,6)"] --> D +``` + +--- + +## chord + +Run a group in parallel, then pass all results to a callback. + +### Constructor + +```python +chord(group_: group, callback: Signature) +``` + +| Parameter | Type | Description | +|---|---|---| +| `group_` | `group` | The group of tasks to run in parallel | +| `callback` | `Signature` | The task to call with all collected results | + +### `chord.apply()` + +```python +chord.apply(queue: Queue | None = None) -> JobResult +``` + +Execute the group, wait for all results, then run the callback with the list of results prepended to its args (unless immutable). Returns the [`JobResult`](result.md) of the callback. + +```python +@queue.task() +def fetch(url): + return requests.get(url).text + +@queue.task() +def merge(results): + return "\n".join(results) + +result = chord( + group(fetch.s("https://a.com"), fetch.s("https://b.com")), + merge.s(), +).apply() + +combined = result.result(timeout=30) +``` + +```mermaid +graph LR + A["fetch(a.com)"] --> C["merge([...])"] + B["fetch(b.com)"] --> C + C --> D["Combined result"] +``` + +--- + +## Complete Example + +An ETL pipeline using all three primitives: + +```python +from quickq import Queue, chain, group, chord + +queue = Queue() + +@queue.task() +def extract(source): + return load_data(source) + +@queue.task() +def transform(data): + return clean(data) + +@queue.task() +def aggregate(results): + return merge_datasets(results) + +@queue.task() +def load(data): + save_to_warehouse(data) + +# Extract from 3 sources in parallel, transform each, +# aggregate all results, then load +pipeline = chain( + chord( + group( + chain(extract.s("db"), transform.s()), + chain(extract.s("api"), transform.s()), + chain(extract.s("csv"), transform.s()), + ), + aggregate.s(), + ), + load.s(), +) + +result = pipeline.apply(queue) +``` diff --git a/docs/api/cli.md b/docs/api/cli.md new file mode 100644 index 00000000..743f5f9b --- /dev/null +++ b/docs/api/cli.md @@ -0,0 +1,132 @@ +# CLI Reference + +quickq provides a command-line interface for running workers and inspecting queue state. + +## Installation + +The CLI is installed automatically with the package: + +```bash +pip install quickq +``` + +The `quickq` command becomes available in your `PATH`. + +## Commands + +### `quickq worker` + +Start a worker process that consumes and executes tasks. + +```bash +quickq worker --app [--queues ] +``` + +| Flag | Required | Description | +|---|---|---| +| `--app` | Yes | Python path to the `Queue` instance in `module:attribute` format | +| `--queues` | No | Comma-separated list of queues to process. Default: all registered queues | + +**Examples:** + +```bash +# Start a worker using the queue defined in myapp/tasks.py +quickq worker --app myapp.tasks:queue + +# Only process the "emails" and "reports" queues +quickq worker --app myapp.tasks:queue --queues emails,reports + +# Use a nested module path +quickq worker --app myproject.workers.tasks:task_queue +``` + +The worker blocks until interrupted with `Ctrl+C`. It performs a graceful shutdown — in-flight tasks are allowed to complete before the process exits. + +### `quickq info` + +Display queue statistics. + +```bash +quickq info --app [--watch] +``` + +| Flag | Required | Description | +|---|---|---| +| `--app` | Yes | Python path to the `Queue` instance | +| `--watch` | No | Continuously refresh stats every 2 seconds | + +**Examples:** + +```bash +# Show stats once +quickq info --app myapp.tasks:queue +``` + +Output: + +``` +quickq queue statistics +------------------------------ + pending 12 + running 4 + completed 1847 + failed 0 + dead 2 + cancelled 0 +------------------------------ + total 1865 +``` + +```bash +# Live monitoring with throughput +quickq info --app myapp.tasks:queue --watch +``` + +Output (refreshes every 2s): + +``` +quickq queue statistics +------------------------------ + pending 3 + running 8 + completed 2104 + failed 0 + dead 2 + cancelled 0 +------------------------------ + total 2117 + + throughput 12.5 jobs/s + +Refreshing every 2s... (Ctrl+C to stop) +``` + +## App Path Format + +The `--app` flag uses `module:attribute` format: + +``` +myapp.tasks:queue +│ │ +│ └── attribute name (the Queue variable) +└── Python module path (dotted, importable) +``` + +The module must be importable from the current working directory. If your module is in a package, make sure the package is installed or the parent directory is in `PYTHONPATH`. + +**Common patterns:** + +| App structure | `--app` value | +|---|---| +| `tasks.py` with `queue = Queue()` | `tasks:queue` | +| `myapp/tasks.py` with `queue = Queue()` | `myapp.tasks:queue` | +| `src/workers/q.py` with `app = Queue()` | `src.workers.q:app` | + +## Error Messages + +| Error | Cause | +|---|---| +| `--app must be in 'module:attribute' format` | Missing `:` separator | +| `could not import module '...'` | Module not found or import error | +| `module '...' has no attribute '...'` | Attribute doesn't exist on the module | +| `'...' is not a Queue instance` | The attribute exists but isn't a `Queue` | diff --git a/docs/api/context.md b/docs/api/context.md new file mode 100644 index 00000000..4389ab87 --- /dev/null +++ b/docs/api/context.md @@ -0,0 +1,105 @@ +# Job Context + +::: quickq.context + +Thread-local context for the currently executing job. Provides access to job metadata and controls from inside a running task. + +## Usage + +```python +from quickq.context import current_job + +# or directly: +from quickq import current_job +``` + +`current_job` is a module-level singleton. It uses thread-local storage internally, so each worker thread sees its own job context. + +!!! warning + `current_job` can only be used inside a running task. Accessing it outside a task raises `RuntimeError`. + +## Properties + +### `current_job.id` + +```python +current_job.id -> str +``` + +The unique ID of the currently executing job. + +```python +@queue.task() +def process(data): + print(f"Running as job {current_job.id}") + ... +``` + +### `current_job.task_name` + +```python +current_job.task_name -> str +``` + +The registered name of the currently executing task. + +### `current_job.retry_count` + +```python +current_job.retry_count -> int +``` + +How many times this job has been retried. `0` on the first attempt. + +```python +@queue.task(max_retries=3) +def flaky_task(): + if current_job.retry_count > 0: + print(f"Retry attempt #{current_job.retry_count}") + call_external_api() +``` + +### `current_job.queue_name` + +```python +current_job.queue_name -> str +``` + +The name of the queue this job is running on. + +## Methods + +### `current_job.update_progress()` + +```python +current_job.update_progress(progress: int) -> None +``` + +Update the job's progress percentage (0–100). The value is written directly to the database and can be read via [`job.progress`](result.md#jobprogress) or [`queue.get_job()`](queue.md#queueget_job). + +```python +@queue.task() +def process_files(file_list): + for i, path in enumerate(file_list): + handle(path) + current_job.update_progress(int((i + 1) / len(file_list) * 100)) +``` + +Read progress from the caller: + +```python +job = process_files.delay(files) + +# Poll progress +import time +while job.status == "running": + print(f"Progress: {job.progress}%") + time.sleep(1) +``` + +## How It Works + +1. Before each task execution, the Rust worker calls `_set_context()` with the job's metadata +2. `current_job` reads from thread-local storage (`threading.local()`) +3. After the task completes (success or failure), `_clear_context()` resets the thread-local +4. Each worker thread has independent context — no cross-thread interference diff --git a/docs/api/queue.md b/docs/api/queue.md new file mode 100644 index 00000000..ca18c4e6 --- /dev/null +++ b/docs/api/queue.md @@ -0,0 +1,226 @@ +# Queue + +::: quickq.app.Queue + +The central class for creating and managing a task queue. + +## Constructor + +```python +Queue( + db_path: str = "quickq.db", + workers: int = 0, + default_retry: int = 3, + default_timeout: int = 300, + default_priority: int = 0, + result_ttl: int | None = None, +) +``` + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `db_path` | `str` | `"quickq.db"` | Path to SQLite database file | +| `workers` | `int` | `0` | Number of worker threads (`0` = auto-detect CPU count) | +| `default_retry` | `int` | `3` | Default max retry attempts for tasks | +| `default_timeout` | `int` | `300` | Default task timeout in seconds | +| `default_priority` | `int` | `0` | Default task priority (higher = more urgent) | +| `result_ttl` | `int \| None` | `None` | Auto-cleanup completed/dead jobs older than this many seconds. `None` disables. | + +## Task Registration + +### `@queue.task()` + +```python +@queue.task( + name: str | None = None, + max_retries: int = 3, + retry_backoff: float = 1.0, + timeout: int = 300, + priority: int = 0, + rate_limit: str | None = None, + queue: str = "default", +) -> TaskWrapper +``` + +Register a function as a background task. Returns a [`TaskWrapper`](task.md). + +### `@queue.periodic()` + +```python +@queue.periodic( + cron: str, + name: str | None = None, + args: tuple = (), + kwargs: dict | None = None, + queue: str = "default", +) -> TaskWrapper +``` + +Register a periodic (cron-scheduled) task. Uses 6-field cron expressions with seconds. + +## Enqueue Methods + +### `queue.enqueue()` + +```python +queue.enqueue( + task_name: str, + args: tuple = (), + kwargs: dict | None = None, + priority: int | None = None, + delay: float | None = None, + queue: str | None = None, + max_retries: int | None = None, + timeout: int | None = None, + unique_key: str | None = None, + metadata: str | None = None, +) -> JobResult +``` + +Enqueue a task for execution. Returns a [`JobResult`](result.md) handle. + +### `queue.enqueue_many()` + +```python +queue.enqueue_many( + task_name: str, + args_list: list[tuple], + kwargs_list: list[dict] | None = None, + priority: int | None = None, + queue: str | None = None, + max_retries: int | None = None, + timeout: int | None = None, +) -> list[JobResult] +``` + +Enqueue multiple jobs in a single SQLite transaction for high throughput. + +## Job Management + +### `queue.get_job()` + +```python +queue.get_job(job_id: str) -> JobResult | None +``` + +Retrieve a job by ID. Returns `None` if not found. + +### `queue.cancel_job()` + +```python +queue.cancel_job(job_id: str) -> bool +``` + +Cancel a pending job. Returns `True` if cancelled, `False` if not pending. + +### `queue.update_progress()` + +```python +queue.update_progress(job_id: str, progress: int) -> None +``` + +Update progress for a running job (0–100). + +### `queue.job_errors()` + +```python +queue.job_errors(job_id: str) -> list[dict] +``` + +Get error history for a job. Returns a list of dicts with `id`, `job_id`, `attempt`, `error`, `failed_at`. + +## Statistics + +### `queue.stats()` + +```python +queue.stats() -> dict[str, int] +``` + +Returns `{"pending": N, "running": N, "completed": N, "failed": N, "dead": N, "cancelled": N}`. + +## Dead Letter Queue + +### `queue.dead_letters()` + +```python +queue.dead_letters(limit: int = 10, offset: int = 0) -> list[dict] +``` + +List dead letter entries. Each dict contains: `id`, `original_job_id`, `queue`, `task_name`, `error`, `retry_count`, `failed_at`, `metadata`. + +### `queue.retry_dead()` + +```python +queue.retry_dead(dead_id: str) -> str +``` + +Re-enqueue a dead letter job. Returns the new job ID. + +### `queue.purge_dead()` + +```python +queue.purge_dead(older_than: int = 86400) -> int +``` + +Purge dead letter entries older than `older_than` seconds. Returns count deleted. + +## Cleanup + +### `queue.purge_completed()` + +```python +queue.purge_completed(older_than: int = 86400) -> int +``` + +Purge completed jobs older than `older_than` seconds. Returns count deleted. + +## Worker + +### `queue.run_worker()` + +```python +queue.run_worker(queues: Sequence[str] | None = None) -> None +``` + +Start the worker loop. **Blocks** until interrupted. Pass `queues` to limit which queues are processed. + +## Hooks + +### `@queue.before_task` + +```python +@queue.before_task +def hook(task_name: str, args: tuple, kwargs: dict) -> None: ... +``` + +### `@queue.after_task` + +```python +@queue.after_task +def hook(task_name: str, args: tuple, kwargs: dict, result: Any, error: Exception | None) -> None: ... +``` + +### `@queue.on_success` + +```python +@queue.on_success +def hook(task_name: str, args: tuple, kwargs: dict, result: Any) -> None: ... +``` + +### `@queue.on_failure` + +```python +@queue.on_failure +def hook(task_name: str, args: tuple, kwargs: dict, error: Exception) -> None: ... +``` + +## Async Methods + +| Sync | Async | +|---|---| +| `queue.stats()` | `await queue.astats()` | +| `queue.dead_letters()` | `await queue.adead_letters()` | +| `queue.retry_dead()` | `await queue.aretry_dead()` | +| `queue.cancel_job()` | `await queue.acancel_job()` | +| `queue.run_worker()` | `await queue.arun_worker()` | diff --git a/docs/api/result.md b/docs/api/result.md new file mode 100644 index 00000000..00596140 --- /dev/null +++ b/docs/api/result.md @@ -0,0 +1,134 @@ +# JobResult + +::: quickq.result.JobResult + +Handle to an enqueued job. Provides methods to check status and retrieve results, both synchronously and asynchronously. + +Returned by [`task.delay()`](task.md#taskdelay), [`task.apply_async()`](task.md#taskapply_async), [`queue.enqueue()`](queue.md#queueenqueue), and canvas operations. + +## Properties + +### `job.id` + +```python +job.id -> str +``` + +The unique job ID (UUIDv7, time-ordered). + +### `job.status` + +```python +job.status -> str +``` + +Current job status. **Fetches fresh from the database** on every access. + +Returns one of: `"pending"`, `"running"`, `"complete"`, `"failed"`, `"dead"`, `"cancelled"`. + +```python +job = add.delay(2, 3) +print(job.status) # "pending" +# ... after worker processes it ... +print(job.status) # "complete" +``` + +### `job.progress` + +```python +job.progress -> int | None +``` + +Current progress (0–100) if reported by the task via [`current_job.update_progress()`](context.md). Returns `None` if no progress has been reported. Refreshes from the database. + +### `job.error` + +```python +job.error -> str | None +``` + +Error message if the job failed. `None` if the job hasn't failed. Refreshes from the database. + +### `job.errors` + +```python +job.errors -> list[dict] +``` + +Full error history for this job — one entry per failed attempt. Each dict contains: + +| Key | Type | Description | +|---|---|---| +| `id` | `str` | Error record ID | +| `job_id` | `str` | Parent job ID | +| `attempt` | `int` | Retry attempt number | +| `error` | `str` | Error message/traceback | +| `failed_at` | `str` | ISO timestamp of the failure | + +```python +job = flaky_task.delay() +# ... after retries ... +for err in job.errors: + print(f"Attempt {err['attempt']}: {err['error']}") +``` + +## Methods + +### `job.result()` + +```python +job.result( + timeout: float = 30.0, + poll_interval: float = 0.05, + max_poll_interval: float = 0.5, +) -> Any +``` + +**Block** until the job completes and return the deserialized result. Uses exponential backoff for polling — starts at `poll_interval` and gradually increases to `max_poll_interval`. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `timeout` | `float` | `30.0` | Maximum seconds to wait | +| `poll_interval` | `float` | `0.05` | Initial seconds between status checks | +| `max_poll_interval` | `float` | `0.5` | Maximum seconds between status checks | + +**Raises:** + +- `TimeoutError` — if the job doesn't complete within `timeout` +- `RuntimeError` — if the job status is `"failed"` or `"dead"` + +```python +job = add.delay(2, 3) +result = job.result(timeout=10) # blocks, returns 5 + +# Custom polling for long-running tasks +result = job.result(timeout=600, poll_interval=1.0, max_poll_interval=5.0) +``` + +### `await job.aresult()` + +```python +async job.aresult( + timeout: float = 30.0, + poll_interval: float = 0.05, + max_poll_interval: float = 0.5, +) -> Any +``` + +Async version of `result()`. Uses `asyncio.sleep()` instead of `time.sleep()`, so it won't block the event loop. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `timeout` | `float` | `30.0` | Maximum seconds to wait | +| `poll_interval` | `float` | `0.05` | Initial seconds between status checks | +| `max_poll_interval` | `float` | `0.5` | Maximum seconds between status checks | + +**Raises:** + +- `TimeoutError` — if the job doesn't complete within `timeout` +- `RuntimeError` — if the job status is `"failed"` or `"dead"` + +```python +job = add.delay(2, 3) +result = await job.aresult(timeout=10) +``` diff --git a/docs/api/task.md b/docs/api/task.md new file mode 100644 index 00000000..73ece781 --- /dev/null +++ b/docs/api/task.md @@ -0,0 +1,128 @@ +# TaskWrapper + +::: quickq.task.TaskWrapper + +Created by `@queue.task()` — not instantiated directly. Wraps a decorated function to provide task submission methods. + +## Properties + +### `task.name` + +```python +task.name -> str +``` + +The registered task name. Either the explicit `name` passed to `@queue.task()` or the function's qualified name. + +## Methods + +### `task.delay()` + +```python +task.delay(*args, **kwargs) -> JobResult +``` + +Enqueue the task for background execution using the decorator's default options. Returns a [`JobResult`](result.md) handle. + +```python +@queue.task(priority=5) +def add(a, b): + return a + b + +job = add.delay(2, 3) +print(job.result(timeout=10)) # 5 +``` + +### `task.apply_async()` + +```python +task.apply_async( + args: tuple = (), + kwargs: dict | None = None, + priority: int | None = None, + delay: float | None = None, + queue: str | None = None, + max_retries: int | None = None, + timeout: int | None = None, + unique_key: str | None = None, + metadata: str | None = None, +) -> JobResult +``` + +Enqueue with full control over submission options. Any parameter not provided falls back to the decorator's default. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `args` | `tuple` | `()` | Positional arguments for the task | +| `kwargs` | `dict \| None` | `None` | Keyword arguments for the task | +| `priority` | `int \| None` | `None` | Override priority (higher = more urgent) | +| `delay` | `float \| None` | `None` | Delay in seconds before the task is eligible | +| `queue` | `str \| None` | `None` | Override queue name | +| `max_retries` | `int \| None` | `None` | Override max retry count | +| `timeout` | `int \| None` | `None` | Override timeout in seconds | +| `unique_key` | `str \| None` | `None` | Deduplicate active jobs with same key | +| `metadata` | `str \| None` | `None` | Arbitrary JSON metadata to attach | + +```python +job = send_email.apply_async( + args=("user@example.com", "Hello"), + priority=10, + delay=3600, + queue="emails", + unique_key="welcome-user@example.com", + metadata='{"campaign": "onboarding"}', +) +``` + +### `task.map()` + +```python +task.map(iterable: list[tuple]) -> list[JobResult] +``` + +Enqueue one job per item in a single batch SQLite transaction. Uses the decorator's default options. + +```python +jobs = add.map([(1, 2), (3, 4), (5, 6)]) +results = [j.result(timeout=10) for j in jobs] +print(results) # [3, 7, 11] +``` + +### `task.s()` + +```python +task.s(*args, **kwargs) -> Signature +``` + +Create a **mutable** [`Signature`](canvas.md). In a [`chain`](canvas.md#chain), the previous task's return value is prepended to `args`. + +```python +sig = add.s(10) +# In a chain, if the previous step returned 5: +# add(5, 10) → 15 +``` + +### `task.si()` + +```python +task.si(*args, **kwargs) -> Signature +``` + +Create an **immutable** [`Signature`](canvas.md#signature). Ignores the previous task's result — arguments are used as-is. + +```python +sig = add.si(2, 3) +# Always calls add(2, 3) regardless of previous result +``` + +### `task()` + +```python +task(*args, **kwargs) -> Any +``` + +Call the underlying function directly (synchronous, not queued). Useful for testing or when you don't need background execution. + +```python +result = add(2, 3) # Direct call, returns 5 +``` diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..73e41cf9 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,208 @@ +# Architecture + +quickq is a hybrid Python/Rust system. Python provides the user-facing API. Rust handles all the heavy lifting: storage, scheduling, dispatch, rate limiting, and worker management. + +## Overview + +```mermaid +graph TB + subgraph Python ["Python Layer"] + A["Queue"] + B["@queue.task()"] + C["TaskWrapper"] + D["JobResult"] + E["current_job"] + end + + subgraph Rust ["Rust Core · PyO3"] + F["PyQueue"] + G["Scheduler
Tokio async runtime"] + H["Worker Pool
OS threads + crossbeam"] + I["Rate Limiter
Token bucket"] + end + + subgraph Storage ["Embedded Storage"] + J[("SQLite · WAL mode
Diesel ORM · r2d2 pool")] + end + + A -->|"enqueue()"| F + F -->|INSERT| J + G -->|"dequeue (poll every 50ms)"| J + G -->|"dispatch via crossbeam"| H + H -->|"acquire GIL → run task"| B + H -->|"JobResult"| G + G -->|"UPDATE status"| J + D -->|"poll status"| F + F -->|SELECT| J + G -.->|"check rate limit"| I + I -.->|"token state"| J +``` + +## Job Lifecycle + +```mermaid +stateDiagram-v2 + [*] --> Pending: enqueue() / delay() + Pending --> Running: dequeued by scheduler + Pending --> Cancelled: cancel_job() + Running --> Complete: task returns successfully + Running --> Failed: task raises exception + Failed --> Pending: retry (count < max_retries)\nwith exponential backoff + Failed --> Dead: retries exhausted\nmoved to DLQ + Dead --> Pending: retry_dead() + Complete --> [*] + Cancelled --> [*] + Dead --> [*]: purge_dead() +``` + +**Status codes in SQLite:** + +| Status | Integer | Description | +|---|---|---| +| Pending | 0 | Waiting to be picked up | +| Running | 1 | Currently executing | +| Complete | 2 | Finished successfully | +| Failed | 3 | Last attempt failed (may retry) | +| Dead | 4 | All retries exhausted, in DLQ | +| Cancelled | 5 | Cancelled before execution | + +## Worker Pool + +```mermaid +graph LR + subgraph Scheduler ["Scheduler Thread"] + S["Tokio async runtime
50ms poll interval"] + end + + S -->|"Job"| JCH["Job Channel
(bounded: workers×2)"] + + subgraph Pool ["Worker Threads"] + W1["Worker 1
GIL per task"] + W2["Worker 2
GIL per task"] + WN["Worker N
GIL per task"] + end + + JCH --> W1 + JCH --> W2 + JCH --> WN + + W1 -->|"Result"| RCH["Result Channel
(bounded: workers×2)"] + W2 -->|"Result"| RCH + WN -->|"Result"| RCH + + RCH --> ML["Main Loop
(py.allow_threads)"] + ML -->|"complete / retry / DLQ"| DB[("SQLite")] +``` + +**Key design decisions:** + +- **OS threads, not Python threads**: Workers are Rust `std::thread` threads. The GIL is only acquired when calling Python task code. +- **Bounded channels**: Both job and result channels are bounded to `workers × 2` to provide backpressure. +- **GIL isolation**: Each worker acquires the GIL independently using `Python::with_gil()`. The scheduler and result handler release the GIL via `py.allow_threads()`. + +## Storage Layer + +### SQLite Configuration + +| Pragma | Value | Why | +|---|---|---| +| `journal_mode` | WAL | Concurrent reads while writing | +| `busy_timeout` | 5000ms | Wait on lock contention instead of failing | +| `synchronous` | NORMAL | Fast writes, safe with WAL | +| `journal_size_limit` | 64MB | Prevent unbounded WAL file growth | + +### Database Schema + +**5 tables:** + +```sql +-- Core job storage +jobs (id, queue, task_name, payload, status, priority, + created_at, scheduled_at, started_at, completed_at, + retry_count, max_retries, result, error, timeout_ms, + unique_key, progress, metadata) + +-- Dead letter queue +dead_letter (id, original_job_id, queue, task_name, + payload, error, retry_count, failed_at, metadata) + +-- Token bucket rate limiting +rate_limits (key, tokens, max_tokens, refill_rate, last_refill) + +-- Cron-scheduled tasks +periodic_tasks (name, task_name, cron_expr, args, kwargs, + queue, enabled, last_run, next_run) + +-- Per-attempt error tracking +job_errors (id, job_id, attempt, error, failed_at) +``` + +**Key indexes:** + +- `idx_jobs_dequeue`: `(queue, status, priority DESC, scheduled_at)` — fast dequeue +- `idx_jobs_status`: `(status)` — fast stats queries +- `idx_jobs_unique_key`: partial unique index on `unique_key` where status is pending/running +- `idx_job_errors_job_id`: `(job_id)` — fast error history lookup + +### Connection Pooling + +Diesel's `r2d2` connection pool with up to 8 connections. In-memory databases use a single connection (SQLite `:memory:` is per-connection). + +## Scheduler Loop + +The scheduler runs in a dedicated Tokio single-threaded async runtime: + +``` +loop { + sleep(50ms) or shutdown signal + + // Try to dequeue and dispatch a job + try_dispatch() + + // Every ~100 iterations (~5s): reap timed-out jobs + reap_stale() + + // Every ~60 iterations (~3s): check periodic tasks + check_periodic() + + // Every ~1200 iterations (~60s): auto-cleanup old jobs + auto_cleanup() +} +``` + +### Dispatch Flow + +1. `dequeue_from()` — atomically SELECT + UPDATE (pending → running) within a transaction +2. Check rate limit — if over limit, reschedule 1s in the future +3. Send job to worker pool via crossbeam channel +4. Worker executes task, sends result back +5. `handle_result()` — mark complete, schedule retry, or move to DLQ + +## Serialization + +- **Arguments**: `cloudpickle.dumps((args, kwargs))` — supports lambdas, closures, and complex objects +- **Results**: `cloudpickle.dumps(return_value)` — stored as BLOB in the `result` column +- **Periodic task args**: Pre-pickled at registration time, stored as BLOBs in `periodic_tasks.args` + +## Rust Crate Structure + +``` +quickq-core/ # Pure Rust, no Python dependency + job.rs # Job, JobStatus, NewJob structs + scheduler.rs # Central async scheduler + retry.rs # Retry policy with exponential backoff + rate_limiter.rs # Token bucket rate limiting + dlq.rs # Dead letter queue operations + periodic.rs # Cron expression parsing + error.rs # Error types + storage/ + sqlite.rs # Diesel-based SQLite driver + schema.rs # Diesel table! macros + models.rs # Queryable/Insertable structs + +quickq-python/ # PyO3 bindings + py_queue.rs # PyQueue class + py_job.rs # PyJob class + py_worker.rs # WorkerPool + task execution + py_config.rs # PyTaskConfig +``` diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 00000000..6cafa3d6 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,37 @@ +# Changelog + +All notable changes to quickq are documented here. + +## 0.1.0 + +*Initial release* + +### Features + +- **Task queue** — `@queue.task()` decorator with `.delay()` and `.apply_async()` +- **Priority queues** — integer priority levels, higher values processed first +- **Retry with exponential backoff** — configurable max retries, backoff multiplier, and jitter +- **Dead letter queue** — failed jobs preserved for inspection and replay +- **Rate limiting** — token bucket algorithm with `"N/s"`, `"N/m"`, `"N/h"` syntax +- **Task workflows** — `chain`, `group`, and `chord` primitives +- **Periodic tasks** — cron-scheduled tasks with 6-field expressions (seconds granularity) +- **Progress tracking** — `current_job.update_progress()` from inside tasks +- **Job cancellation** — cancel pending jobs before execution +- **Unique tasks** — deduplicate active jobs by key +- **Batch enqueue** — `task.map()` and `queue.enqueue_many()` with single-transaction inserts +- **Named queues** — route tasks to isolated queues, subscribe workers selectively +- **Hooks** — `before_task`, `after_task`, `on_success`, `on_failure` +- **Async support** — `aresult()`, `astats()`, `arun_worker()`, and more +- **Job context** — `current_job.id`, `.task_name`, `.retry_count`, `.queue_name` +- **Error history** — per-attempt error tracking via `job.errors` +- **Result TTL** — automatic cleanup of completed/dead jobs +- **CLI** — `quickq worker` and `quickq info --watch` +- **Metadata** — attach arbitrary JSON to jobs + +### Architecture + +- Rust core with PyO3 bindings +- SQLite storage with WAL mode and Diesel ORM +- Tokio async scheduler with 50ms poll interval +- OS thread worker pool with crossbeam channels +- cloudpickle serialization for arguments and results diff --git a/docs/comparison.md b/docs/comparison.md new file mode 100644 index 00000000..f7b22f02 --- /dev/null +++ b/docs/comparison.md @@ -0,0 +1,107 @@ +# Comparison + +How quickq compares to other Python task queues. + +## Feature Matrix + +| Feature | quickq | Celery | RQ | Dramatiq | Huey | +|---|---|---|---|---|---| +| Broker required | **No** | Redis / RabbitMQ | Redis | Redis / RabbitMQ | Redis | +| Core language | **Rust + Python** | Python | Python | Python | Python | +| Priority queues | **Yes** | Yes | No | No | Yes | +| Rate limiting | **Yes** | Yes | No | Yes | No | +| Dead letter queue | **Yes** | No | Yes | No | No | +| Task chaining | **Yes** (chain/group/chord) | Yes (canvas) | No | Yes (pipelines) | No | +| Job cancellation | **Yes** | Yes (revoke) | No | No | Yes | +| Progress tracking | **Yes** | Yes (custom) | No | No | No | +| Unique tasks | **Yes** | No (manual) | No | No | Yes | +| Batch enqueue | **Yes** | No | No | No | No | +| Retry with backoff | **Yes** (exponential + jitter) | Yes | Yes | Yes | Yes | +| Periodic/cron tasks | **Yes** (6-field with seconds) | Yes (celery-beat) | Yes (rq-scheduler) | Yes (APScheduler) | Yes | +| Async support | **Yes** | Yes | No | No | No | +| CLI | **Yes** | Yes | Yes | Yes | Yes | +| Result backend | **Built-in** (SQLite) | Redis / DB / custom | Redis | Redis / custom | Redis / SQLite | +| Setup complexity | **`pip install`** | Broker + backend | Redis server | Broker | Redis server | + +## When to Use quickq + +quickq is ideal when: + +- **Single-machine deployments** — no need for distributed workers across multiple servers +- **Zero infrastructure** — you don't want to install, configure, or manage Redis or RabbitMQ +- **Embedded applications** — CLI tools, desktop apps, or services where simplicity matters +- **Prototyping** — get a task queue running in 5 lines, iterate fast +- **Low-to-medium throughput** — hundreds to thousands of jobs per second is plenty + +## When NOT to Use quickq + +Consider alternatives when: + +- **Multi-server workers** — you need workers on separate machines (use Celery or Dramatiq) +- **Very high throughput** — millions of jobs/sec across a cluster (use Celery + RabbitMQ) +- **Existing Redis infrastructure** — if Redis is already in your stack, RQ or Huey are simple choices +- **Complex routing** — you need topic exchanges, message filtering, or pub/sub patterns (use Celery + RabbitMQ) + +## Detailed Comparison + +### vs Celery + +Celery is the most popular Python task queue — battle-tested, feature-rich, and widely adopted. + +| | quickq | Celery | +|---|---|---| +| **Setup** | `pip install quickq` | Install broker (Redis/RabbitMQ), result backend, Celery itself | +| **Dependencies** | 1 (cloudpickle) | 10+ (kombu, billiard, vine, etc.) | +| **Configuration** | Constructor params | Settings module or app config | +| **Worker model** | Rust OS threads | prefork/eventlet/gevent pools | +| **Distributed** | No (single process) | Yes (multi-server) | +| **Canvas** | chain, group, chord | chain, group, chord, starmap, chunks, and more | + +**Choose quickq** if you want zero-infrastructure simplicity on a single machine. +**Choose Celery** if you need distributed workers, complex routing, or enterprise features. + +### vs RQ (Redis Queue) + +RQ focuses on simplicity — a minimal task queue built on Redis. + +| | quickq | RQ | +|---|---|---| +| **Broker** | None (SQLite) | Redis required | +| **Priority** | Yes (integer levels) | Separate queues for priority | +| **Rate limiting** | Built-in | No | +| **Chaining** | Yes | No | +| **Monitoring** | CLI + progress | rq-dashboard (web) | + +**Choose quickq** if you want similar simplicity without requiring Redis. +**Choose RQ** if you already run Redis and want a web dashboard. + +### vs Dramatiq + +Dramatiq is a reliable, performance-focused alternative to Celery. + +| | quickq | Dramatiq | +|---|---|---| +| **Broker** | None (SQLite) | Redis or RabbitMQ | +| **Priority** | Yes | No (FIFO only) | +| **Rate limiting** | Built-in | Middleware | +| **DLQ** | Built-in | No | +| **Middleware** | Hooks (before/after/success/failure) | Full middleware stack | + +**Choose quickq** if you want built-in DLQ and priority without a broker. +**Choose Dramatiq** if you need a middleware ecosystem and distributed workers. + +### vs Huey + +Huey is a lightweight task queue with Redis or SQLite backends. + +| | quickq | Huey | +|---|---|---| +| **Backend** | SQLite (Rust-native) | Redis or SQLite (Python) | +| **Performance** | Rust scheduler + OS threads | Python threads | +| **Chaining** | chain, group, chord | Pipeline (limited) | +| **Rate limiting** | Built-in token bucket | No | +| **DLQ** | Built-in | No | +| **Progress** | Built-in | No | + +**Choose quickq** if you want higher performance and more features with SQLite. +**Choose Huey** if you need a mature, well-documented SQLite-backed queue. diff --git a/docs/examples/benchmark.md b/docs/examples/benchmark.md new file mode 100644 index 00000000..1a27c3a8 --- /dev/null +++ b/docs/examples/benchmark.md @@ -0,0 +1,186 @@ +# Example: Benchmark + +Measure quickq's throughput by enqueuing and processing a large batch of tasks. + +## benchmark.py + +```python +"""quickq throughput benchmark. + +Measures: +1. Enqueue throughput (jobs/sec) using batch insert +2. Processing throughput (jobs/sec) with N workers +3. End-to-end latency +""" + +import os +import threading +import time + +from quickq import Queue + +# ── Configuration ──────────────────────────────────────── + +NUM_JOBS = 10_000 +NUM_WORKERS = os.cpu_count() or 4 +DB_PATH = ":memory:" # In-memory for pure speed test + +queue = Queue(db_path=DB_PATH, workers=NUM_WORKERS) + +@queue.task() +def noop(x): + """Minimal task — measures framework overhead.""" + return x + +@queue.task() +def cpu_light(x): + """Light CPU work — string formatting.""" + return f"processed-{x}-{'x' * 100}" + +# ── Benchmark Functions ────────────────────────────────── + +def bench_enqueue(task, n): + """Measure batch enqueue throughput.""" + args_list = [(i,) for i in range(n)] + + start = time.perf_counter() + jobs = task.map(args_list) + elapsed = time.perf_counter() - start + + rate = n / elapsed + print(f" Enqueued {n:,} jobs in {elapsed:.2f}s ({rate:,.0f} jobs/s)") + return jobs + +def bench_process(jobs, timeout=120): + """Measure processing throughput by waiting for all jobs.""" + n = len(jobs) + start = time.perf_counter() + + # Wait for the last job (highest ID, enqueued last) + # With FIFO ordering, this means all jobs are done + last = jobs[-1] + try: + last.result(timeout=timeout, poll_interval=0.01, max_poll_interval=0.1) + except TimeoutError: + stats = queue.stats() + print(f" Timed out! Stats: {stats}") + return + + elapsed = time.perf_counter() - start + rate = n / elapsed + print(f" Processed {n:,} jobs in {elapsed:.2f}s ({rate:,.0f} jobs/s)") + +def bench_latency(task, samples=100): + """Measure single-job round-trip latency.""" + latencies = [] + for i in range(samples): + start = time.perf_counter() + job = task.delay(i) + job.result(timeout=10) + latencies.append(time.perf_counter() - start) + + avg = sum(latencies) / len(latencies) + p50 = sorted(latencies)[len(latencies) // 2] + p99 = sorted(latencies)[int(len(latencies) * 0.99)] + print(f" Latency (n={samples}): avg={avg*1000:.1f}ms p50={p50*1000:.1f}ms p99={p99*1000:.1f}ms") + +# ── Main ───────────────────────────────────────────────── + +def main(): + print(f"quickq benchmark") + print(f" Workers: {NUM_WORKERS}") + print(f" Jobs: {NUM_JOBS:,}") + print(f" DB: {DB_PATH}") + print() + + # Start worker in background + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + time.sleep(0.5) # Let worker initialize + + # 1. Noop throughput + print("── noop task (framework overhead) ──") + jobs = bench_enqueue(noop, NUM_JOBS) + bench_process(jobs) + print() + + # 2. Light CPU task throughput + print("── cpu_light task ──") + jobs = bench_enqueue(cpu_light, NUM_JOBS) + bench_process(jobs) + print() + + # 3. Single-job latency + print("── single-job latency ──") + bench_latency(noop) + print() + + # Final stats + stats = queue.stats() + print(f"Final stats: {stats}") + +if __name__ == "__main__": + main() +``` + +## Running + +```bash +python benchmark.py +``` + +## Sample Output + +``` +quickq benchmark + Workers: 8 + Jobs: 10,000 + DB: :memory: + +── noop task (framework overhead) ── + Enqueued 10,000 jobs in 0.18s (55,556 jobs/s) + Processed 10,000 jobs in 2.41s (4,149 jobs/s) + +── cpu_light task ── + Enqueued 10,000 jobs in 0.19s (52,632 jobs/s) + Processed 10,000 jobs in 2.53s (3,953 jobs/s) + +── single-job latency ── + Latency (n=100): avg=1.2ms p50=1.1ms p99=3.4ms + +Final stats: {'pending': 0, 'running': 0, 'completed': 20100, 'failed': 0, 'dead': 0, 'cancelled': 0} +``` + +!!! note + Actual numbers depend on your hardware, Python version, and SQLite configuration. The numbers above are from an 8-core machine with Python 3.12. + +## What Makes quickq Fast + +| Component | How it helps | +|---|---| +| **Batch inserts** | `task.map()` inserts all jobs in a single SQLite transaction | +| **WAL mode** | Concurrent reads while writing — workers don't block enqueue | +| **Rust scheduler** | 50ms poll loop runs in native code, not Python | +| **OS threads** | Workers are Rust `std::thread`, not Python threads | +| **GIL per task** | GIL acquired only during Python task execution, released between tasks | +| **crossbeam channels** | Lock-free job dispatch to workers | +| **r2d2 pool** | Up to 8 concurrent SQLite connections | +| **Diesel ORM** | Compiled SQL queries, no runtime query building | + +## Tuning + +Adjust these for your workload: + +```python +# More workers for I/O-bound tasks +queue = Queue(workers=16) + +# Fewer workers for CPU-bound tasks (limited by GIL) +queue = Queue(workers=4) + +# In-memory DB for maximum throughput (no persistence) +queue = Queue(db_path=":memory:") + +# File DB for durability (slightly slower) +queue = Queue(db_path="tasks.db") +``` diff --git a/docs/examples/web-scraper.md b/docs/examples/web-scraper.md new file mode 100644 index 00000000..c39a966d --- /dev/null +++ b/docs/examples/web-scraper.md @@ -0,0 +1,200 @@ +# Example: Web Scraper Pipeline + +A complete multi-stage web scraper demonstrating rate limiting, retries, workflows, progress tracking, hooks, periodic cleanup, and named queues. + +## Project Structure + +``` +scraper/ + tasks.py # Task definitions + worker.py # Worker entry point + run.py # Enqueue scraping jobs +``` + +## tasks.py + +```python +import json +import time +from quickq import Queue, current_job, chain, group, chord + +queue = Queue( + db_path="scraper.db", + workers=4, + default_retry=3, + default_timeout=60, + result_ttl=3600, # Auto-cleanup results after 1 hour +) + +# ── Hooks ──────────────────────────────────────────────── + +@queue.before_task +def log_start(task_name, args, kwargs): + print(f"[START] {task_name}") + +@queue.on_success +def log_success(task_name, args, kwargs, result): + print(f"[DONE] {task_name}") + +@queue.on_failure +def log_failure(task_name, args, kwargs, error): + print(f"[FAIL] {task_name}: {error}") + +# ── Tasks ──────────────────────────────────────────────── + +@queue.task( + rate_limit="30/m", # Max 30 requests per minute + max_retries=5, + retry_backoff=2.0, + queue="scraping", +) +def fetch_page(url): + """Fetch a single URL. Rate-limited and retried on failure.""" + import urllib.request + with urllib.request.urlopen(url, timeout=10) as resp: + return resp.read().decode("utf-8") + +@queue.task(queue="processing") +def extract_links(html): + """Extract all links from an HTML page.""" + import re + return re.findall(r'href="(https?://[^"]+)"', html) + +@queue.task(queue="processing") +def extract_title(html): + """Extract the page title.""" + import re + match = re.search(r"(.*?)", html, re.IGNORECASE | re.DOTALL) + return match.group(1).strip() if match else "No title" + +@queue.task(queue="storage") +def store_results(results, url=""): + """Store scraped data to a JSON file.""" + data = {"url": url, "results": results, "scraped_at": time.time()} + filename = f"output_{int(time.time())}.json" + with open(filename, "w") as f: + json.dump(data, f, indent=2) + return filename + +@queue.task(queue="processing") +def summarize(pages): + """Aggregate results from multiple pages.""" + total_links = sum(len(p.get("links", [])) for p in pages) + titles = [p.get("title", "?") for p in pages] + return { + "pages_scraped": len(pages), + "total_links": total_links, + "titles": titles, + } + +@queue.task() +def scrape_page(url): + """Full pipeline for a single page: fetch → extract links + title.""" + html = fetch_page(url) # Direct call (not queued) + links = extract_links(html) + title = extract_title(html) + return {"url": url, "title": title, "links": links} + +# ── Periodic cleanup ──────────────────────────────────── + +@queue.periodic(cron="0 0 * * * *") +def hourly_cleanup(): + """Purge completed jobs and dead letters every hour.""" + completed = queue.purge_completed(older_than=3600) + dead = queue.purge_dead(older_than=86400) + print(f"Cleanup: purged {completed} completed, {dead} dead") +``` + +## run.py + +```python +"""Enqueue scraping jobs.""" +from tasks import queue, scrape_page, summarize, store_results +from quickq import group, chord + +urls = [ + "https://example.com", + "https://httpbin.org/html", + "https://jsonplaceholder.typicode.com", +] + +# ── Option 1: Simple parallel scraping ────────────────── + +print("Enqueuing scrape jobs...") +jobs = [scrape_page.delay(url) for url in urls] + +# Wait for all results +for job in jobs: + result = job.result(timeout=30) + print(f" {result['title']} — {len(result['links'])} links") + +# ── Option 2: Chord — scrape in parallel, then summarize ─ + +print("\nRunning chord pipeline...") +result = chord( + group(*[scrape_page.s(url) for url in urls]), + summarize.s(), +).apply(queue) + +summary = result.result(timeout=60) +print(f" Scraped {summary['pages_scraped']} pages") +print(f" Found {summary['total_links']} total links") +print(f" Titles: {summary['titles']}") + +# ── Option 3: Batch enqueue with .map() ───────────────── + +print("\nBatch enqueue with .map()...") +jobs = scrape_page.map([(url,) for url in urls]) +results = [j.result(timeout=30) for j in jobs] +print(f" Scraped {len(results)} pages in batch") + +# ── Check stats ───────────────────────────────────────── + +stats = queue.stats() +print(f"\nQueue stats: {stats}") +``` + +## worker.py + +```python +"""Start the worker.""" +from tasks import queue + +if __name__ == "__main__": + print("Starting scraper worker...") + queue.run_worker(queues=["scraping", "processing", "storage"]) +``` + +## Running It + +=== "Terminal 1 — Worker" + + ```bash + python worker.py + ``` + +=== "Terminal 2 — Enqueue" + + ```bash + python run.py + ``` + +=== "Terminal 3 — Monitor" + + ```bash + quickq info --app tasks:queue --watch + ``` + +## Key Patterns Demonstrated + +| Pattern | Where | +|---|---| +| Rate limiting | `fetch_page` — 30 requests/min | +| Retry with backoff | `fetch_page` — 5 retries, 2.0x backoff | +| Named queues | `scraping`, `processing`, `storage` | +| Hooks | `log_start`, `log_success`, `log_failure` | +| Workflows (chord) | Parallel scrape → summarize | +| Batch enqueue | `.map()` for bulk job creation | +| Periodic tasks | `hourly_cleanup` runs every hour | +| Result TTL | Auto-cleanup completed jobs after 1 hour | +| Direct call | `scrape_page` calls `fetch_page()` directly | diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 00000000..b5715937 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,49 @@ +# Installation + +## From PyPI + +```bash +pip install quickq +``` + +quickq has a single runtime dependency: [`cloudpickle`](https://github.com/cloudpipe/cloudpickle) for serializing task arguments and results. It is installed automatically. + +!!! note "SQLite is bundled" + quickq ships with SQLite compiled in via Rust's `libsqlite3-sys` crate. You do **not** need a system SQLite installation. + +## From Source + +Building from source requires a Rust toolchain (1.70+). + +```bash +# Install Rust (if needed) +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + +# Clone and build +git clone https://github.com/pratyush618/quickq.git +cd quickq +python -m venv .venv +source .venv/bin/activate +pip install maturin +maturin develop --release +``` + +## Development Setup + +```bash +pip install -e ".[dev]" # Tests, linting, type checking +pip install -e ".[docs]" # Documentation (MkDocs Material) +``` + +## Verify Installation + +```python +import quickq +print(quickq.__version__) # 0.1.0 +``` + +## Requirements + +- Python 3.9+ +- Any OS with SQLite support (Linux, macOS, Windows) +- Rust toolchain only needed for building from source diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md new file mode 100644 index 00000000..60104754 --- /dev/null +++ b/docs/getting-started/quickstart.md @@ -0,0 +1,107 @@ +# Quickstart + +Build your first task queue in 5 minutes. + +## 1. Define Tasks + +Create a file called `tasks.py`: + +```python +from quickq import Queue + +# Create a queue backed by SQLite +queue = Queue(db_path="tasks.db") + +@queue.task() +def add(a: int, b: int) -> int: + return a + b + +@queue.task(max_retries=3, retry_backoff=2.0) +def send_email(to: str, subject: str, body: str) -> str: + # Your email sending logic here + print(f"Sending email to {to}: {subject}") + return f"sent to {to}" +``` + +## 2. Enqueue Jobs + +```python +from tasks import add, send_email + +# Enqueue returns a JobResult handle +job = add.delay(2, 3) +print(f"Job ID: {job.id}") # Job ID: 01936... +print(f"Status: {job.status}") # Status: pending +``` + +## 3. Start a Worker + +=== "CLI (Recommended)" + + ```bash + quickq worker --app tasks:queue + ``` + +=== "Threading" + + ```python + import threading + from tasks import queue + + t = threading.Thread(target=queue.run_worker, daemon=True) + t.start() + ``` + +=== "Async" + + ```python + import asyncio + from tasks import queue + + async def main(): + await queue.arun_worker() + + asyncio.run(main()) + ``` + +## 4. Get Results + +```python +from tasks import add + +job = add.delay(2, 3) + +# Block until complete (with exponential backoff polling) +result = job.result(timeout=30) +print(result) # 5 + +# Or use async +result = await job.aresult(timeout=30) +``` + +## 5. Monitor + +```python +from tasks import queue + +stats = queue.stats() +print(stats) +# {'pending': 0, 'running': 0, 'completed': 5, 'failed': 0, 'dead': 0, 'cancelled': 0} +``` + +Or use the CLI: + +```bash +# One-shot stats +quickq info --app tasks:queue + +# Live dashboard (refreshes every 2s) +quickq info --app tasks:queue --watch +``` + +## Next Steps + +- [Tasks](../guide/tasks.md) — decorator options, `.delay()` vs `.apply_async()` +- [Workers](../guide/workers.md) — CLI flags, graceful shutdown, worker count +- [Retries](../guide/retries.md) — exponential backoff, dead letter queue +- [Workflows](../guide/workflows.md) — chain, group, chord diff --git a/docs/guide/advanced.md b/docs/guide/advanced.md new file mode 100644 index 00000000..342180d1 --- /dev/null +++ b/docs/guide/advanced.md @@ -0,0 +1,129 @@ +# Advanced + +## Unique Tasks + +Deduplicate active jobs by key — if a job with the same `unique_key` is already pending or running, the existing job is returned instead of creating a new one: + +```python +job1 = process.apply_async(args=("report",), unique_key="daily-report") +job2 = process.apply_async(args=("report",), unique_key="daily-report") +assert job1.id == job2.id # Same job, not duplicated +``` + +Once the original job completes (or fails to DLQ), the key is released and a new job can be created with the same key. + +!!! info "Implementation" + Deduplication uses a partial unique index: `CREATE UNIQUE INDEX ... ON jobs(unique_key) WHERE unique_key IS NOT NULL AND status IN (0, 1)`. Only pending and running jobs participate. + +## Job Cancellation + +Cancel a pending job before it starts: + +```python +job = send_email.delay("user@example.com", "Hello", "World") +cancelled = queue.cancel_job(job.id) # True if was pending +``` + +- Returns `True` if the job was pending and is now cancelled +- Returns `False` if the job was already running, completed, or in another non-pending state +- Cancelled jobs cannot be un-cancelled + +## Result TTL & Auto-Cleanup + +### Manual Cleanup + +```python +# Purge completed jobs older than 1 hour +deleted = queue.purge_completed(older_than=3600) + +# Purge dead letters older than 24 hours +deleted = queue.purge_dead(older_than=86400) +``` + +### Automatic Cleanup + +Set `result_ttl` on the Queue to automatically purge old jobs while the worker runs: + +```python +queue = Queue( + db_path="myapp.db", + result_ttl=3600, # Auto-purge completed/dead jobs older than 1 hour +) +``` + +The scheduler checks every ~60 seconds and purges: + +- Completed jobs older than `result_ttl` +- Dead letter entries older than `result_ttl` +- Error history records older than `result_ttl` + +Set to `None` (default) to disable auto-cleanup. + +## Async Support + +All inspection methods have async variants that run in a thread pool: + +```python +# Sync +stats = queue.stats() +dead = queue.dead_letters() +new_id = queue.retry_dead(dead_id) +cancelled = queue.cancel_job(job_id) +result = job.result(timeout=30) + +# Async equivalents +stats = await queue.astats() +dead = await queue.adead_letters() +new_id = await queue.aretry_dead(dead_id) +cancelled = await queue.acancel_job(job_id) +result = await job.aresult(timeout=30) +``` + +### Async Worker + +```python +async def main(): + await queue.arun_worker(queues=["default"]) + +asyncio.run(main()) +``` + +## Batch Enqueue + +Insert many jobs in a single SQLite transaction for high throughput: + +### `task.map()` + +```python +@queue.task() +def process(item_id): + return fetch_and_process(item_id) + +# Enqueue 1000 jobs in one transaction +jobs = process.map([(i,) for i in range(1000)]) +``` + +### `queue.enqueue_many()` + +```python +jobs = queue.enqueue_many( + task_name="myapp.process", + args_list=[(i,) for i in range(1000)], + kwargs_list=None, # Optional per-job kwargs + priority=5, # Same priority for all + queue="processing", # Same queue for all +) +``` + +## SQLite Configuration + +quickq configures SQLite for optimal performance: + +| Pragma | Value | Purpose | +|---|---|---| +| `journal_mode` | WAL | Concurrent reads during writes | +| `busy_timeout` | 5000ms | Wait instead of failing on lock contention | +| `synchronous` | NORMAL | Balance between safety and speed | +| `journal_size_limit` | 64MB | Prevent unbounded WAL growth | + +The connection pool uses up to 8 connections via `r2d2`. diff --git a/docs/guide/monitoring.md b/docs/guide/monitoring.md new file mode 100644 index 00000000..d2fd01c2 --- /dev/null +++ b/docs/guide/monitoring.md @@ -0,0 +1,153 @@ +# Monitoring & Hooks + +## Queue Statistics + +Get a snapshot of job counts by status: + +```python +stats = queue.stats() +# {'pending': 12, 'running': 3, 'completed': 450, 'failed': 2, 'dead': 1, 'cancelled': 0} +``` + +Async variant: + +```python +stats = await queue.astats() +``` + +## CLI Monitoring + +### One-Shot Stats + +```bash +quickq info --app myapp:queue +``` + +``` +quickq queue statistics +------------------------------ + pending 12 + running 3 + completed 450 + failed 2 + dead 1 + cancelled 0 +------------------------------ + total 468 +``` + +### Live Dashboard + +```bash +quickq info --app myapp:queue --watch +``` + +Refreshes every 2 seconds with throughput calculation (completed jobs per second). + +## Progress Tracking + +Report progress from inside tasks using `current_job`: + +```python +from quickq import current_job + +@queue.task() +def process_batch(items): + total = len(items) + for i, item in enumerate(items): + process(item) + current_job.update_progress(int((i + 1) / total * 100)) + return f"Processed {total} items" +``` + +Read progress from outside: + +```python +job = process_batch.delay(items) + +# Poll progress +fetched = queue.get_job(job.id) +print(fetched.progress) # 0-100 or None +``` + +### Job Context + +Inside a running task, `current_job` provides: + +| Property | Type | Description | +|---|---|---| +| `current_job.id` | `str` | The current job ID | +| `current_job.task_name` | `str` | The registered task name | +| `current_job.retry_count` | `int` | Current retry attempt (0 = first run) | +| `current_job.queue_name` | `str` | The queue this job is running on | + +```python +from quickq import current_job + +@queue.task() +def my_task(): + print(f"Running job {current_job.id}") + print(f"Task: {current_job.task_name}") + print(f"Attempt: {current_job.retry_count}") + print(f"Queue: {current_job.queue_name}") +``` + +!!! warning + `current_job` properties raise `RuntimeError` when accessed outside of a running task. + +## Hooks + +Run code before/after every task, or on success/failure. + +### `@queue.before_task` + +Called before each task executes: + +```python +@queue.before_task +def log_start(task_name, args, kwargs): + print(f"[START] {task_name}") +``` + +### `@queue.after_task` + +Called after each task, regardless of success or failure: + +```python +@queue.after_task +def log_end(task_name, args, kwargs, result, error): + status = "OK" if error is None else f"FAILED: {error}" + print(f"[END] {task_name} - {status}") +``` + +### `@queue.on_success` + +Called only when a task succeeds: + +```python +@queue.on_success +def track_metrics(task_name, args, kwargs, result): + metrics.increment(f"task.{task_name}.success") +``` + +### `@queue.on_failure` + +Called only when a task raises an exception: + +```python +@queue.on_failure +def alert_on_error(task_name, args, kwargs, error): + sentry_sdk.capture_exception(error) +``` + +### Hook Signatures + +| Hook | Signature | +|---|---| +| `before_task` | `fn(task_name, args, kwargs)` | +| `after_task` | `fn(task_name, args, kwargs, result, error)` | +| `on_success` | `fn(task_name, args, kwargs, result)` | +| `on_failure` | `fn(task_name, args, kwargs, error)` | + +!!! tip "Multiple hooks" + You can register multiple hooks of the same type. They execute in registration order. diff --git a/docs/guide/queues.md b/docs/guide/queues.md new file mode 100644 index 00000000..7265cfe4 --- /dev/null +++ b/docs/guide/queues.md @@ -0,0 +1,100 @@ +# Queues & Priority + +## Named Queues + +Route tasks to different queues for isolation and dedicated processing: + +```python +@queue.task(queue="emails") +def send_email(to, subject, body): + ... + +@queue.task(queue="reports") +def generate_report(report_id): + ... + +@queue.task() # Goes to "default" queue +def process_data(data): + ... +``` + +### Worker Queue Subscription + +Workers can listen to specific queues: + +```bash +# Process only email tasks +quickq worker --app myapp:queue --queues emails + +# Process multiple queues +quickq worker --app myapp:queue --queues emails,reports + +# Process all registered queues (default) +quickq worker --app myapp:queue +``` + +Or programmatically: + +```python +queue.run_worker(queues=["emails", "reports"]) +``` + +!!! tip "Use queues to isolate workloads" + Separate I/O-bound tasks (API calls, emails) from CPU-bound tasks (data processing, report generation) into different queues. Run them on different worker processes for optimal resource usage. + +## Priority + +Higher priority jobs are dequeued first within the same queue. Priority is an integer — higher values mean more urgent. + +### Default Priority + +Set at task registration: + +```python +@queue.task(priority=10) +def urgent_task(data): + ... + +@queue.task(priority=0) # Default +def normal_task(data): + ... +``` + +### Override at Enqueue Time + +```python +# This specific job is extra urgent +urgent_task.apply_async(args=(data,), priority=100) +``` + +### How It Works + +Jobs are dequeued using a compound index: `(queue, status, priority DESC, scheduled_at ASC)`. This means: + +1. Higher priority jobs go first +2. Among equal priority, older jobs (earlier `scheduled_at`) go first +3. Each queue is processed independently + +```python +# These three jobs are in the same queue +low = task.apply_async(args=(1,), priority=1) +mid = task.apply_async(args=(2,), priority=5) +high = task.apply_async(args=(3,), priority=10) + +# Processing order: high (10), mid (5), low (1) +``` + +## Default Queue Settings + +Configure defaults at the Queue level: + +```python +queue = Queue( + db_path="myapp.db", + default_priority=0, # Default priority for all tasks + default_retry=3, # Default max retries + default_timeout=300, # Default timeout in seconds +) +``` + +Individual `@queue.task()` decorators override these defaults. diff --git a/docs/guide/rate-limiting.md b/docs/guide/rate-limiting.md new file mode 100644 index 00000000..dbef1c75 --- /dev/null +++ b/docs/guide/rate-limiting.md @@ -0,0 +1,67 @@ +# Rate Limiting + +quickq uses a **token bucket** algorithm to limit how fast tasks execute. Rate limits are per-task and persisted in SQLite. + +## Usage + +```python +@queue.task(rate_limit="100/m") # 100 per minute +def send_email(to, subject, body): + ... + +@queue.task(rate_limit="10/s") # 10 per second +def api_call(endpoint): + ... + +@queue.task(rate_limit="3600/h") # 3600 per hour +def generate_report(report_id): + ... +``` + +## Syntax + +Rate limits use the format `count/period`: + +| Format | Meaning | +|---|---| +| `"10/s"` | 10 per second | +| `"100/m"` | 100 per minute | +| `"3600/h"` | 3600 per hour | + +## How It Works + +The token bucket algorithm: + +1. Each task name has a bucket with `max_tokens = count` and a `refill_rate = count / period` +2. Before dispatching a job, the scheduler checks if a token is available +3. If a token is available, it's consumed and the job is dispatched +4. If no tokens are available, the job is **rescheduled** 1 second in the future + +!!! info "Rate limit state is persisted" + Token bucket state (current tokens, last refill time) is stored in the `rate_limits` SQLite table. This means rate limits survive worker restarts. + +## Per-Task, Not Per-Queue + +Rate limits apply to the **task name**, regardless of which queue the job is in: + +```python +@queue.task(rate_limit="10/s", queue="emails") +def send_email(to, subject, body): + ... + +# Both of these are rate-limited together (same task name) +send_email.delay("alice@example.com", "Hi", "Body") +send_email.apply_async(args=("bob@example.com", "Hi", "Body"), queue="urgent") +``` + +## Combining with Retries + +Rate limiting and retries work together seamlessly. If a rate-limited task fails and retries, the retry attempt is also subject to the rate limit: + +```python +@queue.task(rate_limit="5/s", max_retries=3, retry_backoff=2.0) +def external_api(url): + response = requests.get(url) + response.raise_for_status() + return response.json() +``` diff --git a/docs/guide/retries.md b/docs/guide/retries.md new file mode 100644 index 00000000..4dc7d35e --- /dev/null +++ b/docs/guide/retries.md @@ -0,0 +1,127 @@ +# Retries & Dead Letters + +quickq automatically retries failed tasks with exponential backoff and moves permanently failed jobs to a dead letter queue. + +## Retry Policy + +Configure retries at the task level: + +```python +@queue.task(max_retries=5, retry_backoff=2.0) +def flaky_api_call(url): + response = requests.get(url) + response.raise_for_status() + return response.json() +``` + +| Parameter | Default | Description | +|---|---|---| +| `max_retries` | `3` | Maximum retry attempts before DLQ | +| `retry_backoff` | `1.0` | Base delay in seconds for exponential backoff | + +### Backoff Formula + +``` +delay = min(max_delay, base_delay * 2^retry_count) + jitter +``` + +- `base_delay` = `retry_backoff` (in seconds) +- `max_delay` = 300 seconds (5 minutes) +- `jitter` = random 0–500ms to prevent thundering herd + +**Example with `retry_backoff=2.0`:** + +| Attempt | Delay | +|---|---| +| 1st retry | ~2s | +| 2nd retry | ~4s | +| 3rd retry | ~8s | +| 4th retry | ~16s | +| 5th retry | ~32s | + +## Retry Flow + +```mermaid +flowchart TD + A["Task Execution"] --> B{Success?} + B -->|Yes| C["Status: Complete
Store result"] + B -->|No| D["Record error in
job_errors table"] + D --> E{"retry_count < max_retries?"} + E -->|Yes| F["Calculate backoff delay"] + F --> G["Status: Pending
retry_count += 1"] + G --> H["Wait for scheduled time"] + H --> A + E -->|No| I["Move to Dead Letter Queue
Status: Dead"] +``` + +## Dead Letter Queue + +Jobs that exhaust all retries are moved to the DLQ for inspection and manual replay. + +### Inspect Dead Letters + +```python +# List the 10 most recent dead letters +dead = queue.dead_letters(limit=10, offset=0) + +for d in dead: + print(f"Job: {d['original_job_id']}") + print(f"Task: {d['task_name']}") + print(f"Error: {d['error']}") + print(f"Retries: {d['retry_count']}") + print() +``` + +### Replay Dead Letters + +```python +# Re-enqueue a dead letter job (creates a new job) +new_job_id = queue.retry_dead(dead[0]["id"]) +``` + +### Purge Old Dead Letters + +```python +# Delete dead letters older than 24 hours +deleted = queue.purge_dead(older_than=86400) +print(f"Purged {deleted} dead letter(s)") +``` + +## Error History + +Every failed attempt is recorded with the error message. Access the full history via `job.errors`: + +```python +@queue.task(max_retries=3) +def unreliable(): + raise ConnectionError("timeout") + +job = unreliable.delay() + +# After the job fails and retries... +for error in job.errors: + print(f"Attempt {error['attempt']}: {error['error']}") + # Attempt 0: timeout + # Attempt 1: timeout + # Attempt 2: timeout +``` + +Each error entry contains: + +| Field | Type | Description | +|---|---|---| +| `id` | `str` | Unique error record ID | +| `job_id` | `str` | The job this error belongs to | +| `attempt` | `int` | Attempt number (0-indexed) | +| `error` | `str` | Error message | +| `failed_at` | `int` | Timestamp in milliseconds | + +## Timeout Reaping + +If a task exceeds its `timeout`, the scheduler automatically detects it (checking every ~5 seconds) and treats it as a failure — triggering the retry/DLQ logic. + +```python +@queue.task(timeout=10) # 10 second timeout +def slow_task(): + time.sleep(60) # Will be reaped after 10s +``` diff --git a/docs/guide/scheduling.md b/docs/guide/scheduling.md new file mode 100644 index 00000000..c574810c --- /dev/null +++ b/docs/guide/scheduling.md @@ -0,0 +1,90 @@ +# Scheduling + +quickq supports both **delayed tasks** (run once in the future) and **periodic tasks** (run on a cron schedule). + +## Delayed Tasks + +Schedule a task to run after a delay: + +```python +# Run 1 hour from now +send_email.apply_async( + args=("user@example.com", "Reminder", "Don't forget!"), + delay=3600, # seconds +) + +# Run 30 seconds from now +cleanup.apply_async(args=(), delay=30) +``` + +The job is created immediately with `status=pending` but won't be picked up by a worker until the `scheduled_at` timestamp is reached. + +## Periodic Tasks + +Register recurring tasks with cron expressions: + +```python +@queue.periodic(cron="0 */5 * * * *") +def health_check(): + """Run every 5 minutes.""" + ping_services() + +@queue.periodic(cron="0 0 0 * * *") +def daily_cleanup(): + """Run at midnight every day.""" + queue.purge_completed(older_than=86400) + +@queue.periodic(cron="0 0 9 * * 1", args=("weekly",)) +def weekly_report(report_type): + """Run every Monday at 9:00 AM.""" + generate_report(report_type) +``` + +### Cron Expression Format + +quickq uses **6-field cron expressions** (with seconds): + +``` +┌─────────── second (0-59) +│ ┌───────── minute (0-59) +│ │ ┌─────── hour (0-23) +│ │ │ ┌───── day of month (1-31) +│ │ │ │ ┌─── month (1-12) +│ │ │ │ │ ┌─ day of week (0-6, Sun=0) +│ │ │ │ │ │ +* * * * * * +``` + +| Expression | Schedule | +|---|---| +| `0 */5 * * * *` | Every 5 minutes | +| `0 0 * * * *` | Every hour | +| `0 0 0 * * *` | Every day at midnight | +| `0 30 9 * * 1-5` | Weekdays at 9:30 AM | +| `0 0 */2 * * *` | Every 2 hours | +| `*/30 * * * * *` | Every 30 seconds | + +### Decorator Options + +```python +@queue.periodic( + cron="0 0 * * * *", # Required: cron expression + name="hourly-cleanup", # Optional: explicit name + args=(3600,), # Optional: positional args + kwargs={"force": True}, # Optional: keyword args + queue="maintenance", # Optional: target queue +) +def cleanup(older_than, force=False): + ... +``` + +### How Periodic Tasks Work + +1. Periodic tasks are registered with the Rust scheduler when the worker starts +2. The scheduler checks for due tasks every ~3 seconds +3. When a task is due, a new job is enqueued automatically +4. The task's `next_run` is computed using the cron expression +5. Periodic task state is persisted in the `periodic_tasks` SQLite table + +!!! note + Periodic tasks are only active while a worker is running. If no worker is running, tasks accumulate and the **next due** job is enqueued when a worker starts. diff --git a/docs/guide/tasks.md b/docs/guide/tasks.md new file mode 100644 index 00000000..3304fa49 --- /dev/null +++ b/docs/guide/tasks.md @@ -0,0 +1,129 @@ +# Tasks + +Tasks are Python functions registered with a queue via the `@queue.task()` decorator. + +## Defining a Task + +```python +from quickq import Queue + +queue = Queue(db_path="myapp.db") + +@queue.task() +def process_data(data: dict) -> str: + # Your logic here + return "done" +``` + +## Decorator Options + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `name` | `str \| None` | Auto-generated | Explicit task name. Defaults to `module.qualname`. | +| `max_retries` | `int` | `3` | Max retry attempts before moving to DLQ. | +| `retry_backoff` | `float` | `1.0` | Base delay in seconds for exponential backoff. | +| `timeout` | `int` | `300` | Max execution time in seconds. | +| `priority` | `int` | `0` | Default priority (higher = more urgent). | +| `rate_limit` | `str \| None` | `None` | Rate limit string, e.g. `"100/m"`. | +| `queue` | `str` | `"default"` | Named queue to submit to. | + +```python +@queue.task( + name="emails.send", + max_retries=5, + retry_backoff=2.0, + timeout=60, + priority=10, + rate_limit="100/m", + queue="emails", +) +def send_email(to: str, subject: str, body: str): + ... +``` + +## Task Naming + +By default, tasks are named using `module.qualname`: + +```python +# In myapp/tasks.py +@queue.task() +def process(): # Named: myapp.tasks.process + ... +``` + +You can override with an explicit name: + +```python +@queue.task(name="my-custom-name") +def process(): # Named: my-custom-name + ... +``` + +## Enqueuing Jobs + +### `.delay()` — Quick Submit + +Submit with default options: + +```python +job = send_email.delay("user@example.com", "Hello", "World") +``` + +### `.apply_async()` — Full Control + +Override any option at enqueue time: + +```python +job = send_email.apply_async( + args=("user@example.com", "Hello", "World"), + priority=100, # Override priority + delay=3600, # Run 1 hour from now + queue="urgent-emails", # Override queue + max_retries=10, # Override retries + timeout=120, # Override timeout + unique_key="welcome-user@example.com", # Deduplicate + metadata='{"source": "signup"}', # Attach JSON metadata +) +``` + +### Direct Call + +Calling a task directly runs it synchronously, bypassing the queue: + +```python +result = send_email("user@example.com", "Hello", "World") # Runs immediately +``` + +## Batch Enqueue + +Enqueue many jobs in a single SQLite transaction: + +```python +# Via task.map() +jobs = send_email.map([ + ("alice@example.com", "Hi", "Body"), + ("bob@example.com", "Hi", "Body"), + ("carol@example.com", "Hi", "Body"), +]) + +# Via queue.enqueue_many() +jobs = queue.enqueue_many( + task_name=send_email.name, + args_list=[("alice@example.com",), ("bob@example.com",)], + kwargs_list=[{"subject": "Hi", "body": "Body"}] * 2, +) +``` + +## Metadata + +Attach arbitrary JSON metadata to jobs: + +```python +job = process.apply_async( + args=(data,), + metadata='{"user_id": 42, "source": "api"}', +) +``` + +Metadata is stored with the job and visible in dead letter queue entries. diff --git a/docs/guide/workers.md b/docs/guide/workers.md new file mode 100644 index 00000000..d6eeae36 --- /dev/null +++ b/docs/guide/workers.md @@ -0,0 +1,109 @@ +# Workers + +Workers process queued jobs. quickq runs workers as OS threads within a single process, managed by a Rust scheduler. + +## Starting a Worker + +=== "CLI (Recommended)" + + ```bash + quickq worker --app myapp.tasks:queue + ``` + + | Flag | Description | + |---|---| + | `--app` | Python path to your Queue instance (`module:attribute`) | + | `--queues` | Comma-separated queue names (default: all registered) | + +=== "Programmatic" + + ```python + # Blocks the current thread + queue.run_worker() + + # With specific queues + queue.run_worker(queues=["emails", "reports"]) + ``` + +=== "Background Thread" + + ```python + import threading + + t = threading.Thread(target=queue.run_worker, daemon=True) + t.start() + + # Your application continues... + ``` + +=== "Async" + + ```python + import asyncio + + async def main(): + # Runs worker in a thread pool, non-blocking + await queue.arun_worker() + + asyncio.run(main()) + ``` + +## Worker Count + +By default, quickq auto-detects the number of CPU cores: + +```python +queue = Queue(db_path="myapp.db", workers=0) # Auto-detect (default) +queue = Queue(db_path="myapp.db", workers=8) # Explicit count +``` + +!!! note + Workers are **OS threads**, not processes. Each worker acquires the Python GIL only during task execution, so the scheduler and dispatch logic run without GIL contention. + +## Graceful Shutdown + +quickq supports graceful shutdown via `Ctrl+C`: + +1. **First `Ctrl+C`**: Stops accepting new jobs, waits up to 30 seconds for in-flight tasks to complete +2. **Second `Ctrl+C`**: Force-kills immediately + +``` +$ quickq worker --app myapp:queue +[quickq] Starting worker... +[quickq] Registered tasks: 3 +[quickq] Queues: default, emails +^C +[quickq] Shutting down gracefully (waiting for in-flight jobs)... +[quickq] Worker stopped. +``` + +### Programmatic Shutdown + +```python +# From another thread or signal handler +queue._inner.request_shutdown() +``` + +## How Workers Work + +```mermaid +graph LR + S["Scheduler
(Tokio async)"] -->|Job| CH["Bounded Channel"] + + CH --> W1["Worker 1"] + CH --> W2["Worker 2"] + CH --> WN["Worker N"] + + W1 -->|Result| RCH["Result Channel"] + W2 -->|Result| RCH + WN -->|Result| RCH + + RCH --> ML["Result Handler"] + ML -->|"complete / retry / DLQ"| DB[("SQLite")] +``` + +1. The **scheduler** runs in a dedicated Tokio async thread, polling SQLite for ready jobs every 50ms +2. Ready jobs are sent to the **worker pool** via a bounded crossbeam channel +3. Each **worker thread** acquires the GIL, deserializes arguments, and runs the Python function +4. Results flow back through a **result channel** to the main loop +5. The main loop updates job status in SQLite (complete, retry, or DLQ) diff --git a/docs/guide/workflows.md b/docs/guide/workflows.md new file mode 100644 index 00000000..1eb2e65c --- /dev/null +++ b/docs/guide/workflows.md @@ -0,0 +1,146 @@ +# Workflows + +quickq provides three composition primitives for building complex task pipelines: **chain**, **group**, and **chord**. + +## Signatures + +A `Signature` wraps a task call for deferred execution. Create them with `.s()` or `.si()`: + +```python +from quickq import chain, group, chord + +# Mutable signature — receives previous result as first argument +sig = add.s(1, 2) + +# Immutable signature — ignores previous result +sig = add.si(1, 2) +``` + +## Chain + +Execute tasks **sequentially**, piping each result as the first argument to the next task: + +```mermaid +graph LR + S1["extract.s(url)"] -->|result| S2["transform.s()"] + S2 -->|result| S3["load.s()"] +``` + +```python +@queue.task() +def extract(url): + return requests.get(url).json() + +@queue.task() +def transform(data): + return [item["name"] for item in data] + +@queue.task() +def load(names): + db.insert_many(names) + return len(names) + +# Build and execute the pipeline +result = chain( + extract.s("https://api.example.com/users"), + transform.s(), + load.s(), +).apply(queue) + +print(result.result(timeout=30)) # Number of records loaded +``` + +!!! tip + Use `.si()` (immutable signatures) when a step should **not** receive the previous result: + + ```python + chain( + step_a.s(input_data), + step_b.si(independent_data), # Ignores step_a's result + step_c.s(), + ).apply(queue) + ``` + +## Group + +Execute tasks **in parallel** (fan-out): + +```mermaid +graph TD + G["group()"] --> S1["process.s(1)"] + G --> S2["process.s(2)"] + G --> S3["process.s(3)"] + + S1 --> R1["Result 1"] + S2 --> R2["Result 2"] + S3 --> R3["Result 3"] +``` + +```python +@queue.task() +def process(item_id): + return fetch_and_process(item_id) + +# Enqueue all three in parallel +jobs = group( + process.s(1), + process.s(2), + process.s(3), +).apply(queue) + +# Collect results +results = [j.result(timeout=30) for j in jobs] +``` + +## Chord + +Fan-out with a **callback** — execute tasks in parallel, then pass all results to a final task: + +```mermaid +graph TD + F1["fetch.s(url1)"] --> C["Collect results"] + F2["fetch.s(url2)"] --> C + F3["fetch.s(url3)"] --> C + C -->|"[r1, r2, r3]"| CB["merge.s()"] +``` + +```python +@queue.task() +def fetch(url): + return requests.get(url).json() + +@queue.task() +def merge(results): + combined = {} + for r in results: + combined.update(r) + return combined + +# Fetch in parallel, then merge +result = chord( + group( + fetch.s("https://api1.example.com"), + fetch.s("https://api2.example.com"), + fetch.s("https://api3.example.com"), + ), + merge.s(), +).apply(queue) + +print(result.result(timeout=60)) +``` + +## Real-World Example: ETL Pipeline + +```python +# Extract from multiple sources in parallel, +# transform each, then load all results +pipeline = chord( + group( + chain(extract.s(source), transform.s()) + for source in data_sources + ), + load.s(), +) + +result = pipeline.apply(queue) +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..cf5e5f2a --- /dev/null +++ b/docs/index.md @@ -0,0 +1,152 @@ +# quickq + +**Rust-powered task queue for Python. No broker required — just SQLite.** + +```bash +pip install quickq +``` + +--- + +## 5-Minute Quickstart + +```python +from quickq import Queue + +queue = Queue(db_path="tasks.db") + +@queue.task() +def add(a: int, b: int) -> int: + return a + b + +job = add.delay(2, 3) + +# Start worker (in production, use the CLI instead) +import threading +t = threading.Thread(target=queue.run_worker, daemon=True) +t.start() + +print(job.result(timeout=10)) # 5 +``` + +[:octicons-arrow-right-24: Get started](getting-started/quickstart.md) + +--- + +## Why quickq? + +Most Python task queues require a separate broker (Redis, RabbitMQ) that you need to install, configure, monitor, and keep running. **quickq** embeds everything into a single SQLite file — the queue, the results, the rate limits, the schedules. Just `pip install` and go. + +The core engine is written in **Rust** for performance: job dispatch, retry scheduling, rate limiting, and storage all happen in compiled native code. Python only runs during actual task execution. + +--- + +## Features + +
+ +- :material-lightning-bolt:{ .lg .middle } **Zero Infrastructure** + + --- + + No Redis, no RabbitMQ — just a SQLite file. Install and start queuing in seconds. + +- :material-sort-ascending:{ .lg .middle } **Priority Queues** + + --- + + Higher priority jobs run first. Override at enqueue time for urgent work. + +- :material-refresh:{ .lg .middle } **Retry with Backoff** + + --- + + Automatic exponential backoff with jitter. Failed jobs land in a dead letter queue for inspection. + +- :material-speedometer:{ .lg .middle } **Rate Limiting** + + --- + + Token bucket rate limiting per task. `"100/m"`, `"10/s"`, `"3600/h"`. + +- :material-link-variant:{ .lg .middle } **Task Workflows** + + --- + + Compose pipelines with `chain`, parallelize with `group`, aggregate with `chord`. + +- :material-clock-outline:{ .lg .middle } **Cron Scheduling** + + --- + + `@queue.periodic(cron="0 */5 * * * *")` for recurring tasks with 6-field cron expressions. + +- :material-progress-check:{ .lg .middle } **Progress Tracking** + + --- + + Report progress from inside tasks. Monitor completion percentage in real time. + +- :material-language-rust:{ .lg .middle } **Rust-Powered** + + --- + + Scheduling, storage, and dispatch in native Rust. Python only runs your task code. + +
+ +--- + +## Architecture + +```mermaid +graph TB + subgraph Python ["Python Layer"] + A["Queue / TaskWrapper"] + D["JobResult"] + end + + subgraph Rust ["Rust Core · PyO3"] + F["PyQueue"] + G["Scheduler · Tokio"] + H["Worker Pool · OS Threads"] + I["Rate Limiter"] + end + + subgraph Storage ["Embedded Storage"] + J[("SQLite · WAL mode
Diesel ORM")] + end + + A -->|enqueue| F + F -->|INSERT| J + G -->|poll & dequeue| J + G -->|crossbeam channel| H + H -->|acquire GIL · run task| A + H -->|result| G + G -->|UPDATE status| J + D -->|poll status| F + F -->|SELECT| J + G -.->|check limit| I + I -.->|token state| J +``` + +[:octicons-arrow-right-24: Architecture deep dive](architecture.md) + +--- + +## Comparison + +| Feature | quickq | Celery | RQ | Dramatiq | Huey | +|---|---|---|---|---|---| +| Broker required | **No** | Redis/RabbitMQ | Redis | Redis/RabbitMQ | Redis | +| Core language | Rust + Python | Python | Python | Python | Python | +| Priority queues | :white_check_mark: | :white_check_mark: | :x: | :x: | :white_check_mark: | +| Rate limiting | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | +| Dead letter queue | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | +| Task workflows | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | +| Job cancellation | :white_check_mark: | :white_check_mark: | :x: | :x: | :white_check_mark: | +| Progress tracking | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | +| Unique tasks | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | +| Setup complexity | `pip install` | Broker + backend | Redis server | Broker | Redis server | + +[:octicons-arrow-right-24: Full comparison](comparison.md) From d6f84a2cebd4b32f420851bf6f103ed65739493b Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Tue, 24 Feb 2026 02:14:43 +0530 Subject: [PATCH 3/3] feat: Introduce job dependencies, FastAPI integration, and a web dashboard, alongside new job listing and dependency retrieval methods. --- docs/api/queue.md | 29 +- docs/api/result.md | 24 + docs/api/task.md | 2 + docs/guide/advanced.md | 179 ++++++ docs/guide/dashboard.md | 176 ++++++ docs/guide/dependencies.md | 245 ++++++++ mkdocs.yml | 2 + pyproject.toml | 1 + python/quickq/_quickq.pyi | 11 + python/quickq/app.py | 44 +- python/quickq/cli.py | 24 + python/quickq/contrib/__init__.py | 1 + python/quickq/contrib/fastapi.py | 277 +++++++++ python/quickq/dashboard.py | 795 +++++++++++++++++++++++++ python/quickq/result.py | 39 ++ python/quickq/task.py | 3 + rust/quickq-core/src/error.rs | 3 + rust/quickq-core/src/job.rs | 1 + rust/quickq-core/src/scheduler.rs | 1 + rust/quickq-core/src/storage/models.rs | 20 +- rust/quickq-core/src/storage/schema.rs | 10 + rust/quickq-core/src/storage/sqlite.rs | 401 +++++++++++-- rust/quickq-python/src/py_queue.rs | 54 +- tests/python/test_dashboard.py | 265 +++++++++ tests/python/test_dependencies.py | 122 ++++ tests/python/test_fastapi.py | 195 ++++++ 26 files changed, 2874 insertions(+), 50 deletions(-) create mode 100644 docs/guide/dashboard.md create mode 100644 docs/guide/dependencies.md create mode 100644 python/quickq/contrib/__init__.py create mode 100644 python/quickq/contrib/fastapi.py create mode 100644 python/quickq/dashboard.py create mode 100644 tests/python/test_dashboard.py create mode 100644 tests/python/test_dependencies.py create mode 100644 tests/python/test_fastapi.py diff --git a/docs/api/queue.md b/docs/api/queue.md index ca18c4e6..b5d91d2f 100644 --- a/docs/api/queue.md +++ b/docs/api/queue.md @@ -74,11 +74,16 @@ queue.enqueue( timeout: int | None = None, unique_key: str | None = None, metadata: str | None = None, + depends_on: str | list[str] | None = None, ) -> JobResult ``` Enqueue a task for execution. Returns a [`JobResult`](result.md) handle. +| Parameter | Type | Default | Description | +|---|---|---|---| +| `depends_on` | `str \| list[str] \| None` | `None` | Job ID(s) this job depends on. See [Dependencies](../guide/dependencies.md). | + ### `queue.enqueue_many()` ```python @@ -105,13 +110,35 @@ queue.get_job(job_id: str) -> JobResult | None Retrieve a job by ID. Returns `None` if not found. +### `queue.list_jobs()` + +```python +queue.list_jobs( + status: str | None = None, + queue: str | None = None, + task_name: str | None = None, + limit: int = 50, + offset: int = 0, +) -> list[JobResult] +``` + +List jobs with optional filters. Returns a list of [`JobResult`](result.md) handles ordered by creation time (newest first). + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `status` | `str \| None` | `None` | Filter by status: `pending`, `running`, `completed`, `failed`, `dead`, `cancelled` | +| `queue` | `str \| None` | `None` | Filter by queue name | +| `task_name` | `str \| None` | `None` | Filter by task name | +| `limit` | `int` | `50` | Maximum results to return | +| `offset` | `int` | `0` | Pagination offset | + ### `queue.cancel_job()` ```python queue.cancel_job(job_id: str) -> bool ``` -Cancel a pending job. Returns `True` if cancelled, `False` if not pending. +Cancel a pending job. Returns `True` if cancelled, `False` if not pending. If the job has dependents, they are cascade-cancelled. ### `queue.update_progress()` diff --git a/docs/api/result.md b/docs/api/result.md index 00596140..f582b8bf 100644 --- a/docs/api/result.md +++ b/docs/api/result.md @@ -72,8 +72,32 @@ for err in job.errors: print(f"Attempt {err['attempt']}: {err['error']}") ``` +### `job.dependencies` + +```python +job.dependencies -> list[str] +``` + +List of job IDs this job depends on. Returns an empty list if the job has no dependencies. See [Dependencies](../guide/dependencies.md). + +### `job.dependents` + +```python +job.dependents -> list[str] +``` + +List of job IDs that depend on this job. Returns an empty list if no other jobs depend on this one. + ## Methods +### `job.to_dict()` + +```python +job.to_dict() -> dict +``` + +Return all job fields as a plain dictionary. Useful for JSON serialization (e.g. in the [dashboard](../guide/dashboard.md) or [FastAPI integration](../guide/advanced.md#fastapi-integration)). + ### `job.result()` ```python diff --git a/docs/api/task.md b/docs/api/task.md index 73ece781..e405bcf9 100644 --- a/docs/api/task.md +++ b/docs/api/task.md @@ -46,6 +46,7 @@ task.apply_async( timeout: int | None = None, unique_key: str | None = None, metadata: str | None = None, + depends_on: str | list[str] | None = None, ) -> JobResult ``` @@ -62,6 +63,7 @@ Enqueue with full control over submission options. Any parameter not provided fa | `timeout` | `int \| None` | `None` | Override timeout in seconds | | `unique_key` | `str \| None` | `None` | Deduplicate active jobs with same key | | `metadata` | `str \| None` | `None` | Arbitrary JSON metadata to attach | +| `depends_on` | `str \| list[str] \| None` | `None` | Job ID(s) this job depends on. See [Dependencies](../guide/dependencies.md). | ```python job = send_email.apply_async( diff --git a/docs/guide/advanced.md b/docs/guide/advanced.md index 342180d1..0f5b72b0 100644 --- a/docs/guide/advanced.md +++ b/docs/guide/advanced.md @@ -127,3 +127,182 @@ quickq configures SQLite for optimal performance: | `journal_size_limit` | 64MB | Prevent unbounded WAL growth | The connection pool uses up to 8 connections via `r2d2`. + +## FastAPI Integration + +quickq provides a first-class FastAPI integration via `quickq.contrib.fastapi`. It gives you a pre-built `APIRouter` with endpoints for job status, progress streaming via SSE, and dead letter queue management. + +### Installation + +```bash +pip install quickq[fastapi] +``` + +This installs `fastapi` and `pydantic` as extras. + +### Quick Setup + +```python +from fastapi import FastAPI +from quickq import Queue +from quickq.contrib.fastapi import QuickQRouter + +queue = Queue(db_path="myapp.db") + +@queue.task() +def process_data(payload: dict) -> str: + return "done" + +app = FastAPI() +app.include_router(QuickQRouter(queue), prefix="/tasks") +``` + +Run with: + +```bash +uvicorn myapp:app --reload +``` + +All quickq endpoints are now available under `/tasks/`. + +### Endpoints + +The `QuickQRouter` exposes the following endpoints: + +| Method | Path | Description | +|---|---|---| +| `GET` | `/stats` | Queue statistics (pending, running, completed, etc.) | +| `GET` | `/jobs/{job_id}` | Job status, progress, and metadata | +| `GET` | `/jobs/{job_id}/errors` | Error history for a job | +| `GET` | `/jobs/{job_id}/result` | Job result (optional blocking with `?timeout=N`) | +| `GET` | `/jobs/{job_id}/progress` | SSE stream of progress updates | +| `POST` | `/jobs/{job_id}/cancel` | Cancel a pending job | +| `GET` | `/dead-letters` | List dead letter entries (paginated) | +| `POST` | `/dead-letters/{dead_id}/retry` | Re-enqueue a dead letter | + +### Blocking Result Fetch + +The `/jobs/{job_id}/result` endpoint supports an optional `timeout` query parameter (0–300 seconds). When `timeout > 0`, the request blocks until the job completes or the timeout elapses: + +```bash +# Non-blocking (default) +curl http://localhost:8000/tasks/jobs/01H5K6X.../result + +# Block up to 30 seconds for the result +curl http://localhost:8000/tasks/jobs/01H5K6X.../result?timeout=30 +``` + +### SSE Progress Streaming + +Stream real-time progress for a running job using Server-Sent Events: + +```python +import httpx + +with httpx.stream("GET", "http://localhost:8000/tasks/jobs/01H5K6X.../progress") as r: + for line in r.iter_lines(): + print(line) + # data: {"progress": 25, "status": "running"} + # data: {"progress": 50, "status": "running"} + # data: {"progress": 100, "status": "completed"} +``` + +From the browser: + +```javascript +const source = new EventSource("/tasks/jobs/01H5K6X.../progress"); +source.onmessage = (event) => { + const data = JSON.parse(event.data); + console.log(`Progress: ${data.progress}%`); + if (data.status === "completed" || data.status === "failed") { + source.close(); + } +}; +``` + +The stream sends a JSON event every 0.5 seconds while the job is active, then a final event when the job reaches a terminal state. + +### Pydantic Response Models + +All endpoints return validated Pydantic models with clean OpenAPI docs. You can import them for type-safe client code: + +```python +from quickq.contrib.fastapi import ( + StatsResponse, + JobResponse, + JobErrorResponse, + JobResultResponse, + CancelResponse, + DeadLetterResponse, + RetryResponse, +) +``` + +### Custom Tags and Dependencies + +Customize the router with FastAPI tags and dependency injection: + +```python +from fastapi import Depends, FastAPI, Header, HTTPException +from quickq.contrib.fastapi import QuickQRouter + +async def verify_api_key(x_api_key: str = Header(...)): + if x_api_key != "secret": + raise HTTPException(status_code=401, detail="Invalid API key") + +app = FastAPI() + +router = QuickQRouter( + queue, + tags=["task-queue"], # OpenAPI tags + dependencies=[Depends(verify_api_key)], # Applied to all endpoints +) + +app.include_router(router, prefix="/tasks") +``` + +### Full Example + +```python +from fastapi import FastAPI, Header, HTTPException, Depends +from quickq import Queue, current_job +from quickq.contrib.fastapi import QuickQRouter + +queue = Queue(db_path="myapp.db") + +@queue.task() +def resize_image(image_url: str, sizes: list[int]) -> dict: + results = {} + for i, size in enumerate(sizes): + results[size] = do_resize(image_url, size) + current_job.update_progress(int((i + 1) / len(sizes) * 100)) + return results + +async def require_auth(authorization: str = Header(...)): + if not authorization.startswith("Bearer "): + raise HTTPException(401) + +app = FastAPI(title="Image Service") +app.include_router( + QuickQRouter(queue, dependencies=[Depends(require_auth)]), + prefix="/tasks", + tags=["tasks"], +) + +# Start worker in a separate process: +# quickq worker --app myapp:queue +``` + +```bash +# Check job status +curl http://localhost:8000/tasks/jobs/01H5K6X... \ + -H "Authorization: Bearer mytoken" + +# Stream progress +curl -N http://localhost:8000/tasks/jobs/01H5K6X.../progress \ + -H "Authorization: Bearer mytoken" + +# Block for result (up to 60s) +curl http://localhost:8000/tasks/jobs/01H5K6X.../result?timeout=60 \ + -H "Authorization: Bearer mytoken" +``` diff --git a/docs/guide/dashboard.md b/docs/guide/dashboard.md new file mode 100644 index 00000000..816e92a2 --- /dev/null +++ b/docs/guide/dashboard.md @@ -0,0 +1,176 @@ +# Web Dashboard + +quickq ships with a built-in web dashboard for monitoring jobs, inspecting dead letters, and managing your task queue in real time. The dashboard is a single-page application served directly from the Rust core -- **zero extra dependencies required**. + +## Launching the Dashboard + +=== "CLI" + + ```bash + quickq dashboard --app myapp:queue + ``` + + The `--app` argument uses the same `module:attribute` format as the worker. + +=== "Programmatic" + + ```python + from quickq.dashboard import serve_dashboard + from myapp import queue + + serve_dashboard(queue, host="0.0.0.0", port=8000) + ``` + +By default the dashboard starts on `http://localhost:8080`. + +### CLI Options + +| Flag | Default | Description | +|---|---|---| +| `--app` | *required* | Module path to your `Queue` instance, e.g. `myapp:queue` | +| `--host` | `127.0.0.1` | Bind address | +| `--port` | `8080` | Bind port | + +```bash +# Bind to all interfaces on port 9000 +quickq dashboard --app myapp:queue --host 0.0.0.0 --port 9000 +``` + +!!! tip "Running alongside the worker" + The dashboard reads directly from the same SQLite database as the worker. You can run them side by side without any coordination: + + ```bash + # Terminal 1 + quickq worker --app myapp:queue + + # Terminal 2 + quickq dashboard --app myapp:queue + ``` + +## SPA Features + +The dashboard is a self-contained single-page application with: + +- **Dark mode** -- Toggle between light and dark themes. Preference is stored in `localStorage`. +- **Auto-refresh** -- Job stats and tables refresh automatically every 2 seconds. Disable with the pause button. +- **Status badges** -- Color-coded pills for each job status: :material-clock-outline: pending, :material-play: running, :material-check: completed, :material-alert: failed, :material-skull: dead, :material-cancel: cancelled. +- **Pagination** -- All job and dead letter tables are paginated for large queues. +- **Job detail view** -- Click any job to see its full payload, error history, retry count, progress, and metadata. +- **Dead letter management** -- Inspect, retry, or purge dead letters directly from the UI. + +!!! info "Zero extra dependencies" + The SPA is embedded directly in the Python package. No Node.js, no npm, no CDN -- just `pip install quickq`. + +## REST API + +The dashboard exposes a JSON API you can use independently of the UI. All endpoints return `application/json`. + +### `GET /api/stats` + +Queue statistics snapshot. + +```json +{ + "pending": 12, + "running": 3, + "completed": 450, + "failed": 2, + "dead": 1, + "cancelled": 0 +} +``` + +### `GET /api/jobs` + +Paginated list of jobs. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `status` | `string` | all | Filter by status: `pending`, `running`, `completed`, `failed`, `dead`, `cancelled` | +| `limit` | `int` | `50` | Page size | +| `offset` | `int` | `0` | Pagination offset | + +```bash +curl http://localhost:8080/api/jobs?status=running&limit=10 +``` + +### `GET /api/jobs/{id}` + +Full detail for a single job, including error history and progress. + +```bash +curl http://localhost:8080/api/jobs/01H5K6X... +``` + +```json +{ + "id": "01H5K6X...", + "task_name": "myapp.tasks.process", + "status": "completed", + "queue": "default", + "priority": 0, + "progress": 100, + "result": "\"done\"", + "retry_count": 0, + "created_at": 1700000000000, + "started_at": 1700000001000, + "completed_at": 1700000005000, + "errors": [], + "metadata": null +} +``` + +### `GET /api/dead-letters` + +Paginated list of dead letter entries. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `limit` | `int` | `50` | Page size | +| `offset` | `int` | `0` | Pagination offset | + +```bash +curl http://localhost:8080/api/dead-letters?limit=5 +``` + +### `POST /api/dead-letters/{id}/retry` + +Re-enqueue a dead letter job. Returns the new job ID. + +```bash +curl -X POST http://localhost:8080/api/dead-letters/01H5K6X.../retry +``` + +```json +{ + "new_job_id": "01H5K7Y..." +} +``` + +### `DELETE /api/dead-letters/{id}` + +Delete a single dead letter entry. + +### `POST /api/jobs/{id}/cancel` + +Cancel a pending job. Returns `204 No Content` on success or `409 Conflict` if the job is not in a cancellable state. + +## Using the API Programmatically + +You can integrate the dashboard API into your own monitoring stack: + +```python +import requests + +# Health check script +stats = requests.get("http://localhost:8080/api/stats").json() + +if stats["dead"] > 0: + print(f"WARNING: {stats['dead']} dead letter(s)") + +if stats["running"] > 100: + print(f"WARNING: {stats['running']} jobs running, possible backlog") +``` + +!!! warning "Authentication" + The dashboard does not include authentication. If you expose it beyond `localhost`, place it behind a reverse proxy with authentication (e.g. nginx with basic auth, or an OAuth2 proxy). diff --git a/docs/guide/dependencies.md b/docs/guide/dependencies.md new file mode 100644 index 00000000..fd1d5aa9 --- /dev/null +++ b/docs/guide/dependencies.md @@ -0,0 +1,245 @@ +# Task Dependencies + +quickq supports declaring dependencies between jobs, allowing you to build DAG-style workflows where a job only runs after its upstream dependencies have completed successfully. + +## Basic Usage + +Pass `depends_on` when enqueuing a job to declare that it should wait for one or more other jobs to finish: + +=== "Single dependency" + + ```python + job_a = extract.delay(url) + + # job_b won't start until job_a completes successfully + job_b = transform.apply_async( + args=(job_a.id,), + depends_on=job_a.id, + ) + ``` + +=== "Multiple dependencies" + + ```python + job_a = fetch.delay("https://api1.example.com") + job_b = fetch.delay("https://api2.example.com") + + # job_c waits for both job_a and job_b + job_c = merge.apply_async( + args=(), + depends_on=[job_a.id, job_b.id], + ) + ``` + +The `depends_on` parameter accepts: + +| Value | Description | +|---|---| +| `str` | A single job ID | +| `list[str]` | Multiple job IDs (all must complete) | +| `None` (default) | No dependencies | + +!!! tip + You can also use `depends_on` with `queue.enqueue()` directly: + + ```python + job_id = queue.enqueue( + task_name="myapp.tasks.merge", + args=(), + depends_on=[job_a.id, job_b.id], + ) + ``` + +## How It Works + +1. When a job with `depends_on` is enqueued, it enters a **waiting** state +2. The scheduler periodically checks waiting jobs to see if all dependencies have completed +3. Once every dependency has `status=completed`, the job transitions to `pending` and becomes eligible for dispatch +4. If any dependency fails, dies, or is cancelled, the dependent job is **cascade cancelled** + +```mermaid +flowchart TD + E["Enqueue with depends_on"] --> W["Status: Waiting"] + W --> CHECK{"All deps completed?"} + CHECK -->|Yes| P["Status: Pending"] + CHECK -->|"Any dep failed/dead/cancelled"| CC["Cascade Cancel"] + P --> R["Dispatched to worker"] + CC --> DONE["Status: Cancelled
reason: dependency_failed"] +``` + +## Cascade Cancel + +When a dependency fails (exhausts retries and moves to DLQ), dies, or is cancelled, all downstream dependents are automatically cancelled. This propagates transitively through the entire dependency graph: + +```python +job_a = step_one.delay() +job_b = step_two.apply_async(args=(), depends_on=job_a.id) +job_c = step_three.apply_async(args=(), depends_on=job_b.id) + +# If job_a fails permanently: +# - job_b is cascade cancelled +# - job_c is cascade cancelled (transitive) +``` + +!!! warning "Cascade is immediate" + As soon as a dependency enters a terminal failure state (`dead` or `cancelled`), all downstream dependents are cancelled in the same scheduler tick. There is no grace period. + +## Inspecting Dependencies + +### `job.dependencies` + +Returns the list of job IDs this job depends on: + +```python +job_c = merge.apply_async( + args=(), + depends_on=[job_a.id, job_b.id], +) + +fetched = queue.get_job(job_c.id) +print(fetched.dependencies) # ['01H5K6X...', '01H5K7Y...'] +``` + +### `job.dependents` + +Returns the list of job IDs that depend on this job: + +```python +fetched_a = queue.get_job(job_a.id) +print(fetched_a.dependents) # ['01H5K8Z...'] (job_c's ID) +``` + +## Error Handling + +### Missing Dependencies + +If you reference a job ID that does not exist, enqueue raises a `ValueError`: + +```python +try: + job = transform.apply_async( + args=(), + depends_on="nonexistent-job-id", + ) +except ValueError as e: + print(e) # "Dependency job 'nonexistent-job-id' not found" +``` + +### Already-Dead Dependencies + +If a dependency is already in a terminal failure state (`dead` or `cancelled`) at enqueue time, the dependent job is immediately cancelled: + +```python +dead_job = queue.get_job(some_dead_id) +assert dead_job.status == "dead" + +# This job is immediately cancelled — it will never run +job = transform.apply_async( + args=(), + depends_on=dead_job.id, +) + +fetched = queue.get_job(job.id) +print(fetched.status) # "cancelled" +``` + +## DAG Workflow Examples + +### Diamond Pattern + +A classic diamond dependency graph where two branches converge: + +```mermaid +graph TD + A["extract.delay()"] --> B["transform_a.apply_async(depends_on=A)"] + A --> C["transform_b.apply_async(depends_on=A)"] + B --> D["load.apply_async(depends_on=[B, C])"] + C --> D +``` + +```python +# Extract +job_a = extract.delay(source_url) + +# Two parallel transforms, each depending on extract +job_b = transform_a.apply_async( + args=("schema_a",), + depends_on=job_a.id, +) +job_c = transform_b.apply_async( + args=("schema_b",), + depends_on=job_a.id, +) + +# Load waits for both transforms +job_d = load.apply_async( + args=(), + depends_on=[job_b.id, job_c.id], +) +``` + +### Multi-Stage Pipeline + +A sequential pipeline with fan-out at one stage: + +```python +# Stage 1: Download +download_jobs = [ + download.delay(url) for url in urls +] + +# Stage 2: Process each download (each depends on its own download) +process_jobs = [ + process.apply_async( + args=(url,), + depends_on=dl.id, + ) + for dl, url in zip(download_jobs, urls) +] + +# Stage 3: Aggregate all results +aggregate_job = aggregate.apply_async( + args=(), + depends_on=[j.id for j in process_jobs], +) +``` + +### Conditional Branching + +Combine dependencies with metadata to build conditional workflows: + +```python +job_a = validate.delay(data) + +# Both branches depend on validation +job_success = on_valid.apply_async( + args=(data,), + depends_on=job_a.id, + metadata='{"branch": "success"}', +) + +# Use a separate task to handle the "validation failed" path +# by inspecting job_a's result in the task body +``` + +!!! note "Dependencies vs. Workflows" + `depends_on` is a lower-level primitive than [chains, groups, and chords](workflows.md). Use `depends_on` when you need fine-grained control over a custom DAG. Use the workflow primitives when your pipeline fits a standard pattern. + +## Combining with Other Features + +Dependencies compose naturally with other quickq features: + +```python +job = transform.apply_async( + args=(data,), + depends_on=job_a.id, + priority=10, # High priority once unblocked + queue="processing", # Target queue + max_retries=5, # Retry policy + delay=60, # Additional delay after deps resolve + unique_key="transform-daily", # Deduplication +) +``` + +!!! info "Delay + depends_on" + When both `delay` and `depends_on` are set, the job first waits for all dependencies to complete, then waits for the additional delay before becoming eligible for dispatch. diff --git a/mkdocs.yml b/mkdocs.yml index 0d366a77..a6de7973 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -89,8 +89,10 @@ nav: - Retries & Dead Letters: guide/retries.md - Rate Limiting: guide/rate-limiting.md - Workflows: guide/workflows.md + - Dependencies: guide/dependencies.md - Scheduling: guide/scheduling.md - Monitoring & Hooks: guide/monitoring.md + - Web Dashboard: guide/dashboard.md - Advanced: guide/advanced.md - Architecture: architecture.md - API Reference: diff --git a/pyproject.toml b/pyproject.toml index 8fabf569..c6aa939f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = ["cloudpickle>=3.0"] [project.optional-dependencies] dev = ["pytest>=7.0", "pytest-asyncio>=0.21", "ruff>=0.8", "mypy>=1.13"] +fastapi = ["fastapi>=0.100.0", "pydantic>=2.0"] docs = ["zensical"] [tool.maturin] diff --git a/python/quickq/_quickq.pyi b/python/quickq/_quickq.pyi index e959a62f..a1387682 100644 --- a/python/quickq/_quickq.pyi +++ b/python/quickq/_quickq.pyi @@ -68,6 +68,7 @@ class PyQueue: timeout: int | None = None, unique_key: str | None = None, metadata: str | None = None, + depends_on: list[str] | None = None, ) -> PyJob: ... def enqueue_batch( self, @@ -78,9 +79,19 @@ class PyQueue: max_retries_list: list[int] | None = None, timeouts: list[int] | None = None, ) -> list[PyJob]: ... + def list_jobs( + self, + status: str | None = None, + queue: str | None = None, + task_name: str | None = None, + limit: int = 50, + offset: int = 0, + ) -> list[PyJob]: ... def get_job(self, job_id: str) -> PyJob | None: ... def get_job_errors(self, job_id: str) -> list[dict[str, Any]]: ... def cancel_job(self, job_id: str) -> bool: ... + def get_dependencies(self, job_id: str) -> list[str]: ... + def get_dependents(self, job_id: str) -> list[str]: ... def update_progress(self, job_id: str, progress: int) -> None: ... def purge_completed(self, older_than_seconds: int) -> int: ... def stats(self) -> dict[str, int]: ... diff --git a/python/quickq/app.py b/python/quickq/app.py index adcc7a90..6fad3354 100644 --- a/python/quickq/app.py +++ b/python/quickq/app.py @@ -254,10 +254,21 @@ def enqueue( timeout: int | None = None, unique_key: str | None = None, metadata: str | None = None, + depends_on: str | list[str] | None = None, ) -> JobResult: - """Enqueue a task for execution.""" + """Enqueue a task for execution. + + Args: + depends_on: Job ID or list of job IDs that must complete + before this job can run. If any dependency fails or is + cancelled, this job will be cascade-cancelled. + """ payload = cloudpickle.dumps((args, kwargs or {})) + dep_ids = None + if depends_on is not None: + dep_ids = [depends_on] if isinstance(depends_on, str) else list(depends_on) + py_job = self._inner.enqueue( task_name=task_name, payload=payload, @@ -268,6 +279,7 @@ def enqueue( timeout=timeout, unique_key=unique_key, metadata=metadata, + depends_on=dep_ids, ) return JobResult(py_job=py_job, queue=self) @@ -324,6 +336,36 @@ def get_job(self, job_id: str) -> JobResult | None: return None return JobResult(py_job=py_job, queue=self) + def list_jobs( + self, + status: str | None = None, + queue: str | None = None, + task_name: str | None = None, + limit: int = 50, + offset: int = 0, + ) -> list[JobResult]: + """List jobs with optional filters and pagination. + + Args: + status: Filter by status ("pending", "running", "complete", + "failed", "dead", "cancelled"). None returns all. + queue: Filter by queue name. None returns all queues. + task_name: Filter by task name. None returns all tasks. + limit: Maximum number of jobs to return. + offset: Number of jobs to skip (for pagination). + + Returns: + List of JobResult handles, ordered by creation time (newest first). + """ + py_jobs = self._inner.list_jobs( + status=status, + queue=queue, + task_name=task_name, + limit=limit, + offset=offset, + ) + return [JobResult(py_job=pj, queue=self) for pj in py_jobs] + # -- Sync inspection methods -- def stats(self) -> dict[str, int]: diff --git a/python/quickq/cli.py b/python/quickq/cli.py index 8f161408..54d1ed62 100644 --- a/python/quickq/cli.py +++ b/python/quickq/cli.py @@ -33,6 +33,20 @@ def main() -> None: help="Comma-separated list of queues to process (default: all registered)", ) + # dashboard subcommand + dash_parser = subparsers.add_parser("dashboard", help="Start the web dashboard") + dash_parser.add_argument( + "--app", + required=True, + help="Python path to the Queue instance (e.g., 'myapp.tasks:queue')", + ) + dash_parser.add_argument( + "--host", default="127.0.0.1", help="Bind address (default: 127.0.0.1)" + ) + dash_parser.add_argument( + "--port", type=int, default=8080, help="Bind port (default: 8080)" + ) + # info subcommand info_parser = subparsers.add_parser("info", help="Show queue statistics") info_parser.add_argument( @@ -51,6 +65,8 @@ def main() -> None: if args.command == "worker": run_worker(args) + elif args.command == "dashboard": + run_dashboard(args) elif args.command == "info": run_info(args) else: @@ -103,6 +119,14 @@ def run_worker(args: argparse.Namespace) -> None: queue.run_worker(queues=queues) +def run_dashboard(args: argparse.Namespace) -> None: + """Start the web dashboard.""" + queue = _load_queue(args.app) + from quickq.dashboard import serve_dashboard + + serve_dashboard(queue, host=args.host, port=args.port) + + def run_info(args: argparse.Namespace) -> None: """Print queue statistics.""" queue = _load_queue(args.app) diff --git a/python/quickq/contrib/__init__.py b/python/quickq/contrib/__init__.py new file mode 100644 index 00000000..33ea8527 --- /dev/null +++ b/python/quickq/contrib/__init__.py @@ -0,0 +1 @@ +"""quickq integrations with third-party frameworks.""" diff --git a/python/quickq/contrib/fastapi.py b/python/quickq/contrib/fastapi.py new file mode 100644 index 00000000..33311963 --- /dev/null +++ b/python/quickq/contrib/fastapi.py @@ -0,0 +1,277 @@ +"""FastAPI integration for quickq. + +Provides a plug-and-play APIRouter with job status, progress SSE streaming, +and dead letter queue management. + +Usage:: + + from fastapi import FastAPI + from quickq import Queue + from quickq.contrib.fastapi import QuickQRouter + + queue = Queue() + app = FastAPI() + app.include_router(QuickQRouter(queue), prefix="/tasks") + +Requires the ``fastapi`` optional dependency:: + + pip install quickq[fastapi] +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +from collections.abc import AsyncGenerator +from typing import TYPE_CHECKING, Any + +try: + from fastapi import APIRouter, HTTPException, Query + from fastapi.responses import StreamingResponse + from pydantic import BaseModel +except ImportError as e: + raise ImportError( + "FastAPI integration requires 'fastapi' and 'pydantic'. " + "Install with: pip install quickq[fastapi]" + ) from e + +if TYPE_CHECKING: + from quickq.app import Queue + + +# ── Pydantic response models ──────────────────────────── + + +class StatsResponse(BaseModel): + """Queue statistics.""" + + pending: int = 0 + running: int = 0 + completed: int = 0 + failed: int = 0 + dead: int = 0 + cancelled: int = 0 + + +class JobResponse(BaseModel): + """Job status and metadata.""" + + id: str + queue: str + task_name: str + status: str + priority: int + progress: int | None = None + retry_count: int + max_retries: int + created_at: int + scheduled_at: int + started_at: int | None = None + completed_at: int | None = None + error: str | None = None + timeout_ms: int + unique_key: str | None = None + metadata: str | None = None + + +class JobErrorResponse(BaseModel): + """A single error from a job attempt.""" + + id: str + job_id: str + attempt: int + error: str + failed_at: int + + +class CancelResponse(BaseModel): + """Response from cancelling a job.""" + + cancelled: bool + + +class RetryResponse(BaseModel): + """Response from retrying a dead letter.""" + + new_job_id: str + + +class DeadLetterResponse(BaseModel): + """A dead letter queue entry.""" + + id: str + original_job_id: str + queue: str + task_name: str + error: str | None = None + retry_count: int + failed_at: int + metadata: str | None = None + + +class JobResultResponse(BaseModel): + """Job result (serialized to string for safety).""" + + id: str + status: str + result: Any = None + error: str | None = None + + +# ── Router factory ─────────────────────────────────────── + + +class QuickQRouter(APIRouter): + """FastAPI APIRouter pre-configured with quickq endpoints. + + Args: + queue: The quickq Queue instance to expose. + **kwargs: Passed to ``APIRouter.__init__()`` (e.g. ``prefix``, + ``tags``, ``dependencies``). + + Example:: + + app.include_router( + QuickQRouter(queue, tags=["tasks"]), + prefix="/tasks", + ) + """ + + def __init__(self, queue: Queue, **kwargs: Any) -> None: + kwargs.setdefault("tags", ["quickq"]) + super().__init__(**kwargs) + self._queue = queue + self._register_routes() + + def _register_routes(self) -> None: + queue = self._queue + + @self.get("/stats", response_model=StatsResponse) + async def get_stats() -> StatsResponse: + """Get queue statistics.""" + stats = await queue.astats() + return StatsResponse(**stats) + + @self.get("/jobs/{job_id}", response_model=JobResponse) + async def get_job(job_id: str) -> JobResponse: + """Get a job by ID.""" + job = queue.get_job(job_id) + if job is None: + raise HTTPException(status_code=404, detail="Job not found") + return JobResponse(**job.to_dict()) + + @self.get("/jobs/{job_id}/errors", response_model=list[JobErrorResponse]) + async def get_job_errors(job_id: str) -> list[JobErrorResponse]: + """Get error history for a job.""" + errors = queue.job_errors(job_id) + return [JobErrorResponse(**e) for e in errors] + + @self.get("/jobs/{job_id}/result", response_model=JobResultResponse) + async def get_job_result( + job_id: str, + timeout: float = Query(default=0, ge=0, le=300), + ) -> JobResultResponse: + """Get job result. Set timeout > 0 for blocking wait.""" + job = queue.get_job(job_id) + if job is None: + raise HTTPException(status_code=404, detail="Job not found") + + if timeout > 0 and job.status not in ("complete", "failed", "dead", "cancelled"): + try: + result = await job.aresult(timeout=timeout) + return JobResultResponse( + id=job_id, + status="complete", + result=_safe_serialize(result), + ) + except TimeoutError: + return JobResultResponse( + id=job_id, + status=job.status, + ) + except RuntimeError as e: + return JobResultResponse( + id=job_id, + status=job.status, + error=str(e), + ) + + d = job.to_dict() + result = None + if d["status"] == "complete": + with contextlib.suppress(Exception): + result = _safe_serialize(job.result(timeout=1)) + + return JobResultResponse( + id=job_id, + status=d["status"], + result=result, + error=d.get("error"), + ) + + @self.get("/jobs/{job_id}/progress") + async def stream_progress(job_id: str) -> StreamingResponse: + """SSE stream of progress updates until job reaches terminal state.""" + job = queue.get_job(job_id) + if job is None: + raise HTTPException(status_code=404, detail="Job not found") + + async def event_stream() -> AsyncGenerator[str, None]: + terminal = {"complete", "failed", "dead", "cancelled"} + while True: + refreshed = queue.get_job(job_id) + if refreshed is None: + yield f"data: {json.dumps({'status': 'not_found'})}\n\n" + return + + d = refreshed.to_dict() + payload = json.dumps( + {"status": d["status"], "progress": d["progress"]} + ) + yield f"data: {payload}\n\n" + + if d["status"] in terminal: + return + + await asyncio.sleep(0.5) + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + @self.post("/jobs/{job_id}/cancel", response_model=CancelResponse) + async def cancel_job(job_id: str) -> CancelResponse: + """Cancel a pending job.""" + ok = await queue.acancel_job(job_id) + return CancelResponse(cancelled=ok) + + @self.get("/dead-letters", response_model=list[DeadLetterResponse]) + async def list_dead_letters( + limit: int = Query(default=20, ge=1, le=100), + offset: int = Query(default=0, ge=0), + ) -> list[DeadLetterResponse]: + """List dead letter queue entries.""" + dead = await queue.adead_letters(limit=limit, offset=offset) + return [DeadLetterResponse(**d) for d in dead] + + @self.post("/dead-letters/{dead_id}/retry", response_model=RetryResponse) + async def retry_dead_letter(dead_id: str) -> RetryResponse: + """Re-enqueue a dead letter job.""" + new_id = await queue.aretry_dead(dead_id) + return RetryResponse(new_job_id=new_id) + + +def _safe_serialize(value: Any) -> Any: + """Convert a value to something JSON-serializable.""" + if value is None: + return None + if isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, (list, tuple)): + return [_safe_serialize(v) for v in value] + if isinstance(value, dict): + return {str(k): _safe_serialize(v) for k, v in value.items()} + return str(value) diff --git a/python/quickq/dashboard.py b/python/quickq/dashboard.py new file mode 100644 index 00000000..059d20d4 --- /dev/null +++ b/python/quickq/dashboard.py @@ -0,0 +1,795 @@ +"""Built-in web dashboard for quickq — zero extra dependencies. + +Usage:: + + quickq dashboard --app myapp:queue + # → http://127.0.0.1:8080 + +Or programmatically:: + + from quickq.dashboard import serve_dashboard + serve_dashboard(queue, host="0.0.0.0", port=8080) +""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import TYPE_CHECKING, Any +from urllib.parse import parse_qs, urlparse + +if TYPE_CHECKING: + from quickq.app import Queue + + +def serve_dashboard( + queue: Queue, + host: str = "127.0.0.1", + port: int = 8080, +) -> None: + """Start the dashboard HTTP server (blocking). + + Args: + queue: The Queue instance to monitor. + host: Bind address. + port: Bind port. + """ + + handler = _make_handler(queue) + server = ThreadingHTTPServer((host, port), handler) + print(f"quickq dashboard → http://{host}:{port}") + print("Press Ctrl+C to stop") + + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + + +def _make_handler(queue: Queue) -> type: + """Create a request handler class bound to the given queue.""" + + class DashboardHandler(BaseHTTPRequestHandler): + + def do_GET(self) -> None: + parsed = urlparse(self.path) + path = parsed.path + qs = parse_qs(parsed.query) + + if path == "/api/stats": + self._json_response(queue.stats()) + elif path == "/api/jobs": + self._handle_list_jobs(qs) + elif path.startswith("/api/jobs/") and path.endswith("/errors"): + job_id = path[len("/api/jobs/"):-len("/errors")] + self._json_response(queue.job_errors(job_id)) + elif path.startswith("/api/jobs/"): + job_id = path[len("/api/jobs/"):] + job = queue.get_job(job_id) + if job is None: + self._json_response({"error": "Job not found"}, status=404) + else: + self._json_response(job.to_dict()) + elif path == "/api/dead-letters": + limit = int(qs.get("limit", ["20"])[0]) + offset = int(qs.get("offset", ["0"])[0]) + self._json_response(queue.dead_letters(limit=limit, offset=offset)) + else: + self._serve_spa() + + def do_POST(self) -> None: + parsed = urlparse(self.path) + path = parsed.path + + if path.startswith("/api/jobs/") and path.endswith("/cancel"): + job_id = path[len("/api/jobs/"):-len("/cancel")] + ok = queue.cancel_job(job_id) + self._json_response({"cancelled": ok}) + elif path.startswith("/api/dead-letters/") and path.endswith("/retry"): + dead_id = path[len("/api/dead-letters/"):-len("/retry")] + new_id = queue.retry_dead(dead_id) + self._json_response({"new_job_id": new_id}) + elif path == "/api/dead-letters/purge": + count = queue.purge_dead(0) + self._json_response({"purged": count}) + else: + self._json_response({"error": "Not found"}, status=404) + + def _handle_list_jobs(self, qs: dict) -> None: + status = qs.get("status", [None])[0] + q = qs.get("queue", [None])[0] + task = qs.get("task", [None])[0] + limit = int(qs.get("limit", ["20"])[0]) + offset = int(qs.get("offset", ["0"])[0]) + + jobs = queue.list_jobs( + status=status, queue=q, task_name=task, + limit=limit, offset=offset, + ) + self._json_response([j.to_dict() for j in jobs]) + + def _json_response(self, data: Any, status: int = 200) -> None: + body = json.dumps(data, default=str).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + self.wfile.write(body) + + def _serve_spa(self) -> None: + body = _SPA_HTML.encode() + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: Any) -> None: + # Suppress default access log noise + pass + + return DashboardHandler + + +# --------------------------------------------------------------------------- +# Embedded SPA — HTML + CSS + JS in a single string +# --------------------------------------------------------------------------- + +_SPA_HTML = """\ + + + + + +quickq dashboard + + + + +
+

quickq dashboard

+ +
+ + +
+
+ +
+ + + + +""" diff --git a/python/quickq/result.py b/python/quickq/result.py index 180952e7..d8646482 100644 --- a/python/quickq/result.py +++ b/python/quickq/result.py @@ -66,6 +66,16 @@ def errors(self) -> list[dict]: """Error history for this job (one entry per failed attempt).""" return self._queue.job_errors(self.id) + @property + def dependencies(self) -> list[str]: + """IDs of jobs this job depends on.""" + return self._queue._inner.get_dependencies(self.id) + + @property + def dependents(self) -> list[str]: + """IDs of jobs that depend on this job.""" + return self._queue._inner.get_dependents(self.id) + def result( self, timeout: float = 30.0, @@ -159,6 +169,35 @@ async def aresult( f"(current status: {self._py_job.status})" ) + def to_dict(self) -> dict[str, Any]: + """Convert to a plain dictionary for JSON serialization. + + Refreshes the job status from the database before building the dict. + Does not include ``result_bytes`` (use :meth:`result` for that). + """ + refreshed = self._queue._inner.get_job(self._py_job.id) + if refreshed is not None: + self._py_job = refreshed + + return { + "id": self._py_job.id, + "queue": self._py_job.queue, + "task_name": self._py_job.task_name, + "status": self._py_job.status, + "priority": self._py_job.priority, + "progress": self._py_job.progress, + "retry_count": self._py_job.retry_count, + "max_retries": self._py_job.max_retries, + "created_at": self._py_job.created_at, + "scheduled_at": self._py_job.scheduled_at, + "started_at": self._py_job.started_at, + "completed_at": self._py_job.completed_at, + "error": self._py_job.error, + "timeout_ms": self._py_job.timeout_ms, + "unique_key": self._py_job.unique_key, + "metadata": self._py_job.metadata, + } + def __repr__(self) -> str: """Return a developer-friendly string representation.""" return f"" diff --git a/python/quickq/task.py b/python/quickq/task.py index 1aece254..af038b14 100644 --- a/python/quickq/task.py +++ b/python/quickq/task.py @@ -81,6 +81,7 @@ def apply_async( timeout: int | None = None, unique_key: str | None = None, metadata: str | None = None, + depends_on: str | list[str] | None = None, ) -> JobResult: """ Enqueue with full control over submission options. @@ -95,6 +96,7 @@ def apply_async( timeout: Override the default timeout in seconds. unique_key: Deduplicate active jobs with the same key. metadata: Arbitrary JSON metadata to attach to the job. + depends_on: Job ID or list of job IDs that must complete first. """ return self._queue.enqueue( task_name=self._task_name, @@ -107,6 +109,7 @@ def apply_async( timeout=timeout if timeout is not None else self._default_timeout, unique_key=unique_key, metadata=metadata, + depends_on=depends_on, ) def map(self, iterable: list[tuple]) -> list[JobResult]: diff --git a/rust/quickq-core/src/error.rs b/rust/quickq-core/src/error.rs index f101c7ce..0a019024 100644 --- a/rust/quickq-core/src/error.rs +++ b/rust/quickq-core/src/error.rs @@ -32,6 +32,9 @@ pub enum QueueError { #[error("config error: {0}")] Config(String), + #[error("dependency not found: {0}")] + DependencyNotFound(String), + #[error("{0}")] Other(String), } diff --git a/rust/quickq-core/src/job.rs b/rust/quickq-core/src/job.rs index 86142cf3..d5acd5e9 100644 --- a/rust/quickq-core/src/job.rs +++ b/rust/quickq-core/src/job.rs @@ -99,6 +99,7 @@ pub struct NewJob { pub timeout_ms: i64, pub unique_key: Option, pub metadata: Option, + pub depends_on: Vec, } impl NewJob { diff --git a/rust/quickq-core/src/scheduler.rs b/rust/quickq-core/src/scheduler.rs index 9e043bc0..dcd99063 100644 --- a/rust/quickq-core/src/scheduler.rs +++ b/rust/quickq-core/src/scheduler.rs @@ -262,6 +262,7 @@ impl Scheduler { timeout_ms: 300_000, unique_key: None, metadata: None, + depends_on: vec![], }; if let Err(e) = self.storage.enqueue(new_job) { diff --git a/rust/quickq-core/src/storage/models.rs b/rust/quickq-core/src/storage/models.rs index c2e827f5..2239d7cc 100644 --- a/rust/quickq-core/src/storage/models.rs +++ b/rust/quickq-core/src/storage/models.rs @@ -1,6 +1,6 @@ use diesel::prelude::*; -use super::schema::{dead_letter, job_errors, jobs, periodic_tasks, rate_limits}; +use super::schema::{dead_letter, job_dependencies, job_errors, jobs, periodic_tasks, rate_limits}; /// A row in the `jobs` table (for SELECT queries). #[derive(Queryable, Selectable, Debug, Clone)] @@ -136,3 +136,21 @@ pub struct NewJobErrorRow<'a> { pub error: &'a str, pub failed_at: i64, } + +/// A row in the `job_dependencies` table. +#[derive(Queryable, Selectable, Debug, Clone)] +#[diesel(table_name = job_dependencies)] +pub struct JobDependencyRow { + pub id: String, + pub job_id: String, + pub depends_on_job_id: String, +} + +/// Insertable struct for job dependency entries. +#[derive(Insertable, Debug)] +#[diesel(table_name = job_dependencies)] +pub struct NewJobDependencyRow<'a> { + pub id: &'a str, + pub job_id: &'a str, + pub depends_on_job_id: &'a str, +} diff --git a/rust/quickq-core/src/storage/schema.rs b/rust/quickq-core/src/storage/schema.rs index d55598d7..3c50149b 100644 --- a/rust/quickq-core/src/storage/schema.rs +++ b/rust/quickq-core/src/storage/schema.rs @@ -68,3 +68,13 @@ diesel::table! { failed_at -> BigInt, } } + +diesel::table! { + job_dependencies (id) { + id -> Text, + job_id -> Text, + depends_on_job_id -> Text, + } +} + +diesel::allow_tables_to_appear_in_same_query!(jobs, job_dependencies); diff --git a/rust/quickq-core/src/storage/sqlite.rs b/rust/quickq-core/src/storage/sqlite.rs index 68ec40c2..ce2d5235 100644 --- a/rust/quickq-core/src/storage/sqlite.rs +++ b/rust/quickq-core/src/storage/sqlite.rs @@ -5,7 +5,7 @@ use diesel::sqlite::SqliteConnection; use crate::error::{QueueError, Result}; use crate::job::{Job, JobStatus, NewJob, now_millis}; use super::models::*; -use super::schema::{dead_letter, job_errors, jobs, periodic_tasks, rate_limits}; +use super::schema::{dead_letter, job_dependencies, job_errors, jobs, periodic_tasks, rate_limits}; type DbPool = Pool>; @@ -147,6 +147,22 @@ impl SqliteStorage { "CREATE INDEX IF NOT EXISTS idx_job_errors_job_id ON job_errors(job_id)" ).execute(&mut conn)?; + diesel::sql_query( + "CREATE TABLE IF NOT EXISTS job_dependencies ( + id TEXT PRIMARY KEY, + job_id TEXT NOT NULL, + depends_on_job_id TEXT NOT NULL + )" + ).execute(&mut conn)?; + + diesel::sql_query( + "CREATE INDEX IF NOT EXISTS idx_job_deps_job_id ON job_dependencies(job_id)" + ).execute(&mut conn)?; + + diesel::sql_query( + "CREATE INDEX IF NOT EXISTS idx_job_deps_depends_on ON job_dependencies(depends_on_job_id)" + ).execute(&mut conn)?; + Ok(()) } @@ -154,28 +170,69 @@ impl SqliteStorage { /// Insert a new job into the queue. Returns the job. pub fn enqueue(&self, new_job: NewJob) -> Result { + let depends_on = new_job.depends_on.clone(); let job = new_job.into_job(); let mut conn = self.conn()?; - let row = NewJobRow { - id: &job.id, - queue: &job.queue, - task_name: &job.task_name, - payload: &job.payload, - status: job.status as i32, - priority: job.priority, - created_at: job.created_at, - scheduled_at: job.scheduled_at, - retry_count: job.retry_count, - max_retries: job.max_retries, - timeout_ms: job.timeout_ms, - unique_key: job.unique_key.as_deref(), - metadata: job.metadata.as_deref(), - }; + conn.transaction(|conn| { + // Validate dependencies exist and aren't dead/cancelled + for dep_id in &depends_on { + let dep: Option = jobs::table + .find(dep_id) + .select(JobRow::as_select()) + .first(conn) + .optional()?; + + match dep { + None => return Err(diesel::result::Error::RollbackTransaction), + Some(d) if d.status == JobStatus::Dead as i32 + || d.status == JobStatus::Cancelled as i32 => + { + return Err(diesel::result::Error::RollbackTransaction); + } + _ => {} + } + } - diesel::insert_into(jobs::table) - .values(&row) - .execute(&mut conn)?; + let row = NewJobRow { + id: &job.id, + queue: &job.queue, + task_name: &job.task_name, + payload: &job.payload, + status: job.status as i32, + priority: job.priority, + created_at: job.created_at, + scheduled_at: job.scheduled_at, + retry_count: job.retry_count, + max_retries: job.max_retries, + timeout_ms: job.timeout_ms, + unique_key: job.unique_key.as_deref(), + metadata: job.metadata.as_deref(), + }; + + diesel::insert_into(jobs::table) + .values(&row) + .execute(conn)?; + + // Insert dependency rows + for dep_id in &depends_on { + let dep_row = NewJobDependencyRow { + id: &uuid::Uuid::now_v7().to_string(), + job_id: &job.id, + depends_on_job_id: dep_id, + }; + diesel::insert_into(job_dependencies::table) + .values(&dep_row) + .execute(conn)?; + } + + Ok(()) + }).map_err(|e| match e { + diesel::result::Error::RollbackTransaction => { + QueueError::DependencyNotFound("dependency not found or already dead/cancelled".to_string()) + } + other => QueueError::Storage(other), + })?; Ok(job) } @@ -213,6 +270,7 @@ impl SqliteStorage { /// Enqueue with unique_key deduplication. Returns existing job if duplicate. pub fn enqueue_unique(&self, new_job: NewJob) -> Result { + let depends_on = new_job.depends_on.clone(); let job = new_job.into_job(); let mut conn = self.conn()?; @@ -253,46 +311,82 @@ impl SqliteStorage { .values(&row) .execute(&mut conn)?; + // Insert dependency rows + for dep_id in &depends_on { + let dep_row = NewJobDependencyRow { + id: &uuid::Uuid::now_v7().to_string(), + job_id: &job.id, + depends_on_job_id: dep_id, + }; + diesel::insert_into(job_dependencies::table) + .values(&dep_row) + .execute(&mut conn)?; + } + Ok(job) } + /// Check if all dependencies for a job are complete. + fn deps_satisfied(conn: &mut SqliteConnection, job_id: &str) -> diesel::result::QueryResult { + let dep_job_ids: Vec = job_dependencies::table + .filter(job_dependencies::job_id.eq(job_id)) + .select(job_dependencies::depends_on_job_id) + .load(conn)?; + + if dep_job_ids.is_empty() { + return Ok(true); + } + + let incomplete: i64 = jobs::table + .filter(jobs::id.eq_any(&dep_job_ids)) + .filter(jobs::status.ne(JobStatus::Complete as i32)) + .count() + .get_result(conn)?; + + Ok(incomplete == 0) + } + /// Atomically dequeue the highest-priority ready job from the given queue. pub fn dequeue(&self, queue_name: &str, now: i64) -> Result> { let mut conn = self.conn()?; conn.transaction(|conn| { - // Find the next ready job - let candidate: Option = jobs::table + // Find candidate jobs (check a few in case some have unmet deps) + let candidates: Vec = jobs::table .filter(jobs::queue.eq(queue_name)) .filter(jobs::status.eq(JobStatus::Pending as i32)) .filter(jobs::scheduled_at.le(now)) .order((jobs::priority.desc(), jobs::scheduled_at.asc())) + .limit(10) .select(JobRow::as_select()) - .first(conn) - .optional()?; + .load(conn)?; - let row = match candidate { - Some(r) => r, - None => return Ok(None), - }; + for row in candidates { + // Check if all dependencies are satisfied + if !Self::deps_satisfied(conn, &row.id)? { + continue; + } - // Atomically claim it - diesel::update(jobs::table) - .filter(jobs::id.eq(&row.id)) - .filter(jobs::status.eq(JobStatus::Pending as i32)) - .set(( - jobs::status.eq(JobStatus::Running as i32), - jobs::started_at.eq(now), - )) - .execute(conn)?; + // Atomically claim it + diesel::update(jobs::table) + .filter(jobs::id.eq(&row.id)) + .filter(jobs::status.eq(JobStatus::Pending as i32)) + .set(( + jobs::status.eq(JobStatus::Running as i32), + jobs::started_at.eq(now), + )) + .execute(conn)?; - // Re-read the updated row - let updated: JobRow = jobs::table - .find(&row.id) - .select(JobRow::as_select()) - .first(conn)?; + // Re-read the updated row + let updated: JobRow = jobs::table + .find(&row.id) + .select(JobRow::as_select()) + .first(conn)?; - Ok(Some(Job::from(updated))) + return Ok(Some(Job::from(updated))); + } + + Ok(None) }) } @@ -370,7 +464,8 @@ impl SqliteStorage { Ok(()) } - /// Cancel a pending job. Returns true if cancelled, false if not pending. + /// Cancel a pending job and cascade-cancel its dependents. + /// Returns true if cancelled, false if not pending. pub fn cancel_job(&self, id: &str) -> Result { let now = now_millis(); let mut conn = self.conn()?; @@ -384,9 +479,80 @@ impl SqliteStorage { )) .execute(&mut conn)?; + drop(conn); + if affected > 0 { + self.cascade_cancel(id, "dependency cancelled")?; + } + Ok(affected > 0) } + /// Cascade-cancel all pending jobs that depend (directly or transitively) + /// on the given job. Uses BFS to handle deep chains. + pub fn cascade_cancel(&self, failed_job_id: &str, reason: &str) -> Result<()> { + let mut conn = self.conn()?; + let now = now_millis(); + + let mut queue: Vec = vec![failed_job_id.to_string()]; + let mut idx = 0; + + while idx < queue.len() { + let current_id = queue[idx].clone(); + idx += 1; + + let dependents: Vec = job_dependencies::table + .filter(job_dependencies::depends_on_job_id.eq(¤t_id)) + .select(job_dependencies::job_id) + .load(&mut conn)?; + + for dep_id in dependents { + if !queue.contains(&dep_id) { + queue.push(dep_id); + } + } + } + + // Remove the original job from the list (only cancel dependents) + if !queue.is_empty() { + queue.remove(0); + } + + if !queue.is_empty() { + let error_msg = format!("{reason}: {failed_job_id}"); + diesel::update(jobs::table) + .filter(jobs::id.eq_any(&queue)) + .filter(jobs::status.eq(JobStatus::Pending as i32)) + .set(( + jobs::status.eq(JobStatus::Cancelled as i32), + jobs::completed_at.eq(now), + jobs::error.eq(&error_msg), + )) + .execute(&mut conn)?; + } + + Ok(()) + } + + /// Get the IDs of jobs that a given job depends on. + pub fn get_dependencies(&self, job_id: &str) -> Result> { + let mut conn = self.conn()?; + let ids: Vec = job_dependencies::table + .filter(job_dependencies::job_id.eq(job_id)) + .select(job_dependencies::depends_on_job_id) + .load(&mut conn)?; + Ok(ids) + } + + /// Get the IDs of jobs that depend on a given job. + pub fn get_dependents(&self, job_id: &str) -> Result> { + let mut conn = self.conn()?; + let ids: Vec = job_dependencies::table + .filter(job_dependencies::depends_on_job_id.eq(job_id)) + .select(job_dependencies::job_id) + .load(&mut conn)?; + Ok(ids) + } + /// Update progress for a running job (0-100). pub fn update_progress(&self, id: &str, progress: i32) -> Result<()> { let mut conn = self.conn()?; @@ -402,11 +568,12 @@ impl SqliteStorage { Ok(()) } - /// Move a job to the dead letter queue. + /// Move a job to the dead letter queue and cascade-cancel dependents. pub fn move_to_dlq(&self, job: &Job, error: &str, metadata: Option<&str>) -> Result<()> { let now = now_millis(); let dlq_id = uuid::Uuid::now_v7().to_string(); let mut conn = self.conn()?; + let job_id = job.id.clone(); conn.transaction(|conn| { let dlq_row = NewDeadLetterRow { @@ -434,8 +601,50 @@ impl SqliteStorage { )) .execute(conn)?; - Ok(()) - }) + Ok::<(), diesel::result::Error>(()) + })?; + + // Drop connection before cascade (needed for single-connection pools) + drop(conn); + + // Cascade cancel dependents + self.cascade_cancel(&job_id, "dependency failed")?; + + Ok(()) + } + + /// List jobs with optional filters and pagination. + pub fn list_jobs( + &self, + status: Option, + queue_name: Option<&str>, + task_name: Option<&str>, + limit: i64, + offset: i64, + ) -> Result> { + let mut conn = self.conn()?; + + let mut query = jobs::table + .into_boxed() + .order(jobs::created_at.desc()); + + if let Some(s) = status { + query = query.filter(jobs::status.eq(s)); + } + if let Some(q) = queue_name { + query = query.filter(jobs::queue.eq(q)); + } + if let Some(t) = task_name { + query = query.filter(jobs::task_name.eq(t)); + } + + let rows: Vec = query + .limit(limit) + .offset(offset) + .select(JobRow::as_select()) + .load(&mut conn)?; + + Ok(rows.into_iter().map(Job::from).collect()) } /// Get a job by ID. @@ -515,6 +724,7 @@ impl SqliteStorage { timeout_ms: 300_000, unique_key: None, metadata: None, + depends_on: vec![], }; let job = self.enqueue(new_job)?; @@ -757,6 +967,7 @@ mod tests { timeout_ms: 300_000, unique_key: None, metadata: None, + depends_on: vec![], } } @@ -1004,4 +1215,102 @@ mod tests { let fetched = storage.get_job(&job.id).unwrap().unwrap(); assert_eq!(fetched.progress, Some(100)); } + + // ── Dependency tests ──────────────────────────────────── + + #[test] + fn test_enqueue_with_dependency() { + let storage = test_storage(); + let job_a = storage.enqueue(make_job("task_a")).unwrap(); + + let mut dep_job = make_job("task_b"); + dep_job.depends_on = vec![job_a.id.clone()]; + let job_b = storage.enqueue(dep_job).unwrap(); + + let deps = storage.get_dependencies(&job_b.id).unwrap(); + assert_eq!(deps, vec![job_a.id.clone()]); + + let dependents = storage.get_dependents(&job_a.id).unwrap(); + assert_eq!(dependents, vec![job_b.id]); + } + + #[test] + fn test_dequeue_blocks_on_unmet_dependency() { + let storage = test_storage(); + let job_a = storage.enqueue(make_job("dep_task")).unwrap(); + + let mut dep_job = make_job("dependent_task"); + dep_job.depends_on = vec![job_a.id.clone()]; + storage.enqueue(dep_job).unwrap(); + + let now = now_millis() + 1000; + + // Dequeue should return job_a first (job_b has unmet dep) + let dequeued = storage.dequeue("default", now).unwrap().unwrap(); + assert_eq!(dequeued.id, job_a.id); + + // job_b should NOT be dequeueable yet (dep not complete) + let none = storage.dequeue("default", now).unwrap(); + assert!(none.is_none()); + + // Complete job_a + storage.complete(&job_a.id, None).unwrap(); + + // Now job_b should be dequeueable + let dequeued = storage.dequeue("default", now).unwrap().unwrap(); + assert_eq!(dequeued.task_name, "dependent_task"); + } + + #[test] + fn test_cascade_cancel_on_job_cancel() { + let storage = test_storage(); + let job_a = storage.enqueue(make_job("root")).unwrap(); + + let mut dep_b = make_job("child"); + dep_b.depends_on = vec![job_a.id.clone()]; + let job_b = storage.enqueue(dep_b).unwrap(); + + let mut dep_c = make_job("grandchild"); + dep_c.depends_on = vec![job_b.id.clone()]; + let job_c = storage.enqueue(dep_c).unwrap(); + + // Cancel root — should cascade to child and grandchild + storage.cancel_job(&job_a.id).unwrap(); + + let b = storage.get_job(&job_b.id).unwrap().unwrap(); + assert_eq!(b.status, JobStatus::Cancelled); + + let c = storage.get_job(&job_c.id).unwrap().unwrap(); + assert_eq!(c.status, JobStatus::Cancelled); + } + + #[test] + fn test_cascade_cancel_on_dlq() { + let storage = test_storage(); + let job_a = storage.enqueue(make_job("parent")).unwrap(); + + let mut dep_b = make_job("child_of_dead"); + dep_b.depends_on = vec![job_a.id.clone()]; + let job_b = storage.enqueue(dep_b).unwrap(); + + // Dequeue and fail job_a to DLQ + let now = now_millis() + 1000; + storage.dequeue("default", now).unwrap(); + let running = storage.get_job(&job_a.id).unwrap().unwrap(); + storage.move_to_dlq(&running, "fatal error", None).unwrap(); + + let b = storage.get_job(&job_b.id).unwrap().unwrap(); + assert_eq!(b.status, JobStatus::Cancelled); + assert!(b.error.unwrap().contains("dependency failed")); + } + + #[test] + fn test_enqueue_rejects_missing_dependency() { + let storage = test_storage(); + + let mut dep_job = make_job("orphan"); + dep_job.depends_on = vec!["nonexistent-id".to_string()]; + let result = storage.enqueue(dep_job); + assert!(result.is_err()); + } } diff --git a/rust/quickq-python/src/py_queue.rs b/rust/quickq-python/src/py_queue.rs index c3b1dec7..451f4c02 100644 --- a/rust/quickq-python/src/py_queue.rs +++ b/rust/quickq-python/src/py_queue.rs @@ -71,7 +71,7 @@ impl PyQueue { } /// Enqueue a job. - #[pyo3(signature = (task_name, payload, queue="default", priority=None, delay_seconds=None, max_retries=None, timeout=None, unique_key=None, metadata=None))] + #[pyo3(signature = (task_name, payload, queue="default", priority=None, delay_seconds=None, max_retries=None, timeout=None, unique_key=None, metadata=None, depends_on=None))] pub fn enqueue( &self, task_name: &str, @@ -83,6 +83,7 @@ impl PyQueue { timeout: Option, unique_key: Option, metadata: Option, + depends_on: Option>, ) -> PyResult { let now = now_millis(); let scheduled_at = match delay_seconds { @@ -100,6 +101,7 @@ impl PyQueue { timeout_ms: timeout.unwrap_or(self.default_timeout) * 1000, unique_key: unique_key.clone(), metadata, + depends_on: depends_on.unwrap_or_default(), }; let job = if unique_key.is_some() { @@ -152,6 +154,7 @@ impl PyQueue { }), unique_key: None, metadata: None, + depends_on: vec![], }); } @@ -163,6 +166,41 @@ impl PyQueue { Ok(jobs.into_iter().map(PyJob::from).collect()) } + /// List jobs with optional filters and pagination. + #[pyo3(signature = (status=None, queue=None, task_name=None, limit=50, offset=0))] + pub fn list_jobs( + &self, + status: Option<&str>, + queue: Option<&str>, + task_name: Option<&str>, + limit: i64, + offset: i64, + ) -> PyResult> { + let status_int = match status { + Some(s) => Some(match s { + "pending" => 0, + "running" => 1, + "complete" | "completed" => 2, + "failed" => 3, + "dead" => 4, + "cancelled" => 5, + _ => { + return Err(pyo3::exceptions::PyValueError::new_err(format!( + "Invalid status: {s}. Use: pending, running, complete, failed, dead, cancelled" + ))) + } + }), + None => None, + }; + + let jobs = self + .storage + .list_jobs(status_int, queue, task_name, limit, offset) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + + Ok(jobs.into_iter().map(PyJob::from).collect()) + } + /// Get a job by ID. pub fn get_job(&self, job_id: &str) -> PyResult> { let job = self @@ -179,6 +217,20 @@ impl PyQueue { .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) } + /// Get the IDs of jobs that a given job depends on. + pub fn get_dependencies(&self, job_id: &str) -> PyResult> { + self.storage + .get_dependencies(job_id) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + + /// Get the IDs of jobs that depend on a given job. + pub fn get_dependents(&self, job_id: &str) -> PyResult> { + self.storage + .get_dependents(job_id) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + /// Update progress for a running job (0-100). pub fn update_progress(&self, job_id: &str, progress: i32) -> PyResult<()> { self.storage diff --git a/tests/python/test_dashboard.py b/tests/python/test_dashboard.py new file mode 100644 index 00000000..1511e1d1 --- /dev/null +++ b/tests/python/test_dashboard.py @@ -0,0 +1,265 @@ +"""Tests for dashboard API endpoints and list_jobs/to_dict.""" + +import json +import threading +import time +import urllib.request +import urllib.error + +import pytest + +from quickq import Queue + + +@pytest.fixture +def queue(tmp_path): + """Create a fresh queue with some test data.""" + db_path = str(tmp_path / "test_dashboard.db") + q = Queue(db_path=db_path, workers=2) + + @q.task(queue="default") + def task_a(x): + return x * 2 + + @q.task(queue="email") + def task_b(x): + return x + 1 + + return q + + +@pytest.fixture +def populated_queue(queue): + """Queue with several jobs enqueued.""" + task_a = None + task_b = None + for name, fn in queue._task_registry.items(): + if "task_a" in name: + task_a = name + elif "task_b" in name: + task_b = name + + jobs = [] + for i in range(5): + jobs.append(queue.enqueue(task_a, args=(i,))) + for i in range(3): + jobs.append(queue.enqueue(task_b, args=(i,), queue="email")) + return queue, jobs + + +# ── list_jobs tests ────────────────────────────────────── + + +def test_list_jobs_returns_all(populated_queue): + """list_jobs() with no filters returns all jobs.""" + queue, jobs = populated_queue + result = queue.list_jobs() + assert len(result) == 8 + + +def test_list_jobs_filter_by_queue(populated_queue): + """list_jobs() can filter by queue name.""" + queue, jobs = populated_queue + result = queue.list_jobs(queue="email") + assert len(result) == 3 + for j in result: + d = j.to_dict() + assert d["queue"] == "email" + + +def test_list_jobs_filter_by_status(populated_queue): + """list_jobs() can filter by status.""" + queue, jobs = populated_queue + result = queue.list_jobs(status="pending") + assert len(result) == 8 # all are pending + + result = queue.list_jobs(status="running") + assert len(result) == 0 + + +def test_list_jobs_filter_by_task_name(populated_queue): + """list_jobs() can filter by task name.""" + queue, jobs = populated_queue + # Find the task_a name + task_a_name = None + for name in queue._task_registry: + if "task_a" in name: + task_a_name = name + break + + result = queue.list_jobs(task_name=task_a_name) + assert len(result) == 5 + + +def test_list_jobs_pagination(populated_queue): + """list_jobs() respects limit and offset.""" + queue, jobs = populated_queue + page1 = queue.list_jobs(limit=3, offset=0) + page2 = queue.list_jobs(limit=3, offset=3) + + assert len(page1) == 3 + assert len(page2) == 3 + + ids1 = {j.id for j in page1} + ids2 = {j.id for j in page2} + assert ids1.isdisjoint(ids2) + + +def test_list_jobs_invalid_status(queue): + """list_jobs() raises on invalid status string.""" + with pytest.raises(ValueError): + queue.list_jobs(status="bogus") + + +# ── to_dict tests ──────────────────────────────────────── + + +def test_to_dict_fields(queue): + """to_dict() returns all expected fields.""" + @queue.task() + def dummy(): + pass + + job = dummy.delay() + d = job.to_dict() + + expected_keys = { + "id", "queue", "task_name", "status", "priority", "progress", + "retry_count", "max_retries", "created_at", "scheduled_at", + "started_at", "completed_at", "error", "timeout_ms", + "unique_key", "metadata", + } + assert set(d.keys()) == expected_keys + assert d["status"] == "pending" + assert d["id"] == job.id + + +def test_to_dict_is_json_serializable(queue): + """to_dict() output can be serialized to JSON.""" + @queue.task() + def dummy(): + pass + + job = dummy.delay() + d = job.to_dict() + serialized = json.dumps(d) + assert isinstance(serialized, str) + + +# ── Dashboard HTTP tests ───────────────────────────────── + + +@pytest.fixture +def dashboard_server(populated_queue): + """Start a dashboard server on a random port.""" + queue, jobs = populated_queue + from quickq.dashboard import serve_dashboard + from http.server import ThreadingHTTPServer + from quickq.dashboard import _make_handler + + handler = _make_handler(queue) + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + port = server.server_address[1] + + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + yield f"http://127.0.0.1:{port}", queue, jobs + + server.shutdown() + + +def _get(url): + """GET request and parse JSON.""" + with urllib.request.urlopen(url) as resp: + return json.loads(resp.read()) + + +def _post(url): + """POST request and parse JSON.""" + req = urllib.request.Request(url, method="POST", data=b"") + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + + +def test_api_stats(dashboard_server): + """GET /api/stats returns valid stats dict.""" + base, queue, jobs = dashboard_server + data = _get(f"{base}/api/stats") + assert "pending" in data + assert data["pending"] == 8 + + +def test_api_jobs_list(dashboard_server): + """GET /api/jobs returns job list.""" + base, queue, jobs = dashboard_server + data = _get(f"{base}/api/jobs") + assert isinstance(data, list) + assert len(data) == 8 + + +def test_api_jobs_filter_status(dashboard_server): + """GET /api/jobs?status=pending filters correctly.""" + base, queue, jobs = dashboard_server + data = _get(f"{base}/api/jobs?status=pending") + assert len(data) == 8 + + data = _get(f"{base}/api/jobs?status=running") + assert len(data) == 0 + + +def test_api_jobs_filter_queue(dashboard_server): + """GET /api/jobs?queue=email filters correctly.""" + base, queue, jobs = dashboard_server + data = _get(f"{base}/api/jobs?queue=email") + assert len(data) == 3 + + +def test_api_jobs_pagination(dashboard_server): + """GET /api/jobs?limit=3&offset=0 paginates.""" + base, queue, jobs = dashboard_server + data = _get(f"{base}/api/jobs?limit=3&offset=0") + assert len(data) == 3 + + +def test_api_job_detail(dashboard_server): + """GET /api/jobs/{id} returns job dict.""" + base, queue, jobs = dashboard_server + job_id = jobs[0].id + data = _get(f"{base}/api/jobs/{job_id}") + assert data["id"] == job_id + assert "status" in data + + +def test_api_job_not_found(dashboard_server): + """GET /api/jobs/nonexistent returns 404.""" + base, queue, jobs = dashboard_server + try: + _get(f"{base}/api/jobs/nonexistent-id") + assert False, "Expected 404" + except urllib.error.HTTPError as e: + assert e.code == 404 + + +def test_api_cancel_job(dashboard_server): + """POST /api/jobs/{id}/cancel cancels a pending job.""" + base, queue, jobs = dashboard_server + job_id = jobs[0].id + data = _post(f"{base}/api/jobs/{job_id}/cancel") + assert data["cancelled"] is True + + +def test_api_dead_letters_empty(dashboard_server): + """GET /api/dead-letters returns empty list initially.""" + base, queue, jobs = dashboard_server + data = _get(f"{base}/api/dead-letters") + assert data == [] + + +def test_spa_html_served(dashboard_server): + """GET / returns the SPA HTML.""" + base, queue, jobs = dashboard_server + with urllib.request.urlopen(base) as resp: + html = resp.read().decode() + assert "quickq dashboard" in html + assert "" in html diff --git a/tests/python/test_dependencies.py b/tests/python/test_dependencies.py new file mode 100644 index 00000000..e02b83b2 --- /dev/null +++ b/tests/python/test_dependencies.py @@ -0,0 +1,122 @@ +"""Tests for task dependency feature.""" + +import threading + +import pytest + +from quickq import Queue + + +@pytest.fixture +def queue(tmp_path): + """Create a fresh queue with a temp database.""" + db_path = str(tmp_path / "test_deps.db") + return Queue(db_path=db_path, workers=2) + + +def test_enqueue_with_depends_on(queue): + """Jobs can declare dependencies on other jobs.""" + + @queue.task() + def step(x): + return x + + job_a = step.delay(1) + job_b = step.apply_async(args=(2,), depends_on=job_a.id) + + assert job_b.dependencies == [job_a.id] + assert job_a.dependents == [job_b.id] + + +def test_enqueue_with_multiple_deps(queue): + """Jobs can depend on multiple other jobs.""" + + @queue.task() + def step(x): + return x + + job_a = step.delay(1) + job_b = step.delay(2) + job_c = step.apply_async(args=(3,), depends_on=[job_a.id, job_b.id]) + + deps = set(job_c.dependencies) + assert deps == {job_a.id, job_b.id} + + +def test_depends_on_string_coercion(queue): + """depends_on accepts a single string ID.""" + + @queue.task() + def step(x): + return x + + job_a = step.delay(1) + # Pass as single string, not list + job_b = queue.enqueue( + task_name=step.name, + args=(2,), + depends_on=job_a.id, + ) + + assert job_b.dependencies == [job_a.id] + + +def test_dependency_blocks_execution(queue): + """Dependent job waits until dependency completes.""" + + @queue.task() + def step(x): + return x * 10 + + job_a = step.delay(1) + job_b = step.apply_async(args=(2,), depends_on=job_a.id) + + worker_thread = threading.Thread(target=queue.run_worker, daemon=True) + worker_thread.start() + + # Both should complete — job_a first, then job_b + result_a = job_a.result(timeout=10) + result_b = job_b.result(timeout=10) + + assert result_a == 10 + assert result_b == 20 + + +def test_cascade_cancel_on_job_cancel(queue): + """Cancelling a job cascades to its dependents.""" + + @queue.task() + def step(x): + return x + + job_a = step.delay(1) + job_b = step.apply_async(args=(2,), depends_on=job_a.id) + job_c = step.apply_async(args=(3,), depends_on=job_b.id) + + queue.cancel_job(job_a.id) + + assert job_b.status == "cancelled" + assert job_c.status == "cancelled" + + +def test_no_dependencies_property_when_none(queue): + """Jobs without dependencies return empty list.""" + + @queue.task() + def step(x): + return x + + job = step.delay(1) + assert job.dependencies == [] + assert job.dependents == [] + + +def test_enqueue_rejects_missing_dependency(queue): + """Enqueuing with a nonexistent dependency raises an error.""" + + @queue.task() + def step(x): + return x + + with pytest.raises(RuntimeError): + step.apply_async(args=(1,), depends_on="nonexistent-id") diff --git a/tests/python/test_fastapi.py b/tests/python/test_fastapi.py new file mode 100644 index 00000000..521901ca --- /dev/null +++ b/tests/python/test_fastapi.py @@ -0,0 +1,195 @@ +"""Tests for FastAPI integration.""" + +import threading + +import pytest + +from quickq import Queue + +# Skip entire module if fastapi is not installed +fastapi = pytest.importorskip("fastapi") +httpx = pytest.importorskip("httpx") + +from fastapi import FastAPI +from fastapi.testclient import TestClient +from quickq.contrib.fastapi import QuickQRouter + + +@pytest.fixture +def queue(tmp_path): + """Create a fresh queue.""" + db_path = str(tmp_path / "test_fastapi.db") + return Queue(db_path=db_path, workers=2) + + +@pytest.fixture +def app(queue): + """Create a FastAPI app with QuickQRouter.""" + app = FastAPI() + app.include_router(QuickQRouter(queue), prefix="/tasks") + return app + + +@pytest.fixture +def client(app): + """Create a TestClient.""" + return TestClient(app) + + +@pytest.fixture +def populated(queue, client): + """Queue with a task and some jobs.""" + + @queue.task() + def add(a, b): + return a + b + + jobs = [add.delay(i, i + 1) for i in range(5)] + return queue, client, jobs, add + + +# ── Stats ──────────────────────────────────────────────── + + +def test_stats(populated): + queue, client, jobs, add = populated + resp = client.get("/tasks/stats") + assert resp.status_code == 200 + data = resp.json() + assert data["pending"] == 5 + assert data["running"] == 0 + + +# ── Job detail ─────────────────────────────────────────── + + +def test_get_job(populated): + queue, client, jobs, add = populated + job_id = jobs[0].id + resp = client.get(f"/tasks/jobs/{job_id}") + assert resp.status_code == 200 + data = resp.json() + assert data["id"] == job_id + assert data["status"] == "pending" + + +def test_get_job_not_found(client): + resp = client.get("/tasks/jobs/nonexistent") + assert resp.status_code == 404 + + +# ── Job errors ─────────────────────────────────────────── + + +def test_get_job_errors_empty(populated): + queue, client, jobs, add = populated + job_id = jobs[0].id + resp = client.get(f"/tasks/jobs/{job_id}/errors") + assert resp.status_code == 200 + assert resp.json() == [] + + +# ── Job result ─────────────────────────────────────────── + + +def test_get_job_result_pending(populated): + queue, client, jobs, add = populated + job_id = jobs[0].id + resp = client.get(f"/tasks/jobs/{job_id}/result") + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "pending" + assert data["result"] is None + + +def test_get_job_result_completed(populated): + queue, client, jobs, add = populated + + worker = threading.Thread(target=queue.run_worker, daemon=True) + worker.start() + + # Wait for first job to complete + jobs[0].result(timeout=10) + + resp = client.get(f"/tasks/jobs/{jobs[0].id}/result") + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "complete" + assert data["result"] == 1 # add(0, 1) = 1 + + +# ── Cancel ─────────────────────────────────────────────── + + +def test_cancel_job(populated): + queue, client, jobs, add = populated + job_id = jobs[0].id + resp = client.post(f"/tasks/jobs/{job_id}/cancel") + assert resp.status_code == 200 + assert resp.json()["cancelled"] is True + + # Cancel again — should be false + resp = client.post(f"/tasks/jobs/{job_id}/cancel") + assert resp.json()["cancelled"] is False + + +# ── Dead letters ───────────────────────────────────────── + + +def test_dead_letters_empty(client): + resp = client.get("/tasks/dead-letters") + assert resp.status_code == 200 + assert resp.json() == [] + + +# ── Progress SSE ───────────────────────────────────────── + + +def test_progress_stream(populated): + queue, client, jobs, add = populated + + # Start worker so the job completes + worker = threading.Thread(target=queue.run_worker, daemon=True) + worker.start() + + job_id = jobs[0].id + # Wait for job to finish first + jobs[0].result(timeout=10) + + with client.stream("GET", f"/tasks/jobs/{job_id}/progress") as resp: + assert resp.status_code == 200 + lines = [] + for line in resp.iter_lines(): + if line.startswith("data:"): + lines.append(line) + break # Just check first event + + assert len(lines) >= 1 + # The first (and only) event should show complete status + import json + data = json.loads(lines[0].replace("data: ", "")) + assert data["status"] == "complete" + + +def test_progress_stream_not_found(client): + resp = client.get("/tasks/jobs/nonexistent/progress") + assert resp.status_code == 404 + + +# ── Router config ──────────────────────────────────────── + + +def test_router_custom_tags(queue): + """QuickQRouter accepts standard APIRouter kwargs.""" + router = QuickQRouter(queue, tags=["my-tasks"]) + assert "my-tasks" in router.tags + + +def test_router_custom_prefix(queue): + """Router can be mounted with a custom prefix.""" + app = FastAPI() + app.include_router(QuickQRouter(queue), prefix="/api/v1/queue") + client = TestClient(app) + + resp = client.get("/api/v1/queue/stats") + assert resp.status_code == 200