Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions .github/workflows/ci-rust.yml
Original file line number Diff line number Diff line change
Expand Up @@ -128,3 +128,71 @@ jobs:
run: LD_LIBRARY_PATH="$pythonLocation/lib" cargo test --workspace --features redis,workflows
env:
TASKITO_REDIS_TEST_URL: redis://localhost:6379/15

# Everything a crates.io release of taskito-core depends on: the declared
# MSRV actually builds, the rustdoc is warning-free, the package passes a
# publish dry-run, and (once a baseline exists on crates.io) the public API
# hasn't broken semver.
publish-readiness:
name: Publish Readiness (taskito-core)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
Comment thread
pratyush618 marked this conversation as resolved.
with:
persist-credentials: false

- name: Read MSRV from workspace manifest
id: msrv
run: |
msrv="$(grep -m1 '^rust-version' Cargo.toml | sed 's/.*"\(.*\)"/\1/')"
echo "version=$msrv" >> "$GITHUB_OUTPUT"

- name: Install MSRV toolchain
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ steps.msrv.outputs.version }}

- name: Check core builds at the MSRV
run: cargo +${{ steps.msrv.outputs.version }} check -p taskito-core --all-features

- name: Install stable toolchain
uses: dtolnay/rust-toolchain@stable

- name: Rustdoc is warning-free
run: RUSTDOCFLAGS="-D warnings" cargo doc -p taskito-core --no-deps --all-features

- name: Package passes a publish dry-run
run: cargo publish --dry-run -p taskito-core

# No baseline exists before the first publish; the check is skipped until
# then, and a breaking change fails CI once one does.
- name: Check for a crates.io baseline
id: baseline
run: |
# Probe the sparse index (index.crates.io), not the REST API: it is
# the registry's sanctioned programmatic path — a static CDN in
# Cargo's own index format, no rate limits, no User-Agent policy
# (the API 403s UA-less requests). Index path scheme for names of
# 4+ chars: /{first-two}/{next-two}/{name}, lowercase.
crate="taskito-core"
prefix="${crate:0:2}/${crate:2:2}"
# Fail closed: only an explicit 404 means "no baseline" — a transient
# network error must not silently skip the semver check. Retries
# absorb CDN blips before that verdict.
if ! status="$(curl -sS -o /dev/null -w '%{http_code}' \
--retry 3 --retry-connrefused \
"https://index.crates.io/${prefix}/${crate}")"; then
echo "Failed to query the crates.io sparse index" >&2
exit 1
fi
Comment thread
coderabbitai[bot] marked this conversation as resolved.
case "$status" in
200) echo "exists=true" >> "$GITHUB_OUTPUT" ;;
404) echo "exists=false" >> "$GITHUB_OUTPUT" ;;
*) echo "Unexpected sparse-index response: HTTP $status" >&2; exit 1 ;;
esac

- name: Semver check against crates.io baseline
if: steps.baseline.outputs.exists == 'true'
uses: obi1kenobi/cargo-semver-checks-action@v2
Comment thread
pratyush618 marked this conversation as resolved.
with:
package: taskito-core
10 changes: 10 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,16 @@
members = ["crates/taskito-core", "crates/taskito-python", "crates/taskito-node", "crates/taskito-java", "crates/taskito-workflows", "crates/taskito-mesh", "crates/taskito-tui"]
resolver = "2"

# Shared crate metadata — every member inherits via `x.workspace = true`, so a
# release bumps `version` in exactly one place. `rust-version` is verified by
# the CI msrv job; sea-query 1.x pins the floor at 1.88.
[workspace.package]
version = "0.20.0"
edition = "2021"
license = "MIT"
repository = "https://github.com/ByteVeda/taskito"
rust-version = "1.88"

[workspace.dependencies]
diesel = { version = "2.2", features = ["sqlite", "r2d2", "returning_clauses_for_sqlite_3_35"] }
libsqlite3-sys = { version = "0.37", features = ["bundled"] }
Expand Down
11 changes: 9 additions & 2 deletions crates/taskito-core/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
[package]
name = "taskito-core"
version = "0.20.0"
edition = "2021"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
description = "Embeddable task queue: SQLite/Postgres/Redis storage, scheduler with retries, rate limits, and workers"
keywords = ["task-queue", "jobs", "scheduler", "background", "worker"]
categories = ["asynchronous", "database"]
readme = "README.md"

[features]
default = []
Expand Down
21 changes: 21 additions & 0 deletions crates/taskito-core/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
MIT License

Copyright (c) 2026 Pratyush Sharma

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
83 changes: 83 additions & 0 deletions crates/taskito-core/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# taskito-core

Embeddable task queue for Rust: durable jobs on SQLite (default), PostgreSQL,
or Redis, a scheduler with retries, rate limits, circuit breakers, cron
periodics, and pub/sub fan-out — and a native worker that runs your handlers.

This crate is the engine behind the Taskito SDKs; it works standalone in any
Rust program.

## Quick start

```rust,no_run
use taskito_core::{
now_millis, Job, NewJob, SqliteStorage, Storage, StorageBackend, Worker,
};

fn main() -> taskito_core::Result<()> {
let storage = StorageBackend::Sqlite(SqliteStorage::new("taskito.db")?);

// A worker executes registered handlers for dequeued jobs.
let handle = Worker::new(storage.clone())
.num_workers(4)
.register("greet", |job: &Job| {
println!("hello, {}!", String::from_utf8_lossy(&job.payload));
Ok(None)
})
.spawn()?;

// Producers enqueue jobs — from this process or any other.
storage.enqueue(NewJob {
queue: "default".to_string(),
task_name: "greet".to_string(),
payload: b"world".to_vec(),
priority: 0,
scheduled_at: now_millis(),
max_retries: 3,
timeout_ms: 30_000,
unique_key: None,
metadata: None,
notes: None,
depends_on: vec![],
expires_at: None,
result_ttl_ms: None,
namespace: None,
})?;

std::thread::sleep(std::time::Duration::from_millis(500));
handle.shutdown()
}
```

Async handlers work too — `register_async` spawns the future on the worker's
runtime. See `examples/hello.rs` for a runnable version.

## Storage backends

| Backend | Feature | Notes |
|---|---|---|
| SQLite | (default) | Zero-config, single file, bundled `libsqlite3` |
| PostgreSQL | `postgres` | `SKIP LOCKED` dequeue, `LISTEN/NOTIFY` push dispatch |
| Redis | `redis` | JSON blobs + sorted-set indexes, Lua-atomic claims |

All three pass one shared contract test suite; behavior is identical up to
backend-documented differences.

## What's in the box

- **Jobs**: priorities, delays, per-job timeouts, uniqueness/idempotency keys,
dependencies, expiry, result TTLs, namespaces.
- **Scheduler**: adaptive polling (or event-driven wakeups via the
`push-dispatch` feature), batch claiming, bounded in-flight dispatch,
stale-job reaping, retention with per-table windows.
- **Resilience**: exponential-backoff retries, per-task rate limits and retry
budgets, circuit breakers, a dead-letter queue with replay.
- **Workers**: `Worker` builder + `TaskRegistry` for sync and async Rust
handlers, heartbeat/registry with elected cluster reaps.
- **Pub/sub**: topics with durable/ephemeral subscriptions and fan-out on
publish.
- **Periodic tasks**: cron expressions with optional time zones.

## License

MIT
17 changes: 17 additions & 0 deletions crates/taskito-core/src/error.rs
Original file line number Diff line number Diff line change
@@ -1,52 +1,69 @@
use thiserror::Error;

/// Every error the queue can produce.
#[derive(Error, Debug)]
pub enum QueueError {
/// A Diesel query or transaction failed.
#[error("storage error: {0}")]
Storage(#[from] diesel::result::Error),

/// The r2d2 connection pool could not hand out a connection.
#[error("connection pool error: {0}")]
Pool(#[from] diesel::r2d2::PoolError),

/// A Redis command or connection failed.
#[cfg(feature = "redis")]
#[error("redis error: {0}")]
Redis(#[from] redis::RedisError),

/// JSON encoding or decoding failed.
#[error("json error: {0}")]
Json(#[from] serde_json::Error),

/// No job exists with the given id.
#[error("job not found: {0}")]
JobNotFound(String),

/// A job referenced a task name with no registered handler.
#[error("task not registered: {0}")]
TaskNotRegistered(String),

/// A payload or result could not be serialized/deserialized.
#[error("serialization error: {0}")]
Serialization(String),

/// A worker-side failure (spawn, dispatch, or pool error).
#[error("worker error: {0}")]
Worker(String),

/// A scheduler-side failure (dispatch, maintenance, or config).
#[error("scheduler error: {0}")]
Scheduler(String),

/// A queue/task rate limit rejected the operation.
#[error("rate limit exceeded for: {0}")]
RateLimitExceeded(String),

/// A job exceeded its execution timeout.
#[error("job timed out: {0}")]
Timeout(String),

/// Invalid or inconsistent configuration.
#[error("config error: {0}")]
Config(String),

/// A job dependency id does not exist.
#[error("dependency not found: {0}")]
DependencyNotFound(String),

/// A distributed lock was already held by another owner.
#[error("lock not acquired: {0}")]
LockNotAcquired(String),

/// Any other failure that fits no specific variant.
#[error("{0}")]
Other(String),
}

/// Crate-wide result alias over [`QueueError`].
pub type Result<T> = std::result::Result<T, QueueError>;
Loading
Loading