diff --git a/CHANGELOG.md b/CHANGELOG.md index 33db8031..957dab98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,46 @@ All notable changes to taskito are documented here. The format is based on [Semantic Versioning](https://semver.org/spec/v2.0.0.html). All SDKs (Python, Node, Java) and the underlying Rust crates are released together, in lock-step. +## 0.20.0 + +Scaling release: the queue survives higher task ingest/dispatch and pub/sub fan-out rates, and +schema management moves to code-first migrations. Worker dispatch now respects each pool's +execution capacity, so a worker no longer claims more than it can run and starve its peers. + +### Added + +- **Code-first migrations.** All schema DDL is built programmatically with `sea-query` (no raw + SQL) and applied by a versioned migrator with path-based auto-discovery — dropping a + `migrations/mXXXX_*.rs` file registers it automatically. Migration recording is atomic and + idempotent, so concurrent startups and interruptions are safe. New scaling indexes ship as + part of the baseline. +- **Batching.** Keyed pub/sub fan-out is committed in a single transaction, Redis subscription + reads are pipelined to constant round trips, execution claims are taken per batch, and + successful results are finalized together — cutting round trips under load across all SDKs. +- **Bounded in-flight dispatch.** A scheduler now caps how many jobs it keeps in flight at the + worker pool's execution parallelism, freeing a slot (and refilling immediately) as each job + finishes. + +### Changed + +- **Drain-until-empty polling.** Each poll wake dispatches all ready work instead of one job per + tick, removing the throughput ceiling imposed by the poll interval. +- **Jittered backoff.** Retry backoff uses full jitter, and the poller's gate-reschedule delays + are jittered, to avoid retry stampedes. +- Topic fan-out releases the GIL during publish/reap/backlog work (Python). + +### Fixed + +- **Worker fairness.** Under drain-until-empty, one worker could claim a whole backlog and + strand it `Running` on itself while peers sharing the database sat idle; in-flight dispatch is + now bounded to the pool size. +- **Workflow step double-execution.** A step now honors the task's `max_retries`, and the + tracker reacts only to a node's terminal job failure — closing an exactly-once violation under + load. +- Jobs stranded by a transient claim error return to `Pending` immediately rather than waiting + out the stale-job reaper. +- Postgres batch claims run in one transaction (no partially-committed claims on error). + ## 0.19.0 Parity release: the Java SDK reaches feature parity with the other SDKs (and its baseline moves diff --git a/Cargo.lock b/Cargo.lock index c8b39747..fb88a519 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1487,7 +1487,7 @@ checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" [[package]] name = "taskito-core" -version = "0.19.0" +version = "0.20.0" dependencies = [ "async-trait", "chrono", @@ -1512,7 +1512,7 @@ dependencies = [ [[package]] name = "taskito-java" -version = "0.19.0" +version = "0.20.0" dependencies = [ "async-trait", "crossbeam-channel", @@ -1531,7 +1531,7 @@ dependencies = [ [[package]] name = "taskito-mesh" -version = "0.19.0" +version = "0.20.0" dependencies = [ "base64", "bincode", @@ -1546,7 +1546,7 @@ dependencies = [ [[package]] name = "taskito-node" -version = "0.19.0" +version = "0.20.0" dependencies = [ "async-trait", "crossbeam-channel", @@ -1565,7 +1565,7 @@ dependencies = [ [[package]] name = "taskito-python" -version = "0.19.0" +version = "0.20.0" dependencies = [ "async-trait", "base64", @@ -1585,7 +1585,7 @@ dependencies = [ [[package]] name = "taskito-workflows" -version = "0.19.0" +version = "0.20.0" dependencies = [ "dagron-core", "diesel", diff --git a/crates/taskito-core/Cargo.toml b/crates/taskito-core/Cargo.toml index b98d9433..9b277e24 100644 --- a/crates/taskito-core/Cargo.toml +++ b/crates/taskito-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "taskito-core" -version = "0.19.0" +version = "0.20.0" edition = "2021" [features] diff --git a/crates/taskito-core/src/scheduler/mod.rs b/crates/taskito-core/src/scheduler/mod.rs index 7c0ec414..5df34012 100644 --- a/crates/taskito-core/src/scheduler/mod.rs +++ b/crates/taskito-core/src/scheduler/mod.rs @@ -274,15 +274,18 @@ impl Scheduler { /// refill. Only ids this scheduler dispatched are tracked, so results for /// recovered or foreign jobs (e.g. from maintenance) are a harmless no-op. fn release_in_flight(&self, job_id: &str) { - if self.config.max_in_flight.is_none() { + let Some(max) = self.config.max_in_flight else { return; - } - let freed = self - .in_flight - .lock() - .unwrap_or_else(|p| p.into_inner()) - .remove(job_id); - if freed { + }; + let mut in_flight = self.in_flight.lock().unwrap_or_else(|p| p.into_inner()); + // Only wake the poller when the pool was saturated — that's the only time + // it's parked on the cap. Below the cap it's already dispatching on its + // normal cadence, so an extra wake per completion just adds needless + // dequeue contention with everything else touching the database. + let was_full = in_flight.len() >= max; + let freed = in_flight.remove(job_id); + drop(in_flight); + if freed && was_full { self.dispatch_wake.notify_one(); } } diff --git a/crates/taskito-java/Cargo.toml b/crates/taskito-java/Cargo.toml index 2063abba..53135ffd 100644 --- a/crates/taskito-java/Cargo.toml +++ b/crates/taskito-java/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "taskito-java" -version = "0.19.0" +version = "0.20.0" edition = "2021" description = "Java (JNI) bindings for the Taskito task-queue core" license = "MIT" diff --git a/crates/taskito-mesh/Cargo.toml b/crates/taskito-mesh/Cargo.toml index 469e63a2..08b76bb0 100644 --- a/crates/taskito-mesh/Cargo.toml +++ b/crates/taskito-mesh/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "taskito-mesh" -version = "0.19.0" +version = "0.20.0" edition = "2021" [dependencies] diff --git a/crates/taskito-node/Cargo.toml b/crates/taskito-node/Cargo.toml index bae54810..ba4817ce 100644 --- a/crates/taskito-node/Cargo.toml +++ b/crates/taskito-node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "taskito-node" -version = "0.19.0" +version = "0.20.0" edition = "2021" description = "Node.js (napi-rs) bindings for the Taskito task-queue core" license = "MIT" diff --git a/crates/taskito-python/Cargo.toml b/crates/taskito-python/Cargo.toml index 03983c64..b9ec638d 100644 --- a/crates/taskito-python/Cargo.toml +++ b/crates/taskito-python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "taskito-python" -version = "0.19.0" +version = "0.20.0" edition = "2021" [features] diff --git a/crates/taskito-workflows/Cargo.toml b/crates/taskito-workflows/Cargo.toml index 7a72eada..bddd958a 100644 --- a/crates/taskito-workflows/Cargo.toml +++ b/crates/taskito-workflows/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "taskito-workflows" -version = "0.19.0" +version = "0.20.0" edition = "2021" [features] diff --git a/docs/app/lib/version.ts b/docs/app/lib/version.ts index ad411799..4ae7b3e5 100644 --- a/docs/app/lib/version.ts +++ b/docs/app/lib/version.ts @@ -1,2 +1,2 @@ // AUTO-GENERATED from /CHANGELOG.md by scripts/sync-changelog.mjs — do not edit directly. -export const VERSION = "0.19.0"; +export const VERSION = "0.20.0"; diff --git a/docs/content/docs/resources/changelog.mdx b/docs/content/docs/resources/changelog.mdx index 54ebe274..d69a0414 100644 --- a/docs/content/docs/resources/changelog.mdx +++ b/docs/content/docs/resources/changelog.mdx @@ -12,6 +12,46 @@ All notable changes to taskito are documented here. The format is based on [Semantic Versioning](https://semver.org/spec/v2.0.0.html). All SDKs (Python, Node, Java) and the underlying Rust crates are released together, in lock-step. +## 0.20.0 + +Scaling release: the queue survives higher task ingest/dispatch and pub/sub fan-out rates, and +schema management moves to code-first migrations. Worker dispatch now respects each pool's +execution capacity, so a worker no longer claims more than it can run and starve its peers. + +### Added + +- **Code-first migrations.** All schema DDL is built programmatically with `sea-query` (no raw + SQL) and applied by a versioned migrator with path-based auto-discovery — dropping a + `migrations/mXXXX_*.rs` file registers it automatically. Migration recording is atomic and + idempotent, so concurrent startups and interruptions are safe. New scaling indexes ship as + part of the baseline. +- **Batching.** Keyed pub/sub fan-out is committed in a single transaction, Redis subscription + reads are pipelined to constant round trips, execution claims are taken per batch, and + successful results are finalized together — cutting round trips under load across all SDKs. +- **Bounded in-flight dispatch.** A scheduler now caps how many jobs it keeps in flight at the + worker pool's execution parallelism, freeing a slot (and refilling immediately) as each job + finishes. + +### Changed + +- **Drain-until-empty polling.** Each poll wake dispatches all ready work instead of one job per + tick, removing the throughput ceiling imposed by the poll interval. +- **Jittered backoff.** Retry backoff uses full jitter, and the poller's gate-reschedule delays + are jittered, to avoid retry stampedes. +- Topic fan-out releases the GIL during publish/reap/backlog work (Python). + +### Fixed + +- **Worker fairness.** Under drain-until-empty, one worker could claim a whole backlog and + strand it `Running` on itself while peers sharing the database sat idle; in-flight dispatch is + now bounded to the pool size. +- **Workflow step double-execution.** A step now honors the task's `max_retries`, and the + tracker reacts only to a node's terminal job failure — closing an exactly-once violation under + load. +- Jobs stranded by a transient claim error return to `Pending` immediately rather than waiting + out the stale-job reaper. +- Postgres batch claims run in one transaction (no partially-committed claims on error). + ## 0.19.0 Parity release: the Java SDK reaches feature parity with the other SDKs (and its baseline moves diff --git a/sdks/java/build.gradle.kts b/sdks/java/build.gradle.kts index eab11896..3832d48a 100644 --- a/sdks/java/build.gradle.kts +++ b/sdks/java/build.gradle.kts @@ -7,7 +7,7 @@ plugins { } group = "org.byteveda" -version = "0.19.0" +version = "0.20.0" java { // Sources + javadoc jars are added by the maven-publish plugin below. diff --git a/sdks/java/graalvm-smoke/build.gradle.kts b/sdks/java/graalvm-smoke/build.gradle.kts index e0f613c7..957221ec 100644 --- a/sdks/java/graalvm-smoke/build.gradle.kts +++ b/sdks/java/graalvm-smoke/build.gradle.kts @@ -9,7 +9,7 @@ plugins { } group = "org.byteveda" -version = "0.19.0" +version = "0.20.0" repositories { mavenCentral() diff --git a/sdks/java/processor/build.gradle.kts b/sdks/java/processor/build.gradle.kts index e21b2cb7..f3b997ec 100644 --- a/sdks/java/processor/build.gradle.kts +++ b/sdks/java/processor/build.gradle.kts @@ -9,7 +9,7 @@ plugins { } group = "org.byteveda" -version = "0.19.0" +version = "0.20.0" mavenPublishing { publishToMavenCentral() diff --git a/sdks/java/spring/build.gradle.kts b/sdks/java/spring/build.gradle.kts index ae028ba0..e5b42e60 100644 --- a/sdks/java/spring/build.gradle.kts +++ b/sdks/java/spring/build.gradle.kts @@ -9,7 +9,7 @@ plugins { group = "org.byteveda" -version = "0.18.0" +version = "0.20.0" mavenPublishing { publishToMavenCentral() diff --git a/sdks/java/test-support/build.gradle.kts b/sdks/java/test-support/build.gradle.kts index 5de72fa8..2b888d48 100644 --- a/sdks/java/test-support/build.gradle.kts +++ b/sdks/java/test-support/build.gradle.kts @@ -9,7 +9,7 @@ plugins { } group = "org.byteveda" -version = "0.19.0" +version = "0.20.0" mavenPublishing { publishToMavenCentral() diff --git a/sdks/node/package.json b/sdks/node/package.json index 71b8dc00..1676053c 100644 --- a/sdks/node/package.json +++ b/sdks/node/package.json @@ -1,6 +1,6 @@ { "name": "@byteveda/taskito", - "version": "0.19.0", + "version": "0.20.0", "description": "Rust-powered task queue for Node.js — no broker required.", "license": "MIT", "type": "module", diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index 6929d106..382f3818 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "taskito" -version = "0.19.0" +version = "0.20.0" description = "Rust-powered task queue for Python. No broker required." requires-python = ">=3.10" license = { file = "LICENSE" } diff --git a/sdks/python/taskito/__init__.py b/sdks/python/taskito/__init__.py index 614f05ff..3f1c26ca 100644 --- a/sdks/python/taskito/__init__.py +++ b/sdks/python/taskito/__init__.py @@ -138,4 +138,4 @@ __version__ = _get_version("taskito") except PackageNotFoundError: - __version__ = "0.19.0" + __version__ = "0.20.0"