From d1c0005b3bf359aaf70818e9056e56dc93c7985f Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:56:41 +0530 Subject: [PATCH 01/12] feat(core): add topic subscription storage registry --- .../src/storage/diesel_common/migrations.rs | 17 ++ .../src/storage/diesel_common/mod.rs | 2 + .../src/storage/diesel_common/pubsub.rs | 92 ++++++ crates/taskito-core/src/storage/mod.rs | 67 +++++ crates/taskito-core/src/storage/models.rs | 38 ++- .../taskito-core/src/storage/postgres/mod.rs | 1 + .../src/storage/postgres/pubsub.rs | 38 +++ .../src/storage/redis_backend/mod.rs | 1 + .../src/storage/redis_backend/pubsub.rs | 261 ++++++++++++++++++ crates/taskito-core/src/storage/schema.rs | 13 + crates/taskito-core/src/storage/sqlite/mod.rs | 1 + .../taskito-core/src/storage/sqlite/pubsub.rs | 24 ++ .../taskito-core/src/storage/sqlite/tests.rs | 191 +++++++++++++ crates/taskito-core/src/storage/traits.rs | 22 ++ .../taskito-core/tests/rust/storage_tests.rs | 107 +++++++ 15 files changed, 874 insertions(+), 1 deletion(-) create mode 100644 crates/taskito-core/src/storage/diesel_common/pubsub.rs create mode 100644 crates/taskito-core/src/storage/postgres/pubsub.rs create mode 100644 crates/taskito-core/src/storage/redis_backend/pubsub.rs create mode 100644 crates/taskito-core/src/storage/sqlite/pubsub.rs diff --git a/crates/taskito-core/src/storage/diesel_common/migrations.rs b/crates/taskito-core/src/storage/diesel_common/migrations.rs index 01d1c797..23dd94e5 100644 --- a/crates/taskito-core/src/storage/diesel_common/migrations.rs +++ b/crates/taskito-core/src/storage/diesel_common/migrations.rs @@ -265,6 +265,19 @@ pub fn create_tables(d: &Dialect) -> Vec { updated_at {bi} NOT NULL )" ), + format!( + "CREATE TABLE IF NOT EXISTS topic_subscriptions ( + topic TEXT NOT NULL, + subscription_name TEXT NOT NULL, + task_name TEXT NOT NULL, + queue TEXT NOT NULL DEFAULT 'default', + active {bool_true}, + durable {bool_true}, + owner_worker_id TEXT, + created_at {bi} NOT NULL, + PRIMARY KEY (topic, subscription_name) + )" + ), ] } @@ -305,6 +318,10 @@ pub fn create_indexes() -> &'static [&'static str] { // list_archived_filtered orders by created_at; namespace filter likewise. "CREATE INDEX IF NOT EXISTS idx_archived_jobs_created_at ON archived_jobs(created_at)", "CREATE INDEX IF NOT EXISTS idx_archived_jobs_namespace ON archived_jobs(namespace)", + // reap_ephemeral_subscriptions scans only owner-bound (ephemeral) rows; + // the partial index keeps durable rows (owner NULL) out of the reaper's way. + "CREATE INDEX IF NOT EXISTS idx_topic_subs_owner + ON topic_subscriptions(owner_worker_id) WHERE owner_worker_id IS NOT NULL", ] } diff --git a/crates/taskito-core/src/storage/diesel_common/mod.rs b/crates/taskito-core/src/storage/diesel_common/mod.rs index a7b8f9ad..20cb34f3 100644 --- a/crates/taskito-core/src/storage/diesel_common/mod.rs +++ b/crates/taskito-core/src/storage/diesel_common/mod.rs @@ -10,6 +10,7 @@ mod locks; mod logs; mod metrics; pub(crate) mod migrations; +mod pubsub; mod workers; pub(crate) use archival::impl_diesel_archival_ops; @@ -18,4 +19,5 @@ pub(crate) use jobs::impl_diesel_job_ops; pub(crate) use locks::impl_diesel_lock_ops; pub(crate) use logs::impl_diesel_log_ops; pub(crate) use metrics::impl_diesel_metric_ops; +pub(crate) use pubsub::impl_diesel_pubsub_ops; pub(crate) use workers::impl_diesel_worker_ops; diff --git a/crates/taskito-core/src/storage/diesel_common/pubsub.rs b/crates/taskito-core/src/storage/diesel_common/pubsub.rs new file mode 100644 index 00000000..0b1ca576 --- /dev/null +++ b/crates/taskito-core/src/storage/diesel_common/pubsub.rs @@ -0,0 +1,92 @@ +/// Generates shared topic-subscription operation methods for Diesel-backed +/// storage backends. +/// +/// `register_subscription` differs between SQLite (`replace_into`) and Postgres +/// (`on_conflict...do_update`), so it stays backend-specific. The remaining five +/// read/update/reap methods are identical and live here. +macro_rules! impl_diesel_pubsub_ops { + ($storage_type:ty) => { + impl $storage_type { + /// Active subscriptions for a topic, in registration order. + /// + /// Ordering is `created_at` then `subscription_name` so every backend + /// agrees on a stable, registration-time order. + pub fn list_subscriptions_for_topic( + &self, + topic: &str, + ) -> Result> { + let mut conn = self.conn()?; + + let rows = topic_subscriptions::table + .filter(topic_subscriptions::topic.eq(topic)) + .filter(topic_subscriptions::active.eq(true)) + .order(( + topic_subscriptions::created_at.asc(), + topic_subscriptions::subscription_name.asc(), + )) + .select(SubscriptionRow::as_select()) + .load(&mut conn)?; + + Ok(rows) + } + + /// Every registered subscription, active or paused, across all topics. + pub fn list_subscriptions(&self) -> Result> { + let mut conn = self.conn()?; + + let rows = topic_subscriptions::table + .select(SubscriptionRow::as_select()) + .load(&mut conn)?; + + Ok(rows) + } + + /// Remove a subscription. Returns false if none matched. + pub fn unsubscribe(&self, topic: &str, subscription_name: &str) -> Result { + let mut conn = self.conn()?; + + let affected = + diesel::delete(topic_subscriptions::table.find((topic, subscription_name))) + .execute(&mut conn)?; + + Ok(affected > 0) + } + + /// Pause (false) or resume (true) a subscription without removing its + /// registration. Returns false if none matched. + pub fn set_subscription_active( + &self, + topic: &str, + subscription_name: &str, + active: bool, + ) -> Result { + let mut conn = self.conn()?; + + let affected = + diesel::update(topic_subscriptions::table.find((topic, subscription_name))) + .set(topic_subscriptions::active.eq(active)) + .execute(&mut conn)?; + + Ok(affected > 0) + } + + /// Remove ephemeral subscriptions whose owner is not in + /// `live_worker_ids`. Durable rows (owner NULL) are excluded by the + /// `is_not_null` filter and never touched. Returns the count removed. + pub fn reap_ephemeral_subscriptions(&self, live_worker_ids: &[String]) -> Result { + let mut conn = self.conn()?; + + let affected = diesel::delete( + topic_subscriptions::table + .filter(topic_subscriptions::owner_worker_id.is_not_null()) + .filter(topic_subscriptions::owner_worker_id.ne_all(live_worker_ids)), + ) + .execute(&mut conn)?; + + Ok(affected as u64) + } + } + }; +} + +pub(crate) use impl_diesel_pubsub_ops; diff --git a/crates/taskito-core/src/storage/mod.rs b/crates/taskito-core/src/storage/mod.rs index 6c3139d0..417a3593 100644 --- a/crates/taskito-core/src/storage/mod.rs +++ b/crates/taskito-core/src/storage/mod.rs @@ -342,6 +342,44 @@ macro_rules! impl_storage { ) -> $crate::error::Result { self.set_periodic_enabled(name, enabled) } + fn register_subscription( + &self, + sub: &$crate::storage::models::NewSubscriptionRow, + ) -> $crate::error::Result<()> { + self.register_subscription(sub) + } + fn list_subscriptions_for_topic( + &self, + topic: &str, + ) -> $crate::error::Result> { + self.list_subscriptions_for_topic(topic) + } + fn list_subscriptions( + &self, + ) -> $crate::error::Result> { + self.list_subscriptions() + } + fn unsubscribe( + &self, + topic: &str, + subscription_name: &str, + ) -> $crate::error::Result { + self.unsubscribe(topic, subscription_name) + } + fn set_subscription_active( + &self, + topic: &str, + subscription_name: &str, + active: bool, + ) -> $crate::error::Result { + self.set_subscription_active(topic, subscription_name, active) + } + fn reap_ephemeral_subscriptions( + &self, + live_worker_ids: &[String], + ) -> $crate::error::Result { + self.reap_ephemeral_subscriptions(live_worker_ids) + } fn record_metric( &self, task_name: &str, @@ -876,6 +914,35 @@ impl Storage for StorageBackend { fn set_periodic_enabled(&self, name: &str, enabled: bool) -> Result { delegate!(self, set_periodic_enabled, name, enabled) } + fn register_subscription(&self, sub: &models::NewSubscriptionRow) -> Result<()> { + delegate!(self, register_subscription, sub) + } + fn list_subscriptions_for_topic(&self, topic: &str) -> Result> { + delegate!(self, list_subscriptions_for_topic, topic) + } + fn list_subscriptions(&self) -> Result> { + delegate!(self, list_subscriptions) + } + fn unsubscribe(&self, topic: &str, subscription_name: &str) -> Result { + delegate!(self, unsubscribe, topic, subscription_name) + } + fn set_subscription_active( + &self, + topic: &str, + subscription_name: &str, + active: bool, + ) -> Result { + delegate!( + self, + set_subscription_active, + topic, + subscription_name, + active + ) + } + fn reap_ephemeral_subscriptions(&self, live_worker_ids: &[String]) -> Result { + delegate!(self, reap_ephemeral_subscriptions, live_worker_ids) + } fn record_metric( &self, task_name: &str, diff --git a/crates/taskito-core/src/storage/models.rs b/crates/taskito-core/src/storage/models.rs index 0e05447f..c8365b7d 100644 --- a/crates/taskito-core/src/storage/models.rs +++ b/crates/taskito-core/src/storage/models.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use super::schema::{ archived_jobs, circuit_breakers, dashboard_settings, dead_letter, distributed_locks, execution_claims, job_dependencies, job_errors, jobs, periodic_tasks, queue_state, rate_limits, - replay_history, task_logs, task_metrics, workers, + replay_history, task_logs, task_metrics, topic_subscriptions, workers, }; /// A row in the `jobs` table (for SELECT queries). @@ -401,6 +401,42 @@ pub struct DashboardSettingRow { pub updated_at: i64, } +// ── Topic Subscriptions ───────────────────────────────────────── + +/// A row in the `topic_subscriptions` registry table. +/// +/// The natural composite key is `(topic, subscription_name)`; there is no +/// surrogate id, matching the `periodic_tasks` registry precedent. A NULL +/// `owner_worker_id` marks a durable subscription that persists until an +/// explicit unsubscribe; a set owner marks an ephemeral one that is reaped +/// when its worker dies. +#[derive(Queryable, Selectable, Debug, Clone)] +#[diesel(table_name = topic_subscriptions)] +pub struct SubscriptionRow { + pub topic: String, + pub subscription_name: String, + pub task_name: String, + pub queue: String, + pub active: bool, + pub durable: bool, + pub owner_worker_id: Option, + pub created_at: i64, +} + +/// Insertable/updatable struct for subscription registrations. +#[derive(Insertable, AsChangeset, Debug)] +#[diesel(table_name = topic_subscriptions)] +pub struct NewSubscriptionRow<'a> { + pub topic: &'a str, + pub subscription_name: &'a str, + pub task_name: &'a str, + pub queue: &'a str, + pub active: bool, + pub durable: bool, + pub owner_worker_id: Option<&'a str>, + pub created_at: i64, +} + // ── Distributed Locks ─────────────────────────────────────────── #[derive(Queryable, Selectable, QueryableByName, Debug, Clone)] diff --git a/crates/taskito-core/src/storage/postgres/mod.rs b/crates/taskito-core/src/storage/postgres/mod.rs index fb4d2091..38158ec8 100644 --- a/crates/taskito-core/src/storage/postgres/mod.rs +++ b/crates/taskito-core/src/storage/postgres/mod.rs @@ -7,6 +7,7 @@ mod locks; mod logs; mod metrics; mod periodic; +mod pubsub; mod queue_state; mod rate_limits; mod trait_impl; diff --git a/crates/taskito-core/src/storage/postgres/pubsub.rs b/crates/taskito-core/src/storage/postgres/pubsub.rs new file mode 100644 index 00000000..35c42b77 --- /dev/null +++ b/crates/taskito-core/src/storage/postgres/pubsub.rs @@ -0,0 +1,38 @@ +use diesel::prelude::*; + +use super::super::models::*; +use super::super::schema::topic_subscriptions; +use super::PostgresStorage; +use crate::error::Result; + +crate::storage::diesel_common::impl_diesel_pubsub_ops!(PostgresStorage); + +impl PostgresStorage { + /// Insert or update a subscription. Idempotent on (topic, subscription_name). + /// + /// The `do_update` sets each mutable column explicitly rather than via + /// `AsChangeset`, so re-registering with `owner_worker_id = None` writes SQL + /// NULL (clearing a previously-ephemeral owner) instead of leaving it stale. + pub fn register_subscription(&self, sub: &NewSubscriptionRow) -> Result<()> { + let mut conn = self.conn()?; + + diesel::insert_into(topic_subscriptions::table) + .values(sub) + .on_conflict(( + topic_subscriptions::topic, + topic_subscriptions::subscription_name, + )) + .do_update() + .set(( + topic_subscriptions::task_name.eq(sub.task_name), + topic_subscriptions::queue.eq(sub.queue), + topic_subscriptions::active.eq(sub.active), + topic_subscriptions::durable.eq(sub.durable), + topic_subscriptions::owner_worker_id.eq(sub.owner_worker_id), + topic_subscriptions::created_at.eq(sub.created_at), + )) + .execute(&mut conn)?; + + Ok(()) + } +} diff --git a/crates/taskito-core/src/storage/redis_backend/mod.rs b/crates/taskito-core/src/storage/redis_backend/mod.rs index 017a00f4..6fa5f17b 100644 --- a/crates/taskito-core/src/storage/redis_backend/mod.rs +++ b/crates/taskito-core/src/storage/redis_backend/mod.rs @@ -7,6 +7,7 @@ mod locks; mod logs; mod metrics; mod periodic; +mod pubsub; mod queue_state; mod rate_limits; mod trait_impl; diff --git a/crates/taskito-core/src/storage/redis_backend/pubsub.rs b/crates/taskito-core/src/storage/redis_backend/pubsub.rs new file mode 100644 index 00000000..2c7368ee --- /dev/null +++ b/crates/taskito-core/src/storage/redis_backend/pubsub.rs @@ -0,0 +1,261 @@ +use std::collections::HashSet; + +use redis::Commands; +use serde::{Deserialize, Serialize}; + +use super::{map_err, RedisStorage}; +use crate::error::{QueueError, Result}; +use crate::storage::models::{NewSubscriptionRow, SubscriptionRow}; + +/// JSON blob stored at `sub::`. Mirrors +/// [`SubscriptionRow`] so the CRUD path needs no Lua — index sets alone keep +/// lookups and the reaper off a keyspace `SCAN`. +#[derive(Serialize, Deserialize)] +struct SubEntry { + topic: String, + subscription_name: String, + task_name: String, + queue: String, + active: bool, + durable: bool, + owner_worker_id: Option, + created_at: i64, +} + +impl From for SubscriptionRow { + fn from(e: SubEntry) -> Self { + Self { + topic: e.topic, + subscription_name: e.subscription_name, + task_name: e.task_name, + queue: e.queue, + active: e.active, + durable: e.durable, + owner_worker_id: e.owner_worker_id, + created_at: e.created_at, + } + } +} + +impl RedisStorage { + /// The `topic|subscription_name` member string used in the index sets. + fn sub_composite(topic: &str, subscription_name: &str) -> String { + format!("{topic}|{subscription_name}") + } + + /// Load the entry a composite `topic|name` member points at, or `None` when + /// the blob is gone (a stale index member). Splits on the first `|`, matching + /// how [`Self::sub_composite`] builds it. + fn load_sub_by_composite( + &self, + conn: &mut redis::Connection, + composite: &str, + ) -> Result> { + let Some((topic, name)) = composite.split_once('|') else { + return Ok(None); + }; + let blob_key = self.key(&["sub", topic, name]); + let data: Option = conn.get(&blob_key).map_err(map_err)?; + match data { + Some(d) => { + let entry: SubEntry = + serde_json::from_str(&d).map_err(|e| QueueError::Other(e.to_string()))?; + Ok(Some(entry)) + } + None => Ok(None), + } + } + + /// Insert or update a subscription. Idempotent on (topic, subscription_name). + /// + /// Maintains three index sets — `subs:by_topic:`, `subs:all`, and + /// `subs:by_owner:` (ephemeral rows only). When a re-registration + /// changes the owner (including clearing it to durable), the stale + /// `by_owner` membership is dropped so the reaper's view stays exact. + pub fn register_subscription(&self, sub: &NewSubscriptionRow) -> Result<()> { + let mut conn = self.conn()?; + + let entry = SubEntry { + topic: sub.topic.to_string(), + subscription_name: sub.subscription_name.to_string(), + task_name: sub.task_name.to_string(), + queue: sub.queue.to_string(), + active: sub.active, + durable: sub.durable, + owner_worker_id: sub.owner_worker_id.map(|s| s.to_string()), + created_at: sub.created_at, + }; + let json = serde_json::to_string(&entry).map_err(|e| QueueError::Other(e.to_string()))?; + + let blob_key = self.key(&["sub", sub.topic, sub.subscription_name]); + let by_topic = self.key(&["subs", "by_topic", sub.topic]); + let all = self.key(&["subs", "all"]); + let composite = Self::sub_composite(sub.topic, sub.subscription_name); + + // A prior row may have carried a different (or NULL) owner; drop its + // stale by_owner entry so ownership never lingers across re-registration. + let prior_owner = self + .load_sub_by_composite(&mut conn, &composite)? + .and_then(|e| e.owner_worker_id); + + let pipe = &mut redis::pipe(); + pipe.set(&blob_key, &json); + pipe.sadd(&by_topic, sub.subscription_name); + pipe.sadd(&all, &composite); + if let Some(prev) = prior_owner.as_deref() { + if Some(prev) != sub.owner_worker_id { + pipe.srem(self.key(&["subs", "by_owner", prev]), &composite); + } + } + if let Some(owner) = sub.owner_worker_id { + pipe.sadd(self.key(&["subs", "by_owner", owner]), &composite); + } + pipe.query::<()>(&mut conn).map_err(map_err)?; + + Ok(()) + } + + /// Active subscriptions for a topic, in registration order (`created_at` + /// then `subscription_name`, matching the Diesel backends). + pub fn list_subscriptions_for_topic(&self, topic: &str) -> Result> { + let mut conn = self.conn()?; + let by_topic = self.key(&["subs", "by_topic", topic]); + + let names: Vec = conn.smembers(&by_topic).map_err(map_err)?; + + let mut rows = Vec::new(); + for name in names { + let blob_key = self.key(&["sub", topic, &name]); + let data: Option = conn.get(&blob_key).map_err(map_err)?; + if let Some(d) = data { + let entry: SubEntry = + serde_json::from_str(&d).map_err(|e| QueueError::Other(e.to_string()))?; + if entry.active { + rows.push(SubscriptionRow::from(entry)); + } + } + } + + rows.sort_by(|a, b| { + a.created_at + .cmp(&b.created_at) + .then_with(|| a.subscription_name.cmp(&b.subscription_name)) + }); + + Ok(rows) + } + + /// Every registered subscription, active or paused, across all topics. + pub fn list_subscriptions(&self) -> Result> { + let mut conn = self.conn()?; + let all = self.key(&["subs", "all"]); + + let composites: Vec = conn.smembers(&all).map_err(map_err)?; + + let mut rows = Vec::new(); + for composite in composites { + if let Some(entry) = self.load_sub_by_composite(&mut conn, &composite)? { + rows.push(SubscriptionRow::from(entry)); + } + } + + Ok(rows) + } + + /// Remove a subscription. Returns false if none matched. + pub fn unsubscribe(&self, topic: &str, subscription_name: &str) -> Result { + let mut conn = self.conn()?; + let blob_key = self.key(&["sub", topic, subscription_name]); + + // Read the row first so its owner index entry can be dropped too. + let data: Option = conn.get(&blob_key).map_err(map_err)?; + let Some(d) = data else { + return Ok(false); + }; + let entry: SubEntry = + serde_json::from_str(&d).map_err(|e| QueueError::Other(e.to_string()))?; + + let by_topic = self.key(&["subs", "by_topic", topic]); + let all = self.key(&["subs", "all"]); + let composite = Self::sub_composite(topic, subscription_name); + + let pipe = &mut redis::pipe(); + pipe.del(&blob_key); + pipe.srem(&by_topic, subscription_name); + pipe.srem(&all, &composite); + if let Some(owner) = entry.owner_worker_id.as_deref() { + pipe.srem(self.key(&["subs", "by_owner", owner]), &composite); + } + pipe.query::<()>(&mut conn).map_err(map_err)?; + + Ok(true) + } + + /// Pause (false) or resume (true) a subscription without removing its + /// registration. Returns false if none matched. + pub fn set_subscription_active( + &self, + topic: &str, + subscription_name: &str, + active: bool, + ) -> Result { + let mut conn = self.conn()?; + let blob_key = self.key(&["sub", topic, subscription_name]); + + let data: Option = conn.get(&blob_key).map_err(map_err)?; + let Some(d) = data else { + return Ok(false); + }; + + let mut entry: SubEntry = + serde_json::from_str(&d).map_err(|e| QueueError::Other(e.to_string()))?; + entry.active = active; + + let json = serde_json::to_string(&entry).map_err(|e| QueueError::Other(e.to_string()))?; + conn.set::<_, _, ()>(&blob_key, &json).map_err(map_err)?; + + Ok(true) + } + + /// Remove ephemeral subscriptions whose owner is not in `live_worker_ids`. + /// Durable rows (owner NULL) are never touched. Returns the count removed. + /// + /// Iterates the `subs:all` index set rather than a keyspace `SCAN`; each + /// removed row is also pulled from its `by_topic` and `by_owner` sets. + pub fn reap_ephemeral_subscriptions(&self, live_worker_ids: &[String]) -> Result { + let mut conn = self.conn()?; + let all = self.key(&["subs", "all"]); + + let composites: Vec = conn.smembers(&all).map_err(map_err)?; + let live: HashSet<&str> = live_worker_ids.iter().map(String::as_str).collect(); + + let mut removed = 0u64; + for composite in composites { + let Some(entry) = self.load_sub_by_composite(&mut conn, &composite)? else { + continue; + }; + // Durable rows (owner NULL) and live-owned rows survive. + let Some(owner) = entry.owner_worker_id.as_deref() else { + continue; + }; + if live.contains(owner) { + continue; + } + + let blob_key = self.key(&["sub", &entry.topic, &entry.subscription_name]); + let by_topic = self.key(&["subs", "by_topic", &entry.topic]); + let by_owner = self.key(&["subs", "by_owner", owner]); + + let pipe = &mut redis::pipe(); + pipe.del(&blob_key); + pipe.srem(&by_topic, &entry.subscription_name); + pipe.srem(&all, &composite); + pipe.srem(&by_owner, &composite); + pipe.query::<()>(&mut conn).map_err(map_err)?; + + removed += 1; + } + + Ok(removed) + } +} diff --git a/crates/taskito-core/src/storage/schema.rs b/crates/taskito-core/src/storage/schema.rs index b3d90b20..bc0ea3ae 100644 --- a/crates/taskito-core/src/storage/schema.rs +++ b/crates/taskito-core/src/storage/schema.rs @@ -225,4 +225,17 @@ diesel::table! { } } +diesel::table! { + topic_subscriptions (topic, subscription_name) { + topic -> Text, + subscription_name -> Text, + task_name -> Text, + queue -> Text, + active -> Bool, + durable -> Bool, + owner_worker_id -> Nullable, + created_at -> BigInt, + } +} + diesel::allow_tables_to_appear_in_same_query!(jobs, job_dependencies); diff --git a/crates/taskito-core/src/storage/sqlite/mod.rs b/crates/taskito-core/src/storage/sqlite/mod.rs index b4fdd879..e00ff1c5 100644 --- a/crates/taskito-core/src/storage/sqlite/mod.rs +++ b/crates/taskito-core/src/storage/sqlite/mod.rs @@ -7,6 +7,7 @@ mod locks; mod logs; mod metrics; mod periodic; +mod pubsub; mod queue_state; mod rate_limits; mod trait_impl; diff --git a/crates/taskito-core/src/storage/sqlite/pubsub.rs b/crates/taskito-core/src/storage/sqlite/pubsub.rs new file mode 100644 index 00000000..be7bd16a --- /dev/null +++ b/crates/taskito-core/src/storage/sqlite/pubsub.rs @@ -0,0 +1,24 @@ +use diesel::prelude::*; + +use super::super::models::*; +use super::super::schema::topic_subscriptions; +use super::SqliteStorage; +use crate::error::Result; + +crate::storage::diesel_common::impl_diesel_pubsub_ops!(SqliteStorage); + +impl SqliteStorage { + /// Insert or update a subscription. Idempotent on (topic, subscription_name). + /// + /// `replace_into` deletes any prior row before inserting, so re-registering + /// with `owner_worker_id = None` correctly clears a previously-set owner. + pub fn register_subscription(&self, sub: &NewSubscriptionRow) -> Result<()> { + let mut conn = self.conn()?; + + diesel::replace_into(topic_subscriptions::table) + .values(sub) + .execute(&mut conn)?; + + Ok(()) + } +} diff --git a/crates/taskito-core/src/storage/sqlite/tests.rs b/crates/taskito-core/src/storage/sqlite/tests.rs index cb332269..38b661da 100644 --- a/crates/taskito-core/src/storage/sqlite/tests.rs +++ b/crates/taskito-core/src/storage/sqlite/tests.rs @@ -1366,3 +1366,194 @@ fn test_periodic_pause_resume_and_delete() { // Toggling an unknown task reports nothing changed. assert!(!storage.set_periodic_enabled("ghost", false).unwrap()); } + +// -- Topic pub/sub -- + +fn make_sub<'a>( + topic: &'a str, + name: &'a str, + task_name: &'a str, + owner: Option<&'a str>, + created_at: i64, +) -> crate::storage::models::NewSubscriptionRow<'a> { + crate::storage::models::NewSubscriptionRow { + topic, + subscription_name: name, + task_name, + queue: "default", + active: true, + durable: owner.is_none(), + owner_worker_id: owner, + created_at, + } +} + +#[test] +fn test_register_subscription_is_idempotent_upsert() { + let storage = test_storage(); + let now = now_millis(); + + storage + .register_subscription(&make_sub("orders", "emailer", "send_email", None, now)) + .unwrap(); + // Re-registering the same (topic, name) with a different task_name updates in + // place rather than inserting a duplicate. + storage + .register_subscription(&make_sub("orders", "emailer", "send_email_v2", None, now)) + .unwrap(); + + let subs = storage.list_subscriptions_for_topic("orders").unwrap(); + assert_eq!(subs.len(), 1, "upsert must not duplicate the composite key"); + assert_eq!(subs[0].task_name, "send_email_v2"); +} + +#[test] +fn test_list_subscriptions_for_topic_active_only_in_order() { + let storage = test_storage(); + let now = now_millis(); + + storage + .register_subscription(&make_sub("orders", "first", "task_a", None, now)) + .unwrap(); + storage + .register_subscription(&make_sub("orders", "second", "task_b", None, now + 1)) + .unwrap(); + storage + .register_subscription(&make_sub("orders", "third", "task_c", None, now + 2)) + .unwrap(); + // A subscription on another topic must not leak into the topic listing. + storage + .register_subscription(&make_sub("shipments", "other", "task_d", None, now)) + .unwrap(); + + // Pausing "second" drops it from the active listing. + assert!(storage + .set_subscription_active("orders", "second", false) + .unwrap()); + + let names: Vec = storage + .list_subscriptions_for_topic("orders") + .unwrap() + .into_iter() + .map(|s| s.subscription_name) + .collect(); + assert_eq!( + names, + vec!["first".to_string(), "third".to_string()], + "active subscriptions only, in registration order" + ); + + // list_subscriptions returns every row across topics, active or paused. + assert_eq!(storage.list_subscriptions().unwrap().len(), 4); +} + +#[test] +fn test_unsubscribe_unknown_returns_false() { + let storage = test_storage(); + assert!(!storage.unsubscribe("orders", "ghost").unwrap()); + + storage + .register_subscription(&make_sub( + "orders", + "emailer", + "send_email", + None, + now_millis(), + )) + .unwrap(); + assert!(storage.unsubscribe("orders", "emailer").unwrap()); + assert!(storage + .list_subscriptions_for_topic("orders") + .unwrap() + .is_empty()); + // Removing it a second time reports nothing removed. + assert!(!storage.unsubscribe("orders", "emailer").unwrap()); +} + +#[test] +fn test_set_subscription_active_pause_resume_roundtrip() { + let storage = test_storage(); + storage + .register_subscription(&make_sub( + "orders", + "emailer", + "send_email", + None, + now_millis(), + )) + .unwrap(); + + // Pause: gone from the active listing but still registered. + assert!(storage + .set_subscription_active("orders", "emailer", false) + .unwrap()); + assert!(storage + .list_subscriptions_for_topic("orders") + .unwrap() + .is_empty()); + assert_eq!(storage.list_subscriptions().unwrap().len(), 1); + + // Resume: back in the active listing. + assert!(storage + .set_subscription_active("orders", "emailer", true) + .unwrap()); + assert_eq!( + storage + .list_subscriptions_for_topic("orders") + .unwrap() + .len(), + 1 + ); + + // Toggling an unknown subscription reports nothing changed. + assert!(!storage + .set_subscription_active("orders", "ghost", true) + .unwrap()); +} + +#[test] +fn test_reap_ephemeral_subscriptions_spares_durable_and_live() { + let storage = test_storage(); + let now = now_millis(); + + // Durable (owner NULL) — must never be reaped. + storage + .register_subscription(&make_sub("orders", "durable", "task_a", None, now)) + .unwrap(); + // Ephemeral, owner alive — survives. + storage + .register_subscription(&make_sub("orders", "live", "task_b", Some("worker-1"), now)) + .unwrap(); + // Ephemeral, owner dead — reaped. + storage + .register_subscription(&make_sub("orders", "dead", "task_c", Some("worker-2"), now)) + .unwrap(); + + let live = vec!["worker-1".to_string()]; + let removed = storage.reap_ephemeral_subscriptions(&live).unwrap(); + assert_eq!(removed, 1, "only the dead-owner ephemeral row is reaped"); + + let remaining: Vec = storage + .list_subscriptions() + .unwrap() + .into_iter() + .map(|s| s.subscription_name) + .collect(); + assert!(remaining.contains(&"durable".to_string())); + assert!(remaining.contains(&"live".to_string())); + assert!(!remaining.contains(&"dead".to_string())); + + // Re-reaping with the same live set removes nothing more. + assert_eq!(storage.reap_ephemeral_subscriptions(&live).unwrap(), 0); + + // With no live workers, every ephemeral row is reaped but the durable one stays. + let removed = storage.reap_ephemeral_subscriptions(&[]).unwrap(); + assert_eq!(removed, 1); + let names: Vec = storage + .list_subscriptions() + .unwrap() + .into_iter() + .map(|s| s.subscription_name) + .collect(); + assert_eq!(names, vec!["durable".to_string()]); +} diff --git a/crates/taskito-core/src/storage/traits.rs b/crates/taskito-core/src/storage/traits.rs index 21489623..66aa3878 100644 --- a/crates/taskito-core/src/storage/traits.rs +++ b/crates/taskito-core/src/storage/traits.rs @@ -126,6 +126,28 @@ pub trait Storage: Send + Sync + Clone { /// Returns false if no task had that name. fn set_periodic_enabled(&self, name: &str, enabled: bool) -> Result; + // ── Topic pub/sub ─────────────────────────────────────────────── + + /// Insert or update a subscription. Idempotent on (topic, subscription_name). + fn register_subscription(&self, sub: &NewSubscriptionRow) -> Result<()>; + /// Active subscriptions for a topic (active = true only). + fn list_subscriptions_for_topic(&self, topic: &str) -> Result>; + /// Every registered subscription (active or paused), all topics. + fn list_subscriptions(&self) -> Result>; + /// Remove a subscription. Returns false if none matched. + fn unsubscribe(&self, topic: &str, subscription_name: &str) -> Result; + /// Pause/resume without removing registration. Returns false if none matched. + fn set_subscription_active( + &self, + topic: &str, + subscription_name: &str, + active: bool, + ) -> Result; + /// Remove ephemeral subscriptions (owner_worker_id set) whose owner is not in + /// `live_worker_ids`. Durable rows (owner NULL) are never touched. Returns the + /// count removed. + fn reap_ephemeral_subscriptions(&self, live_worker_ids: &[String]) -> Result; + // ── Metrics operations ────────────────────────────────────────── fn record_metric( diff --git a/crates/taskito-core/tests/rust/storage_tests.rs b/crates/taskito-core/tests/rust/storage_tests.rs index 81841f76..022168b0 100644 --- a/crates/taskito-core/tests/rust/storage_tests.rs +++ b/crates/taskito-core/tests/rust/storage_tests.rs @@ -852,6 +852,112 @@ fn test_periodic_crud(s: &impl Storage) { assert!(!s.delete_periodic("pc-a").unwrap()); } +fn test_topic_subscriptions_crud(s: &impl Storage) { + use taskito_core::storage::models::NewSubscriptionRow; + let now = now_millis(); + let sub = |topic: &'static str, + name: &'static str, + task_name: &'static str, + owner: Option<&'static str>, + created_at: i64| NewSubscriptionRow { + topic, + subscription_name: name, + task_name, + queue: "default", + active: true, + durable: owner.is_none(), + owner_worker_id: owner, + created_at, + }; + + // Upsert idempotency: re-registering (topic, name) updates in place. + s.register_subscription(&sub("ts-orders", "emailer", "send_email", None, now)) + .unwrap(); + s.register_subscription(&sub("ts-orders", "emailer", "send_email_v2", None, now)) + .unwrap(); + s.register_subscription(&sub("ts-orders", "analytics", "track", None, now + 1)) + .unwrap(); + + let listed = s.list_subscriptions_for_topic("ts-orders").unwrap(); + assert_eq!( + listed.len(), + 2, + "upsert must not duplicate the composite key" + ); + // Registration order (created_at, then name). + assert_eq!( + listed + .iter() + .map(|r| r.subscription_name.as_str()) + .collect::>(), + vec!["emailer", "analytics"] + ); + assert_eq!(listed[0].task_name, "send_email_v2"); + + // Pausing drops from the active listing but keeps the registration. + assert!(s + .set_subscription_active("ts-orders", "emailer", false) + .unwrap()); + let active_names: Vec = s + .list_subscriptions_for_topic("ts-orders") + .unwrap() + .into_iter() + .map(|r| r.subscription_name) + .collect(); + assert_eq!(active_names, vec!["analytics".to_string()]); + assert!(s + .list_subscriptions() + .unwrap() + .iter() + .any(|r| r.topic == "ts-orders" && r.subscription_name == "emailer")); + + // Resuming brings it back. + assert!(s + .set_subscription_active("ts-orders", "emailer", true) + .unwrap()); + assert_eq!( + s.list_subscriptions_for_topic("ts-orders").unwrap().len(), + 2 + ); + + // Toggling / unsubscribing an unknown row reports "not found". + assert!(!s + .set_subscription_active("ts-orders", "ghost", true) + .unwrap()); + assert!(!s.unsubscribe("ts-orders", "ghost").unwrap()); + + // Reaper: only dead-owner ephemeral rows go; durable rows never do. + s.register_subscription(&sub("ts-live", "live", "task_b", Some("ts-worker-1"), now)) + .unwrap(); + s.register_subscription(&sub("ts-live", "dead", "task_c", Some("ts-worker-2"), now)) + .unwrap(); + let removed = s + .reap_ephemeral_subscriptions(&["ts-worker-1".to_string()]) + .unwrap(); + assert_eq!(removed, 1, "only the dead-owner ephemeral row is reaped"); + let live_topic: Vec = s + .list_subscriptions_for_topic("ts-live") + .unwrap() + .into_iter() + .map(|r| r.subscription_name) + .collect(); + assert_eq!(live_topic, vec!["live".to_string()]); + // Durable rows on ts-orders untouched by the reaper. + assert_eq!( + s.list_subscriptions_for_topic("ts-orders").unwrap().len(), + 2 + ); + + // Unsubscribe removes the row. + assert!(s.unsubscribe("ts-orders", "emailer").unwrap()); + assert!(s.unsubscribe("ts-orders", "analytics").unwrap()); + assert!(s + .list_subscriptions_for_topic("ts-orders") + .unwrap() + .is_empty()); + assert!(s.unsubscribe("ts-live", "live").unwrap()); +} + /// Two workers draining one queue concurrently must claim disjoint jobs — every /// enqueued job is handed out exactly once, never twice. Exercises the Postgres /// `FOR UPDATE SKIP LOCKED` dequeue path and the SQLite `BEGIN IMMEDIATE` / @@ -911,6 +1017,7 @@ fn run_storage_tests(s: &impl Storage) { test_workers(s); test_pause_resume_queue(s); test_periodic_crud(s); + test_topic_subscriptions_crud(s); test_circuit_breakers(s); test_execution_claims_purge(s); test_reap_stale_jobs(s); From 1a5ba485616453cfe406a63fa000db2f89d2f638 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:57:03 +0530 Subject: [PATCH 02/12] feat(core): add publish_to_topic fan-out --- crates/taskito-core/src/lib.rs | 1 + crates/taskito-core/src/pubsub.rs | 272 ++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+) create mode 100644 crates/taskito-core/src/pubsub.rs diff --git a/crates/taskito-core/src/lib.rs b/crates/taskito-core/src/lib.rs index d32576b2..7ca19157 100644 --- a/crates/taskito-core/src/lib.rs +++ b/crates/taskito-core/src/lib.rs @@ -1,6 +1,7 @@ pub mod error; pub mod job; pub mod periodic; +pub mod pubsub; pub mod resilience; pub mod scheduler; pub mod storage; diff --git a/crates/taskito-core/src/pubsub.rs b/crates/taskito-core/src/pubsub.rs new file mode 100644 index 00000000..2d03fce2 --- /dev/null +++ b/crates/taskito-core/src/pubsub.rs @@ -0,0 +1,272 @@ +//! Topic pub/sub fan-out: one job per active subscription. +//! +//! Lives in the core (not the language shells) so every binding shares the +//! same fan-out semantics — in particular the idempotency-key salting, which +//! silently drops deliveries if a shell gets it wrong. + +use std::collections::HashMap; + +use crate::error::Result; +use crate::job::{Job, NewJob}; +use crate::storage::models::SubscriptionRow; +use crate::storage::Storage; + +/// Per-task delivery settings a subscriber's task declared at registration. +/// Shells pass what their task registry knows; unknown tasks fall back to +/// the queue-level defaults. +#[derive(Clone, Copy)] +pub struct DeliveryDefaults { + pub priority: i32, + pub max_retries: i32, + pub timeout_ms: i64, +} + +/// Everything a publish shares across the jobs it fans out to. Per-subscriber +/// routing (task name, queue) comes from the subscription registry; delivery +/// settings resolve per field as explicit publish override, then the +/// subscriber task's own defaults, then the queue defaults. +pub struct PublishRequest { + pub topic: String, + /// Wire-envelope payload bytes; every subscriber receives the same body. + pub payload: Vec, + /// Salted per subscription (`key::subscription_name`) before becoming a + /// job `unique_key`: the jobs table's unique index is global, so reusing + /// the raw key across the fan-out would dedup away all but one delivery. + pub idempotency_key: Option, + pub metadata: Option, + /// Pre-validated canonical notes JSON object (see `Job::notes`); the + /// topic and subscription name are stamped in per delivery. + pub notes: Option, + pub priority: Option, + pub scheduled_at: i64, + pub max_retries: Option, + pub timeout_ms: Option, + pub expires_at: Option, + pub result_ttl_ms: Option, + pub namespace: Option, + pub queue_defaults: DeliveryDefaults, + pub task_defaults: HashMap, +} + +/// Fan a message out to every active subscription of `topic` as ordinary +/// jobs. Returns the created jobs — empty when the topic has no active +/// subscriptions (a valid pub/sub no-op, not an error). +/// +/// Keyed publishes go through `enqueue_unique` per delivery so a re-published +/// event dedups per subscriber instead of failing the whole batch on the +/// unique index; unkeyed publishes use one batch insert. +pub fn publish_to_topic(storage: &S, request: &PublishRequest) -> Result> { + let subscriptions = storage.list_subscriptions_for_topic(&request.topic)?; + if subscriptions.is_empty() { + return Ok(Vec::new()); + } + let jobs: Vec = subscriptions + .iter() + .map(|sub| delivery_job(request, sub)) + .collect(); + if request.idempotency_key.is_none() { + return storage.enqueue_batch(jobs); + } + jobs.into_iter() + .map(|job| storage.enqueue_unique(job)) + .collect() +} + +fn delivery_job(request: &PublishRequest, sub: &SubscriptionRow) -> NewJob { + let task = request + .task_defaults + .get(&sub.task_name) + .copied() + .unwrap_or(request.queue_defaults); + NewJob { + queue: sub.queue.clone(), + task_name: sub.task_name.clone(), + payload: request.payload.clone(), + priority: request.priority.unwrap_or(task.priority), + scheduled_at: request.scheduled_at, + max_retries: request.max_retries.unwrap_or(task.max_retries), + timeout_ms: request.timeout_ms.unwrap_or(task.timeout_ms), + unique_key: request + .idempotency_key + .as_ref() + .map(|key| format!("{key}::{}", sub.subscription_name)), + metadata: request.metadata.clone(), + notes: Some(delivery_notes(request, sub)), + depends_on: Vec::new(), + expires_at: request.expires_at, + result_ttl_ms: request.result_ttl_ms, + namespace: request.namespace.clone(), + } +} + +/// Stamp `topic` and `subscription` into the caller's notes object so every +/// delivery is filterable per subscriber without a schema change. +fn delivery_notes(request: &PublishRequest, sub: &SubscriptionRow) -> String { + let mut notes = request + .notes + .as_deref() + .and_then(|raw| { + serde_json::from_str::>(raw).ok() + }) + .unwrap_or_default(); + notes.insert( + "topic".to_string(), + serde_json::Value::String(request.topic.clone()), + ); + notes.insert( + "subscription".to_string(), + serde_json::Value::String(sub.subscription_name.clone()), + ); + serde_json::Value::Object(notes).to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::job::now_millis; + use crate::storage::models::NewSubscriptionRow; + use crate::SqliteStorage; + + fn request(topic: &str, idempotency_key: Option<&str>) -> PublishRequest { + PublishRequest { + topic: topic.to_string(), + payload: vec![0x02, 0xf5], + idempotency_key: idempotency_key.map(str::to_string), + metadata: None, + notes: None, + priority: None, + scheduled_at: now_millis(), + max_retries: None, + timeout_ms: None, + expires_at: None, + result_ttl_ms: None, + namespace: None, + queue_defaults: DeliveryDefaults { + priority: 0, + max_retries: 3, + timeout_ms: 300_000, + }, + task_defaults: HashMap::new(), + } + } + + fn subscribe(storage: &SqliteStorage, topic: &str, name: &str, task: &str) { + storage + .register_subscription(&NewSubscriptionRow { + topic, + subscription_name: name, + task_name: task, + queue: "default", + active: true, + durable: true, + owner_worker_id: None, + created_at: now_millis(), + }) + .unwrap(); + } + + #[test] + fn publish_with_no_subscriptions_is_a_no_op() { + let storage = SqliteStorage::in_memory().unwrap(); + let jobs = publish_to_topic(&storage, &request("orders", None)).unwrap(); + assert!(jobs.is_empty()); + } + + #[test] + fn publish_fans_out_one_job_per_subscription() { + let storage = SqliteStorage::in_memory().unwrap(); + subscribe(&storage, "orders", "email", "send_email"); + subscribe(&storage, "orders", "analytics", "track_order"); + + let jobs = publish_to_topic(&storage, &request("orders", None)).unwrap(); + + assert_eq!(jobs.len(), 2); + let mut tasks: Vec<_> = jobs.iter().map(|j| j.task_name.as_str()).collect(); + tasks.sort_unstable(); + assert_eq!(tasks, ["send_email", "track_order"]); + assert!(jobs.iter().all(|j| j.payload == vec![0x02, 0xf5])); + } + + #[test] + fn idempotency_key_is_salted_per_subscription() { + // The jobs unique index is global: an unsalted key would let only the + // first fan-out insert win and silently drop the other deliveries. + let storage = SqliteStorage::in_memory().unwrap(); + subscribe(&storage, "orders", "email", "send_email"); + subscribe(&storage, "orders", "analytics", "track_order"); + + let first = publish_to_topic(&storage, &request("orders", Some("evt-42"))).unwrap(); + assert_eq!(first.len(), 2); + let mut keys: Vec<_> = first.iter().filter_map(|j| j.unique_key.clone()).collect(); + keys.sort_unstable(); + assert_eq!(keys, ["evt-42::analytics", "evt-42::email"]); + + // Same event published twice: every subscriber still has exactly one + // delivery (per-subscriber dedup), not zero and not two. + let second = publish_to_topic(&storage, &request("orders", Some("evt-42"))).unwrap(); + assert_eq!(second.len(), 2); + assert_eq!( + first.iter().map(|j| &j.id).collect::>(), + second.iter().map(|j| &j.id).collect::>(), + ); + } + + #[test] + fn delivery_settings_resolve_override_then_task_then_queue() { + let storage = SqliteStorage::in_memory().unwrap(); + subscribe(&storage, "orders", "email", "send_email"); + subscribe(&storage, "orders", "audit", "audit_log"); + + let mut req = request("orders", None); + req.task_defaults.insert( + "send_email".to_string(), + DeliveryDefaults { + priority: 5, + max_retries: 0, + timeout_ms: 60_000, + }, + ); + let jobs = publish_to_topic(&storage, &req).unwrap(); + let by_task: HashMap<_, _> = jobs.iter().map(|j| (j.task_name.as_str(), j)).collect(); + // send_email declared its own settings; audit_log falls back to queue defaults. + assert_eq!(by_task["send_email"].max_retries, 0); + assert_eq!(by_task["send_email"].priority, 5); + assert_eq!(by_task["audit_log"].max_retries, 3); + + // An explicit publish-level override beats both. + let mut req = request("orders", None); + req.max_retries = Some(9); + let jobs = publish_to_topic(&storage, &req).unwrap(); + assert!(jobs.iter().all(|j| j.max_retries == 9)); + } + + #[test] + fn paused_subscriptions_are_skipped() { + let storage = SqliteStorage::in_memory().unwrap(); + subscribe(&storage, "orders", "email", "send_email"); + subscribe(&storage, "orders", "analytics", "track_order"); + storage + .set_subscription_active("orders", "analytics", false) + .unwrap(); + + let jobs = publish_to_topic(&storage, &request("orders", None)).unwrap(); + assert_eq!(jobs.len(), 1); + assert_eq!(jobs[0].task_name, "send_email"); + } + + #[test] + fn deliveries_carry_topic_and_subscription_notes() { + let storage = SqliteStorage::in_memory().unwrap(); + subscribe(&storage, "orders", "email", "send_email"); + + let mut req = request("orders", None); + req.notes = Some(r#"{"tenant":"acme"}"#.to_string()); + let jobs = publish_to_topic(&storage, &req).unwrap(); + + let notes: serde_json::Value = + serde_json::from_str(jobs[0].notes.as_deref().unwrap()).unwrap(); + assert_eq!(notes["topic"], "orders"); + assert_eq!(notes["subscription"], "email"); + assert_eq!(notes["tenant"], "acme"); + } +} From 39601afdce7771a56fa1df1b2a60c75819b4fdd1 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:57:15 +0530 Subject: [PATCH 03/12] feat(python): expose pub/sub bindings --- crates/taskito-python/src/py_queue/mod.rs | 1 + crates/taskito-python/src/py_queue/pubsub.rs | 200 +++++++++++++++++++ sdks/python/taskito/_taskito.pyi | 32 +++ 3 files changed, 233 insertions(+) create mode 100644 crates/taskito-python/src/py_queue/pubsub.rs diff --git a/crates/taskito-python/src/py_queue/mod.rs b/crates/taskito-python/src/py_queue/mod.rs index a107c6b5..81e8d26e 100644 --- a/crates/taskito-python/src/py_queue/mod.rs +++ b/crates/taskito-python/src/py_queue/mod.rs @@ -2,6 +2,7 @@ #![allow(clippy::useless_conversion)] mod inspection; +mod pubsub; mod worker; #[cfg(feature = "workflows")] mod workflow_ops; diff --git a/crates/taskito-python/src/py_queue/pubsub.rs b/crates/taskito-python/src/py_queue/pubsub.rs new file mode 100644 index 00000000..9337f35e --- /dev/null +++ b/crates/taskito-python/src/py_queue/pubsub.rs @@ -0,0 +1,200 @@ +use std::collections::HashMap; + +use pyo3::prelude::*; + +use taskito_core::job::now_millis; +use taskito_core::pubsub::{publish_to_topic, DeliveryDefaults, PublishRequest}; +use taskito_core::storage::models::NewSubscriptionRow; +use taskito_core::storage::Storage; + +use super::PyQueue; +use crate::py_job::PyJob; + +/// A subscription row surfaced to Python: +/// `(topic, subscription_name, task_name, queue, active, durable)`. +type SubscriptionTuple = (String, String, String, String, bool, bool); + +#[pymethods] +impl PyQueue { + /// Insert or update a topic subscription (idempotent on topic + name). + #[pyo3(signature = (topic, subscription_name, task_name, queue="default", durable=true, owner_worker_id=None))] + pub fn register_subscription( + &self, + topic: &str, + subscription_name: &str, + task_name: &str, + queue: &str, + durable: bool, + owner_worker_id: Option<&str>, + ) -> PyResult<()> { + let row = NewSubscriptionRow { + topic, + subscription_name, + task_name, + queue, + active: true, + durable, + owner_worker_id, + created_at: now_millis(), + }; + self.storage + .register_subscription(&row) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + + /// List subscriptions — all of them, or only a topic's active ones. + #[pyo3(signature = (topic=None))] + pub fn list_subscriptions(&self, topic: Option<&str>) -> PyResult> { + let rows = match topic { + Some(t) => self.storage.list_subscriptions_for_topic(t), + None => self.storage.list_subscriptions(), + } + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + Ok(rows + .into_iter() + .map(|row| { + ( + row.topic, + row.subscription_name, + row.task_name, + row.queue, + row.active, + row.durable, + ) + }) + .collect()) + } + + /// Remove a subscription. Returns false if none matched. + pub fn unsubscribe(&self, topic: &str, subscription_name: &str) -> PyResult { + self.storage + .unsubscribe(topic, subscription_name) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + + /// Pause (false) or resume (true) a subscription without unregistering it. + pub fn set_subscription_active( + &self, + topic: &str, + subscription_name: &str, + active: bool, + ) -> PyResult { + self.storage + .set_subscription_active(topic, subscription_name, active) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + + /// Drop ephemeral subscriptions whose owning worker is gone. Runs on the + /// heartbeat cadence, after `reap_dead_workers` has pruned the registry. + pub fn reap_ephemeral_subscriptions(&self) -> PyResult { + let live: Vec = self + .storage + .list_workers() + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))? + .into_iter() + .map(|w| w.worker_id) + .collect(); + self.storage + .reap_ephemeral_subscriptions(&live) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string())) + } + + /// Publish a payload to a topic: one job per active subscription. + /// Returns the created jobs — empty when nothing is subscribed. + #[pyo3(signature = (topic, payload, idempotency_key=None, metadata=None, notes=None, priority=None, delay_seconds=None, max_retries=None, timeout=None, expires=None, result_ttl=None, task_defaults=None))] + #[allow(clippy::too_many_arguments)] + pub fn publish( + &self, + topic: &str, + payload: Vec, + idempotency_key: Option, + metadata: Option, + notes: Option, + priority: Option, + delay_seconds: Option, + max_retries: Option, + timeout: Option, + expires: Option, + result_ttl: Option, + task_defaults: Option>, + ) -> PyResult> { + let now = now_millis(); + let scheduled_at = match delay_seconds { + Some(d) => { + if !d.is_finite() || d < 0.0 { + return Err(pyo3::exceptions::PyValueError::new_err( + "delay_seconds must be a finite non-negative number", + )); + } + now.checked_add((d * 1000.0) as i64).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err( + "delay_seconds too large, would overflow", + ) + })? + } + None => now, + }; + let expires_at = match expires { + Some(e) => { + if !e.is_finite() || e < 0.0 { + return Err(pyo3::exceptions::PyValueError::new_err( + "expires must be a finite non-negative number", + )); + } + Some(now.checked_add((e * 1000.0) as i64).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err("expires too large, would overflow") + })?) + } + None => None, + }; + let result_ttl_ms = match result_ttl { + Some(s) => Some(s.checked_mul(1000).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err("result_ttl too large, would overflow") + })?), + None => None, + }; + let timeout_ms = match timeout { + Some(t) => Some(t.checked_mul(1000).ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err("timeout too large, would overflow") + })?), + None => None, + }; + + let request = PublishRequest { + topic: topic.to_string(), + payload, + idempotency_key, + metadata, + notes, + priority, + scheduled_at, + max_retries, + timeout_ms, + expires_at, + result_ttl_ms, + namespace: self.namespace.clone(), + queue_defaults: DeliveryDefaults { + priority: self.default_priority, + max_retries: self.default_retry, + timeout_ms: self.default_timeout.saturating_mul(1000), + }, + task_defaults: task_defaults + .unwrap_or_default() + .into_iter() + .map(|(name, (priority, max_retries, timeout_ms))| { + ( + name, + DeliveryDefaults { + priority, + max_retries, + timeout_ms, + }, + ) + }) + .collect(), + }; + let jobs = publish_to_topic(&self.storage, &request) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + Ok(jobs.into_iter().map(PyJob::from).collect()) + } +} diff --git a/sdks/python/taskito/_taskito.pyi b/sdks/python/taskito/_taskito.pyi index 78204319..d30b2799 100644 --- a/sdks/python/taskito/_taskito.pyi +++ b/sdks/python/taskito/_taskito.pyi @@ -186,6 +186,38 @@ class PyQueue: def delete_periodic(self, name: str) -> bool: ... def pause_periodic(self, name: str) -> bool: ... def resume_periodic(self, name: str) -> bool: ... + def register_subscription( + self, + topic: str, + subscription_name: str, + task_name: str, + queue: str = "default", + durable: bool = True, + owner_worker_id: str | None = None, + ) -> None: ... + def list_subscriptions( + self, topic: str | None = None + ) -> list[tuple[str, str, str, str, bool, bool]]: ... + def unsubscribe(self, topic: str, subscription_name: str) -> bool: ... + def set_subscription_active( + self, topic: str, subscription_name: str, active: bool + ) -> bool: ... + def reap_ephemeral_subscriptions(self) -> int: ... + def publish( + self, + topic: str, + payload: bytes, + idempotency_key: str | None = None, + metadata: str | None = None, + notes: str | None = None, + priority: int | None = None, + delay_seconds: float | None = None, + max_retries: int | None = None, + timeout: int | None = None, + expires: float | None = None, + result_ttl: int | None = None, + task_defaults: dict[str, tuple[int, int, int]] | None = None, + ) -> list[PyJob]: ... def run_worker( self, task_registry: dict[str, Any], From be80a1111b6a32ff14e53f029d56f6e22e03474d Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:59:02 +0530 Subject: [PATCH 04/12] feat(python): add topic subscriber and publish API --- sdks/python/taskito/app.py | 3 + sdks/python/taskito/mixins/__init__.py | 2 + sdks/python/taskito/mixins/lifecycle.py | 7 + sdks/python/taskito/mixins/pubsub.py | 192 ++++++++++++++++++++++ sdks/python/tests/core/test_pubsub.py | 201 ++++++++++++++++++++++++ 5 files changed, 405 insertions(+) create mode 100644 sdks/python/taskito/mixins/pubsub.py create mode 100644 sdks/python/tests/core/test_pubsub.py diff --git a/sdks/python/taskito/app.py b/sdks/python/taskito/app.py index 4ea7785d..2fd518b1 100644 --- a/sdks/python/taskito/app.py +++ b/sdks/python/taskito/app.py @@ -45,6 +45,7 @@ QueueOperationsMixin, QueueOverridesMixin, QueuePredicateMixin, + QueuePubSubMixin, QueueResourceMixin, QueueRuntimeConfigMixin, QueueSettingsMixin, @@ -87,6 +88,7 @@ class QueueWorkflowMixin: # type: ignore[no-redef] class Queue( QueueDecoratorMixin, QueuePredicateMixin, + QueuePubSubMixin, QueueRuntimeConfigMixin, QueueResourceMixin, QueueEventsMixin, @@ -254,6 +256,7 @@ def __init__( self._task_configs: list[PyTaskConfig] = [] self._executor = ThreadPoolExecutor(max_workers=2) self._periodic_configs: list[dict[str, Any]] = [] + self._subscription_configs: list[dict[str, Any]] = [] self._hooks: dict[str, list[Callable]] = { "before_task": [], "after_task": [], diff --git a/sdks/python/taskito/mixins/__init__.py b/sdks/python/taskito/mixins/__init__.py index 2d54c056..c3df886d 100644 --- a/sdks/python/taskito/mixins/__init__.py +++ b/sdks/python/taskito/mixins/__init__.py @@ -9,6 +9,7 @@ from taskito.mixins.operations import QueueOperationsMixin from taskito.mixins.overrides import QueueOverridesMixin from taskito.mixins.predicates import QueuePredicateMixin +from taskito.mixins.pubsub import QueuePubSubMixin from taskito.mixins.resources import QueueResourceMixin from taskito.mixins.runtime_config import QueueRuntimeConfigMixin from taskito.mixins.settings import QueueSettingsMixin @@ -23,6 +24,7 @@ "QueueOperationsMixin", "QueueOverridesMixin", "QueuePredicateMixin", + "QueuePubSubMixin", "QueueResourceMixin", "QueueRuntimeConfigMixin", "QueueSettingsMixin", diff --git a/sdks/python/taskito/mixins/lifecycle.py b/sdks/python/taskito/mixins/lifecycle.py index ef8f69ec..363ec9c7 100644 --- a/sdks/python/taskito/mixins/lifecycle.py +++ b/sdks/python/taskito/mixins/lifecycle.py @@ -228,6 +228,10 @@ def sighup_handler(signum: int, frame: Any) -> None: # Generate worker ID and start Python-side heartbeat thread worker_id = str(uuid.uuid4()) + + # Flush topic subscriptions now that the ephemeral ones have an owner. + self._declare_worker_subscriptions(worker_id) # type: ignore[attr-defined] + stop_heartbeat = threading.Event() heartbeat_thread = threading.Thread( target=self._run_heartbeat, @@ -331,6 +335,9 @@ def _run_heartbeat( # Emit WORKER_OFFLINE events for reaped dead workers for rid in reaped_ids: self._emit_event(EventType.WORKER_OFFLINE, {"worker_id": rid}) # type: ignore[attr-defined] + # Dead workers gone from the registry → drop their ephemeral + # topic subscriptions on the same cadence. + self._inner.reap_ephemeral_subscriptions() except Exception: logger.debug("Heartbeat failed", exc_info=True) diff --git a/sdks/python/taskito/mixins/pubsub.py b/sdks/python/taskito/mixins/pubsub.py new file mode 100644 index 00000000..7845df22 --- /dev/null +++ b/sdks/python/taskito/mixins/pubsub.py @@ -0,0 +1,192 @@ +"""Topic pub/sub: N independent subscribers, each delivered its own job.""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +from taskito.notes import validate_and_encode_notes +from taskito.result import JobResult + +if TYPE_CHECKING: + from taskito._taskito import PyQueue + from taskito.serializers import Serializer + from taskito.task import TaskWrapper + + +class QueuePubSubMixin: + """Publish/subscribe over topics. + + A subscriber is a normal task plus a registry row mapping + ``(topic, subscription_name) -> (task_name, queue)``. ``publish()`` fans a + message out as one ordinary job per active subscription, so every task + feature — retries, DLQ, middleware, rate limits — applies per subscriber, + and a failing subscriber never affects its siblings. + + Deliveries use the queue-level serializer (there is one payload for all + subscribers); per-task serializer overrides don't apply to topic tasks. + """ + + _inner: PyQueue + _serializer: Serializer + _subscription_configs: list[dict[str, Any]] + + def subscriber( + self, + topic: str, + name: str | None = None, + queue: str = "default", + durable: bool = True, + **task_kwargs: Any, + ) -> Callable[[Callable[..., Any]], TaskWrapper]: + """Decorator registering ``fn`` as an independent subscriber of ``topic``. + + The function becomes a normal task (``task_kwargs`` are forwarded to + :meth:`task`), and the subscription is written to storage when + ``run_worker()`` starts — or via :meth:`declare_subscriptions` in a + producer-only process. ``durable=False`` ties the subscription to one + worker process: it only registers inside ``run_worker()`` and is + reaped automatically once its worker stops heartbeating. + + Args: + topic: Topic to subscribe to. + name: Stable subscription identity. Defaults to the task name; + re-registering the same ``(topic, name)`` updates the routing + target instead of duplicating. + queue: Queue the subscriber's delivery jobs go to. + durable: Persist across restarts (default). ``False`` = ephemeral. + **task_kwargs: Any :meth:`task` option (``max_retries``, + ``timeout``, ``middleware``, ``idempotent``, ...). + """ + + def decorator(fn: Callable[..., Any]) -> TaskWrapper: + wrapper: TaskWrapper = self.task(queue=queue, **task_kwargs)(fn) # type: ignore[attr-defined] + self._subscription_configs.append( + { + "topic": topic, + "subscription_name": name or wrapper.name, + "task_name": wrapper.name, + "queue": queue, + "durable": durable, + } + ) + return wrapper + + return decorator + + def declare_subscriptions(self) -> None: + """Write pending durable subscriptions to storage. + + Called automatically at ``run_worker()`` startup. Call it explicitly + in a producer-only process (one that imports subscriber modules but + never runs a worker) so ``publish()`` sees the subscriptions. + Ephemeral subscriptions are skipped — they need an owning worker. + """ + for config in self._subscription_configs: + if config["durable"]: + self._register_subscription(config, owner_worker_id=None) + + def publish( + self, + topic: str, + *args: Any, + idempotency_key: str | None = None, + metadata: dict[str, Any] | None = None, + notes: dict[str, Any] | None = None, + priority: int | None = None, + delay_seconds: float | None = None, + max_retries: int | None = None, + timeout: int | None = None, + expires: float | None = None, + result_ttl: int | None = None, + **kwargs: Any, + ) -> list[JobResult]: + """Publish a message to ``topic``. + + Every active subscription receives an independent job carrying the + same ``(args, kwargs)`` payload (at-least-once per subscriber). + Returns one :class:`JobResult` per delivery — empty when the topic + has no active subscribers, which is a valid pub/sub no-op. + + ``idempotency_key`` dedupes per subscriber: republishing the same key + yields no new deliveries, and a subscription added later still gets + its own copy. Each delivery's ``notes`` carry ``topic`` and + ``subscription`` for filtering. + """ + payload = self._serializer.dumps((args, kwargs)) + self._check_payload_size(topic, len(payload)) # type: ignore[attr-defined] + py_jobs = self._inner.publish( + topic=topic, + payload=payload, + idempotency_key=idempotency_key, + metadata=json.dumps(metadata) if metadata is not None else None, + notes=validate_and_encode_notes(notes) if notes is not None else None, + priority=priority, + delay_seconds=delay_seconds, + max_retries=max_retries, + timeout=timeout, + expires=expires, + result_ttl=result_ttl, + task_defaults=self._delivery_task_defaults(), + ) + return [JobResult(py_job=py_job, queue=self) for py_job in py_jobs] # type: ignore[arg-type] + + def unsubscribe(self, topic: str, name: str) -> bool: + """Remove a subscription. Returns False if none matched.""" + return self._inner.unsubscribe(topic, name) + + def pause_subscription(self, topic: str, name: str) -> bool: + """Stop deliveries without unregistering. Returns False if unknown.""" + return self._inner.set_subscription_active(topic, name, False) + + def resume_subscription(self, topic: str, name: str) -> bool: + """Resume a paused subscription. Returns False if unknown.""" + return self._inner.set_subscription_active(topic, name, True) + + def list_subscriptions(self, topic: str | None = None) -> list[dict[str, Any]]: + """List subscriptions — all of them, or one topic's active ones.""" + rows = self._inner.list_subscriptions(topic) + return [ + { + "topic": row[0], + "name": row[1], + "task_name": row[2], + "queue": row[3], + "active": row[4], + "durable": row[5], + } + for row in rows + ] + + def list_topics(self) -> list[str]: + """Distinct topics that currently have at least one subscription.""" + seen: dict[str, None] = {} + for row in self._inner.list_subscriptions(None): + seen.setdefault(row[0], None) + return list(seen) + + def _delivery_task_defaults(self) -> dict[str, tuple[int, int, int]]: + """Per-task ``(priority, max_retries, timeout_ms)`` from this process's + task registry, so deliveries honor each subscriber's own settings. + Tasks registered elsewhere fall back to queue defaults.""" + return { + config.name: (config.priority, config.max_retries, config.timeout * 1000) + for config in self._task_configs # type: ignore[attr-defined] + } + + def _register_subscription(self, config: dict[str, Any], owner_worker_id: str | None) -> None: + self._inner.register_subscription( + topic=config["topic"], + subscription_name=config["subscription_name"], + task_name=config["task_name"], + queue=config["queue"], + durable=config["durable"], + owner_worker_id=owner_worker_id, + ) + + def _declare_worker_subscriptions(self, worker_id: str) -> None: + """Flush all subscriptions at worker startup, owning the ephemeral ones.""" + for config in self._subscription_configs: + owner = None if config["durable"] else worker_id + self._register_subscription(config, owner_worker_id=owner) diff --git a/sdks/python/tests/core/test_pubsub.py b/sdks/python/tests/core/test_pubsub.py new file mode 100644 index 00000000..6eab8809 --- /dev/null +++ b/sdks/python/tests/core/test_pubsub.py @@ -0,0 +1,201 @@ +"""Topic pub/sub: fan-out, per-subscriber isolation, subscription lifecycle.""" + +import threading +from typing import Any + +from taskito import Queue + +PollUntil = Any # the conftest fixture's runtime type + + +class TestSubscriptionRegistry: + def test_declare_and_list(self, queue: Queue) -> None: + @queue.subscriber("orders", name="email") + def send_email(order_id: int) -> str: + return f"email {order_id}" + + @queue.subscriber("orders", name="analytics") + def track(order_id: int) -> str: + return f"track {order_id}" + + queue.declare_subscriptions() + + subs = queue.list_subscriptions("orders") + assert {s["name"] for s in subs} == {"email", "analytics"} + assert all(s["active"] and s["durable"] for s in subs) + assert queue.list_topics() == ["orders"] + + def test_subscriber_name_defaults_to_task_name(self, queue: Queue) -> None: + @queue.subscriber("orders") + def handle(order_id: int) -> None: ... + + queue.declare_subscriptions() + (sub,) = queue.list_subscriptions("orders") + assert sub["name"] == sub["task_name"] == handle.name + + def test_redeclare_updates_instead_of_duplicating(self, queue: Queue) -> None: + @queue.subscriber("orders", name="email", queue="mail") + def send_email(order_id: int) -> None: ... + + queue.declare_subscriptions() + queue.declare_subscriptions() + + subs = queue.list_subscriptions() + assert len(subs) == 1 + assert subs[0]["queue"] == "mail" + + def test_unsubscribe(self, queue: Queue) -> None: + @queue.subscriber("orders", name="email") + def send_email(order_id: int) -> None: ... + + queue.declare_subscriptions() + assert queue.unsubscribe("orders", "email") is True + assert queue.unsubscribe("orders", "email") is False + assert queue.list_subscriptions("orders") == [] + + def test_pause_resume(self, queue: Queue) -> None: + @queue.subscriber("orders", name="email") + def send_email(order_id: int) -> None: ... + + queue.declare_subscriptions() + assert queue.pause_subscription("orders", "email") is True + assert queue.publish("orders", 1) == [] + assert queue.resume_subscription("orders", "email") is True + assert len(queue.publish("orders", 2)) == 1 + + +class TestPublish: + def test_publish_without_subscribers_is_noop(self, queue: Queue) -> None: + assert queue.publish("ghost-topic", 1, key="value") == [] + + def test_fan_out_delivers_to_every_subscriber( + self, queue: Queue, run_worker: threading.Thread, poll_until: PollUntil + ) -> None: + received: dict[str, int] = {} + lock = threading.Lock() + + @queue.subscriber("orders", name="email") + def send_email(order_id: int) -> None: + with lock: + received["email"] = order_id + + @queue.subscriber("orders", name="analytics") + def track(order_id: int) -> None: + with lock: + received["analytics"] = order_id + + queue.declare_subscriptions() + results = queue.publish("orders", 42) + assert len(results) == 2 + + poll_until(lambda: len(received) == 2, message="both subscribers should run") + assert received == {"email": 42, "analytics": 42} + + def test_failing_subscriber_does_not_affect_sibling( + self, queue: Queue, run_worker: threading.Thread, poll_until: PollUntil + ) -> None: + outcomes: list[str] = [] + + @queue.subscriber("orders", name="flaky", max_retries=0) + def flaky(order_id: int) -> None: + raise RuntimeError("boom") + + @queue.subscriber("orders", name="steady") + def steady(order_id: int) -> None: + outcomes.append("steady") + + queue.declare_subscriptions() + deliveries = queue.publish("orders", 7) + assert len(deliveries) == 2 + + poll_until(lambda: outcomes == ["steady"], message="healthy subscriber should complete") + jobs = [queue.get_job(r.id) for r in deliveries] + by_subscription = { + job.notes["subscription"]: delivery + for job, delivery in zip(jobs, deliveries, strict=True) + if job is not None and job.notes is not None + } + assert set(by_subscription) == {"flaky", "steady"} + + def status_of(subscription: str) -> str: + delivery = by_subscription[subscription] + delivery.refresh() + return delivery.status + + poll_until( + lambda: status_of("steady") == "complete", + message="steady delivery should complete", + ) + flaky_id = by_subscription["flaky"].id + poll_until( + lambda: any(dead["original_job_id"] == flaky_id for dead in queue.dead_letters()), + message="flaky delivery should dead-letter independently", + ) + + def test_idempotency_key_dedupes_per_subscriber(self, queue: Queue) -> None: + """Regression: the jobs unique index is global — an unsalted key + would silently drop delivery to all but one subscriber.""" + + @queue.subscriber("orders", name="email") + def send_email(order_id: int) -> None: ... + + @queue.subscriber("orders", name="analytics") + def track(order_id: int) -> None: ... + + queue.declare_subscriptions() + first = queue.publish("orders", 1, idempotency_key="evt-9") + assert len(first) == 2 + + second = queue.publish("orders", 1, idempotency_key="evt-9") + assert sorted(r.id for r in second) == sorted(r.id for r in first) + + def test_deliveries_carry_topic_and_subscription_notes(self, queue: Queue) -> None: + @queue.subscriber("orders", name="email") + def send_email(order_id: int) -> None: ... + + queue.declare_subscriptions() + (result,) = queue.publish("orders", 5, notes={"tenant": "acme"}) + + job = queue.get_job(result.id) + assert job is not None and job.notes is not None + assert job.notes["topic"] == "orders" + assert job.notes["subscription"] == "email" + assert job.notes["tenant"] == "acme" + + def test_late_join_gets_only_later_publishes(self, queue: Queue) -> None: + @queue.subscriber("orders", name="email") + def send_email(order_id: int) -> None: ... + + queue.declare_subscriptions() + assert len(queue.publish("orders", 1)) == 1 + + queue._inner.register_subscription( + topic="orders", subscription_name="late", task_name=send_email.name + ) + deliveries = queue.publish("orders", 2) + assert len(deliveries) == 2 + + +class TestEphemeralSubscriptions: + def test_ephemeral_registers_only_with_worker(self, queue: Queue) -> None: + @queue.subscriber("orders", name="debug-tail", durable=False) + def tail(order_id: int) -> None: ... + + queue.declare_subscriptions() + assert queue.list_subscriptions("orders") == [] + + def test_reap_removes_dead_owner_subscriptions(self, queue: Queue) -> None: + queue._inner.register_subscription( + topic="orders", + subscription_name="ghost", + task_name="some.task", + durable=False, + owner_worker_id="worker-that-never-heartbeats", + ) + queue._inner.register_subscription( + topic="orders", subscription_name="durable", task_name="some.task" + ) + + assert queue._inner.reap_ephemeral_subscriptions() == 1 + names = {s["name"] for s in queue.list_subscriptions("orders")} + assert names == {"durable"} From f5e7e426fd689b3dcddb0e77587d98684b18e7f2 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:35:25 +0530 Subject: [PATCH 05/12] feat(node): add topic subscriber and publish API --- crates/taskito-node/src/config.rs | 40 ++++ crates/taskito-node/src/convert/job.rs | 8 +- crates/taskito-node/src/convert/mod.rs | 3 + crates/taskito-node/src/convert/pubsub.rs | 28 +++ crates/taskito-node/src/queue/mod.rs | 1 + crates/taskito-node/src/queue/pubsub.rs | 221 ++++++++++++++++++++++ sdks/node/src/index.ts | 3 + sdks/node/src/native.ts | 3 + sdks/node/src/queue.ts | 151 +++++++++++++++ sdks/node/src/types.ts | 24 +++ sdks/node/src/worker.ts | 14 ++ sdks/node/test/core/pubsub.test.ts | 197 +++++++++++++++++++ 12 files changed, 690 insertions(+), 3 deletions(-) create mode 100644 crates/taskito-node/src/convert/pubsub.rs create mode 100644 crates/taskito-node/src/queue/pubsub.rs create mode 100644 sdks/node/test/core/pubsub.test.ts diff --git a/crates/taskito-node/src/config.rs b/crates/taskito-node/src/config.rs index f05eb66f..f38bc688 100644 --- a/crates/taskito-node/src/config.rs +++ b/crates/taskito-node/src/config.rs @@ -1,6 +1,8 @@ //! Plain option objects passed from JavaScript. napi maps snake_case Rust //! fields to camelCase JS keys (`maxRetries`, `timeoutMs`). +use std::collections::HashMap; + use napi::bindgen_prelude::Buffer; use napi_derive::napi; @@ -52,6 +54,44 @@ pub struct EnqueueJob { pub options: Option, } +/// A subscriber task's own delivery settings, keyed by task name in +/// [`PublishOptions::task_defaults`]. Unset fields fall back to the queue +/// defaults, so the shell never has to duplicate them. +#[napi(object)] +#[derive(Default)] +pub struct DeliveryDefaultsInput { + pub priority: Option, + pub max_retries: Option, + pub timeout_ms: Option, +} + +/// Options for [`crate::queue::JsQueue::publish`]. All optional — omitted +/// delivery settings resolve per subscriber (task defaults, then queue +/// defaults) in the core. +#[napi(object)] +#[derive(Default)] +pub struct PublishOptions { + /// Dedup key, salted per subscription in the core so republishing the + /// same key yields no new delivery for any subscriber. + pub idempotency_key: Option, + /// Free-form metadata string stored on every delivery. + pub metadata: Option, + /// Pre-encoded canonical notes JSON object; the core stamps `topic` and + /// `subscription` into it per delivery. + pub notes: Option, + pub priority: Option, + /// Deliver no earlier than `now + delayMs`. + pub delay_ms: Option, + pub max_retries: Option, + pub timeout_ms: Option, + /// Expire undelivered jobs at `now + expiresMs`. + pub expires_ms: Option, + pub result_ttl_ms: Option, + /// Per-task delivery defaults from the publisher's task registry, so a + /// subscriber's own registration options are honored. + pub task_defaults: Option>, +} + /// Filter for [`crate::queue::JsQueue::list_jobs`]. All fields optional. #[napi(object)] #[derive(Default)] diff --git a/crates/taskito-node/src/convert/job.rs b/crates/taskito-node/src/convert/job.rs index 4135bc85..fd215da4 100644 --- a/crates/taskito-node/src/convert/job.rs +++ b/crates/taskito-node/src/convert/job.rs @@ -10,8 +10,10 @@ use crate::config::EnqueueOptions; use crate::error::non_negative; const DEFAULT_QUEUE: &str = "default"; -const DEFAULT_MAX_RETRIES: i32 = 3; -const DEFAULT_TIMEOUT_MS: i64 = 300_000; +// Shared with publish fan-out (queue-level delivery defaults). +pub(crate) const DEFAULT_PRIORITY: i32 = 0; +pub(crate) const DEFAULT_MAX_RETRIES: i32 = 3; +pub(crate) const DEFAULT_TIMEOUT_MS: i64 = 300_000; /// Build a [`NewJob`] from a task name, opaque payload bytes, and JS options. /// The core never interprets `payload` — the shell owns (de)serialization. @@ -38,7 +40,7 @@ pub fn build_new_job( queue: opts.queue.unwrap_or_else(|| DEFAULT_QUEUE.to_string()), task_name, payload, - priority: opts.priority.unwrap_or(0), + priority: opts.priority.unwrap_or(DEFAULT_PRIORITY), // Saturate so an extreme delay can't overflow into a past schedule. scheduled_at: now_millis().saturating_add(delay), max_retries, diff --git a/crates/taskito-node/src/convert/mod.rs b/crates/taskito-node/src/convert/mod.rs index 05d3382c..16b9b06b 100644 --- a/crates/taskito-node/src/convert/mod.rs +++ b/crates/taskito-node/src/convert/mod.rs @@ -7,12 +7,14 @@ mod log; mod ops; mod outcome; mod periodic; +mod pubsub; mod stats; mod task_config; #[cfg(feature = "workflows")] mod workflow; pub use job::{build_new_job, job_to_js, JsJob, JsTaskInvocation}; +pub(crate) use job::{DEFAULT_MAX_RETRIES, DEFAULT_PRIORITY, DEFAULT_TIMEOUT_MS}; pub use lock::{lock_info_to_js, JsLockInfo}; pub use log::{log_to_js, JsTaskLog}; pub use ops::{ @@ -20,6 +22,7 @@ pub use ops::{ }; pub use outcome::{outcome_to_js, JsOutcome}; pub use periodic::{periodic_to_js, JsPeriodicTask}; +pub use pubsub::{subscription_to_js, JsSubscription}; pub use stats::{ dead_job_to_js, job_error_to_js, metric_to_js, stats_to_js, status_code, worker_to_js, JsDeadJob, JsJobError, JsMetric, JsStats, JsWorkerRow, diff --git a/crates/taskito-node/src/convert/pubsub.rs b/crates/taskito-node/src/convert/pubsub.rs new file mode 100644 index 00000000..2e8972f2 --- /dev/null +++ b/crates/taskito-node/src/convert/pubsub.rs @@ -0,0 +1,28 @@ +//! JS-facing shape of a topic subscription. + +use napi_derive::napi; +use taskito_core::storage::models::SubscriptionRow; + +/// A topic subscription: routes messages published to `topic` to `taskName` +/// jobs on `queue`, one delivery per active subscription. +#[napi(object)] +pub struct JsSubscription { + pub topic: String, + pub subscription_name: String, + pub task_name: String, + pub queue: String, + pub active: bool, + pub durable: bool, +} + +/// Convert a core [`SubscriptionRow`] into its JS-facing shape. +pub fn subscription_to_js(row: SubscriptionRow) -> JsSubscription { + JsSubscription { + topic: row.topic, + subscription_name: row.subscription_name, + task_name: row.task_name, + queue: row.queue, + active: row.active, + durable: row.durable, + } +} diff --git a/crates/taskito-node/src/queue/mod.rs b/crates/taskito-node/src/queue/mod.rs index 7025e08b..0833eafe 100644 --- a/crates/taskito-node/src/queue/mod.rs +++ b/crates/taskito-node/src/queue/mod.rs @@ -15,6 +15,7 @@ mod inspect; mod locks; mod logs; mod periodic; +mod pubsub; #[cfg(feature = "workflows")] mod workflows; diff --git a/crates/taskito-node/src/queue/pubsub.rs b/crates/taskito-node/src/queue/pubsub.rs new file mode 100644 index 00000000..f0ce9844 --- /dev/null +++ b/crates/taskito-node/src/queue/pubsub.rs @@ -0,0 +1,221 @@ +//! Topic pub/sub methods on `JsQueue`. Fan-out and registry semantics live in +//! the core (`taskito_core::pubsub`); every method touches storage, so each is +//! async and offloads the blocking I/O to the blocking pool. + +use std::collections::HashMap; + +use napi::bindgen_prelude::{spawn_blocking, Buffer, Result}; +use napi_derive::napi; +use taskito_core::job::now_millis; +use taskito_core::pubsub::{publish_to_topic, DeliveryDefaults, PublishRequest}; +use taskito_core::storage::models::NewSubscriptionRow; +use taskito_core::Storage; + +use super::JsQueue; +use crate::config::{DeliveryDefaultsInput, PublishOptions}; +use crate::convert::{ + job_to_js, subscription_to_js, JsJob, JsSubscription, DEFAULT_MAX_RETRIES, DEFAULT_PRIORITY, + DEFAULT_TIMEOUT_MS, +}; +use crate::error::{join_to_napi_err, non_negative, to_napi_err}; + +#[napi] +impl JsQueue { + /// Insert or update a topic subscription (idempotent on topic + name). + /// Ephemeral subscriptions (`durable: false`) carry the owning worker's id + /// so they can be reaped once that worker is gone. + #[napi] + pub async fn register_subscription( + &self, + topic: String, + subscription_name: String, + task_name: String, + queue: String, + durable: bool, + owner_worker_id: Option, + ) -> Result<()> { + let storage = self.storage.clone(); + spawn_blocking(move || { + let row = NewSubscriptionRow { + topic: &topic, + subscription_name: &subscription_name, + task_name: &task_name, + queue: &queue, + active: true, + durable, + owner_worker_id: owner_worker_id.as_deref(), + created_at: now_millis(), + }; + storage.register_subscription(&row).map_err(to_napi_err) + }) + .await + .map_err(join_to_napi_err)? + } + + /// List subscriptions — all of them, or only a topic's active ones. + #[napi] + pub async fn list_subscriptions(&self, topic: Option) -> Result> { + let storage = self.storage.clone(); + spawn_blocking(move || { + let rows = match topic.as_deref() { + Some(t) => storage.list_subscriptions_for_topic(t), + None => storage.list_subscriptions(), + } + .map_err(to_napi_err)?; + Ok(rows.into_iter().map(subscription_to_js).collect()) + }) + .await + .map_err(join_to_napi_err)? + } + + /// Remove a subscription. Returns false if none matched. + #[napi] + pub async fn unsubscribe(&self, topic: String, subscription_name: String) -> Result { + let storage = self.storage.clone(); + spawn_blocking(move || { + storage + .unsubscribe(&topic, &subscription_name) + .map_err(to_napi_err) + }) + .await + .map_err(join_to_napi_err)? + } + + /// Pause (false) or resume (true) a subscription without unregistering it. + /// Returns false if none matched. + #[napi] + pub async fn set_subscription_active( + &self, + topic: String, + subscription_name: String, + active: bool, + ) -> Result { + let storage = self.storage.clone(); + spawn_blocking(move || { + storage + .set_subscription_active(&topic, &subscription_name, active) + .map_err(to_napi_err) + }) + .await + .map_err(join_to_napi_err)? + } + + /// Drop ephemeral subscriptions whose owning worker is gone. Runs on the + /// heartbeat cadence, after dead workers have been reaped. Returns the + /// number of subscriptions removed. + #[napi] + pub async fn reap_ephemeral_subscriptions(&self) -> Result { + let storage = self.storage.clone(); + spawn_blocking(move || { + let live: Vec = storage + .list_workers() + .map_err(to_napi_err)? + .into_iter() + .map(|worker| worker.worker_id) + .collect(); + storage + .reap_ephemeral_subscriptions(&live) + .map(|n| n as i64) + .map_err(to_napi_err) + }) + .await + .map_err(join_to_napi_err)? + } + + /// Publish an opaque payload to a topic: one job per active subscription. + /// Returns the created jobs — empty when nothing is subscribed (a valid + /// pub/sub no-op, not an error). + #[napi] + pub async fn publish( + &self, + topic: String, + payload: Buffer, + options: Option, + ) -> Result> { + let request = build_publish_request( + topic, + payload.to_vec(), + options.unwrap_or_default(), + self.namespace.clone(), + )?; + let storage = self.storage.clone(); + spawn_blocking(move || { + let jobs = publish_to_topic(&storage, &request).map_err(to_napi_err)?; + Ok(jobs.into_iter().map(job_to_js).collect()) + }) + .await + .map_err(join_to_napi_err)? + } +} + +/// Build the core [`PublishRequest`]: validate signed JS inputs (like +/// `build_new_job` does for enqueue) and leave omitted settings `None` so the +/// core's per-subscriber resolution applies. +fn build_publish_request( + topic: String, + payload: Vec, + opts: PublishOptions, + namespace: Option, +) -> Result { + let now = now_millis(); + let delay = non_negative(opts.delay_ms.unwrap_or(0), "delayMs")?; + let max_retries = match opts.max_retries { + Some(n) => Some(non_negative(n as i64, "maxRetries")? as i32), + None => None, + }; + let timeout_ms = match opts.timeout_ms { + Some(n) => Some(non_negative(n, "timeoutMs")?), + None => None, + }; + let expires_at = match opts.expires_ms { + Some(n) => Some(now.saturating_add(non_negative(n, "expiresMs")?)), + None => None, + }; + let result_ttl_ms = match opts.result_ttl_ms { + Some(n) => Some(non_negative(n, "resultTtlMs")?), + None => None, + }; + let queue_defaults = DeliveryDefaults { + priority: DEFAULT_PRIORITY, + max_retries: DEFAULT_MAX_RETRIES, + timeout_ms: DEFAULT_TIMEOUT_MS, + }; + Ok(PublishRequest { + topic, + payload, + idempotency_key: opts.idempotency_key, + metadata: opts.metadata, + notes: opts.notes, + priority: opts.priority, + // Saturate so an extreme delay can't overflow into a past schedule. + scheduled_at: now.saturating_add(delay), + max_retries, + timeout_ms, + expires_at, + result_ttl_ms, + namespace, + queue_defaults, + task_defaults: task_defaults(opts.task_defaults.unwrap_or_default(), queue_defaults), + }) +} + +/// Resolve per-task delivery defaults, filling unset fields from the queue +/// defaults so the shell never duplicates them. +fn task_defaults( + input: HashMap, + queue_defaults: DeliveryDefaults, +) -> HashMap { + input + .into_iter() + .map(|(name, defaults)| { + ( + name, + DeliveryDefaults { + priority: defaults.priority.unwrap_or(queue_defaults.priority), + max_retries: defaults.max_retries.unwrap_or(queue_defaults.max_retries), + timeout_ms: defaults.timeout_ms.unwrap_or(queue_defaults.timeout_ms), + }, + ) + }) + .collect() +} diff --git a/sdks/node/src/index.ts b/sdks/node/src/index.ts index f50987f2..05cb438f 100644 --- a/sdks/node/src/index.ts +++ b/sdks/node/src/index.ts @@ -87,11 +87,14 @@ export type { Metric, PeriodicOptions, PeriodicTask, + PublishOptions, QueueLimits, RateLimit, ResultOptions, Stats, StreamOptions, + SubscriberOptions, + Subscription, TaskHandler, TaskLog, TaskMap, diff --git a/sdks/node/src/native.ts b/sdks/node/src/native.ts index 094b1d8c..fe1bf35c 100644 --- a/sdks/node/src/native.ts +++ b/sdks/node/src/native.ts @@ -17,6 +17,7 @@ export type NativeWorker = InstanceType; export type { CircuitBreakerInput, + DeliveryDefaultsInput, EnqueueOptions, JobFilter, JsCircuitBreaker, @@ -30,6 +31,7 @@ export type { JsOutcome, JsReplayEntry, JsStats, + JsSubscription, JsTaskInvocation, JsTaskLog, JsWorkerRow, @@ -38,6 +40,7 @@ export type { JsWorkflowRun, MeshWorkerConfig, OpenOptions, + PublishOptions, QueueConfigInput, TaskConfigInput, WorkerOptions, diff --git a/sdks/node/src/queue.ts b/sdks/node/src/queue.ts index 599c75b3..dd76d0c1 100644 --- a/sdks/node/src/queue.ts +++ b/sdks/node/src/queue.ts @@ -25,6 +25,7 @@ import { type Interception, InterceptionMetrics, type Interceptor } from "./inte import { Lock, type LockOptions } from "./locks"; import type { EnqueueContext, Middleware } from "./middleware"; import { + type DeliveryDefaultsInput, JsQueue, type EnqueueOptions as NativeEnqueueOptions, type NativeQueue, @@ -59,12 +60,15 @@ import type { Metric, PeriodicOptions, PeriodicTask, + PublishOptions, QueueLimits, RegisteredTask, ReplayEntry, ResultOptions, Stats, StreamOptions, + SubscriberOptions, + Subscription, TaskLog, TaskMap, TaskOptions, @@ -119,6 +123,7 @@ export class Queue { private readonly serializer: Serializer; private readonly codecs: ReadonlyMap; private readonly tasks = new Map(); + private readonly pendingSubscriptions: PendingSubscription[] = []; private readonly queueLimits = new Map(); private readonly middleware: Middleware[] = []; private readonly interceptors: Interceptor[] = []; @@ -267,6 +272,32 @@ export class Queue { return this as unknown as Queue; } + /** + * Register `handler` as an independent subscriber of `topic`. It becomes a + * normal task named `name` (so retries, DLQ, middleware, and rate limits all + * apply per subscriber), and the subscription is written to storage when a + * worker starts — or via {@link Queue.declareSubscriptions} in a + * producer-only process. `durable: false` ties the subscription to one + * worker: it only registers inside a running worker and is reaped once that + * worker stops heartbeating. + */ + subscriber( + topic: string, + name: Name, + handler: Handler, + options?: SubscriberOptions, + ): Queue> { + const { subscriptionName, queue, durable, ...taskOptions } = options ?? {}; + this.pendingSubscriptions.push({ + topic, + subscriptionName: subscriptionName ?? name, + taskName: name, + queue: queue ?? "default", + durable: durable ?? true, + }); + return this.task(name, handler, taskOptions); + } + /** * Register an injectable resource. Worker-scoped (default) values are built * once and shared across the worker's lifetime; task-scoped values are built @@ -398,6 +429,116 @@ export class Queue { return this.native.enqueueMany(name, prepared); } + // ── Topic pub/sub ───────────────────────────────────────────────── + + /** + * Publish a message to `topic`: every active subscription receives an + * independent job carrying the same serialized `args` (at-least-once per + * subscriber). Returns the created jobs — empty when the topic has no + * active subscribers, a valid pub/sub no-op. `idempotencyKey` dedupes per + * subscriber. Deliveries use the queue-level serializer; per-task codecs + * do not apply. + */ + publish(topic: string, args: unknown[] = [], options?: PublishOptions): Promise { + const { notes, ...rest } = options ?? {}; + const payload = Buffer.from(serializeCall(this.serializer, args)); + return this.native.publish(topic, payload, { + ...rest, + notes: notes === undefined ? undefined : encodeNotes(notes), + taskDefaults: this.deliveryTaskDefaults(), + }); + } + + /** + * Write pending durable subscriptions to storage. Runs automatically at + * worker startup; call it explicitly in a producer-only process (one that + * registers subscribers but never runs a worker) so `publish()` sees them. + * Ephemeral subscriptions are skipped — they need an owning worker. + */ + async declareSubscriptions(): Promise { + for (const subscription of this.pendingSubscriptions) { + if (subscription.durable) { + await this.registerSubscription(subscription, undefined); + } + } + } + + /** Remove a subscription. Resolves false if none matched. */ + unsubscribe(topic: string, name: string): Promise { + return this.native.unsubscribe(topic, name); + } + + /** Stop deliveries without unregistering. Resolves false if unknown. */ + pauseSubscription(topic: string, name: string): Promise { + return this.native.setSubscriptionActive(topic, name, false); + } + + /** Resume a paused subscription. Resolves false if unknown. */ + resumeSubscription(topic: string, name: string): Promise { + return this.native.setSubscriptionActive(topic, name, true); + } + + /** List subscriptions — all of them, or one topic's active ones. */ + listSubscriptions(topic?: string): Promise { + return this.native.listSubscriptions(topic); + } + + /** + * Drop ephemeral subscriptions whose owning worker is gone. Workers run + * this on their heartbeat cadence; exposed for operational tooling. + * Resolves to the number of subscriptions removed. + */ + reapEphemeralSubscriptions(): Promise { + return this.native.reapEphemeralSubscriptions(); + } + + /** Distinct topics that currently have at least one subscription. */ + async listTopics(): Promise { + const topics = new Set(); + for (const subscription of await this.native.listSubscriptions(undefined)) { + topics.add(subscription.topic); + } + return [...topics]; + } + + /** Flush every pending subscription at worker startup, owning the ephemeral ones. */ + private async declareWorkerSubscriptions(workerId: string): Promise { + for (const subscription of this.pendingSubscriptions) { + await this.registerSubscription(subscription, subscription.durable ? undefined : workerId); + } + } + + private registerSubscription( + subscription: PendingSubscription, + ownerWorkerId: string | undefined, + ): Promise { + return this.native.registerSubscription( + subscription.topic, + subscription.subscriptionName, + subscription.taskName, + subscription.queue, + subscription.durable, + ownerWorkerId, + ); + } + + /** + * Per-task delivery defaults from this process's task registry so a publish + * honors each subscriber's own registration options. Unset fields and tasks + * registered elsewhere fall back to queue defaults in the core. + */ + private deliveryTaskDefaults(): Record { + const defaults: Record = {}; + for (const [name, task] of this.tasks) { + const { maxRetries, timeoutMs } = task.options ?? {}; + if (maxRetries === undefined && timeoutMs === undefined) { + continue; + } + defaults[name] = { maxRetries, timeoutMs }; + } + return defaults; + } + /** * Run enqueue interceptors, merge per-task defaults, run the middleware * `onEnqueue` hooks, then serialize the args and encode the options — the @@ -918,11 +1059,21 @@ export class Queue { emitter: this.emitter, resources: this.resources, workflowTracker: this.trackerIfSupported(), + declareSubscriptions: (workerId) => this.declareWorkerSubscriptions(workerId), run: options, }); } } +/** A subscription recorded by {@link Queue.subscriber}, pending storage registration. */ +interface PendingSubscription { + topic: string; + subscriptionName: string; + taskName: string; + queue: string; + durable: boolean; +} + /** Log level used for published partial results (matches the cross-SDK contract). */ const STREAM_LEVEL = "result"; /** Job statuses at which a stream stops. */ diff --git a/sdks/node/src/types.ts b/sdks/node/src/types.ts index 79118ea3..c3474b0e 100644 --- a/sdks/node/src/types.ts +++ b/sdks/node/src/types.ts @@ -2,6 +2,7 @@ import type { CircuitBreakerInput, MeshWorkerConfig, EnqueueOptions as NativeEnqueueOptions, + PublishOptions as NativePublishOptions, } from "./native"; export type { @@ -16,6 +17,7 @@ export type { JsMetric as Metric, JsReplayEntry as ReplayEntry, JsStats as Stats, + JsSubscription as Subscription, JsTaskLog as TaskLog, JsWorkerRow as WorkerInfo, MeshWorkerConfig, @@ -31,6 +33,28 @@ export interface EnqueueOptions extends Omit { notes?: Record; } +/** + * Per-publish options. Mirrors the native options, but `notes` is a structured + * object here (validated and JSON-encoded before it reaches the core) and + * `taskDefaults` is built internally from the queue's task registry. + */ +export interface PublishOptions extends Omit { + /** Structured annotations stamped on every delivery (plus `topic`/`subscription`). */ + notes?: Record; +} + +/** Options for {@link Queue.subscriber}: task options plus subscription routing. */ +export interface SubscriberOptions extends TaskOptions { + /** Stable subscription identity — re-registering the same `(topic, name)` + * updates the routing target instead of duplicating. Defaults to the task name. */ + subscriptionName?: string; + /** Queue the subscriber's delivery jobs go to (default `"default"`). */ + queue?: string; + /** Persist across restarts (default true). `false` = ephemeral: registered + * only by a running worker and reaped once that worker stops heartbeating. */ + durable?: boolean; +} + /** Options for {@link Queue.result}. */ export interface ResultOptions { /** Max time to wait for a terminal state (ms). Default 30000. */ diff --git a/sdks/node/src/worker.ts b/sdks/node/src/worker.ts index 44c34a46..0d032798 100644 --- a/sdks/node/src/worker.ts +++ b/sdks/node/src/worker.ts @@ -53,6 +53,8 @@ export interface WorkerStartParams { resources: ResourceRuntime; /** The queue's shared tracker (undefined on addons without workflows). */ workflowTracker?: WorkflowTracker; + /** Flushes the queue's pending topic subscriptions under this worker's id. */ + declareSubscriptions?: (workerId: string) => Promise; run?: WorkerRunOptions; } @@ -242,7 +244,19 @@ export class Worker { void queue.workerHeartbeat(native.id, snapshot && JSON.stringify(snapshot)).catch((error) => { log.debug(() => "worker heartbeat failed", error); }); + // Same cadence: prune ephemeral subscriptions whose owner is gone. + // Per-tick failures are swallowed like the heartbeat's — the next + // beat retries. + void queue.reapEphemeralSubscriptions().catch((error) => { + log.debug(() => "ephemeral subscription reap failed", error); + }); }; + // Register this worker's topic subscriptions (ephemeral ones under its id) + // now that the id exists. Fire-and-forget like the heartbeat: a failure is + // logged and the worker still runs. + void params.declareSubscriptions?.(native.id).catch((error) => { + log.debug(() => "subscription registration failed", error); + }); sendHeartbeat(); const heartbeat = setInterval(sendHeartbeat, HEARTBEAT_INTERVAL_MS); heartbeat.unref(); diff --git a/sdks/node/test/core/pubsub.test.ts b/sdks/node/test/core/pubsub.test.ts new file mode 100644 index 00000000..b6ec3700 --- /dev/null +++ b/sdks/node/test/core/pubsub.test.ts @@ -0,0 +1,197 @@ +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { Queue, type Worker } from "../../src/index"; + +let worker: Worker | undefined; +afterEach(() => { + worker?.stop(); + worker = undefined; +}); + +function newQueue(): Queue { + return new Queue({ dbPath: join(mkdtempSync(join(tmpdir(), "taskito-pubsub-")), "q.db") }); +} + +async function waitFor( + predicate: () => boolean | Promise, + timeoutMs = 4000, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await predicate()) { + return true; + } + await new Promise((r) => setTimeout(r, 20)); + } + return false; +} + +it("declares and lists subscriptions", async () => { + const queue = newQueue(); + queue.subscriber("orders", "send_email", () => undefined, { subscriptionName: "email" }); + queue.subscriber("orders", "track_order", () => undefined, { subscriptionName: "analytics" }); + await queue.declareSubscriptions(); + + const subs = await queue.listSubscriptions("orders"); + expect(new Set(subs.map((s) => s.subscriptionName))).toEqual(new Set(["email", "analytics"])); + expect(subs.every((s) => s.active && s.durable)).toBe(true); + expect(await queue.listTopics()).toEqual(["orders"]); +}); + +it("subscription name defaults to the task name", async () => { + const queue = newQueue(); + queue.subscriber("orders", "handle_order", () => undefined); + await queue.declareSubscriptions(); + + const [sub] = await queue.listSubscriptions("orders"); + expect(sub?.subscriptionName).toBe("handle_order"); + expect(sub?.taskName).toBe("handle_order"); +}); + +it("redeclaring updates the subscription instead of duplicating", async () => { + const queue = newQueue(); + queue.subscriber("orders", "send_email", () => undefined, { + subscriptionName: "email", + queue: "mail", + }); + await queue.declareSubscriptions(); + await queue.declareSubscriptions(); + + const subs = await queue.listSubscriptions(); + expect(subs).toHaveLength(1); + expect(subs[0]?.queue).toBe("mail"); +}); + +it("unsubscribe removes the subscription and reports a missing one", async () => { + const queue = newQueue(); + queue.subscriber("orders", "send_email", () => undefined, { subscriptionName: "email" }); + await queue.declareSubscriptions(); + + expect(await queue.unsubscribe("orders", "email")).toBe(true); + expect(await queue.unsubscribe("orders", "email")).toBe(false); + expect(await queue.listSubscriptions("orders")).toEqual([]); +}); + +it("pause blocks deliveries and resume restores them", async () => { + const queue = newQueue(); + queue.subscriber("orders", "send_email", () => undefined, { subscriptionName: "email" }); + await queue.declareSubscriptions(); + + expect(await queue.pauseSubscription("orders", "email")).toBe(true); + expect(await queue.publish("orders", [1])).toEqual([]); + expect(await queue.resumeSubscription("orders", "email")).toBe(true); + expect(await queue.publish("orders", [2])).toHaveLength(1); +}); + +it("publishing to a topic without subscribers is a no-op", async () => { + const queue = newQueue(); + expect(await queue.publish("ghost-topic", [1])).toEqual([]); +}); + +it("fans a publish out to every subscriber", async () => { + const queue = newQueue(); + const received = new Map(); + queue.subscriber("orders", "send_email", (orderId: number) => { + received.set("email", orderId); + }); + queue.subscriber("orders", "track_order", (orderId: number) => { + received.set("analytics", orderId); + }); + await queue.declareSubscriptions(); + + const jobs = await queue.publish("orders", [42]); + expect(jobs).toHaveLength(2); + + worker = queue.runWorker(); + expect(await waitFor(() => received.size === 2)).toBe(true); + expect(received.get("email")).toBe(42); + expect(received.get("analytics")).toBe(42); +}); + +it("a failing subscriber does not affect its sibling", async () => { + const queue = newQueue(); + const outcomes: string[] = []; + queue.subscriber( + "orders", + "flaky", + () => { + throw new Error("boom"); + }, + { maxRetries: 0 }, + ); + queue.subscriber("orders", "steady", () => { + outcomes.push("steady"); + }); + await queue.declareSubscriptions(); + + const jobs = await queue.publish("orders", [7]); + expect(jobs).toHaveLength(2); + + worker = queue.runWorker(); + expect(await waitFor(() => outcomes.length === 1)).toBe(true); + + const statusOf = (taskName: string): string | undefined => + queue.getJob(jobs.find((job) => job.taskName === taskName)?.id ?? "")?.status; + expect(await waitFor(() => statusOf("steady") === "complete")).toBe(true); + expect(await waitFor(() => ["failed", "dead"].includes(statusOf("flaky") ?? ""))).toBe(true); +}); + +it("idempotencyKey dedupes per subscriber across republishes", async () => { + const queue = newQueue(); + queue.subscriber("orders", "send_email", () => undefined, { subscriptionName: "email" }); + queue.subscriber("orders", "track_order", () => undefined, { subscriptionName: "analytics" }); + await queue.declareSubscriptions(); + + const first = await queue.publish("orders", [42], { idempotencyKey: "evt-42" }); + const second = await queue.publish("orders", [42], { idempotencyKey: "evt-42" }); + expect(first).toHaveLength(2); + expect(second).toHaveLength(2); + expect(new Set(second.map((job) => job.id))).toEqual(new Set(first.map((job) => job.id))); +}); + +it("deliveries carry topic and subscription notes", async () => { + const queue = newQueue(); + queue.subscriber("orders", "send_email", () => undefined, { subscriptionName: "email" }); + await queue.declareSubscriptions(); + + const [job] = await queue.publish("orders", [1], { notes: { tenant: "acme" } }); + const notes = JSON.parse(job?.notes ?? "{}"); + expect(notes.topic).toBe("orders"); + expect(notes.subscription).toBe("email"); + expect(notes.tenant).toBe("acme"); +}); + +it("declareSubscriptions skips ephemeral subscriptions", async () => { + const queue = newQueue(); + queue.subscriber("orders", "durable_task", () => undefined); + queue.subscriber("orders", "ephemeral_task", () => undefined, { durable: false }); + await queue.declareSubscriptions(); + + const subs = await queue.listSubscriptions("orders"); + expect(subs.map((s) => s.subscriptionName)).toEqual(["durable_task"]); +}); + +it("reapEphemeralSubscriptions removes dead-owner rows only", async () => { + const queue = newQueue(); + queue.subscriber("orders", "durable_task", () => undefined); + queue.subscriber("orders", "ephemeral_task", () => undefined, { durable: false }); + + // A running worker flushes both, owning the ephemeral one. + worker = queue.runWorker(); + expect(await waitFor(async () => (await queue.listSubscriptions("orders")).length === 2)).toBe( + true, + ); + // Owner alive: the reap must not touch its ephemeral subscription. + expect(await queue.reapEphemeralSubscriptions()).toBe(0); + + // Stop the worker (unregisters it) — the orphaned ephemeral row is reaped. + worker.stop(); + worker = undefined; + expect(await waitFor(async () => (await queue.listWorkers()).length === 0)).toBe(true); + expect(await queue.reapEphemeralSubscriptions()).toBe(1); + + const remaining = await queue.listSubscriptions("orders"); + expect(remaining.map((s) => s.subscriptionName)).toEqual(["durable_task"]); +}); From 01859138b941d72fb3e10868df20c9c85fc40c8c Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:35:44 +0530 Subject: [PATCH 06/12] feat(java): add topic subscriber and publish API --- crates/taskito-java/src/backend.rs | 13 + crates/taskito-java/src/convert.rs | 126 +++++++++- crates/taskito-java/src/queue/mod.rs | 1 + crates/taskito-java/src/queue/pubsub.rs | 165 ++++++++++++ crates/taskito-java/src/worker.rs | 54 +++- .../org/byteveda/taskito/DefaultTaskito.java | 122 ++++++++- .../java/org/byteveda/taskito/Taskito.java | 55 ++++ .../taskito/internal/JniQueueBackend.java | 40 +++ .../taskito/internal/NativeQueue.java | 27 ++ .../byteveda/taskito/model/Subscription.java | 39 +++ .../taskito/pubsub/PublishOptions.java | 192 ++++++++++++++ .../taskito/pubsub/SubscriptionConfig.java | 72 ++++++ .../taskito/pubsub/SubscriptionOptions.java | 76 ++++++ .../byteveda/taskito/pubsub/package-info.java | 9 + .../byteveda/taskito/spi/QueueBackend.java | 44 ++++ .../byteveda/taskito/task/EnqueueOptions.java | 20 ++ .../org/byteveda/taskito/worker/Worker.java | 30 +++ .../org/byteveda/taskito/core/PubSubTest.java | 238 ++++++++++++++++++ .../taskito/test/InMemoryQueueBackend.java | 224 ++++++++++++++++- .../taskito/test/InMemoryPubSubTest.java | 178 +++++++++++++ 20 files changed, 1717 insertions(+), 8 deletions(-) create mode 100644 crates/taskito-java/src/queue/pubsub.rs create mode 100644 sdks/java/src/main/java/org/byteveda/taskito/model/Subscription.java create mode 100644 sdks/java/src/main/java/org/byteveda/taskito/pubsub/PublishOptions.java create mode 100644 sdks/java/src/main/java/org/byteveda/taskito/pubsub/SubscriptionConfig.java create mode 100644 sdks/java/src/main/java/org/byteveda/taskito/pubsub/SubscriptionOptions.java create mode 100644 sdks/java/src/main/java/org/byteveda/taskito/pubsub/package-info.java create mode 100644 sdks/java/src/test/java/org/byteveda/taskito/core/PubSubTest.java create mode 100644 sdks/java/test-support/src/test/java/org/byteveda/taskito/test/InMemoryPubSubTest.java diff --git a/crates/taskito-java/src/backend.rs b/crates/taskito-java/src/backend.rs index 955927e2..f7b96d07 100644 --- a/crates/taskito-java/src/backend.rs +++ b/crates/taskito-java/src/backend.rs @@ -107,6 +107,19 @@ pub fn enqueue_batch_dedup( Ok(ids.into_iter().flatten().collect()) } +/// Drop ephemeral topic subscriptions whose owning worker is no longer in the +/// worker registry. Durable rows and rows owned by a live worker are untouched. +/// Callers prune dead workers first — a stale registry row keeps its +/// subscriptions alive. Returns the count removed. +pub fn reap_ephemeral_subscriptions(storage: &StorageBackend) -> Result { + let live: Vec = storage + .list_workers()? + .into_iter() + .map(|worker| worker.worker_id) + .collect(); + Ok(storage.reap_ephemeral_subscriptions(&live)?) +} + /// Error for a backend that is unknown or whose cargo feature is not compiled in. fn unknown_backend(name: &str) -> BindingError { BindingError::new(format!( diff --git a/crates/taskito-java/src/convert.rs b/crates/taskito-java/src/convert.rs index b06d742f..b93844af 100644 --- a/crates/taskito-java/src/convert.rs +++ b/crates/taskito-java/src/convert.rs @@ -3,12 +3,15 @@ //! Option and filter structs cross as JSON strings (decoded here); opaque job //! payloads cross as raw `byte[]` and are never interpreted by the core. +use std::collections::HashMap; + use serde::{Deserialize, Serialize}; use taskito_core::job::{now_millis, Job, NewJob}; +use taskito_core::pubsub::{DeliveryDefaults, PublishRequest}; use taskito_core::resilience::circuit_breaker::CircuitState; use taskito_core::storage::models::{ - CircuitBreakerRow, JobErrorRow, LockInfoRow, PeriodicTaskRow, ReplayHistoryRow, TaskLogRow, - TaskMetricRow, WorkerRow, + CircuitBreakerRow, JobErrorRow, LockInfoRow, PeriodicTaskRow, ReplayHistoryRow, + SubscriptionRow, TaskLogRow, TaskMetricRow, WorkerRow, }; use taskito_core::storage::{DeadJob, QueueStats}; @@ -85,6 +88,82 @@ pub fn build_new_job( } } +/// Options accepted by `NativeQueue.publish`. Delivery fields left unset +/// resolve per subscriber (its own task defaults, then the queue defaults). +#[derive(Deserialize, Default)] +#[serde(rename_all = "camelCase", default)] +pub struct PublishOptions { + pub idempotency_key: Option, + pub metadata: Option, + /// Pre-encoded canonical notes JSON; the core stamps topic/subscription in. + pub notes: Option, + pub priority: Option, + pub delay_ms: Option, + pub max_retries: Option, + pub timeout_ms: Option, + pub expires_ms: Option, + pub result_ttl_ms: Option, + /// Per-task delivery settings from the SDK's task registry, keyed by task name. + pub task_defaults: Option>, +} + +/// A subscriber task's own delivery settings. Absent fields mirror +/// [`build_new_job`]'s zero defaults so a delivery resolves like a plain enqueue. +#[derive(Deserialize, Default, Clone, Copy)] +#[serde(rename_all = "camelCase", default)] +pub struct TaskDeliveryDefaults { + pub priority: i32, + pub max_retries: i32, + pub timeout_ms: i64, +} + +/// Build a core [`PublishRequest`] from a publish call. The queue-level +/// fallback is [`build_new_job`]'s zero defaults — the Java shell keeps no +/// queue-wide delivery config, so enqueue and publish resolve identically. +pub fn build_publish_request( + topic: String, + payload: Vec, + options: PublishOptions, + default_namespace: Option<&str>, +) -> PublishRequest { + let now = now_millis(); + PublishRequest { + topic, + payload, + idempotency_key: options.idempotency_key, + metadata: options.metadata, + notes: options.notes, + priority: options.priority, + // Saturate rather than wrap/panic on an absurd delay or expiry. + scheduled_at: now.saturating_add(options.delay_ms.unwrap_or(0).max(0)), + max_retries: options.max_retries, + timeout_ms: options.timeout_ms, + expires_at: options.expires_ms.map(|ms| now.saturating_add(ms.max(0))), + result_ttl_ms: options.result_ttl_ms, + namespace: default_namespace.map(str::to_string), + queue_defaults: DeliveryDefaults { + priority: 0, + max_retries: 0, + timeout_ms: 0, + }, + task_defaults: options + .task_defaults + .unwrap_or_default() + .into_iter() + .map(|(name, task)| { + ( + name, + DeliveryDefaults { + priority: task.priority, + max_retries: task.max_retries, + timeout_ms: task.timeout_ms, + }, + ) + }) + .collect(), + } +} + /// Job-count snapshot returned by `NativeQueue.stats`. #[derive(Serialize)] #[serde(rename_all = "camelCase")] @@ -163,6 +242,35 @@ impl<'a> From<&'a Job> for JobView<'a> { } } +/// Java-facing view of a topic subscription. `created_at` is Unix milliseconds. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SubscriptionView<'a> { + pub topic: &'a str, + pub subscription_name: &'a str, + pub task_name: &'a str, + pub queue: &'a str, + pub active: bool, + pub durable: bool, + pub owner_worker_id: Option<&'a str>, + pub created_at: i64, +} + +impl<'a> From<&'a SubscriptionRow> for SubscriptionView<'a> { + fn from(r: &'a SubscriptionRow) -> Self { + Self { + topic: &r.topic, + subscription_name: &r.subscription_name, + task_name: &r.task_name, + queue: &r.queue, + active: r.active, + durable: r.durable, + owner_worker_id: r.owner_worker_id.as_deref(), + created_at: r.created_at, + } + } +} + /// Java-facing view of a task's circuit-breaker state. `state` is the lowercase wire /// string (`closed`/`open`/`half_open`); timestamps are Unix milliseconds. #[derive(Serialize)] @@ -212,6 +320,20 @@ pub struct WorkerOptions { /// dispatcher; otherwise this is ignored. #[cfg_attr(not(feature = "mesh"), allow(dead_code))] pub mesh_config: Option, + /// Topic subscriptions written at worker start; ephemeral entries bind to + /// the started worker's id. + pub subscriptions: Option>, +} + +/// One topic subscription declared by the SDK, registered at worker start. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SubscriptionSpec { + pub topic: String, + pub subscription_name: String, + pub task_name: String, + pub queue: String, + pub durable: bool, } /// A task's retry-backoff curve. Fields left unset fall back to the core's diff --git a/crates/taskito-java/src/queue/mod.rs b/crates/taskito-java/src/queue/mod.rs index c5fc0a70..75eb528a 100644 --- a/crates/taskito-java/src/queue/mod.rs +++ b/crates/taskito-java/src/queue/mod.rs @@ -11,6 +11,7 @@ mod inspect; mod locks; mod logs; mod periodic; +mod pubsub; use jni::objects::{JByteArray, JClass, JObjectArray, JString}; use jni::sys::{jboolean, jbyteArray, jint, jlong, jobjectArray, jstring, JNI_FALSE, JNI_TRUE}; diff --git a/crates/taskito-java/src/queue/pubsub.rs b/crates/taskito-java/src/queue/pubsub.rs new file mode 100644 index 00000000..85551b61 --- /dev/null +++ b/crates/taskito-java/src/queue/pubsub.rs @@ -0,0 +1,165 @@ +//! Pub/sub JNI entry points: the topic-subscription registry and `publish`. +//! +//! Fan-out semantics (per-subscriber idempotency-key salting, notes stamping, +//! delivery-settings resolution) live in the core's `publish_to_topic`; this +//! module only marshals arguments and views. + +use jni::objects::{JByteArray, JClass, JString}; +use jni::sys::{jboolean, jlong, jstring, JNI_FALSE}; +use jni::JNIEnv; +use taskito_core::job::now_millis; +use taskito_core::pubsub::publish_to_topic; +use taskito_core::storage::models::NewSubscriptionRow; +use taskito_core::Storage; + +use crate::backend; +use crate::convert::{ + build_publish_request, parse_json, to_json, JobView, PublishOptions, SubscriptionView, +}; +use crate::ffi::{guard, new_string, read_bytes, read_optional_string, read_string}; + +use super::{borrow_queue, to_jboolean}; + +/// `void registerSubscription(long handle, String topic, String subscriptionName, +/// String taskName, String queue, boolean durable, String ownerWorkerIdOrNull)` +/// — insert or update a subscription (idempotent on topic + name). +#[no_mangle] +pub extern "system" fn Java_org_byteveda_taskito_internal_NativeQueue_registerSubscription< + 'local, +>( + mut env: JNIEnv<'local>, + _class: JClass<'local>, + handle: jlong, + topic: JString<'local>, + subscription_name: JString<'local>, + task_name: JString<'local>, + queue: JString<'local>, + durable: jboolean, + owner_worker_id: JString<'local>, +) { + guard(&mut env, (), |env| { + let queue_handle = unsafe { borrow_queue(handle) }; + let topic = read_string(env, &topic)?; + let subscription_name = read_string(env, &subscription_name)?; + let task_name = read_string(env, &task_name)?; + let queue = read_string(env, &queue)?; + let owner_worker_id = read_optional_string(env, &owner_worker_id)?; + let row = NewSubscriptionRow { + topic: &topic, + subscription_name: &subscription_name, + task_name: &task_name, + queue: &queue, + active: true, + durable: durable != 0, + owner_worker_id: owner_worker_id.as_deref(), + created_at: now_millis(), + }; + queue_handle.storage.register_subscription(&row)?; + Ok(()) + }) +} + +/// `String listSubscriptions(long handle, String topicOrNull)` — a JSON array +/// of subscriptions: all of them, or only a topic's active ones. +#[no_mangle] +pub extern "system" fn Java_org_byteveda_taskito_internal_NativeQueue_listSubscriptions<'local>( + mut env: JNIEnv<'local>, + _class: JClass<'local>, + handle: jlong, + topic: JString<'local>, +) -> jstring { + guard(&mut env, std::ptr::null_mut(), |env| { + let queue = unsafe { borrow_queue(handle) }; + let rows = match read_optional_string(env, &topic)? { + Some(topic) => queue.storage.list_subscriptions_for_topic(&topic)?, + None => queue.storage.list_subscriptions()?, + }; + let views: Vec = rows.iter().map(SubscriptionView::from).collect(); + new_string(env, to_json(&views)?) + }) +} + +/// `boolean unsubscribe(long handle, String topic, String subscriptionName)` — +/// remove a subscription; false if none matched. +#[no_mangle] +pub extern "system" fn Java_org_byteveda_taskito_internal_NativeQueue_unsubscribe<'local>( + mut env: JNIEnv<'local>, + _class: JClass<'local>, + handle: jlong, + topic: JString<'local>, + subscription_name: JString<'local>, +) -> jboolean { + guard(&mut env, JNI_FALSE, |env| { + let queue = unsafe { borrow_queue(handle) }; + let topic = read_string(env, &topic)?; + let subscription_name = read_string(env, &subscription_name)?; + Ok(to_jboolean( + queue.storage.unsubscribe(&topic, &subscription_name)?, + )) + }) +} + +/// `boolean setSubscriptionActive(long handle, String topic, String subscriptionName, +/// boolean active)` — pause (false) or resume (true) without unregistering; +/// false if none matched. +#[no_mangle] +pub extern "system" fn Java_org_byteveda_taskito_internal_NativeQueue_setSubscriptionActive< + 'local, +>( + mut env: JNIEnv<'local>, + _class: JClass<'local>, + handle: jlong, + topic: JString<'local>, + subscription_name: JString<'local>, + active: jboolean, +) -> jboolean { + guard(&mut env, JNI_FALSE, |env| { + let queue = unsafe { borrow_queue(handle) }; + let topic = read_string(env, &topic)?; + let subscription_name = read_string(env, &subscription_name)?; + Ok(to_jboolean(queue.storage.set_subscription_active( + &topic, + &subscription_name, + active != 0, + )?)) + }) +} + +/// `long reapEphemeralSubscriptions(long handle)` — drop ephemeral subscriptions +/// whose owning worker is gone; returns the count removed. +#[no_mangle] +pub extern "system" fn Java_org_byteveda_taskito_internal_NativeQueue_reapEphemeralSubscriptions( + mut env: JNIEnv, + _class: JClass, + handle: jlong, +) -> jlong { + guard(&mut env, 0, |_env| { + let queue = unsafe { borrow_queue(handle) }; + Ok(backend::reap_ephemeral_subscriptions(&queue.storage)? as jlong) + }) +} + +/// `String publish(long handle, String topic, byte[] payload, String optionsJson)` +/// — fan a payload out to every active subscription of the topic. Returns the +/// created jobs as a JSON array — empty when nothing is subscribed. +#[no_mangle] +pub extern "system" fn Java_org_byteveda_taskito_internal_NativeQueue_publish<'local>( + mut env: JNIEnv<'local>, + _class: JClass<'local>, + handle: jlong, + topic: JString<'local>, + payload: JByteArray<'local>, + options_json: JString<'local>, +) -> jstring { + guard(&mut env, std::ptr::null_mut(), |env| { + let queue = unsafe { borrow_queue(handle) }; + let topic = read_string(env, &topic)?; + let payload = read_bytes(env, &payload)?; + let raw_opts = read_string(env, &options_json)?; + let options: PublishOptions = parse_json(&raw_opts, "publish options")?; + let request = build_publish_request(topic, payload, options, queue.namespace.as_deref()); + let jobs = publish_to_topic(&queue.storage, &request)?; + let views: Vec = jobs.iter().map(JobView::from).collect(); + new_string(env, to_json(&views)?) + }) +} diff --git a/crates/taskito-java/src/worker.rs b/crates/taskito-java/src/worker.rs index c785d87d..66c9528d 100644 --- a/crates/taskito-java/src/worker.rs +++ b/crates/taskito-java/src/worker.rs @@ -19,8 +19,11 @@ use taskito_core::worker::WorkerDispatcher; use taskito_core::{Scheduler, SchedulerConfig, Storage, StorageBackend}; use tokio::sync::Notify; +use taskito_core::job::now_millis; +use taskito_core::storage::models::NewSubscriptionRow; + use crate::backend::QueueHandle; -use crate::convert::{parse_json, TaskRetryConfig, WorkerOptions}; +use crate::convert::{parse_json, SubscriptionSpec, TaskRetryConfig, WorkerOptions}; use crate::dispatcher::{JavaDispatcher, Registry, TaskOutcome}; use crate::ffi::{guard, read_bytes, read_string}; use crate::handle::{self, drop_handle, into_handle}; @@ -50,7 +53,7 @@ pub struct WorkerHandle { fn start_worker( storage: StorageBackend, namespace: Option, - options: WorkerOptions, + mut options: WorkerOptions, callbacks: GlobalRef, ) -> Result { let runtime = tokio::runtime::Builder::new_multi_thread() @@ -82,6 +85,11 @@ fn start_worker( #[cfg(feature = "mesh")] let mesh_worker_id = worker_id.clone(); + // Write topic subscriptions before the scheduler starts, so its first poll + // can already see deliveries. A registration failure aborts the start — + // a silently missing subscription would drop deliveries. + register_subscriptions(&storage, &worker_id, options.subscriptions.take())?; + let mut scheduler = Scheduler::new(storage, queues, config, namespace); // Claim execution under this worker's id so dead-worker recovery can // attribute orphaned jobs (matches the register_worker id below). @@ -159,6 +167,34 @@ fn start_worker( }) } +/// Register the worker's declared topic subscriptions. Durable rows carry no +/// owner; ephemeral rows bind to this worker's id and are reaped once the +/// worker leaves the registry. +fn register_subscriptions( + storage: &StorageBackend, + worker_id: &str, + specs: Option>, +) -> Result<(), crate::error::BindingError> { + let Some(specs) = specs else { + return Ok(()); + }; + let created_at = now_millis(); + for spec in &specs { + let row = NewSubscriptionRow { + topic: &spec.topic, + subscription_name: &spec.subscription_name, + task_name: &spec.task_name, + queue: &spec.queue, + active: true, + durable: spec.durable, + owner_worker_id: (!spec.durable).then_some(worker_id), + created_at, + }; + storage.register_subscription(&row)?; + } + Ok(()) +} + /// Register each task's retry-backoff curve with the scheduler. Only the curve /// is set here — the core resolves the retry budget from the job's `max_retries` /// — so unset fields keep the core [`RetryPolicy`] defaults. @@ -351,12 +387,26 @@ fn spawn_lifecycle( if let Err(e) = storage.heartbeat(&worker_id, None) { log::warn!("[taskito-java] worker heartbeat failed: {e}"); } + // Prune stale workers first: the ephemeral-subscription reap + // treats every registry row as live, so a lingering dead + // worker would keep its subscriptions receiving deliveries. + if let Err(e) = storage.reap_dead_workers() { + log::warn!("[taskito-java] dead-worker reap failed: {e}"); + } + if let Err(e) = crate::backend::reap_ephemeral_subscriptions(&storage) { + log::warn!("[taskito-java] ephemeral subscription reap failed: {e}"); + } } } } if let Err(e) = storage.unregister_worker(&worker_id) { log::warn!("[taskito-java] worker unregister failed: {e}"); } + // This worker just left the registry, so its own ephemeral subscriptions + // are now dead-owned; reap immediately instead of waiting for a peer. + if let Err(e) = crate::backend::reap_ephemeral_subscriptions(&storage) { + log::warn!("[taskito-java] ephemeral subscription reap failed: {e}"); + } }); } diff --git a/sdks/java/src/main/java/org/byteveda/taskito/DefaultTaskito.java b/sdks/java/src/main/java/org/byteveda/taskito/DefaultTaskito.java index a993ade1..571cf914 100644 --- a/sdks/java/src/main/java/org/byteveda/taskito/DefaultTaskito.java +++ b/sdks/java/src/main/java/org/byteveda/taskito/DefaultTaskito.java @@ -8,6 +8,7 @@ import java.util.Base64; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -38,6 +39,7 @@ import org.byteveda.taskito.model.PeriodicInfo; import org.byteveda.taskito.model.QueueStats; import org.byteveda.taskito.model.ReplayEntry; +import org.byteveda.taskito.model.Subscription; import org.byteveda.taskito.model.TaskLog; import org.byteveda.taskito.model.TaskMetric; import org.byteveda.taskito.model.WorkerInfo; @@ -46,6 +48,9 @@ import org.byteveda.taskito.predicates.EnqueueGate; import org.byteveda.taskito.predicates.Predicate; import org.byteveda.taskito.predicates.PredicateContext; +import org.byteveda.taskito.pubsub.PublishOptions; +import org.byteveda.taskito.pubsub.SubscriptionConfig; +import org.byteveda.taskito.pubsub.SubscriptionOptions; import org.byteveda.taskito.resources.PoolConfig; import org.byteveda.taskito.resources.ResourceContext; import org.byteveda.taskito.resources.ResourceDefinition; @@ -83,6 +88,7 @@ final class DefaultTaskito implements Taskito { private final ResourceRuntime resources = new ResourceRuntime(); private final Map> gates = new ConcurrentHashMap<>(); private final List interceptors = new CopyOnWriteArrayList<>(); + private final List subscriptions = new CopyOnWriteArrayList<>(); DefaultTaskito(QueueBackend backend, Serializer serializer, Map codecs) { this.backend = backend; @@ -659,6 +665,117 @@ public boolean resumePeriodic(String name) { return backend.setPeriodicEnabled(name, true); } + // ── Pub/Sub ───────────────────────────────────────────────────── + + @Override + public Taskito subscribe(String topic, Task task) { + return subscribe(topic, task, SubscriptionOptions.none()); + } + + @Override + public Taskito subscribe(String topic, Task task, SubscriptionOptions options) { + String name = options.name() != null ? options.name() : task.name(); + EnqueueOptions taskDefaults = task.options(); + subscriptions.add(new SubscriptionConfig( + topic, + name, + task.name(), + options.queue(), + options.durable(), + taskDefaults.priority(), + taskDefaults.maxRetries(), + taskDefaults.timeoutMs())); + // A durable subscription registers now so producer-only processes see it; + // an ephemeral one waits for a worker start to bind to that worker's id. + if (options.durable()) { + backend.registerSubscription(topic, name, task.name(), options.queue(), true, null); + } + return this; + } + + @Override + public List publish(String topic, Object payload) { + return publish(topic, payload, PublishOptions.none()); + } + + @Override + public List publish(String topic, Object payload, PublishOptions options) { + // Deliveries use the queue-level serializer: there is one payload for all + // subscribers, so per-task codec chains don't apply to topic tasks. + byte[] payloadBytes = serializer.serializeCall(payload); + return decodeList(backend.publishJson(topic, payloadBytes, encode(publishRequest(options))), Job.class); + } + + /** + * The wire shape of a publish: the caller's options plus each locally + * subscribed task's own delivery settings, so deliveries honor per-task + * defaults. Tasks subscribed elsewhere fall back to the core defaults. + */ + private Map publishRequest(PublishOptions options) { + Map request = new LinkedHashMap<>(); + putIfSet(request, "idempotencyKey", options.idempotencyKey()); + putIfSet(request, "metadata", options.metadata()); + putIfSet(request, "notes", options.notes()); + putIfSet(request, "priority", options.priority()); + putIfSet(request, "delayMs", options.delayMs()); + putIfSet(request, "maxRetries", options.maxRetries()); + putIfSet(request, "timeoutMs", options.timeoutMs()); + putIfSet(request, "expiresMs", options.expiresMs()); + putIfSet(request, "resultTtlMs", options.resultTtlMs()); + Map> taskDefaults = new LinkedHashMap<>(); + for (SubscriptionConfig config : subscriptions) { + Map defaults = new LinkedHashMap<>(); + defaults.put("priority", config.taskPriority() == null ? 0 : config.taskPriority()); + defaults.put("maxRetries", config.taskMaxRetries() == null ? 0 : config.taskMaxRetries()); + defaults.put("timeoutMs", config.taskTimeoutMs() == null ? 0L : config.taskTimeoutMs()); + taskDefaults.put(config.taskName(), defaults); + } + if (!taskDefaults.isEmpty()) { + request.put("taskDefaults", taskDefaults); + } + return request; + } + + private static void putIfSet(Map request, String key, Object value) { + if (value != null) { + request.put(key, value); + } + } + + @Override + public boolean unsubscribe(String topic, String name) { + return backend.unsubscribe(topic, name); + } + + @Override + public boolean pauseSubscription(String topic, String name) { + return backend.setSubscriptionActive(topic, name, false); + } + + @Override + public boolean resumeSubscription(String topic, String name) { + return backend.setSubscriptionActive(topic, name, true); + } + + @Override + public List listSubscriptions() { + return decodeList(backend.listSubscriptionsJson(null), Subscription.class); + } + + @Override + public List listSubscriptions(String topic) { + return decodeList(backend.listSubscriptionsJson(topic), Subscription.class); + } + + @Override + public List listTopics() { + Set topics = new LinkedHashSet<>(); + for (Subscription subscription : listSubscriptions()) { + topics.add(subscription.topic); + } + return new ArrayList<>(topics); + } + // ── Workflows ─────────────────────────────────────────────────── @Override @@ -901,7 +1018,10 @@ private static String encodeGate(GateConfig gate) { @Override public Worker.Builder worker() { // Each worker gets its own runtime (own WORKER-scoped cache) over shared definitions. - return Worker.builder(backend, serializer, middleware, resources.forWorker(), codecs); + // The live subscription list is shared so subscribe() calls made before + // start() are registered under the started worker's id. + return Worker.builder(backend, serializer, middleware, resources.forWorker(), codecs) + .subscriptions(subscriptions); } @Override diff --git a/sdks/java/src/main/java/org/byteveda/taskito/Taskito.java b/sdks/java/src/main/java/org/byteveda/taskito/Taskito.java index d4b59f3c..00873c3e 100644 --- a/sdks/java/src/main/java/org/byteveda/taskito/Taskito.java +++ b/sdks/java/src/main/java/org/byteveda/taskito/Taskito.java @@ -31,12 +31,15 @@ import org.byteveda.taskito.model.PeriodicInfo; import org.byteveda.taskito.model.QueueStats; import org.byteveda.taskito.model.ReplayEntry; +import org.byteveda.taskito.model.Subscription; import org.byteveda.taskito.model.TaskLog; import org.byteveda.taskito.model.TaskMetric; import org.byteveda.taskito.model.WorkerInfo; import org.byteveda.taskito.model.WorkflowRunInfo; import org.byteveda.taskito.predicates.EnqueueGate; import org.byteveda.taskito.predicates.Predicate; +import org.byteveda.taskito.pubsub.PublishOptions; +import org.byteveda.taskito.pubsub.SubscriptionOptions; import org.byteveda.taskito.resources.PoolConfig; import org.byteveda.taskito.resources.ResourceContext; import org.byteveda.taskito.resources.ResourceScope; @@ -275,6 +278,58 @@ static Builder builder() { /** Resume a paused periodic task; false if none had that name. */ boolean resumePeriodic(String name); + // ── Pub/Sub ───────────────────────────────────────────────────── + + /** + * Subscribe {@code task} to {@code topic} as an independent, durable + * subscriber named after the task. Every {@link #publish} to the topic then + * enqueues one ordinary job of this task; register a handler for it on the + * worker as usual. Returns {@code this}. + */ + Taskito subscribe(String topic, Task task); + + /** + * As {@link #subscribe(String, Task)} with explicit {@link SubscriptionOptions}. + * A durable subscription registers immediately; an ephemeral one + * ({@code durable(false)}) binds to a worker and registers when that worker + * starts, disappearing once it stops heartbeating. + */ + Taskito subscribe(String topic, Task task, SubscriptionOptions options); + + /** + * Publish a message to {@code topic}: one job per active subscription, each + * carrying the same serialized payload. Returns the created jobs — empty + * when the topic has no active subscribers (a valid no-op). Each delivery's + * notes carry {@code topic} and {@code subscription} for filtering. + */ + List publish(String topic, Object payload); + + /** + * As {@link #publish(String, Object)} with {@link PublishOptions}. An + * {@code idempotencyKey} dedupes per subscriber: republishing the same key + * yields no new deliveries, and a subscription added later still gets its + * own copy. + */ + List publish(String topic, Object payload, PublishOptions options); + + /** Remove a subscription; false if none matched. */ + boolean unsubscribe(String topic, String name); + + /** Stop deliveries without unregistering; false if none matched. */ + boolean pauseSubscription(String topic, String name); + + /** Resume a paused subscription; false if none matched. */ + boolean resumeSubscription(String topic, String name); + + /** Every registered subscription (active or paused), across all topics. */ + List listSubscriptions(); + + /** One topic's active subscriptions. */ + List listSubscriptions(String topic); + + /** Distinct topics that currently have at least one subscription. */ + List listTopics(); + // ── Workflows ─────────────────────────────────────────────────── /** Submit a workflow DAG; returns a handle to the run. */ diff --git a/sdks/java/src/main/java/org/byteveda/taskito/internal/JniQueueBackend.java b/sdks/java/src/main/java/org/byteveda/taskito/internal/JniQueueBackend.java index 730d01dd..7690e07b 100644 --- a/sdks/java/src/main/java/org/byteveda/taskito/internal/JniQueueBackend.java +++ b/sdks/java/src/main/java/org/byteveda/taskito/internal/JniQueueBackend.java @@ -286,6 +286,46 @@ public boolean setPeriodicEnabled(String name, boolean enabled) { return withOpenHandle(() -> NativeQueue.setPeriodicEnabled(handle, name, enabled)); } + @Override + public void registerSubscription( + String topic, + String subscriptionName, + String taskName, + String queue, + boolean durable, + String ownerWorkerIdOrNull) { + withOpenHandle(() -> { + NativeQueue.registerSubscription( + handle, topic, subscriptionName, taskName, queue, durable, ownerWorkerIdOrNull); + return null; + }); + } + + @Override + public String listSubscriptionsJson(String topicOrNull) { + return withOpenHandle(() -> NativeQueue.listSubscriptions(handle, topicOrNull)); + } + + @Override + public boolean unsubscribe(String topic, String subscriptionName) { + return withOpenHandle(() -> NativeQueue.unsubscribe(handle, topic, subscriptionName)); + } + + @Override + public boolean setSubscriptionActive(String topic, String subscriptionName, boolean active) { + return withOpenHandle(() -> NativeQueue.setSubscriptionActive(handle, topic, subscriptionName, active)); + } + + @Override + public long reapEphemeralSubscriptions() { + return withOpenHandle(() -> NativeQueue.reapEphemeralSubscriptions(handle)); + } + + @Override + public String publishJson(String topic, byte[] payload, String optionsJson) { + return withOpenHandle(() -> NativeQueue.publish(handle, topic, payload, optionsJson)); + } + @Override public String submitWorkflow( String name, diff --git a/sdks/java/src/main/java/org/byteveda/taskito/internal/NativeQueue.java b/sdks/java/src/main/java/org/byteveda/taskito/internal/NativeQueue.java index c84bb6d7..d7f36855 100644 --- a/sdks/java/src/main/java/org/byteveda/taskito/internal/NativeQueue.java +++ b/sdks/java/src/main/java/org/byteveda/taskito/internal/NativeQueue.java @@ -135,6 +135,33 @@ public static native long registerPeriodic( /** Pause (false) or resume (true) a periodic task; false if none had that name. */ public static native boolean setPeriodicEnabled(long handle, String name, boolean enabled); + // ── Pub/Sub ───────────────────────────────────────────────────── + /** Insert or update a topic subscription (idempotent on topic + name). */ + public static native void registerSubscription( + long handle, + String topic, + String subscriptionName, + String taskName, + String queue, + boolean durable, + String ownerWorkerIdOrNull); + + /** A JSON array of subscriptions — all of them, or only a topic's active ones. */ + public static native String listSubscriptions(long handle, String topicOrNull); + + /** Remove a subscription; false if none matched. */ + public static native boolean unsubscribe(long handle, String topic, String subscriptionName); + + /** Pause (false) or resume (true) a subscription; false if none matched. */ + public static native boolean setSubscriptionActive( + long handle, String topic, String subscriptionName, boolean active); + + /** Drop ephemeral subscriptions whose owning worker is gone; returns the count removed. */ + public static native long reapEphemeralSubscriptions(long handle); + + /** Publish to a topic; returns the created delivery jobs as a JSON array. */ + public static native String publish(long handle, String topic, byte[] payload, String optionsJson); + // ── Worker ────────────────────────────────────────────────────── /** Start a worker; returns its handle. {@code bridge} is a {@code WorkerBridge}. */ public static native long runWorker(long handle, Object bridge, String optionsJson); diff --git a/sdks/java/src/main/java/org/byteveda/taskito/model/Subscription.java b/sdks/java/src/main/java/org/byteveda/taskito/model/Subscription.java new file mode 100644 index 00000000..47747c1e --- /dev/null +++ b/sdks/java/src/main/java/org/byteveda/taskito/model/Subscription.java @@ -0,0 +1,39 @@ +package org.byteveda.taskito.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** A topic subscription: routes messages published to {@link #topic} to {@link #taskName}. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public final class Subscription { + public final String topic; + + /** Stable subscription identity; unique per topic. */ + public final String name; + + public final String taskName; + public final String queue; + + /** Whether the subscription currently receives deliveries (false = paused). */ + public final boolean active; + + /** Whether the registration persists across restarts (false = ephemeral). */ + public final boolean durable; + + @JsonCreator + public Subscription( + @JsonProperty("topic") String topic, + @JsonProperty("subscriptionName") String name, + @JsonProperty("taskName") String taskName, + @JsonProperty("queue") String queue, + @JsonProperty("active") boolean active, + @JsonProperty("durable") boolean durable) { + this.topic = topic; + this.name = name; + this.taskName = taskName; + this.queue = queue; + this.active = active; + this.durable = durable; + } +} diff --git a/sdks/java/src/main/java/org/byteveda/taskito/pubsub/PublishOptions.java b/sdks/java/src/main/java/org/byteveda/taskito/pubsub/PublishOptions.java new file mode 100644 index 00000000..32c049b3 --- /dev/null +++ b/sdks/java/src/main/java/org/byteveda/taskito/pubsub/PublishOptions.java @@ -0,0 +1,192 @@ +package org.byteveda.taskito.pubsub; + +import java.time.Duration; +import java.util.Map; +import org.byteveda.taskito.serialization.Notes; + +/** + * Options for {@code Taskito.publish(...)}. Every field is optional; unset + * delivery settings resolve per subscriber (the subscriber task's own defaults, + * then the core defaults). All durations are milliseconds. + */ +public final class PublishOptions { + private final String idempotencyKey; + private final String metadata; + private final String notes; + private final Integer priority; + private final Long delayMs; + private final Integer maxRetries; + private final Long timeoutMs; + private final Long expiresMs; + private final Long resultTtlMs; + + private PublishOptions(Builder b) { + this.idempotencyKey = b.idempotencyKey; + this.metadata = b.metadata; + this.notes = b.notes; + this.priority = b.priority; + this.delayMs = b.delayMs; + this.maxRetries = b.maxRetries; + this.timeoutMs = b.timeoutMs; + this.expiresMs = b.expiresMs; + this.resultTtlMs = b.resultTtlMs; + } + + public static PublishOptions none() { + return builder().build(); + } + + public static Builder builder() { + return new Builder(); + } + + /** The per-subscriber dedup key, or {@code null} when the publish is unkeyed. */ + public String idempotencyKey() { + return idempotencyKey; + } + + /** The opaque metadata blob every delivery carries, or {@code null}. */ + public String metadata() { + return metadata; + } + + /** Canonical notes JSON (validated at build time), or {@code null}. */ + public String notes() { + return notes; + } + + public Integer priority() { + return priority; + } + + public Long delayMs() { + return delayMs; + } + + public Integer maxRetries() { + return maxRetries; + } + + public Long timeoutMs() { + return timeoutMs; + } + + public Long expiresMs() { + return expiresMs; + } + + public Long resultTtlMs() { + return resultTtlMs; + } + + public static final class Builder { + private String idempotencyKey; + private String metadata; + private String notes; + private Integer priority; + private Long delayMs; + private Integer maxRetries; + private Long timeoutMs; + private Long expiresMs; + private Long resultTtlMs; + + /** + * Dedupe per subscriber: republishing the same key yields no new deliveries, + * while a subscription added later still gets its own copy. + */ + public Builder idempotencyKey(String idempotencyKey) { + this.idempotencyKey = idempotencyKey; + return this; + } + + public Builder metadata(String metadata) { + this.metadata = metadata; + return this; + } + + /** + * Attach a bounded, user-readable annotation map to every delivery + * (validated and canonically encoded now). Each delivery additionally + * carries {@code topic} and {@code subscription} keys. + * + * @throws org.byteveda.taskito.errors.NotesValidationException if the map + * breaks the {@link Notes} contract + */ + public Builder notes(Map notes) { + this.notes = Notes.encode(notes); + return this; + } + + /** Override every delivery's priority, beating the subscriber tasks' own defaults. */ + public Builder priority(int priority) { + this.priority = priority; + return this; + } + + public Builder delayMs(long delayMs) { + if (delayMs < 0) { + throw new IllegalArgumentException("delayMs must be >= 0"); + } + this.delayMs = delayMs; + return this; + } + + /** Schedule the deliveries after {@code delay} (Duration form of {@link #delayMs}). */ + public Builder delay(Duration delay) { + return delayMs(delay.toMillis()); + } + + public Builder maxRetries(int maxRetries) { + if (maxRetries < 0) { + throw new IllegalArgumentException("maxRetries must be >= 0"); + } + this.maxRetries = maxRetries; + return this; + } + + public Builder timeoutMs(long timeoutMs) { + if (timeoutMs < 0) { + throw new IllegalArgumentException("timeoutMs must be >= 0"); + } + this.timeoutMs = timeoutMs; + return this; + } + + /** Per-delivery timeout (Duration form of {@link #timeoutMs}). */ + public Builder timeout(Duration timeout) { + return timeoutMs(timeout.toMillis()); + } + + /** Deliveries not started within this window are discarded. */ + public Builder expiresMs(long expiresMs) { + if (expiresMs < 0) { + throw new IllegalArgumentException("expiresMs must be >= 0"); + } + this.expiresMs = expiresMs; + return this; + } + + /** Expiry window (Duration form of {@link #expiresMs}). */ + public Builder expires(Duration expires) { + return expiresMs(expires.toMillis()); + } + + /** How long each delivery's result is retained after completion. */ + public Builder resultTtlMs(long resultTtlMs) { + if (resultTtlMs < 0) { + throw new IllegalArgumentException("resultTtlMs must be >= 0"); + } + this.resultTtlMs = resultTtlMs; + return this; + } + + /** Result retention (Duration form of {@link #resultTtlMs}). */ + public Builder resultTtl(Duration resultTtl) { + return resultTtlMs(resultTtl.toMillis()); + } + + public PublishOptions build() { + return new PublishOptions(this); + } + } +} diff --git a/sdks/java/src/main/java/org/byteveda/taskito/pubsub/SubscriptionConfig.java b/sdks/java/src/main/java/org/byteveda/taskito/pubsub/SubscriptionConfig.java new file mode 100644 index 00000000..5ddb01d8 --- /dev/null +++ b/sdks/java/src/main/java/org/byteveda/taskito/pubsub/SubscriptionConfig.java @@ -0,0 +1,72 @@ +package org.byteveda.taskito.pubsub; + +/** + * A resolved subscription declaration recorded by {@code Taskito.subscribe(...)}. + * Workers register these at start (ephemeral entries bind to the started + * worker's id), and {@code publish(...)} reads the task delivery defaults so + * deliveries honor each subscriber's own settings. + */ +public final class SubscriptionConfig { + private final String topic; + private final String name; + private final String taskName; + private final String queue; + private final boolean durable; + private final Integer taskPriority; + private final Integer taskMaxRetries; + private final Long taskTimeoutMs; + + public SubscriptionConfig( + String topic, + String name, + String taskName, + String queue, + boolean durable, + Integer taskPriority, + Integer taskMaxRetries, + Long taskTimeoutMs) { + this.topic = topic; + this.name = name; + this.taskName = taskName; + this.queue = queue; + this.durable = durable; + this.taskPriority = taskPriority; + this.taskMaxRetries = taskMaxRetries; + this.taskTimeoutMs = taskTimeoutMs; + } + + public String topic() { + return topic; + } + + public String name() { + return name; + } + + public String taskName() { + return taskName; + } + + public String queue() { + return queue; + } + + public boolean durable() { + return durable; + } + + /** The subscriber task's default priority, or {@code null} for the core default. */ + public Integer taskPriority() { + return taskPriority; + } + + /** The subscriber task's default retry budget, or {@code null} for the core default. */ + public Integer taskMaxRetries() { + return taskMaxRetries; + } + + /** The subscriber task's default timeout, or {@code null} for the core default. */ + public Long taskTimeoutMs() { + return taskTimeoutMs; + } +} diff --git a/sdks/java/src/main/java/org/byteveda/taskito/pubsub/SubscriptionOptions.java b/sdks/java/src/main/java/org/byteveda/taskito/pubsub/SubscriptionOptions.java new file mode 100644 index 00000000..25f8a3b2 --- /dev/null +++ b/sdks/java/src/main/java/org/byteveda/taskito/pubsub/SubscriptionOptions.java @@ -0,0 +1,76 @@ +package org.byteveda.taskito.pubsub; + +import java.util.Objects; + +/** + * Options for {@code Taskito.subscribe(...)}. The subscription name defaults to + * the task name, the delivery queue to {@code "default"}, and durability to + * {@code true} (the registration persists across restarts). + */ +public final class SubscriptionOptions { + private final String name; + private final String queue; + private final boolean durable; + + private SubscriptionOptions(Builder b) { + this.name = b.name; + this.queue = b.queue; + this.durable = b.durable; + } + + public static SubscriptionOptions none() { + return builder().build(); + } + + public static Builder builder() { + return new Builder(); + } + + /** The explicit subscription name, or {@code null} to default to the task name. */ + public String name() { + return name; + } + + /** The queue the subscriber's delivery jobs go to. */ + public String queue() { + return queue; + } + + /** Whether the registration persists across restarts; {@code false} = ephemeral. */ + public boolean durable() { + return durable; + } + + public static final class Builder { + private String name; + private String queue = "default"; + private boolean durable = true; + + /** + * Stable subscription identity. Re-registering the same {@code (topic, name)} + * updates the routing target instead of duplicating the subscription. + */ + public Builder name(String name) { + this.name = name; + return this; + } + + public Builder queue(String queue) { + this.queue = Objects.requireNonNull(queue, "queue must not be null"); + return this; + } + + /** + * {@code false} ties the subscription to one worker process: it registers + * when that worker starts and is reaped once the worker stops heartbeating. + */ + public Builder durable(boolean durable) { + this.durable = durable; + return this; + } + + public SubscriptionOptions build() { + return new SubscriptionOptions(this); + } + } +} diff --git a/sdks/java/src/main/java/org/byteveda/taskito/pubsub/package-info.java b/sdks/java/src/main/java/org/byteveda/taskito/pubsub/package-info.java new file mode 100644 index 00000000..fe11a87f --- /dev/null +++ b/sdks/java/src/main/java/org/byteveda/taskito/pubsub/package-info.java @@ -0,0 +1,9 @@ +/** + * Topic pub/sub: N independent subscribers, each delivered its own job. A + * subscriber is a normal task plus a registry row mapping {@code (topic, name)} + * to {@code (taskName, queue)}; {@code Taskito.publish(...)} fans a message out + * as one ordinary job per active subscription, so every task feature — retries, + * dead-lettering, middleware — applies per subscriber, and a failing subscriber + * never affects its siblings. + */ +package org.byteveda.taskito.pubsub; diff --git a/sdks/java/src/main/java/org/byteveda/taskito/spi/QueueBackend.java b/sdks/java/src/main/java/org/byteveda/taskito/spi/QueueBackend.java index 30e2f7ad..8792a4ba 100644 --- a/sdks/java/src/main/java/org/byteveda/taskito/spi/QueueBackend.java +++ b/sdks/java/src/main/java/org/byteveda/taskito/spi/QueueBackend.java @@ -162,6 +162,50 @@ default boolean setPeriodicEnabled(String name, boolean enabled) { throw new UnsupportedOperationException("periodic tasks not supported by this backend"); } + // ── Pub/Sub ───────────────────────────────────────────────────── + // Optional capability: default to throwing so existing custom backends keep + // compiling and fail explicitly only when pub/sub is actually used. + String PUBSUB_UNSUPPORTED = "pub/sub not supported by this backend"; + + /** Insert or update a topic subscription (idempotent on topic + name). */ + default void registerSubscription( + String topic, + String subscriptionName, + String taskName, + String queue, + boolean durable, + String ownerWorkerIdOrNull) { + throw new UnsupportedOperationException(PUBSUB_UNSUPPORTED); + } + + /** A JSON array of subscriptions — all of them, or only a topic's active ones. */ + default String listSubscriptionsJson(String topicOrNull) { + throw new UnsupportedOperationException(PUBSUB_UNSUPPORTED); + } + + /** Remove a subscription; false if none matched. */ + default boolean unsubscribe(String topic, String subscriptionName) { + throw new UnsupportedOperationException(PUBSUB_UNSUPPORTED); + } + + /** Pause (false) or resume (true) a subscription; false if none matched. */ + default boolean setSubscriptionActive(String topic, String subscriptionName, boolean active) { + throw new UnsupportedOperationException(PUBSUB_UNSUPPORTED); + } + + /** Drop ephemeral subscriptions whose owning worker is gone; returns the count removed. */ + default long reapEphemeralSubscriptions() { + throw new UnsupportedOperationException(PUBSUB_UNSUPPORTED); + } + + /** + * Fan a payload out to every active subscription of {@code topic}. Returns + * the created jobs as a JSON array — empty when nothing is subscribed. + */ + default String publishJson(String topic, byte[] payload, String optionsJson) { + throw new UnsupportedOperationException(PUBSUB_UNSUPPORTED); + } + // ── Workflows ─────────────────────────────────────────────────── // Optional capability: default to throwing so existing custom backends keep // compiling and fail explicitly only when workflows are actually used. diff --git a/sdks/java/src/main/java/org/byteveda/taskito/task/EnqueueOptions.java b/sdks/java/src/main/java/org/byteveda/taskito/task/EnqueueOptions.java index c19a6c0f..c6b24f59 100644 --- a/sdks/java/src/main/java/org/byteveda/taskito/task/EnqueueOptions.java +++ b/sdks/java/src/main/java/org/byteveda/taskito/task/EnqueueOptions.java @@ -88,6 +88,26 @@ public Builder toBuilder() { return b; } + /** The target queue, or {@code null} for the default. */ + public String queue() { + return queue; + } + + /** The job priority, or {@code null} for the core default. */ + public Integer priority() { + return priority; + } + + /** The retry budget, or {@code null} for the core default. */ + public Integer maxRetries() { + return maxRetries; + } + + /** The per-job timeout in milliseconds, or {@code null} for the core default. */ + public Long timeoutMs() { + return timeoutMs; + } + /** Job ids this enqueue waits on before it can be dequeued, or {@code null} when none. */ public List dependsOn() { return dependsOn; diff --git a/sdks/java/src/main/java/org/byteveda/taskito/worker/Worker.java b/sdks/java/src/main/java/org/byteveda/taskito/worker/Worker.java index 3c932e15..4630e979 100644 --- a/sdks/java/src/main/java/org/byteveda/taskito/worker/Worker.java +++ b/sdks/java/src/main/java/org/byteveda/taskito/worker/Worker.java @@ -29,6 +29,7 @@ import org.byteveda.taskito.events.OutcomeEvent; import org.byteveda.taskito.logging.TaskitoLogger; import org.byteveda.taskito.middleware.Middleware; +import org.byteveda.taskito.pubsub.SubscriptionConfig; import org.byteveda.taskito.resources.ResourceRuntime; import org.byteveda.taskito.serialization.PayloadCodec; import org.byteveda.taskito.serialization.Serializer; @@ -184,6 +185,7 @@ public static final class Builder { private final Map taskPolicies = new HashMap<>(); private final Map taskCircuitBreakers = new HashMap<>(); private final Map>> listeners = new EnumMap<>(EventName.class); + private List subscriptions = List.of(); private List queues; private int concurrency; private Integer channelCapacity; @@ -252,6 +254,16 @@ public Builder register(HandlerRegistry registry) { return this; } + /** + * Topic subscriptions to register at {@code start()} (wired by + * {@code Taskito.worker()}). Ephemeral entries bind to the started + * worker's id and are reaped once it stops heartbeating. + */ + public Builder subscriptions(List subscriptions) { + this.subscriptions = subscriptions; + return this; + } + public Builder queues(String... queues) { this.queues = Arrays.asList(queues); return this; @@ -418,6 +430,9 @@ private String encodeOptions() { if (mesh != null) { options.put("meshConfig", mesh.toConfigJson()); } + if (!subscriptions.isEmpty()) { + options.put("subscriptions", encodeSubscriptions()); + } try { return JSON.writeValueAsString(options); } catch (Exception e) { @@ -425,6 +440,21 @@ private String encodeOptions() { } } + /** Serialize the declared topic subscriptions into the wire shape the binding reads. */ + private List> encodeSubscriptions() { + List> specs = new ArrayList<>(subscriptions.size()); + for (SubscriptionConfig config : subscriptions) { + Map spec = new LinkedHashMap<>(); + spec.put("topic", config.topic()); + spec.put("subscriptionName", config.name()); + spec.put("taskName", config.taskName()); + spec.put("queue", config.queue()); + spec.put("durable", config.durable()); + specs.add(spec); + } + return specs; + } + /** * Serialize each task's per-task config (retry curve and/or circuit breaker) into the * wire shape the binding reads — one entry per task, merging both sources by name. diff --git a/sdks/java/src/test/java/org/byteveda/taskito/core/PubSubTest.java b/sdks/java/src/test/java/org/byteveda/taskito/core/PubSubTest.java new file mode 100644 index 00000000..c222ce7d --- /dev/null +++ b/sdks/java/src/test/java/org/byteveda/taskito/core/PubSubTest.java @@ -0,0 +1,238 @@ +package org.byteveda.taskito.core; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.internal.JniQueueBackend; +import org.byteveda.taskito.model.Job; +import org.byteveda.taskito.model.JobStatus; +import org.byteveda.taskito.model.Subscription; +import org.byteveda.taskito.pubsub.PublishOptions; +import org.byteveda.taskito.pubsub.SubscriptionOptions; +import org.byteveda.taskito.task.Task; +import org.byteveda.taskito.worker.Worker; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +class PubSubTest { + + private static final Task SEND_EMAIL = Task.of("pubsub.send_email", String.class); + private static final Task TRACK_ORDER = Task.of("pubsub.track_order", String.class); + + private Taskito open(Path dir) { + return Taskito.builder() + .backend("sqlite") + .url(dir.resolve("t.db").toString()) + .open(); + } + + @Test + void subscriptionNameDefaultsToTaskName(@TempDir Path dir) { + try (Taskito queue = open(dir)) { + queue.subscribe("orders", SEND_EMAIL); + List subs = queue.listSubscriptions("orders"); + assertEquals(1, subs.size()); + Subscription sub = subs.get(0); + assertEquals("orders", sub.topic); + assertEquals(SEND_EMAIL.name(), sub.name); + assertEquals(SEND_EMAIL.name(), sub.taskName); + assertEquals("default", sub.queue); + assertTrue(sub.active); + assertTrue(sub.durable); + assertEquals(List.of("orders"), queue.listTopics()); + } + } + + @Test + void redeclareUpdatesRoutingInsteadOfDuplicating(@TempDir Path dir) { + try (Taskito queue = open(dir)) { + queue.subscribe( + "orders", + SEND_EMAIL, + SubscriptionOptions.builder().name("email").build()); + queue.subscribe( + "orders", + TRACK_ORDER, + SubscriptionOptions.builder().name("email").queue("emails").build()); + List subs = queue.listSubscriptions("orders"); + assertEquals(1, subs.size()); + assertEquals(TRACK_ORDER.name(), subs.get(0).taskName); + assertEquals("emails", subs.get(0).queue); + } + } + + @Test + void unsubscribeReportsWhetherAnythingWasRemoved(@TempDir Path dir) { + try (Taskito queue = open(dir)) { + queue.subscribe("orders", SEND_EMAIL); + assertTrue(queue.unsubscribe("orders", SEND_EMAIL.name())); + assertFalse(queue.unsubscribe("orders", SEND_EMAIL.name())); + assertTrue(queue.listSubscriptions().isEmpty()); + } + } + + @Test + void publishWithoutSubscribersIsANoOp(@TempDir Path dir) { + try (Taskito queue = open(dir)) { + assertTrue(queue.publish("orders", "o-1").isEmpty()); + assertEquals(0, queue.stats().pending); + } + } + + @Test + void publishFansOutOnePerSubscriberWithStampedNotes(@TempDir Path dir) { + try (Taskito queue = open(dir)) { + queue.subscribe("orders", SEND_EMAIL); + queue.subscribe("orders", TRACK_ORDER); + + List deliveries = queue.publish( + "orders", + "o-1", + PublishOptions.builder().notes(Map.of("tenant", "acme")).build()); + + assertEquals(2, deliveries.size()); + for (Job job : deliveries) { + Map notes = job.notesMap().orElseThrow(); + assertEquals("orders", notes.get("topic")); + assertEquals("acme", notes.get("tenant")); + } + List tasks = new ArrayList<>(); + deliveries.forEach(job -> tasks.add(job.taskName)); + Collections.sort(tasks); + assertEquals(List.of(SEND_EMAIL.name(), TRACK_ORDER.name()), tasks); + } + } + + @Test + void pauseBlocksDeliveriesAndResumeRestoresThem(@TempDir Path dir) { + try (Taskito queue = open(dir)) { + queue.subscribe("orders", SEND_EMAIL); + queue.subscribe("orders", TRACK_ORDER); + + assertTrue(queue.pauseSubscription("orders", TRACK_ORDER.name())); + List paused = queue.publish("orders", "o-1"); + assertEquals(1, paused.size()); + assertEquals(SEND_EMAIL.name(), paused.get(0).taskName); + // The paused row stays registered (visible in the all-topics listing). + assertEquals(1, queue.listSubscriptions("orders").size()); + assertEquals(2, queue.listSubscriptions().size()); + + assertTrue(queue.resumeSubscription("orders", TRACK_ORDER.name())); + assertEquals(2, queue.publish("orders", "o-2").size()); + assertFalse(queue.pauseSubscription("orders", "unknown")); + } + } + + @Test + void idempotencyKeyDedupesPerSubscriber(@TempDir Path dir) { + try (Taskito queue = open(dir)) { + queue.subscribe("orders", SEND_EMAIL); + queue.subscribe("orders", TRACK_ORDER); + PublishOptions keyed = + PublishOptions.builder().idempotencyKey("evt-42").build(); + + List first = queue.publish("orders", "o-1", keyed); + List second = queue.publish("orders", "o-1", keyed); + assertEquals(2, first.size()); + assertEquals(2, second.size()); + assertEquals(ids(first), ids(second)); + assertEquals(2, queue.stats().pending); + + // A subscriber added after the first publish still gets its own copy. + queue.subscribe( + "orders", + SEND_EMAIL, + SubscriptionOptions.builder().name("audit").build()); + List third = queue.publish("orders", "o-1", keyed); + assertEquals(3, third.size()); + assertEquals(3, queue.stats().pending); + } + } + + @Test + void deliveriesHonorSubscriberTaskDefaultsAndPublishOverrides(@TempDir Path dir) { + try (Taskito queue = open(dir)) { + queue.subscribe("orders", SEND_EMAIL.priority(7).maxRetries(2)); + queue.subscribe("orders", TRACK_ORDER); + + for (Job job : queue.publish("orders", "o-1")) { + if (job.taskName.equals(SEND_EMAIL.name())) { + assertEquals(7, job.priority); + assertEquals(2, job.maxRetries); + } else { + assertEquals(0, job.priority); + } + } + + // An explicit publish-level override beats the subscriber's defaults. + List overridden = queue.publish( + "orders", "o-2", PublishOptions.builder().priority(9).build()); + overridden.forEach(job -> assertEquals(9, job.priority)); + } + } + + @Test + @Timeout(30) + void ephemeralSubscriptionRegistersWithARunningWorker(@TempDir Path dir) throws Exception { + try (Taskito queue = open(dir)) { + queue.subscribe( + "orders", + SEND_EMAIL, + SubscriptionOptions.builder().durable(false).build()); + // No owning worker yet, so the subscription is not registered. + assertTrue(queue.publish("orders", "o-1").isEmpty()); + + try (Worker worker = + queue.worker().handle(SEND_EMAIL, payload -> payload).start()) { + List deliveries = queue.publish("orders", "o-2"); + assertEquals(1, deliveries.size()); + Job done = queue.awaitJob(deliveries.get(0).id, Duration.ofSeconds(10)) + .orElseThrow(); + assertEquals(JobStatus.COMPLETE, done.status); + assertEquals("o-2", queue.getResult(done.id, String.class).orElseThrow()); + } + } + } + + @Test + @Timeout(30) + void reapRemovesDeadOwnedSubscriptionsOnly(@TempDir Path dir) throws Exception { + String options = new ObjectMapper() + .writeValueAsString( + Map.of("backend", "sqlite", "dsn", dir.resolve("t.db").toString())); + JniQueueBackend backend = JniQueueBackend.open(options); + try (Taskito queue = Taskito.builder().open(backend)) { + try (Worker worker = queue.worker().start()) { + String liveWorkerId = queue.listWorkers().get(0).workerId; + backend.registerSubscription("orders", "live", SEND_EMAIL.name(), "default", false, liveWorkerId); + backend.registerSubscription("orders", "ghost", SEND_EMAIL.name(), "default", false, "gone-worker"); + backend.registerSubscription("orders", "durable", SEND_EMAIL.name(), "default", true, null); + + // The running worker's own heartbeat may reap concurrently, so + // assert the converged state rather than this call's count. + backend.reapEphemeralSubscriptions(); + List names = new ArrayList<>(); + queue.listSubscriptions("orders").forEach(sub -> names.add(sub.name)); + Collections.sort(names); + assertEquals(List.of("durable", "live"), names); + } + } + } + + private static List ids(List jobs) { + List ids = new ArrayList<>(); + jobs.forEach(job -> ids.add(job.id)); + Collections.sort(ids); + return ids; + } +} diff --git a/sdks/java/test-support/src/main/java/org/byteveda/taskito/test/InMemoryQueueBackend.java b/sdks/java/test-support/src/main/java/org/byteveda/taskito/test/InMemoryQueueBackend.java index 80bf8e90..1bf453b4 100644 --- a/sdks/java/test-support/src/main/java/org/byteveda/taskito/test/InMemoryQueueBackend.java +++ b/sdks/java/test-support/src/main/java/org/byteveda/taskito/test/InMemoryQueueBackend.java @@ -2,7 +2,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.ArrayList; +import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -18,7 +20,8 @@ /** * A pure-Java, in-memory {@link QueueBackend} for fast unit tests — no JNI, no * disk. It runs jobs on a small polling worker (dispatch → complete/fail → retry - * or dead-letter), and supports inspection, admin, settings, and locks. + * or dead-letter), and supports inspection, admin, settings, locks, and topic + * pub/sub (fan-out, per-subscriber dedup, ephemeral reaping). * *

Not supported: workflows (use a native backend). Periodic registration is * recorded but never fires. Behaviour approximates the core for testing handlers, @@ -36,6 +39,8 @@ public final class InMemoryQueueBackend implements QueueBackend { private final Map>> logs = new ConcurrentHashMap<>(); private final java.util.Set paused = ConcurrentHashMap.newKeySet(); private final List workers = new CopyOnWriteArrayList<>(); + private final Map subscriptions = new ConcurrentHashMap<>(); + private final java.util.Set liveWorkers = ConcurrentHashMap.newKeySet(); private final AtomicLong seq = new AtomicLong(); private static long now() { @@ -84,6 +89,7 @@ private synchronized String enqueueOne(String taskName, byte[] payload, JsonNode job.uniqueKey = uniqueKey; job.metadata = text(opts, "metadata"); job.namespace = text(opts, "namespace"); + job.notes = text(opts, "notes"); long createdAt = now(); job.createdAt = createdAt; job.scheduledAt = createdAt + optLong(opts, "delayMs", 0); @@ -432,16 +438,191 @@ public boolean setPeriodicEnabled(String name, boolean enabled) { return true; } + // ── Pub/Sub ───────────────────────────────────────────────────── + + @Override + public synchronized void registerSubscription( + String topic, + String subscriptionName, + String taskName, + String queue, + boolean durable, + String ownerWorkerIdOrNull) { + // Full replacement mirrors the core's upsert: re-registering updates the + // routing target and clears any previously set owner. + subscriptions.put( + subscriptionKey(topic, subscriptionName), + new SubscriptionRec( + topic, subscriptionName, taskName, queue, durable, ownerWorkerIdOrNull, seq.incrementAndGet())); + } + + @Override + public String listSubscriptionsJson(String topicOrNull) { + List> views = new ArrayList<>(); + for (SubscriptionRec sub : subscriptionsInOrder()) { + if (topicOrNull != null && !(topicOrNull.equals(sub.topic) && sub.active)) { + continue; + } + views.add(subscriptionView(sub)); + } + return toJson(views); + } + + @Override + public boolean unsubscribe(String topic, String subscriptionName) { + return subscriptions.remove(subscriptionKey(topic, subscriptionName)) != null; + } + + @Override + public boolean setSubscriptionActive(String topic, String subscriptionName, boolean active) { + SubscriptionRec sub = subscriptions.get(subscriptionKey(topic, subscriptionName)); + if (sub == null) { + return false; + } + sub.active = active; + return true; + } + + @Override + public long reapEphemeralSubscriptions() { + long removed = 0; + for (Map.Entry entry : subscriptions.entrySet()) { + String owner = entry.getValue().ownerWorkerId; + if (owner != null && !liveWorkers.contains(owner) && subscriptions.remove(entry.getKey()) != null) { + removed++; + } + } + return removed; + } + + @Override + public synchronized String publishJson(String topic, byte[] payload, String optionsJson) { + JsonNode opts = readNode(optionsJson); + List> views = new ArrayList<>(); + for (SubscriptionRec sub : subscriptionsInOrder()) { + if (!topic.equals(sub.topic) || !sub.active) { + continue; + } + // enqueueOne's unique-key scan gives keyed publishes per-subscriber + // dedup: a republished key resolves to the existing delivery. + String id = enqueueOne(sub.taskName, payload, deliveryOptions(opts, sub)); + views.add(jobView(jobs.get(id))); + } + return toJson(views); + } + + private static String subscriptionKey(String topic, String name) { + return topic + "\0" + name; + } + + /** Registration order (creation seq), matching the core's stable listing order. */ + private List subscriptionsInOrder() { + List ordered = new ArrayList<>(subscriptions.values()); + ordered.sort(Comparator.comparingLong(sub -> sub.createdSeq)); + return ordered; + } + + private static Map subscriptionView(SubscriptionRec sub) { + Map view = new LinkedHashMap<>(); + view.put("topic", sub.topic); + view.put("subscriptionName", sub.name); + view.put("taskName", sub.taskName); + view.put("queue", sub.queue); + view.put("active", sub.active); + view.put("durable", sub.durable); + view.put("ownerWorkerId", sub.ownerWorkerId); + return view; + } + + /** + * One delivery's enqueue options: routing from the subscription, delivery + * settings resolving publish override → the subscriber task's defaults → + * zero, the idempotency key salted per subscriber (the unique-key scan is + * global, so a raw key would dedup away all but one delivery), and the + * caller's notes stamped with {@code topic}/{@code subscription}. + */ + private static ObjectNode deliveryOptions(JsonNode opts, SubscriptionRec sub) { + JsonNode taskDefaults = opts == null ? null : opts.path("taskDefaults").get(sub.taskName); + ObjectNode delivery = JSON.createObjectNode(); + delivery.put("queue", sub.queue); + delivery.put("priority", resolveDeliveryInt(opts, taskDefaults, "priority")); + delivery.put("maxRetries", resolveDeliveryInt(opts, taskDefaults, "maxRetries")); + delivery.put("timeoutMs", resolveDeliveryLong(opts, taskDefaults, "timeoutMs")); + delivery.put("delayMs", optLong(opts, "delayMs", 0)); + String idempotencyKey = text(opts, "idempotencyKey"); + if (idempotencyKey != null) { + delivery.put("uniqueKey", idempotencyKey + "::" + sub.name); + } + String metadata = text(opts, "metadata"); + if (metadata != null) { + delivery.put("metadata", metadata); + } + delivery.put("notes", deliveryNotes(text(opts, "notes"), sub)); + return delivery; + } + + private static int resolveDeliveryInt(JsonNode opts, JsonNode taskDefaults, String field) { + JsonNode override = opts == null ? null : opts.get(field); + if (override != null && !override.isNull()) { + return override.asInt(); + } + return optInt(taskDefaults, field, 0); + } + + private static long resolveDeliveryLong(JsonNode opts, JsonNode taskDefaults, String field) { + JsonNode override = opts == null ? null : opts.get(field); + if (override != null && !override.isNull()) { + return override.asLong(); + } + return optLong(taskDefaults, field, 0); + } + + /** The caller's notes (if any) with {@code topic} and {@code subscription} stamped in. */ + private static String deliveryNotes(String callerNotes, SubscriptionRec sub) { + ObjectNode notes = JSON.createObjectNode(); + if (callerNotes != null) { + JsonNode parsed = readNode(callerNotes); + if (parsed.isObject()) { + notes.setAll((ObjectNode) parsed); + } + } + notes.put("topic", sub.topic); + notes.put("subscription", sub.name); + return notes.toString(); + } + // ── Worker ────────────────────────────────────────────────────── @Override public WorkerControl startWorker(WorkerBridge bridge, String optionsJson) { - InMemoryWorker worker = new InMemoryWorker(bridge, parseQueues(optionsJson)); + JsonNode options = readNode(optionsJson); + String workerId = "im-worker-" + seq.incrementAndGet(); + registerWorkerSubscriptions(workerId, options); + liveWorkers.add(workerId); + InMemoryWorker worker = new InMemoryWorker(bridge, parseQueues(optionsJson), workerId); workers.add(worker); worker.start(); return worker; } + /** Register the worker's declared subscriptions; ephemeral ones bind to its id. */ + private void registerWorkerSubscriptions(String workerId, JsonNode options) { + JsonNode specs = options == null ? null : options.get("subscriptions"); + if (specs == null || !specs.isArray()) { + return; + } + for (JsonNode spec : specs) { + boolean durable = spec.path("durable").asBoolean(true); + registerSubscription( + text(spec, "topic"), + text(spec, "subscriptionName"), + text(spec, "taskName"), + optText(spec, "queue", DEFAULT_QUEUE), + durable, + durable ? null : workerId); + } + } + /** The worker's queue filter from its options JSON; empty = consume every queue. */ private java.util.Set parseQueues(String optionsJson) { java.util.Set queues = new java.util.HashSet<>(); @@ -658,6 +839,7 @@ private static Map jobView(JobRec job) { view.put("uniqueKey", job.uniqueKey); view.put("namespace", job.namespace); view.put("metadata", job.metadata); + view.put("notes", job.notes); return view; } @@ -722,9 +904,39 @@ private static final class JobRec { String uniqueKey; String namespace; String metadata; + String notes; volatile boolean cancelRequested; } + /** A topic subscription row. Keyed by {@link #subscriptionKey}. */ + private static final class SubscriptionRec { + final String topic; + final String name; + final String taskName; + final String queue; + final boolean durable; + final String ownerWorkerId; + final long createdSeq; + volatile boolean active = true; + + SubscriptionRec( + String topic, + String name, + String taskName, + String queue, + boolean durable, + String ownerWorkerId, + long createdSeq) { + this.topic = topic; + this.name = name; + this.taskName = taskName; + this.queue = queue; + this.durable = durable; + this.ownerWorkerId = ownerWorkerId; + this.createdSeq = createdSeq; + } + } + private static final class LockRec { final String name; final String ownerId; @@ -743,13 +955,15 @@ private static final class LockRec { private final class InMemoryWorker implements WorkerControl { private final WorkerBridge bridge; private final java.util.Set queues; + private final String workerId; private final Map inFlight = new ConcurrentHashMap<>(); private volatile boolean running = true; private Thread loop; - InMemoryWorker(WorkerBridge bridge, java.util.Set queues) { + InMemoryWorker(WorkerBridge bridge, java.util.Set queues, String workerId) { this.bridge = bridge; this.queues = queues; + this.workerId = workerId; } void start() { @@ -819,6 +1033,10 @@ public void stop() { if (loop != null) { loop.interrupt(); } + // This worker just went away: its ephemeral subscriptions are now + // dead-owned, so reap immediately (mirrors the native shutdown path). + liveWorkers.remove(workerId); + reapEphemeralSubscriptions(); } @Override diff --git a/sdks/java/test-support/src/test/java/org/byteveda/taskito/test/InMemoryPubSubTest.java b/sdks/java/test-support/src/test/java/org/byteveda/taskito/test/InMemoryPubSubTest.java new file mode 100644 index 00000000..4b0f440b --- /dev/null +++ b/sdks/java/test-support/src/test/java/org/byteveda/taskito/test/InMemoryPubSubTest.java @@ -0,0 +1,178 @@ +package org.byteveda.taskito.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.model.Job; +import org.byteveda.taskito.model.JobStatus; +import org.byteveda.taskito.model.Subscription; +import org.byteveda.taskito.pubsub.PublishOptions; +import org.byteveda.taskito.pubsub.SubscriptionOptions; +import org.byteveda.taskito.task.Task; +import org.byteveda.taskito.worker.Worker; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +class InMemoryPubSubTest { + + private static final Task EMAIL = Task.of("im.pubsub.email", String.class); + private static final Task AUDIT = Task.of("im.pubsub.audit", String.class); + + @Test + @Timeout(20) + void publishFansOutAndDeliversToEachSubscriberHandler() throws Exception { + try (Taskito queue = InMemoryTaskito.open()) { + queue.subscribe("orders", EMAIL); + queue.subscribe("orders", AUDIT); + try (Worker worker = queue.worker() + .handle(EMAIL, payload -> "email:" + payload) + .handle(AUDIT, payload -> "audit:" + payload) + .start()) { + List deliveries = queue.publish("orders", "o-1"); + assertEquals(2, deliveries.size()); + for (Job delivery : deliveries) { + Job done = + queue.awaitJob(delivery.id, Duration.ofSeconds(10)).orElseThrow(); + assertEquals(JobStatus.COMPLETE, done.status); + assertEquals( + done.taskName.equals(EMAIL.name()) ? "email:o-1" : "audit:o-1", + queue.getResult(done.id, String.class).orElseThrow()); + } + } + } + } + + @Test + void publishWithoutSubscribersIsANoOp() { + try (Taskito queue = InMemoryTaskito.open()) { + assertTrue(queue.publish("orders", "o-1").isEmpty()); + } + } + + @Test + void redeclareUpsertsOnTopicAndName() { + try (Taskito queue = InMemoryTaskito.open()) { + queue.subscribe( + "orders", EMAIL, SubscriptionOptions.builder().name("sink").build()); + queue.subscribe( + "orders", AUDIT, SubscriptionOptions.builder().name("sink").build()); + List subs = queue.listSubscriptions("orders"); + assertEquals(1, subs.size()); + assertEquals(AUDIT.name(), subs.get(0).taskName); + } + } + + @Test + void unsubscribeReportsWhetherAnythingWasRemoved() { + try (Taskito queue = InMemoryTaskito.open()) { + queue.subscribe("orders", EMAIL); + assertTrue(queue.unsubscribe("orders", EMAIL.name())); + assertFalse(queue.unsubscribe("orders", EMAIL.name())); + } + } + + @Test + void pauseBlocksDeliveriesAndResumeRestoresThem() { + try (Taskito queue = InMemoryTaskito.open()) { + queue.subscribe("orders", EMAIL); + queue.subscribe("orders", AUDIT); + + assertTrue(queue.pauseSubscription("orders", AUDIT.name())); + List paused = queue.publish("orders", "o-1"); + assertEquals(1, paused.size()); + assertEquals(EMAIL.name(), paused.get(0).taskName); + assertEquals(1, queue.listSubscriptions("orders").size()); + assertEquals(2, queue.listSubscriptions().size()); + + assertTrue(queue.resumeSubscription("orders", AUDIT.name())); + assertEquals(2, queue.publish("orders", "o-2").size()); + } + } + + @Test + void idempotencyKeyDedupesPerSubscriberButNotAcrossThem() { + try (Taskito queue = InMemoryTaskito.open()) { + queue.subscribe("orders", EMAIL); + queue.subscribe("orders", AUDIT); + PublishOptions keyed = + PublishOptions.builder().idempotencyKey("evt-1").build(); + + List first = queue.publish("orders", "o-1", keyed); + List second = queue.publish("orders", "o-1", keyed); + assertEquals(2, first.size()); + assertEquals(2, second.size()); + assertEquals(first.get(0).id, second.get(0).id); + assertEquals(first.get(1).id, second.get(1).id); + assertEquals(2, queue.stats().pending); + } + } + + @Test + void deliveriesCarryStampedNotesAndTaskDefaults() { + try (Taskito queue = InMemoryTaskito.open()) { + queue.subscribe("orders", EMAIL.priority(7).maxRetries(2)); + List deliveries = queue.publish( + "orders", + "o-1", + PublishOptions.builder().notes(Map.of("tenant", "acme")).build()); + + assertEquals(1, deliveries.size()); + Job delivery = deliveries.get(0); + assertEquals(7, delivery.priority); + assertEquals(2, delivery.maxRetries); + Map notes = delivery.notesMap().orElseThrow(); + assertEquals("orders", notes.get("topic")); + assertEquals(EMAIL.name(), notes.get("subscription")); + assertEquals("acme", notes.get("tenant")); + + // An explicit publish-level override beats the subscriber's defaults. + List overridden = queue.publish( + "orders", "o-2", PublishOptions.builder().priority(9).build()); + assertEquals(9, overridden.get(0).priority); + } + } + + @Test + @Timeout(20) + void ephemeralSubscriptionBindsToAWorkerAndIsReapedWhenItStops() { + try (Taskito queue = InMemoryTaskito.open()) { + queue.subscribe("orders", EMAIL); + queue.subscribe( + "orders", + AUDIT, + SubscriptionOptions.builder().durable(false).build()); + + // The ephemeral subscriber has no owning worker yet: durable only. + assertEquals(1, queue.publish("orders", "o-1").size()); + + try (Worker worker = queue.worker() + .handle(EMAIL, payload -> payload) + .handle(AUDIT, payload -> payload) + .start()) { + assertEquals(2, queue.publish("orders", "o-2").size()); + } + + // Worker stopped: its ephemeral subscription is reaped, durable survives. + List after = queue.publish("orders", "o-3"); + assertEquals(1, after.size()); + assertEquals(EMAIL.name(), after.get(0).taskName); + } + } + + @Test + void reapRemovesDeadOwnedSubscriptionsOnly() { + InMemoryQueueBackend backend = new InMemoryQueueBackend(); + backend.registerSubscription("orders", "ghost", EMAIL.name(), "default", false, "gone-worker"); + backend.registerSubscription("orders", "durable", EMAIL.name(), "default", true, null); + + assertEquals(1, backend.reapEphemeralSubscriptions()); + assertEquals(0, backend.reapEphemeralSubscriptions()); + assertTrue(backend.listSubscriptionsJson("orders").contains("durable")); + assertFalse(backend.listSubscriptionsJson("orders").contains("ghost")); + } +} From 12f19bd957b055fde2ed953134d05d4d3c6a7416 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:08:51 +0530 Subject: [PATCH 07/12] docs: document topic pub/sub across SDKs --- .../content/docs/java/api-reference/queue.mdx | 14 + .../java/getting-started/capabilities.mdx | 1 + docs/content/docs/java/guides/core/index.mdx | 5 + docs/content/docs/java/guides/core/meta.json | 2 +- .../content/docs/node/api-reference/queue.mdx | 16 + .../node/getting-started/capabilities.mdx | 1 + docs/content/docs/node/guides/core/index.mdx | 5 + docs/content/docs/node/guides/core/meta.json | 2 +- .../docs/python/api-reference/queue/index.mdx | 1 + .../docs/python/api-reference/queue/meta.json | 2 +- .../python/api-reference/queue/pubsub.mdx | 120 +++++ .../content/docs/python/guides/core/index.mdx | 5 + .../content/docs/python/guides/core/meta.json | 1 + .../docs/shared/guides/core/pubsub.mdx | 415 ++++++++++++++++++ 14 files changed, 587 insertions(+), 3 deletions(-) create mode 100644 docs/content/docs/python/api-reference/queue/pubsub.mdx create mode 100644 docs/content/docs/shared/guides/core/pubsub.mdx diff --git a/docs/content/docs/java/api-reference/queue.mdx b/docs/content/docs/java/api-reference/queue.mdx index e81848eb..b0a5fde6 100644 --- a/docs/content/docs/java/api-reference/queue.mdx +++ b/docs/content/docs/java/api-reference/queue.mdx @@ -84,6 +84,20 @@ try (Taskito taskito = Taskito.builder().sqlite("taskito.db").open()) { | `registerPeriodic(PeriodicTask)` | Register (or replace) a cron task → next fire time (Unix ms). | | `listPeriodic()` / `deletePeriodic(name)` / `pausePeriodic(name)` / `resumePeriodic(name)` | Periodic task management. | +## Pub/Sub + +See [Pub/Sub](/java/guides/core/pubsub) for the full guide (delivery +semantics, lifecycle, cross-SDK topics). + +| Method | Description | +|---|---| +| `subscribe(topic, Task)` / `subscribe(topic, Task, SubscriptionOptions)` | Wire `Task` into `topic`'s routing (durable by default). Register the handler on the worker separately, as for any task. | +| `publish(topic, payload)` / `publish(topic, payload, PublishOptions)` → `List` | Fan a message out to every active subscription — one job each. Empty list when nothing is subscribed. | +| `unsubscribe(topic, name)` | Remove a subscription; `false` if none matched. | +| `pauseSubscription(topic, name)` / `resumeSubscription(topic, name)` | Stop/resume deliveries without unregistering; `false` if none matched. | +| `listSubscriptions()` / `listSubscriptions(topic)` | Every subscription, or one topic's active ones. | +| `listTopics()` | Distinct topics with at least one subscription. | + ## Subsystems | Method | Description | diff --git a/docs/content/docs/java/getting-started/capabilities.mdx b/docs/content/docs/java/getting-started/capabilities.mdx index 092567a9..02201f7a 100644 --- a/docs/content/docs/java/getting-started/capabilities.mdx +++ b/docs/content/docs/java/getting-started/capabilities.mdx @@ -17,6 +17,7 @@ core — pick a backend, describe tasks, run workers. | Cooperative cancellation & progress | [Cancellation](/java/guides/core/cancellation) | | Partial results via task logs | [Streaming](/java/guides/core/streaming) | | Execution model (thread pools, autoscaling) | [Execution model](/java/guides/core/execution-model) | +| Topic pub/sub fan-out to independent subscribers | [Pub/Sub](/java/guides/core/pubsub) | | Compile-time `TaskHandler` bindings (no reflection) | [Tasks](/java/guides/core/tasks#typed-tasks-from-taskhandler) | ## Reliability diff --git a/docs/content/docs/java/guides/core/index.mdx b/docs/content/docs/java/guides/core/index.mdx index b3caa304..d753054f 100644 --- a/docs/content/docs/java/guides/core/index.mdx +++ b/docs/content/docs/java/guides/core/index.mdx @@ -54,4 +54,9 @@ description: "The building blocks of every Java taskito application." href="/java/guides/core/streaming" description="Emit progress from a running task" /> + diff --git a/docs/content/docs/java/guides/core/meta.json b/docs/content/docs/java/guides/core/meta.json index 65bcba5c..e761d084 100644 --- a/docs/content/docs/java/guides/core/meta.json +++ b/docs/content/docs/java/guides/core/meta.json @@ -1 +1 @@ -{ "title": "Core", "pages": ["tasks", "queues", "workers", "execution-model", "enqueue-options", "batching", "predicates", "scheduling", "cancellation", "streaming"] } +{ "title": "Core", "pages": ["tasks", "queues", "workers", "execution-model", "enqueue-options", "batching", "predicates", "scheduling", "cancellation", "streaming", "pubsub"] } diff --git a/docs/content/docs/node/api-reference/queue.mdx b/docs/content/docs/node/api-reference/queue.mdx index 0b008c25..ed1c1362 100644 --- a/docs/content/docs/node/api-reference/queue.mdx +++ b/docs/content/docs/node/api-reference/queue.mdx @@ -62,6 +62,22 @@ background thread pool. | `pauseQueue(q)` / `resumeQueue(q)` / `listPausedQueues()` | Queue control. | | `listWorkers()` → `Promise` | Registered workers. | +## Pub/Sub + +See [Pub/Sub](/node/guides/core/pubsub) for the full guide (delivery +semantics, lifecycle, cross-SDK topics). + +| Method | Description | +|---|---| +| `subscriber(topic, name, fn, options?)` | Register `fn` as `name`, subscribed to `topic`. Returns the queue (chainable, like `task`). | +| `publish(topic, args?, options?)` → `Promise` | Fan a message out to every active subscription — one job each. Empty array when nothing is subscribed. | +| `declareSubscriptions()` → `Promise` | Write pending durable subscriptions to storage. Call in a producer-only process; `runWorker()` does this automatically. | +| `unsubscribe(topic, name)` → `Promise` | Remove a subscription. | +| `pauseSubscription(topic, name)` / `resumeSubscription(topic, name)` → `Promise` | Stop/resume deliveries without unregistering. | +| `listSubscriptions(topic?)` → `Promise` | All subscriptions, or one topic's active ones. | +| `listTopics()` → `Promise` | Distinct topics with at least one subscription. | +| `reapEphemeralSubscriptions()` → `Promise` | Drop ephemeral subscriptions whose owning worker is gone. Runs automatically on the worker heartbeat cadence; exposed for operational tooling. | + ## Subsystems | Member | Description | diff --git a/docs/content/docs/node/getting-started/capabilities.mdx b/docs/content/docs/node/getting-started/capabilities.mdx index 5e2d850e..71fb4cf8 100644 --- a/docs/content/docs/node/getting-started/capabilities.mdx +++ b/docs/content/docs/node/getting-started/capabilities.mdx @@ -17,6 +17,7 @@ with zero Python — pick a backend, register tasks, run workers. | Cooperative cancellation (`AbortSignal`) | [Cancellation](/node/guides/core/cancellation) | | Streaming partial results (`publish` / `stream`) | [Streaming](/node/guides/core/streaming) | | Execution model (async, concurrency) | [Execution model](/node/guides/core/execution-model) | +| Topic pub/sub fan-out to independent subscribers | [Pub/Sub](/node/guides/core/pubsub) | ## Reliability diff --git a/docs/content/docs/node/guides/core/index.mdx b/docs/content/docs/node/guides/core/index.mdx index c69593a0..4509f0b5 100644 --- a/docs/content/docs/node/guides/core/index.mdx +++ b/docs/content/docs/node/guides/core/index.mdx @@ -49,4 +49,9 @@ description: "The building blocks of every Node.js taskito application." href="/node/guides/core/streaming" description="Emit progress from a running task" /> + diff --git a/docs/content/docs/node/guides/core/meta.json b/docs/content/docs/node/guides/core/meta.json index 13277466..1d1a3655 100644 --- a/docs/content/docs/node/guides/core/meta.json +++ b/docs/content/docs/node/guides/core/meta.json @@ -1 +1 @@ -{ "title": "Core", "pages": ["tasks", "queues", "workers", "execution-model", "enqueue-options", "predicates", "scheduling", "cancellation", "streaming"] } +{ "title": "Core", "pages": ["tasks", "queues", "workers", "execution-model", "enqueue-options", "predicates", "scheduling", "cancellation", "streaming", "pubsub"] } diff --git a/docs/content/docs/python/api-reference/queue/index.mdx b/docs/content/docs/python/api-reference/queue/index.mdx index fdb75106..0d3b9c91 100644 --- a/docs/content/docs/python/api-reference/queue/index.mdx +++ b/docs/content/docs/python/api-reference/queue/index.mdx @@ -13,6 +13,7 @@ import { Callout } from "fumadocs-ui/components/callout"; - **[Workers & Hooks](/python/api-reference/queue/workers)** — run workers, lifecycle hooks, circuit breakers, async methods - **[Resources & Locking](/python/api-reference/queue/resources)** — resource system, distributed locks - **[Events & Logs](/python/api-reference/queue/events)** — event callbacks, webhooks, structured logs + - **[Pub/Sub](/python/api-reference/queue/pubsub)** — topic subscriptions, fan-out publish ## Constructor diff --git a/docs/content/docs/python/api-reference/queue/meta.json b/docs/content/docs/python/api-reference/queue/meta.json index 3b4e43d9..f258a2f3 100644 --- a/docs/content/docs/python/api-reference/queue/meta.json +++ b/docs/content/docs/python/api-reference/queue/meta.json @@ -1,4 +1,4 @@ { "title": "Queue", - "pages": ["jobs", "queues", "workers", "resources", "events"] + "pages": ["jobs", "queues", "workers", "resources", "events", "pubsub"] } diff --git a/docs/content/docs/python/api-reference/queue/pubsub.mdx b/docs/content/docs/python/api-reference/queue/pubsub.mdx new file mode 100644 index 00000000..ac17a301 --- /dev/null +++ b/docs/content/docs/python/api-reference/queue/pubsub.mdx @@ -0,0 +1,120 @@ +--- +title: Pub/Sub +description: "Topic subscriptions and fan-out publish. See the Pub/Sub guide for delivery semantics and lifecycle." +--- + +See [Pub/Sub](/python/guides/core/pubsub) for the full guide — delivery +semantics, subscription lifecycle, and cross-SDK topics. + +## Subscribing + +### `@queue.subscriber()` + +```python +@queue.subscriber( + topic: str, + name: str | None = None, + queue: str = "default", + durable: bool = True, + **task_kwargs: Any, +) -> Callable[[Callable], TaskWrapper] +``` + +Register `fn` as an independent subscriber of `topic`. The function becomes a +normal task — `task_kwargs` are forwarded to +[`@queue.task()`](/python/api-reference/queue) — and the subscription is +written to storage when `run_worker()` starts, or via +`declare_subscriptions()` in a producer-only process. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `topic` | `str` | — | Topic to subscribe to. | +| `name` | `str \| None` | Task name | Stable subscription identity. Re-registering the same `(topic, name)` updates the routing target instead of duplicating. | +| `queue` | `str` | `"default"` | Queue the subscriber's delivery jobs go to. | +| `durable` | `bool` | `True` | Persist across restarts. `False` ties the subscription to one worker process — it only registers inside `run_worker()` and is reaped once that worker stops heartbeating. | +| `**task_kwargs` | `Any` | — | Any `@queue.task()` option (`max_retries`, `timeout`, `middleware`, `idempotent`, ...). | + +### `queue.declare_subscriptions()` + +```python +queue.declare_subscriptions() -> None +``` + +Write pending durable subscriptions to storage. Called automatically at +`run_worker()` startup — call it explicitly in a producer-only process (one +that imports subscriber modules but never runs a worker) so `publish()` sees +the subscriptions. Ephemeral subscriptions are skipped; they need an owning +worker. + +## Publishing + +### `queue.publish()` + +```python +queue.publish( + topic: str, + *args: Any, + idempotency_key: str | None = None, + metadata: dict[str, Any] | None = None, + notes: dict[str, Any] | None = None, + priority: int | None = None, + delay_seconds: float | None = None, + max_retries: int | None = None, + timeout: int | None = None, + expires: float | None = None, + result_ttl: int | None = None, + **kwargs: Any, +) -> list[JobResult] +``` + +Publish a message to `topic`. Every active subscription receives an +independent job carrying the same `(args, kwargs)` payload (at-least-once +per subscriber). Returns one [`JobResult`](/python/api-reference/result) per +delivery — empty when the topic has no active subscribers, a valid pub/sub +no-op. + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `idempotency_key` | `str \| None` | `None` | Dedupes per subscriber — republishing the same key yields no new deliveries for a subscriber that already received it. Salted internally with the subscription name. | +| `notes` | `dict \| None` | `None` | Structured annotations, validated against the same 15-field cap as `enqueue()`'s `notes` — `topic` and `subscription` are then stamped into every delivery's notes on top of that. | +| `priority` / `max_retries` / `timeout` / `expires` / `result_ttl` | — | `None` | Override every delivery's setting. Unset fields resolve to the subscriber task's own registered default, then the queue default. | + +Unrecognized keyword arguments are forwarded as part of the task payload's +`kwargs`, same as `enqueue()`. + +## Managing subscriptions + +### `queue.unsubscribe()` + +```python +queue.unsubscribe(topic: str, name: str) -> bool +``` + +Remove a subscription. Returns `False` if none matched. + +### `queue.pause_subscription()` / `queue.resume_subscription()` + +```python +queue.pause_subscription(topic: str, name: str) -> bool +queue.resume_subscription(topic: str, name: str) -> bool +``` + +Stop or resume deliveries without unregistering. Returns `False` if the +subscription is unknown. + +### `queue.list_subscriptions()` + +```python +queue.list_subscriptions(topic: str | None = None) -> list[dict[str, Any]] +``` + +List subscriptions — all of them, or one topic's active ones. Each dict +contains `topic`, `name`, `task_name`, `queue`, `active`, and `durable`. + +### `queue.list_topics()` + +```python +queue.list_topics() -> list[str] +``` + +Distinct topics that currently have at least one subscription. diff --git a/docs/content/docs/python/guides/core/index.mdx b/docs/content/docs/python/guides/core/index.mdx index 9d4ae772..41be3559 100644 --- a/docs/content/docs/python/guides/core/index.mdx +++ b/docs/content/docs/python/guides/core/index.mdx @@ -39,6 +39,11 @@ description: "The building blocks of every taskito application." href="/python/guides/core/batching" description="Collect many small calls into a single job" /> + Subscribe with the @queue.subscriber() decorator.} node={<>Subscribe with queue.subscriber().} java={<>Subscribe with taskito.subscribe().} /> + + T{{"Topic: orders"}} + T --> S1["Subscription: email"] + T --> S2["Subscription: analytics"] + S1 --> J1["Job: send_confirmation_email"] + S2 --> J2["Job: track_order"]`} +/> + +## Quick start + +Register a subscriber like any other task, then publish a message to the +topic — every active subscription gets its own delivery: + + + + +```python +from taskito import Queue + +queue = Queue() + +@queue.subscriber("orders", name="email") +def send_confirmation_email(order_id: int) -> None: + ... + +@queue.subscriber("orders", name="analytics") +def track_order(order_id: int) -> None: + ... + +queue.declare_subscriptions() # only needed before run_worker() has started +queue.publish("orders", 42) # -> [JobResult, JobResult] — one per subscriber +``` + + + + +```ts +import { Queue } from "@byteveda/taskito"; + +const queue = new Queue({ dbPath: "orders.db" }); + +queue.subscriber("orders", "sendConfirmationEmail", (orderId: number) => { + // ... +}, { subscriptionName: "email" }); + +queue.subscriber("orders", "trackOrder", (orderId: number) => { + // ... +}, { subscriptionName: "analytics" }); + +await queue.declareSubscriptions(); // only needed before runWorker() has started +await queue.publish("orders", [42]); // -> Job[] — one per subscriber +``` + + + + +```java +import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.pubsub.SubscriptionOptions; +import org.byteveda.taskito.task.Task; + +Task sendConfirmationEmail = Task.of("send_confirmation_email", Integer.class); +Task trackOrder = Task.of("track_order", Integer.class); + +try (Taskito taskito = Taskito.builder().sqlite("orders.db").open()) { + taskito.subscribe("orders", sendConfirmationEmail, + SubscriptionOptions.builder().name("email").build()); + taskito.subscribe("orders", trackOrder, + SubscriptionOptions.builder().name("analytics").build()); + + taskito.publish("orders", 42); // -> List — one per subscriber +} +``` + + + + + + +`subscribe()` only wires the routing — it doesn't take a handler. Register the +handler itself on the worker the same way you would for any other task: + +```java +taskito.worker() + .handle(sendConfirmationEmail, orderId -> mailer.sendConfirmation(orderId)) + .handle(trackOrder, orderId -> analytics.record(orderId)) + .start(); +``` + + + + + +`declare_subscriptions()` writes durable subscriptions to storage immediately — +call it in a producer-only process (one that imports subscriber modules but +never runs a worker) so `publish()` sees them. `run_worker()` does this +automatically at startup, so a process that both subscribes and runs a worker +doesn't need the explicit call. + + + + + +`declareSubscriptions()` writes durable subscriptions to storage immediately — +call it in a producer-only process (one that registers subscribers but never +runs a worker) so `publish()` sees them. `runWorker()` does this automatically +at startup, so a process that both subscribes and runs a worker doesn't need +the explicit call. + + + + + +A durable `subscribe()` call registers immediately — there's no separate +"declare" step. A producer-only process just needs to call `subscribe()` +before it calls `publish()`. + + + +## Subscription lifecycle + +### Durable vs. ephemeral + +Subscriptions are **durable** by default: the registry row persists until you +unsubscribe, and a worker that's down just lets its deliveries pile up as +ordinary pending jobs. An **ephemeral** subscription (`durable=False` / +`durable: false` / `SubscriptionOptions.durable(false)`) ties the subscription +to one worker process instead — it's only registered while that worker is +running, and it's automatically reaped once that worker stops heartbeating. +Use ephemeral subscriptions for throwaway consumers (a debug tail, a +short-lived fan-out watcher) that shouldn't leave a stale registration behind +if the process never comes back. + + + + +```python +@queue.subscriber("orders", name="debug-tail", durable=False) +def tail(order_id: int) -> None: + print(f"order {order_id}") + +queue.run_worker() # the subscription registers here, owned by this worker +``` + + + + +```ts +queue.subscriber("orders", "tail", (orderId: number) => { + console.log(`order ${orderId}`); +}, { durable: false }); + +queue.runWorker(); // the subscription registers here, owned by this worker +``` + + + + +```java +Task tail = Task.of("tail", Integer.class); +taskito.subscribe("orders", tail, SubscriptionOptions.builder().durable(false).build()); + +// The subscription registers once a worker actually starts and handles it. +try (Worker worker = taskito.worker().handle(tail, orderId -> log(orderId)).start()) { + ... +} +``` + + + + + + Every SDK reaps ephemeral subscriptions on the worker heartbeat cadence, so + a crashed or stopped worker's ephemeral rows disappear without operator + intervention. + + +### Re-registering updates instead of duplicating + +Registering the same `(topic, subscription name)` again updates its routing +target (task, queue) in place instead of creating a second row — safe to call +on every process restart. + +### Pause, resume, unsubscribe + +Pausing stops deliveries without forgetting the subscription — useful for a +maintenance window on one consumer while the others keep receiving messages. +Unsubscribing removes the registry row entirely. + + + + +```python +queue.pause_subscription("orders", "email") # stop deliveries to "email" +queue.publish("orders", 1) # only "analytics" is delivered +queue.resume_subscription("orders", "email") + +queue.unsubscribe("orders", "email") # remove it entirely +queue.list_subscriptions("orders") # active subscriptions for a topic +queue.list_topics() # distinct topics with a subscriber +``` + + + + +```ts +await queue.pauseSubscription("orders", "email"); +await queue.publish("orders", [1]); // only "analytics" is delivered +await queue.resumeSubscription("orders", "email"); + +await queue.unsubscribe("orders", "email"); // remove it entirely +await queue.listSubscriptions("orders"); // active subscriptions for a topic +await queue.listTopics(); // distinct topics with a subscriber +``` + + + + +```java +taskito.pauseSubscription("orders", "email"); +taskito.publish("orders", 1); // only "analytics" is delivered +taskito.resumeSubscription("orders", "email"); + +taskito.unsubscribe("orders", "email"); // remove it entirely +taskito.listSubscriptions("orders"); // active subscriptions for a topic +taskito.listTopics(); // distinct topics with a subscriber +``` + + + + +### Late-join boundary + + + There's no history to replay. A subscription only receives messages + published **after** it was registered — publishing before a subscriber + exists, or while it's unsubscribed, permanently misses that subscriber. If + you need durable event history, that's a different problem than pub/sub + fan-out; this feature is about delivering *live* messages to every current + subscriber, each with full task semantics. + + +## Delivery semantics + +### At-least-once, per subscriber + +Each delivery is an ordinary job, so it inherits the same +at-least-once delivery +guarantee as everything else: a delivery is never silently lost, but a crash +between running the handler and recording the result can run it twice. A +subscriber that exhausts its retries dead-letters **independently** — the +`send_confirmation_email` delivery failing and dead-lettering has no effect +on the `track_order` delivery for the same message. + +### Delivery settings resolve per subscriber + +An unset delivery setting (priority, max retries, timeout) resolves in this +order: + +1. An explicit option passed to `publish()`. +2. The subscriber task's own registered default. +3. The queue's default. + +So one high-priority subscriber and one best-effort subscriber can consume +the exact same published message, each retried and prioritized according to +its own task registration — unless the publisher explicitly overrides a +setting for every delivery. + +### Idempotency-key dedup is per subscriber + +`idempotency_key` / `idempotencyKey` dedupes **per subscriber**, not per +publish call — internally it's salted with the subscription name before +becoming the delivery's dedup key. Republishing the same key creates no new +deliveries for subscribers that already received it, but a subscription added +*after* the first publish still gets its own copy the next time that key (or +any key) is published, because its salted key hasn't been used yet. + + + + +```python +first = queue.publish("orders", 1, idempotency_key="evt-42") # 2 subscribers -> 2 jobs +second = queue.publish("orders", 1, idempotency_key="evt-42") # same jobs, no new rows + +@queue.subscriber("orders", name="audit") +def audit_order(order_id: int) -> None: ... +queue.declare_subscriptions() + +third = queue.publish("orders", 1, idempotency_key="evt-42") # "audit" gets its own delivery +``` + + + + +```ts +const first = await queue.publish("orders", [1], { idempotencyKey: "evt-42" }); // 2 jobs +const second = await queue.publish("orders", [1], { idempotencyKey: "evt-42" }); // same jobs + +queue.subscriber("orders", "auditOrder", (orderId: number) => {}, { subscriptionName: "audit" }); +await queue.declareSubscriptions(); + +const third = await queue.publish("orders", [1], { idempotencyKey: "evt-42" }); // "audit" delivered +``` + + + + +```java +PublishOptions keyed = PublishOptions.builder().idempotencyKey("evt-42").build(); +List first = taskito.publish("orders", 1, keyed); // 2 subscribers -> 2 jobs +List second = taskito.publish("orders", 1, keyed); // same jobs, no new rows + +Task auditOrder = Task.of("audit_order", Integer.class); +taskito.subscribe("orders", auditOrder, SubscriptionOptions.builder().name("audit").build()); +List third = taskito.publish("orders", 1, keyed); // "audit" gets its own delivery +``` + + + + +### Every delivery is stamped with `topic` and `subscription` + +Every delivery's structured notes carry the topic and the subscription name +it was routed through, merged with any notes the publisher supplied — filter +or group deliveries on the dashboard, or in `list_jobs()`/`listJobs()` +queries, by which subscriber received them. + + + + +```python +(result,) = queue.publish("orders", 5, notes={"tenant": "acme"}) +job = queue.get_job(result.id) +job.notes # {"topic": "orders", "subscription": "email", "tenant": "acme"} +``` + + + + +```ts +const [job] = await queue.publish("orders", [5], { notes: { tenant: "acme" } }); +JSON.parse(job.notes ?? "{}"); +// { topic: "orders", subscription: "email", tenant: "acme" } +``` + + + + +```java +List jobs = taskito.publish("orders", 5, + PublishOptions.builder().notes(Map.of("tenant", "acme")).build()); +Map notes = jobs.get(0).notesMap().orElseThrow(); +// {topic=orders, subscription=email, tenant=acme} +``` + + + + +## Cross-SDK topics + +A publish writes exactly **one** serialized payload, shared by every +delivery — it's encoded once with the queue-level serializer, not +per-subscriber, so per-task serializer overrides don't apply to topic +deliveries. + + + When a topic's publisher and its subscribers aren't all the same language, + configure the `CborSerializer` + on every process that touches the topic — it's the wire format the + cross-SDK contract is built on, and it round-trips the types JSON can't + (big integers, `datetime`/`Date`, bytes, decimals) losslessly. A single + object/dict argument maps most cleanly onto every language's handler + signature. + + +## Operational notes + +- **A down durable subscriber doesn't lose messages.** Its deliveries just + accumulate as ordinary pending jobs in its queue until a worker resumes + consuming it — the same backlog you'd see from any other stalled queue, and + visible the same way (`stats()`, the dashboard). +- **Retention is normal job retention.** There's no separate pub/sub log or + replay buffer — a delivery is purged, replayed, or expired exactly like any + other job, via `result_ttl`, `purge_completed()`, and the dead-letter queue. +- **Publish multiplies writes by subscriber count.** A topic with 20 active + subscribers turns every `publish()` call into 20 job rows. For a high-fanout + topic, set an aggressive `result_ttl` on the publish (or on the subscriber + tasks themselves) so completed deliveries don't pile up in storage between + cleanup passes. + +## See also + +- Delivery guarantees — + the at-least-once contract every delivery inherits. +- Idempotency — how + dedup keys are derived and prioritized for a single enqueue. +- Serializers — the + `CborSerializer` cross-SDK wire format used for cross-language topics. From b617b3f464fe5fa9e237b690a875cf2ebec253de Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sun, 12 Jul 2026 10:22:34 +0530 Subject: [PATCH 08/12] fix(core): include topic in pub/sub idempotency salt Subscription identity is (topic, name); salting on name alone let the same key dedup across two topics sharing a subscription name. Length-prefix both parts so the encoding is injective. --- crates/taskito-core/src/pubsub.rs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/crates/taskito-core/src/pubsub.rs b/crates/taskito-core/src/pubsub.rs index 2d03fce2..ba0f46a2 100644 --- a/crates/taskito-core/src/pubsub.rs +++ b/crates/taskito-core/src/pubsub.rs @@ -29,7 +29,8 @@ pub struct PublishRequest { pub topic: String, /// Wire-envelope payload bytes; every subscriber receives the same body. pub payload: Vec, - /// Salted per subscription (`key::subscription_name`) before becoming a + /// Salted per subscription identity (topic + name, length-prefixed so + /// delimiter characters in either part cannot collide) before becoming a /// job `unique_key`: the jobs table's unique index is global, so reusing /// the raw key across the fan-out would dedup away all but one delivery. pub idempotency_key: Option, @@ -72,6 +73,17 @@ pub fn publish_to_topic(storage: &S, request: &PublishRequest) -> Re .collect() } +/// `key::::` — length prefixes make the +/// encoding injective, so distinct (topic, name) pairs can never produce the +/// same salted key even when the parts contain the delimiter characters. +fn salted_unique_key(key: &str, topic: &str, subscription_name: &str) -> String { + format!( + "{key}::{}:{}:{topic}{subscription_name}", + topic.len(), + subscription_name.len() + ) +} + fn delivery_job(request: &PublishRequest, sub: &SubscriptionRow) -> NewJob { let task = request .task_defaults @@ -89,7 +101,7 @@ fn delivery_job(request: &PublishRequest, sub: &SubscriptionRow) -> NewJob { unique_key: request .idempotency_key .as_ref() - .map(|key| format!("{key}::{}", sub.subscription_name)), + .map(|key| salted_unique_key(key, &request.topic, &sub.subscription_name)), metadata: request.metadata.clone(), notes: Some(delivery_notes(request, sub)), depends_on: Vec::new(), @@ -199,7 +211,10 @@ mod tests { assert_eq!(first.len(), 2); let mut keys: Vec<_> = first.iter().filter_map(|j| j.unique_key.clone()).collect(); keys.sort_unstable(); - assert_eq!(keys, ["evt-42::analytics", "evt-42::email"]); + assert_eq!( + keys, + ["evt-42::6:5:ordersemail", "evt-42::6:9:ordersanalytics"] + ); // Same event published twice: every subscriber still has exactly one // delivery (per-subscriber dedup), not zero and not two. From 7cd672ec92ab429fc957e6ce911e2b822b3e3f67 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sun, 12 Jul 2026 10:22:50 +0530 Subject: [PATCH 09/12] fix(core): harden subscription registry lifecycle and atomicity Re-registering a subscription now preserves its paused state and created_at (worker restart no longer resumes a paused sub). Ephemeral reaping skips rows inside a grace window so a starting worker's subs aren't reaped before its first heartbeat. Redis registry mutations run in atomic pipelines. --- .../src/storage/diesel_common/pubsub.rs | 11 ++++-- crates/taskito-core/src/storage/mod.rs | 7 ++++ .../src/storage/postgres/pubsub.rs | 4 +-- .../src/storage/redis_backend/pubsub.rs | 30 ++++++++++------ .../taskito-core/src/storage/sqlite/pubsub.rs | 19 ++++++++-- .../taskito-core/src/storage/sqlite/tests.rs | 4 ++- .../taskito-core/tests/rust/storage_tests.rs | 35 ++++++++++++++++++- 7 files changed, 91 insertions(+), 19 deletions(-) diff --git a/crates/taskito-core/src/storage/diesel_common/pubsub.rs b/crates/taskito-core/src/storage/diesel_common/pubsub.rs index 0b1ca576..3b135820 100644 --- a/crates/taskito-core/src/storage/diesel_common/pubsub.rs +++ b/crates/taskito-core/src/storage/diesel_common/pubsub.rs @@ -72,14 +72,21 @@ macro_rules! impl_diesel_pubsub_ops { /// Remove ephemeral subscriptions whose owner is not in /// `live_worker_ids`. Durable rows (owner NULL) are excluded by the - /// `is_not_null` filter and never touched. Returns the count removed. + /// `is_not_null` filter and never touched. Rows younger than the + /// registration grace window survive: a starting worker inserts its + /// ephemeral subscriptions before its first heartbeat lands, and + /// another worker's reap tick must not race that gap. + /// Returns the count removed. pub fn reap_ephemeral_subscriptions(&self, live_worker_ids: &[String]) -> Result { let mut conn = self.conn()?; + let cutoff = + $crate::job::now_millis() - $crate::storage::EPHEMERAL_SUBSCRIPTION_GRACE_MS; let affected = diesel::delete( topic_subscriptions::table .filter(topic_subscriptions::owner_worker_id.is_not_null()) - .filter(topic_subscriptions::owner_worker_id.ne_all(live_worker_ids)), + .filter(topic_subscriptions::owner_worker_id.ne_all(live_worker_ids)) + .filter(topic_subscriptions::created_at.lt(cutoff)), ) .execute(&mut conn)?; diff --git a/crates/taskito-core/src/storage/mod.rs b/crates/taskito-core/src/storage/mod.rs index 417a3593..b983a4d9 100644 --- a/crates/taskito-core/src/storage/mod.rs +++ b/crates/taskito-core/src/storage/mod.rs @@ -23,6 +23,13 @@ use crate::job::{Job, NewJob}; /// cutoff. Kept in one place so SQLite, Postgres, and Redis can never drift. pub const DEAD_WORKER_THRESHOLD_MS: i64 = 30_000; +/// Ephemeral subscriptions younger than this survive the reaper even when +/// their owner is not (yet) in the live-worker set: a starting worker inserts +/// its ephemeral subscriptions before its first heartbeat registers it live, +/// and another worker's reap tick must not race that gap. Twice the dead-worker +/// threshold keeps one full failure-detection cycle of headroom. +pub const EPHEMERAL_SUBSCRIPTION_GRACE_MS: i64 = 2 * DEAD_WORKER_THRESHOLD_MS; + // ── Shared helper types ──────────────────────────────────────────────── #[derive(Debug, Clone, Default)] diff --git a/crates/taskito-core/src/storage/postgres/pubsub.rs b/crates/taskito-core/src/storage/postgres/pubsub.rs index 35c42b77..ea579ae3 100644 --- a/crates/taskito-core/src/storage/postgres/pubsub.rs +++ b/crates/taskito-core/src/storage/postgres/pubsub.rs @@ -13,6 +13,8 @@ impl PostgresStorage { /// The `do_update` sets each mutable column explicitly rather than via /// `AsChangeset`, so re-registering with `owner_worker_id = None` writes SQL /// NULL (clearing a previously-ephemeral owner) instead of leaving it stale. + /// `active` and `created_at` are deliberately excluded: re-declaring must + /// not resume a paused subscription or reset its registration time. pub fn register_subscription(&self, sub: &NewSubscriptionRow) -> Result<()> { let mut conn = self.conn()?; @@ -26,10 +28,8 @@ impl PostgresStorage { .set(( topic_subscriptions::task_name.eq(sub.task_name), topic_subscriptions::queue.eq(sub.queue), - topic_subscriptions::active.eq(sub.active), topic_subscriptions::durable.eq(sub.durable), topic_subscriptions::owner_worker_id.eq(sub.owner_worker_id), - topic_subscriptions::created_at.eq(sub.created_at), )) .execute(&mut conn)?; diff --git a/crates/taskito-core/src/storage/redis_backend/pubsub.rs b/crates/taskito-core/src/storage/redis_backend/pubsub.rs index 2c7368ee..852dd688 100644 --- a/crates/taskito-core/src/storage/redis_backend/pubsub.rs +++ b/crates/taskito-core/src/storage/redis_backend/pubsub.rs @@ -85,8 +85,6 @@ impl RedisStorage { owner_worker_id: sub.owner_worker_id.map(|s| s.to_string()), created_at: sub.created_at, }; - let json = serde_json::to_string(&entry).map_err(|e| QueueError::Other(e.to_string()))?; - let blob_key = self.key(&["sub", sub.topic, sub.subscription_name]); let by_topic = self.key(&["subs", "by_topic", sub.topic]); let all = self.key(&["subs", "all"]); @@ -94,11 +92,19 @@ impl RedisStorage { // A prior row may have carried a different (or NULL) owner; drop its // stale by_owner entry so ownership never lingers across re-registration. - let prior_owner = self - .load_sub_by_composite(&mut conn, &composite)? - .and_then(|e| e.owner_worker_id); + // Re-declaring also must not resume a paused subscription or reset its + // registration time, so both survive from the prior entry. + let prior = self.load_sub_by_composite(&mut conn, &composite)?; + let prior_owner = prior.as_ref().and_then(|e| e.owner_worker_id.clone()); + let mut entry = entry; + if let Some(prior) = &prior { + entry.active = prior.active; + entry.created_at = prior.created_at; + } + let json = serde_json::to_string(&entry).map_err(|e| QueueError::Other(e.to_string()))?; - let pipe = &mut redis::pipe(); + let pipe = redis::pipe().atomic().to_owned(); + let mut pipe = pipe; pipe.set(&blob_key, &json); pipe.sadd(&by_topic, sub.subscription_name); pipe.sadd(&all, &composite); @@ -179,7 +185,7 @@ impl RedisStorage { let all = self.key(&["subs", "all"]); let composite = Self::sub_composite(topic, subscription_name); - let pipe = &mut redis::pipe(); + let mut pipe = redis::pipe().atomic().to_owned(); pipe.del(&blob_key); pipe.srem(&by_topic, subscription_name); pipe.srem(&all, &composite); @@ -218,13 +224,17 @@ impl RedisStorage { } /// Remove ephemeral subscriptions whose owner is not in `live_worker_ids`. - /// Durable rows (owner NULL) are never touched. Returns the count removed. + /// Durable rows (owner NULL) are never touched, and rows younger than the + /// registration grace window survive (a starting worker inserts its + /// ephemeral subscriptions before its first heartbeat lands). + /// Returns the count removed. /// /// Iterates the `subs:all` index set rather than a keyspace `SCAN`; each /// removed row is also pulled from its `by_topic` and `by_owner` sets. pub fn reap_ephemeral_subscriptions(&self, live_worker_ids: &[String]) -> Result { let mut conn = self.conn()?; let all = self.key(&["subs", "all"]); + let cutoff = crate::job::now_millis() - crate::storage::EPHEMERAL_SUBSCRIPTION_GRACE_MS; let composites: Vec = conn.smembers(&all).map_err(map_err)?; let live: HashSet<&str> = live_worker_ids.iter().map(String::as_str).collect(); @@ -238,7 +248,7 @@ impl RedisStorage { let Some(owner) = entry.owner_worker_id.as_deref() else { continue; }; - if live.contains(owner) { + if live.contains(owner) || entry.created_at >= cutoff { continue; } @@ -246,7 +256,7 @@ impl RedisStorage { let by_topic = self.key(&["subs", "by_topic", &entry.topic]); let by_owner = self.key(&["subs", "by_owner", owner]); - let pipe = &mut redis::pipe(); + let mut pipe = redis::pipe().atomic().to_owned(); pipe.del(&blob_key); pipe.srem(&by_topic, &entry.subscription_name); pipe.srem(&all, &composite); diff --git a/crates/taskito-core/src/storage/sqlite/pubsub.rs b/crates/taskito-core/src/storage/sqlite/pubsub.rs index be7bd16a..f932f087 100644 --- a/crates/taskito-core/src/storage/sqlite/pubsub.rs +++ b/crates/taskito-core/src/storage/sqlite/pubsub.rs @@ -10,13 +10,26 @@ crate::storage::diesel_common::impl_diesel_pubsub_ops!(SqliteStorage); impl SqliteStorage { /// Insert or update a subscription. Idempotent on (topic, subscription_name). /// - /// `replace_into` deletes any prior row before inserting, so re-registering - /// with `owner_worker_id = None` correctly clears a previously-set owner. + /// The update set deliberately excludes `active`: re-declaring a + /// subscription (every worker start does) must not resume one an operator + /// paused. `owner_worker_id` is written explicitly so a durable + /// re-registration clears a previously-ephemeral owner to SQL NULL. pub fn register_subscription(&self, sub: &NewSubscriptionRow) -> Result<()> { let mut conn = self.conn()?; - diesel::replace_into(topic_subscriptions::table) + diesel::insert_into(topic_subscriptions::table) .values(sub) + .on_conflict(( + topic_subscriptions::topic, + topic_subscriptions::subscription_name, + )) + .do_update() + .set(( + topic_subscriptions::task_name.eq(sub.task_name), + topic_subscriptions::queue.eq(sub.queue), + topic_subscriptions::durable.eq(sub.durable), + topic_subscriptions::owner_worker_id.eq(sub.owner_worker_id), + )) .execute(&mut conn)?; Ok(()) diff --git a/crates/taskito-core/src/storage/sqlite/tests.rs b/crates/taskito-core/src/storage/sqlite/tests.rs index 38b661da..29b58b51 100644 --- a/crates/taskito-core/src/storage/sqlite/tests.rs +++ b/crates/taskito-core/src/storage/sqlite/tests.rs @@ -1514,7 +1514,9 @@ fn test_set_subscription_active_pause_resume_roundtrip() { #[test] fn test_reap_ephemeral_subscriptions_spares_durable_and_live() { let storage = test_storage(); - let now = now_millis(); + // Aged past the registration grace window so the reaper may act on them; + // a fresh row must survive even with a dead owner (startup race guard). + let now = now_millis() - crate::storage::EPHEMERAL_SUBSCRIPTION_GRACE_MS - 1_000; // Durable (owner NULL) — must never be reaped. storage diff --git a/crates/taskito-core/tests/rust/storage_tests.rs b/crates/taskito-core/tests/rust/storage_tests.rs index 022168b0..13793853 100644 --- a/crates/taskito-core/tests/rust/storage_tests.rs +++ b/crates/taskito-core/tests/rust/storage_tests.rs @@ -854,7 +854,9 @@ fn test_periodic_crud(s: &impl Storage) { fn test_topic_subscriptions_crud(s: &impl Storage) { use taskito_core::storage::models::NewSubscriptionRow; - let now = now_millis(); + // Aged past the registration grace window so the reaper may act on the + // ephemeral rows created below; freshness is covered by the grace test. + let now = now_millis() - taskito_core::storage::EPHEMERAL_SUBSCRIPTION_GRACE_MS - 1_000; let sub = |topic: &'static str, name: &'static str, task_name: &'static str, @@ -926,6 +928,37 @@ fn test_topic_subscriptions_crud(s: &impl Storage) { .unwrap()); assert!(!s.unsubscribe("ts-orders", "ghost").unwrap()); + // Re-registering must not resume a paused subscription. + assert!(s + .set_subscription_active("ts-orders", "emailer", false) + .unwrap()); + s.register_subscription(&sub("ts-orders", "emailer", "send_email_v3", None, now)) + .unwrap(); + assert!( + !s.list_subscriptions() + .unwrap() + .iter() + .any(|r| r.subscription_name == "emailer" && r.active), + "re-registration must preserve the paused state" + ); + assert!(s + .set_subscription_active("ts-orders", "emailer", true) + .unwrap()); + + // A fresh ephemeral row (inside the grace window) survives a reap even + // with a dead owner — startup registers subscriptions before the first + // heartbeat lands. + s.register_subscription(&sub( + "ts-live", + "fresh", + "task_a", + Some("ts-worker-gone"), + now_millis(), + )) + .unwrap(); + assert_eq!(s.reap_ephemeral_subscriptions(&[]).unwrap(), 0); + assert!(s.unsubscribe("ts-live", "fresh").unwrap()); + // Reaper: only dead-owner ephemeral rows go; durable rows never do. s.register_subscription(&sub("ts-live", "live", "task_b", Some("ts-worker-1"), now)) .unwrap(); From dc9ca2eb427c4c6622b6d4cd0eb34c9f41f5fae8 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sun, 12 Jul 2026 10:23:03 +0530 Subject: [PATCH 10/12] fix(python): validate and clean up subscription registration Reject unowned ephemeral subscriptions and per-task codecs on subscribers (publish encodes one shared payload). Re-declaring replaces the pending entry instead of appending, and unsubscribe drops the local declaration so a later worker start can't resurrect it. --- crates/taskito-python/src/py_queue/pubsub.rs | 7 +++ sdks/python/taskito/mixins/pubsub.py | 43 +++++++++++++----- sdks/python/tests/core/test_pubsub.py | 46 ++++++++++++++++++-- 3 files changed, 83 insertions(+), 13 deletions(-) diff --git a/crates/taskito-python/src/py_queue/pubsub.rs b/crates/taskito-python/src/py_queue/pubsub.rs index 9337f35e..98968774 100644 --- a/crates/taskito-python/src/py_queue/pubsub.rs +++ b/crates/taskito-python/src/py_queue/pubsub.rs @@ -27,6 +27,13 @@ impl PyQueue { durable: bool, owner_worker_id: Option<&str>, ) -> PyResult<()> { + // An unowned ephemeral row could never be reaped (cleanup keys off + // live worker ids), so it would stay active forever. + if !durable && owner_worker_id.is_none() { + return Err(pyo3::exceptions::PyValueError::new_err( + "an ephemeral subscription (durable=false) requires owner_worker_id", + )); + } let row = NewSubscriptionRow { topic, subscription_name, diff --git a/sdks/python/taskito/mixins/pubsub.py b/sdks/python/taskito/mixins/pubsub.py index 7845df22..fdd9ee4e 100644 --- a/sdks/python/taskito/mixins/pubsub.py +++ b/sdks/python/taskito/mixins/pubsub.py @@ -60,17 +60,31 @@ def subscriber( ``timeout``, ``middleware``, ``idempotent``, ...). """ + if "codecs" in task_kwargs: + raise ValueError( + "subscriber() does not accept per-task codecs: publish() encodes one " + "shared payload with the queue serializer, so a per-subscriber codec " + "chain could never be applied on the producer side" + ) + def decorator(fn: Callable[..., Any]) -> TaskWrapper: wrapper: TaskWrapper = self.task(queue=queue, **task_kwargs)(fn) # type: ignore[attr-defined] - self._subscription_configs.append( - { - "topic": topic, - "subscription_name": name or wrapper.name, - "task_name": wrapper.name, - "queue": queue, - "durable": durable, - } - ) + config = { + "topic": topic, + "subscription_name": name or wrapper.name, + "task_name": wrapper.name, + "queue": queue, + "durable": durable, + } + # Replace, don't append: re-decorating the same (topic, name) (module + # reloads, test fixtures) must not accumulate stale declarations. + identity = (config["topic"], config["subscription_name"]) + self._subscription_configs[:] = [ + c + for c in self._subscription_configs + if (c["topic"], c["subscription_name"]) != identity + ] + self._subscription_configs.append(config) return wrapper return decorator @@ -133,7 +147,16 @@ def publish( return [JobResult(py_job=py_job, queue=self) for py_job in py_jobs] # type: ignore[arg-type] def unsubscribe(self, topic: str, name: str) -> bool: - """Remove a subscription. Returns False if none matched.""" + """Remove a subscription. Returns False if none matched. + + Also drops any matching local declaration so a later worker start + doesn't re-register what was just removed. + """ + self._subscription_configs[:] = [ + c + for c in self._subscription_configs + if (c["topic"], c["subscription_name"]) != (topic, name) + ] return self._inner.unsubscribe(topic, name) def pause_subscription(self, topic: str, name: str) -> bool: diff --git a/sdks/python/tests/core/test_pubsub.py b/sdks/python/tests/core/test_pubsub.py index 6eab8809..5e8480f1 100644 --- a/sdks/python/tests/core/test_pubsub.py +++ b/sdks/python/tests/core/test_pubsub.py @@ -3,6 +3,8 @@ import threading from typing import Any +import pytest + from taskito import Queue PollUntil = Any # the conftest fixture's runtime type @@ -184,7 +186,12 @@ def tail(order_id: int) -> None: ... queue.declare_subscriptions() assert queue.list_subscriptions("orders") == [] - def test_reap_removes_dead_owner_subscriptions(self, queue: Queue) -> None: + def test_fresh_ephemeral_rows_survive_reap_grace_window(self, queue: Queue) -> None: + """A just-registered ephemeral row must survive the reaper even with a + dead owner: workers insert subscriptions before their first heartbeat + lands, and another worker's reap tick must not race that gap. Aged-row + reaping is covered by the Rust storage tests (created_at is not + settable through the public API).""" queue._inner.register_subscription( topic="orders", subscription_name="ghost", @@ -196,6 +203,39 @@ def test_reap_removes_dead_owner_subscriptions(self, queue: Queue) -> None: topic="orders", subscription_name="durable", task_name="some.task" ) - assert queue._inner.reap_ephemeral_subscriptions() == 1 + assert queue._inner.reap_ephemeral_subscriptions() == 0 names = {s["name"] for s in queue.list_subscriptions("orders")} - assert names == {"durable"} + assert names == {"ghost", "durable"} + + def test_ephemeral_registration_requires_owner(self, queue: Queue) -> None: + with pytest.raises(ValueError, match="owner_worker_id"): + queue._inner.register_subscription( + topic="orders", + subscription_name="unowned", + task_name="some.task", + durable=False, + ) + + def test_subscriber_rejects_codecs(self, queue: Queue) -> None: + with pytest.raises(ValueError, match="codecs"): + + @queue.subscriber("orders", name="bad", codecs=["gzip"]) + def handler(order_id: int) -> None: ... + + def test_redeclare_after_pause_stays_paused(self, queue: Queue) -> None: + @queue.subscriber("orders", name="email") + def send_email(order_id: int) -> None: ... + + queue.declare_subscriptions() + assert queue.pause_subscription("orders", "email") is True + queue.declare_subscriptions() + assert queue.publish("orders", 1) == [] + + def test_unsubscribe_drops_local_declaration(self, queue: Queue) -> None: + @queue.subscriber("orders", name="email") + def send_email(order_id: int) -> None: ... + + queue.declare_subscriptions() + assert queue.unsubscribe("orders", "email") is True + queue.declare_subscriptions() + assert queue.list_subscriptions("orders") == [] From 56ebd0c4a64e628406f8fbad05438577666deb6f Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sun, 12 Jul 2026 10:23:20 +0530 Subject: [PATCH 11/12] fix(node): validate and clean up subscription registration Reject unowned ephemeral subscriptions, validate per-task delivery defaults, and reject per-task codecs on subscribers. Pending declarations are now idempotent and cleared on unsubscribe; the reaper prunes dead workers first, runs a final sweep on worker stop, and registration retries until it succeeds. --- crates/taskito-node/src/queue/pubsub.rs | 43 +++++++++++---- sdks/node/src/queue.ts | 31 ++++++++++- sdks/node/src/types.ts | 3 +- sdks/node/src/worker.ts | 39 ++++++++++--- sdks/node/test/core/pubsub.test.ts | 73 +++++++++++++++++++++++-- 5 files changed, 164 insertions(+), 25 deletions(-) diff --git a/crates/taskito-node/src/queue/pubsub.rs b/crates/taskito-node/src/queue/pubsub.rs index f0ce9844..ef5a44da 100644 --- a/crates/taskito-node/src/queue/pubsub.rs +++ b/crates/taskito-node/src/queue/pubsub.rs @@ -17,7 +17,7 @@ use crate::convert::{ job_to_js, subscription_to_js, JsJob, JsSubscription, DEFAULT_MAX_RETRIES, DEFAULT_PRIORITY, DEFAULT_TIMEOUT_MS, }; -use crate::error::{join_to_napi_err, non_negative, to_napi_err}; +use crate::error::{invalid_arg, join_to_napi_err, non_negative, to_napi_err}; #[napi] impl JsQueue { @@ -34,6 +34,13 @@ impl JsQueue { durable: bool, owner_worker_id: Option, ) -> Result<()> { + // An owner-less ephemeral row could never be reaped — reject it before + // it reaches storage. + if !durable && owner_worker_id.is_none() { + return Err(invalid_arg( + "an ephemeral subscription (durable=false) requires ownerWorkerId", + )); + } let storage = self.storage.clone(); spawn_blocking(move || { let row = NewSubscriptionRow { @@ -100,13 +107,16 @@ impl JsQueue { .map_err(join_to_napi_err)? } - /// Drop ephemeral subscriptions whose owning worker is gone. Runs on the - /// heartbeat cadence, after dead workers have been reaped. Returns the - /// number of subscriptions removed. + /// Drop ephemeral subscriptions whose owning worker is gone. Prunes dead + /// worker rows first so the live set is current. Runs on the heartbeat + /// cadence. Returns the number of subscriptions removed. #[napi] pub async fn reap_ephemeral_subscriptions(&self) -> Result { let storage = self.storage.clone(); spawn_blocking(move || { + // Prune stale worker rows first so a crashed owner doesn't keep its + // ephemeral subscriptions alive; opportunistic like the heartbeat's. + let _ = storage.reap_dead_workers(); let live: Vec = storage .list_workers() .map_err(to_napi_err)? @@ -195,27 +205,38 @@ fn build_publish_request( result_ttl_ms, namespace, queue_defaults, - task_defaults: task_defaults(opts.task_defaults.unwrap_or_default(), queue_defaults), + task_defaults: task_defaults(opts.task_defaults.unwrap_or_default(), queue_defaults)?, }) } /// Resolve per-task delivery defaults, filling unset fields from the queue -/// defaults so the shell never duplicates them. +/// defaults so the shell never duplicates them. Entries get the same +/// non-negative checks as the top-level options, naming the offending task. fn task_defaults( input: HashMap, queue_defaults: DeliveryDefaults, -) -> HashMap { +) -> Result> { input .into_iter() .map(|(name, defaults)| { - ( + let max_retries = match defaults.max_retries { + Some(n) => { + non_negative(n as i64, &format!("taskDefaults[\"{name}\"].maxRetries"))? as i32 + } + None => queue_defaults.max_retries, + }; + let timeout_ms = match defaults.timeout_ms { + Some(n) => non_negative(n, &format!("taskDefaults[\"{name}\"].timeoutMs"))?, + None => queue_defaults.timeout_ms, + }; + Ok(( name, DeliveryDefaults { priority: defaults.priority.unwrap_or(queue_defaults.priority), - max_retries: defaults.max_retries.unwrap_or(queue_defaults.max_retries), - timeout_ms: defaults.timeout_ms.unwrap_or(queue_defaults.timeout_ms), + max_retries, + timeout_ms, }, - ) + )) }) .collect() } diff --git a/sdks/node/src/queue.ts b/sdks/node/src/queue.ts index dd76d0c1..a022f4d7 100644 --- a/sdks/node/src/queue.ts +++ b/sdks/node/src/queue.ts @@ -288,13 +288,32 @@ export class Queue { options?: SubscriberOptions, ): Queue> { const { subscriptionName, queue, durable, ...taskOptions } = options ?? {}; - this.pendingSubscriptions.push({ + // publish() encodes one shared payload with the queue serializer only, but + // the worker would reverse a per-task codec chain — a guaranteed decode + // failure, so reject it up front. + if (taskOptions.codecs && taskOptions.codecs.length > 0) { + throw new QueueError( + `subscriber "${name}": per-task codecs do not apply to topic deliveries — ` + + "published payloads use the queue-level serializer only", + ); + } + const pending: PendingSubscription = { topic, subscriptionName: subscriptionName ?? name, taskName: name, queue: queue ?? "default", durable: durable ?? true, - }); + }; + // Redeclaring the same (topic, subscriptionName) replaces the pending + // entry — declareSubscriptions must stay idempotent. + const existing = this.pendingSubscriptions.findIndex( + (sub) => sub.topic === topic && sub.subscriptionName === pending.subscriptionName, + ); + if (existing >= 0) { + this.pendingSubscriptions[existing] = pending; + } else { + this.pendingSubscriptions.push(pending); + } return this.task(name, handler, taskOptions); } @@ -465,6 +484,14 @@ export class Queue { /** Remove a subscription. Resolves false if none matched. */ unsubscribe(topic: string, name: string): Promise { + // Drop any matching pending entry too, so a later declareSubscriptions() + // or worker start doesn't resurrect the removed subscription. + const pending = this.pendingSubscriptions.findIndex( + (sub) => sub.topic === topic && sub.subscriptionName === name, + ); + if (pending >= 0) { + this.pendingSubscriptions.splice(pending, 1); + } return this.native.unsubscribe(topic, name); } diff --git a/sdks/node/src/types.ts b/sdks/node/src/types.ts index c3474b0e..fc0bce23 100644 --- a/sdks/node/src/types.ts +++ b/sdks/node/src/types.ts @@ -43,7 +43,8 @@ export interface PublishOptions extends Omit; } -/** Options for {@link Queue.subscriber}: task options plus subscription routing. */ +/** Options for {@link Queue.subscriber}: task options plus subscription routing. + * `codecs` is not supported — published payloads use the queue-level serializer. */ export interface SubscriberOptions extends TaskOptions { /** Stable subscription identity — re-registering the same `(topic, name)` * updates the routing target instead of duplicating. Defaults to the task name. */ diff --git a/sdks/node/src/worker.ts b/sdks/node/src/worker.ts index 0d032798..73ef2568 100644 --- a/sdks/node/src/worker.ts +++ b/sdks/node/src/worker.ts @@ -62,6 +62,7 @@ export interface WorkerStartParams { export class Worker { private constructor( private readonly native: NativeWorker, + private readonly queue: NativeQueue, private readonly resources: ResourceRuntime, private readonly heartbeat: ReturnType, ) {} @@ -236,6 +237,30 @@ export class Worker { // only) — recreation of failing resources is per runtime, not per worker. resources.acquireWorker(); + // Register this worker's topic subscriptions (ephemeral ones under its id) + // now that the id exists. Registration is idempotent, so a failed flush is + // retried whole on every heartbeat tick until it succeeds — a silently + // missing subscription would drop deliveries for the worker's lifetime. + const flushSubscriptions = params.declareSubscriptions; + let subscriptionsDeclared = flushSubscriptions === undefined; + let declarationInFlight = false; + const declareSubscriptions = (): void => { + if (subscriptionsDeclared || declarationInFlight || !flushSubscriptions) { + return; + } + declarationInFlight = true; + void flushSubscriptions(native.id) + .then(() => { + subscriptionsDeclared = true; + }) + .catch((error) => { + log.error(() => "subscription registration failed; retrying on next heartbeat", error); + }) + .finally(() => { + declarationInFlight = false; + }); + }; + // Heartbeat with current resource health so inspection (and dead-worker // reaping) see this worker as alive. Failures are logged, never thrown — // the next beat retries. First beat goes out immediately. @@ -250,22 +275,22 @@ export class Worker { void queue.reapEphemeralSubscriptions().catch((error) => { log.debug(() => "ephemeral subscription reap failed", error); }); + declareSubscriptions(); }; - // Register this worker's topic subscriptions (ephemeral ones under its id) - // now that the id exists. Fire-and-forget like the heartbeat: a failure is - // logged and the worker still runs. - void params.declareSubscriptions?.(native.id).catch((error) => { - log.debug(() => "subscription registration failed", error); - }); sendHeartbeat(); const heartbeat = setInterval(sendHeartbeat, HEARTBEAT_INTERVAL_MS); heartbeat.unref(); - return new Worker(native, resources, heartbeat); + return new Worker(native, queue, resources, heartbeat); } /** Stop the worker; in-flight results drain before background tasks exit. */ stop(): void { + // One last sweep for orphaned ephemeral subscriptions before this worker's + // reap cadence goes away. Best effort — stopping must never throw. + void this.queue.reapEphemeralSubscriptions().catch((error) => { + log.debug(() => "final ephemeral subscription reap failed", error); + }); clearInterval(this.heartbeat); this.native.stop(); // Dispose worker-scoped resources after the native worker quiesces (the diff --git a/sdks/node/test/core/pubsub.test.ts b/sdks/node/test/core/pubsub.test.ts index b6ec3700..e1b7a1e3 100644 --- a/sdks/node/test/core/pubsub.test.ts +++ b/sdks/node/test/core/pubsub.test.ts @@ -3,6 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, it } from "vitest"; import { Queue, type Worker } from "../../src/index"; +import type { NativeQueue } from "../../src/native"; let worker: Worker | undefined; afterEach(() => { @@ -64,6 +65,45 @@ it("redeclaring updates the subscription instead of duplicating", async () => { expect(subs[0]?.queue).toBe("mail"); }); +it("re-registering the same (topic, name) replaces the pending entry", async () => { + const queue = newQueue(); + queue.subscriber("orders", "send_email", () => undefined, { + subscriptionName: "email", + queue: "mail", + }); + queue.subscriber("orders", "send_email_v2", () => undefined, { + subscriptionName: "email", + queue: "mail-v2", + }); + await queue.declareSubscriptions(); + + const subs = await queue.listSubscriptions("orders"); + expect(subs).toHaveLength(1); + expect(subs[0]?.taskName).toBe("send_email_v2"); + expect(subs[0]?.queue).toBe("mail-v2"); +}); + +it("subscriber rejects per-task codecs", () => { + const queue = newQueue(); + expect(() => + queue.subscriber("orders", "send_email", () => undefined, { codecs: ["gzip"] }), + ).toThrow(/per-task codecs do not apply/); +}); + +it("registering an ephemeral subscription without an owner rejects", async () => { + const queue = newQueue(); + const native = (queue as unknown as { native: NativeQueue }).native; + await expect( + native.registerSubscription("orders", "eph", "eph_task", "default", false, undefined), + ).rejects.toThrow(/ephemeral subscription \(durable=false\) requires ownerWorkerId/); +}); + +it("publish rejects negative per-task delivery defaults naming the task", async () => { + const queue = newQueue(); + queue.task("bad_task", () => undefined, { maxRetries: -1 }); + await expect(queue.publish("orders", [1])).rejects.toThrow('taskDefaults["bad_task"].maxRetries'); +}); + it("unsubscribe removes the subscription and reports a missing one", async () => { const queue = newQueue(); queue.subscriber("orders", "send_email", () => undefined, { subscriptionName: "email" }); @@ -74,6 +114,26 @@ it("unsubscribe removes the subscription and reports a missing one", async () => expect(await queue.listSubscriptions("orders")).toEqual([]); }); +it("a later declareSubscriptions does not resurrect an unsubscribed subscriber", async () => { + const queue = newQueue(); + queue.subscriber("orders", "send_email", () => undefined, { subscriptionName: "email" }); + await queue.declareSubscriptions(); + + expect(await queue.unsubscribe("orders", "email")).toBe(true); + await queue.declareSubscriptions(); + expect(await queue.listSubscriptions("orders")).toEqual([]); +}); + +it("redeclaring keeps a paused subscription paused", async () => { + const queue = newQueue(); + queue.subscriber("orders", "send_email", () => undefined, { subscriptionName: "email" }); + await queue.declareSubscriptions(); + + expect(await queue.pauseSubscription("orders", "email")).toBe(true); + await queue.declareSubscriptions(); + expect(await queue.publish("orders", [1])).toEqual([]); +}); + it("pause blocks deliveries and resume restores them", async () => { const queue = newQueue(); queue.subscriber("orders", "send_email", () => undefined, { subscriptionName: "email" }); @@ -173,7 +233,7 @@ it("declareSubscriptions skips ephemeral subscriptions", async () => { expect(subs.map((s) => s.subscriptionName)).toEqual(["durable_task"]); }); -it("reapEphemeralSubscriptions removes dead-owner rows only", async () => { +it("reapEphemeralSubscriptions spares fresh rows even with a dead owner", async () => { const queue = newQueue(); queue.subscriber("orders", "durable_task", () => undefined); queue.subscriber("orders", "ephemeral_task", () => undefined, { durable: false }); @@ -186,12 +246,17 @@ it("reapEphemeralSubscriptions removes dead-owner rows only", async () => { // Owner alive: the reap must not touch its ephemeral subscription. expect(await queue.reapEphemeralSubscriptions()).toBe(0); - // Stop the worker (unregisters it) — the orphaned ephemeral row is reaped. + // Stop the worker (unregisters it). The orphaned row is younger than the + // startup grace window, so it survives — a starting worker inserts its + // ephemeral subscriptions before its first heartbeat, and another worker's + // reap tick must not race that gap. Aged reaping is covered in core tests. worker.stop(); worker = undefined; expect(await waitFor(async () => (await queue.listWorkers()).length === 0)).toBe(true); - expect(await queue.reapEphemeralSubscriptions()).toBe(1); + expect(await queue.reapEphemeralSubscriptions()).toBe(0); const remaining = await queue.listSubscriptions("orders"); - expect(remaining.map((s) => s.subscriptionName)).toEqual(["durable_task"]); + expect(new Set(remaining.map((s) => s.subscriptionName))).toEqual( + new Set(["durable_task", "ephemeral_task"]), + ); }); From 4e74c3d89ad0657deb099f976a454ac42e209074 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sun, 12 Jul 2026 10:24:20 +0530 Subject: [PATCH 12/12] fix(java): validate and clean up subscription registration Reject unowned ephemeral subscriptions, register the worker before its ephemeral subs, and remove local declarations on unsubscribe. The in-memory backend now preserves paused state on re-registration, applies the reap grace window, uses conditional removal, and rejects unsupported publish options. --- crates/taskito-java/src/queue/pubsub.rs | 9 ++ crates/taskito-java/src/worker.rs | 67 ++++++---- .../org/byteveda/taskito/DefaultTaskito.java | 20 ++- .../byteveda/taskito/spi/QueueBackend.java | 6 +- .../org/byteveda/taskito/core/PubSubTest.java | 62 ++++++++- .../taskito/test/InMemoryQueueBackend.java | 123 ++++++++++++++++-- .../taskito/test/InMemoryPubSubTest.java | 113 +++++++++++++++- 7 files changed, 348 insertions(+), 52 deletions(-) diff --git a/crates/taskito-java/src/queue/pubsub.rs b/crates/taskito-java/src/queue/pubsub.rs index 85551b61..0c11e59d 100644 --- a/crates/taskito-java/src/queue/pubsub.rs +++ b/crates/taskito-java/src/queue/pubsub.rs @@ -44,11 +44,20 @@ pub extern "system" fn Java_org_byteveda_taskito_internal_NativeQueue_registerSu let task_name = read_string(env, &task_name)?; let queue = read_string(env, &queue)?; let owner_worker_id = read_optional_string(env, &owner_worker_id)?; + // An ownerless ephemeral row would never be reaped (the reaper matches + // on owner) yet keeps receiving deliveries — reject it up front. + if durable == 0 && owner_worker_id.is_none() { + return Err(crate::error::BindingError::new( + "an ephemeral subscription (durable=false) requires ownerWorkerId", + )); + } let row = NewSubscriptionRow { topic: &topic, subscription_name: &subscription_name, task_name: &task_name, queue: &queue, + // Insert default only: the core upsert preserves an existing row's + // `active` (and `created_at`), so re-declaring never resumes a pause. active: true, durable: durable != 0, owner_worker_id: owner_worker_id.as_deref(), diff --git a/crates/taskito-java/src/worker.rs b/crates/taskito-java/src/worker.rs index 66c9528d..2429880e 100644 --- a/crates/taskito-java/src/worker.rs +++ b/crates/taskito-java/src/worker.rs @@ -85,14 +85,19 @@ fn start_worker( #[cfg(feature = "mesh")] let mesh_worker_id = worker_id.clone(); - // Write topic subscriptions before the scheduler starts, so its first poll - // can already see deliveries. A registration failure aborts the start — - // a silently missing subscription would drop deliveries. + // Create the live worker row before its ephemeral subscriptions exist: + // the reaper only spares owned rows whose owner is registered, so this + // ordering (plus the core's registration grace window) keeps a concurrent + // reap from racing the start. Then write topic subscriptions before the + // scheduler starts, so its first poll can already see deliveries. Either + // failure aborts the start — a silently missing subscription would drop + // deliveries. + register_live_worker(&lifecycle_storage, &worker_id, &queues_csv, capacity)?; register_subscriptions(&storage, &worker_id, options.subscriptions.take())?; let mut scheduler = Scheduler::new(storage, queues, config, namespace); // Claim execution under this worker's id so dead-worker recovery can - // attribute orphaned jobs (matches the register_worker id below). + // attribute orphaned jobs (matches the worker id registered above). scheduler.set_claim_owner(worker_id.clone()); register_task_policies(&mut scheduler, options.task_configs); let scheduler = Arc::new(scheduler); @@ -147,13 +152,11 @@ fn start_worker( let drain_callbacks = callbacks; runtime.spawn_blocking(move || drain_results(result_rx, drain_scheduler, drain_callbacks)); - // Lifecycle loop: register, heartbeat, unregister. + // Lifecycle loop: heartbeat until stopped, then unregister. spawn_lifecycle( &runtime, lifecycle_storage, worker_id, - queues_csv, - capacity, heartbeat_stop.clone(), ); @@ -167,9 +170,36 @@ fn start_worker( }) } +/// Create this worker's registry row. Runs before the worker's ephemeral +/// subscriptions are written, and a failure aborts the start: those rows bind +/// to this id, so a missing registry row would leave them reap-able as soon +/// as the registration grace window lapses. +fn register_live_worker( + storage: &StorageBackend, + worker_id: &str, + queues_csv: &str, + capacity: usize, +) -> Result<(), crate::error::BindingError> { + let hostname = gethostname::gethostname().to_string_lossy().to_string(); + let pid = std::process::id() as i32; + storage.register_worker( + worker_id, + queues_csv, + None, + None, + None, + capacity as i32, + Some(&hostname), + Some(pid), + Some("java"), + )?; + Ok(()) +} + /// Register the worker's declared topic subscriptions. Durable rows carry no /// owner; ephemeral rows bind to this worker's id and are reaped once the -/// worker leaves the registry. +/// worker leaves the registry. `active` is an insert default — the core upsert +/// preserves an existing row's paused state across a restart. fn register_subscriptions( storage: &StorageBackend, worker_id: &str, @@ -355,30 +385,15 @@ fn describe(outcome: &ResultOutcome) -> (&str, &str, &str, Option<&str>, i32, bo } } -/// Register the worker, heartbeat every 5s until stopped, then unregister. +/// Heartbeat the already-registered worker every 5s until stopped, then +/// unregister it. Registration itself happens synchronously at start (see +/// [`register_live_worker`]) so ephemeral subscriptions never precede it. fn spawn_lifecycle( runtime: &tokio::runtime::Runtime, storage: StorageBackend, worker_id: String, - queues_csv: String, - capacity: usize, stop: Arc, ) { - let hostname = gethostname::gethostname().to_string_lossy().to_string(); - let pid = std::process::id() as i32; - if let Err(e) = storage.register_worker( - &worker_id, - &queues_csv, - None, - None, - None, - capacity as i32, - Some(&hostname), - Some(pid), - Some("java"), - ) { - log::warn!("[taskito-java] worker registration failed: {e}"); - } runtime.spawn(async move { loop { tokio::select! { diff --git a/sdks/java/src/main/java/org/byteveda/taskito/DefaultTaskito.java b/sdks/java/src/main/java/org/byteveda/taskito/DefaultTaskito.java index 571cf914..2bba3195 100644 --- a/sdks/java/src/main/java/org/byteveda/taskito/DefaultTaskito.java +++ b/sdks/java/src/main/java/org/byteveda/taskito/DefaultTaskito.java @@ -676,6 +676,17 @@ public Taskito subscribe(String topic, Task task) { public Taskito subscribe(String topic, Task task, SubscriptionOptions options) { String name = options.name() != null ? options.name() : task.name(); EnqueueOptions taskDefaults = task.options(); + // A durable subscription registers now so producer-only processes see it; + // an ephemeral one waits for a worker start to bind to that worker's id. + // Register before recording locally, so a failed registration leaves no + // orphaned local declaration behind. + if (options.durable()) { + backend.registerSubscription(topic, name, task.name(), options.queue(), true, null); + } + // Re-declaring (topic, name) replaces the previous local entry, mirroring + // the backend's upsert instead of accumulating duplicates. + subscriptions.removeIf( + existing -> existing.topic().equals(topic) && existing.name().equals(name)); subscriptions.add(new SubscriptionConfig( topic, name, @@ -685,11 +696,6 @@ public Taskito subscribe(String topic, Task task, SubscriptionOptions opt taskDefaults.priority(), taskDefaults.maxRetries(), taskDefaults.timeoutMs())); - // A durable subscription registers now so producer-only processes see it; - // an ephemeral one waits for a worker start to bind to that worker's id. - if (options.durable()) { - backend.registerSubscription(topic, name, task.name(), options.queue(), true, null); - } return this; } @@ -744,6 +750,10 @@ private static void putIfSet(Map request, String key, Object val @Override public boolean unsubscribe(String topic, String name) { + // Drop the local declaration too, or a later worker start would + // re-register the subscription from the shared worker state. + subscriptions.removeIf( + config -> config.topic().equals(topic) && config.name().equals(name)); return backend.unsubscribe(topic, name); } diff --git a/sdks/java/src/main/java/org/byteveda/taskito/spi/QueueBackend.java b/sdks/java/src/main/java/org/byteveda/taskito/spi/QueueBackend.java index 8792a4ba..e5506773 100644 --- a/sdks/java/src/main/java/org/byteveda/taskito/spi/QueueBackend.java +++ b/sdks/java/src/main/java/org/byteveda/taskito/spi/QueueBackend.java @@ -167,7 +167,11 @@ default boolean setPeriodicEnabled(String name, boolean enabled) { // compiling and fail explicitly only when pub/sub is actually used. String PUBSUB_UNSUPPORTED = "pub/sub not supported by this backend"; - /** Insert or update a topic subscription (idempotent on topic + name). */ + /** + * Insert or update a topic subscription (idempotent on topic + name). + * Re-registering updates routing but preserves a paused state; an ephemeral + * subscription ({@code durable=false}) requires {@code ownerWorkerIdOrNull}. + */ default void registerSubscription( String topic, String subscriptionName, diff --git a/sdks/java/src/test/java/org/byteveda/taskito/core/PubSubTest.java b/sdks/java/src/test/java/org/byteveda/taskito/core/PubSubTest.java index c222ce7d..68d454ac 100644 --- a/sdks/java/src/test/java/org/byteveda/taskito/core/PubSubTest.java +++ b/sdks/java/src/test/java/org/byteveda/taskito/core/PubSubTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.ObjectMapper; @@ -12,6 +13,7 @@ import java.util.List; import java.util.Map; import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.TaskitoException; import org.byteveda.taskito.internal.JniQueueBackend; import org.byteveda.taskito.model.Job; import org.byteveda.taskito.model.JobStatus; @@ -206,7 +208,7 @@ void ephemeralSubscriptionRegistersWithARunningWorker(@TempDir Path dir) throws @Test @Timeout(30) - void reapRemovesDeadOwnedSubscriptionsOnly(@TempDir Path dir) throws Exception { + void reapSparesFreshEphemeralRowsWithinTheRegistrationGrace(@TempDir Path dir) throws Exception { String options = new ObjectMapper() .writeValueAsString( Map.of("backend", "sqlite", "dsn", dir.resolve("t.db").toString())); @@ -218,13 +220,63 @@ void reapRemovesDeadOwnedSubscriptionsOnly(@TempDir Path dir) throws Exception { backend.registerSubscription("orders", "ghost", SEND_EMAIL.name(), "default", false, "gone-worker"); backend.registerSubscription("orders", "durable", SEND_EMAIL.name(), "default", true, null); - // The running worker's own heartbeat may reap concurrently, so - // assert the converged state rather than this call's count. - backend.reapEphemeralSubscriptions(); + // Even the dead-owned row is inside the core's registration + // grace window, so nothing is eligible yet — a starting + // worker's rows must not be raced away before its liveness + // becomes visible. (Post-grace reaping is covered by the + // core's storage tests, which can backdate rows.) + assertEquals(0, backend.reapEphemeralSubscriptions()); List names = new ArrayList<>(); queue.listSubscriptions("orders").forEach(sub -> names.add(sub.name)); Collections.sort(names); - assertEquals(List.of("durable", "live"), names); + assertEquals(List.of("durable", "ghost", "live"), names); + } + } + } + + @Test + void ephemeralRegistrationWithoutAnOwnerIsRejected(@TempDir Path dir) throws Exception { + String options = new ObjectMapper() + .writeValueAsString( + Map.of("backend", "sqlite", "dsn", dir.resolve("t.db").toString())); + JniQueueBackend backend = JniQueueBackend.open(options); + try (Taskito queue = Taskito.builder().open(backend)) { + TaskitoException rejected = assertThrows( + TaskitoException.class, + () -> backend.registerSubscription("orders", "sink", SEND_EMAIL.name(), "default", false, null)); + assertTrue(rejected.getMessage().contains("requires ownerWorkerId")); + } + } + + @Test + @Timeout(30) + void pauseSurvivesRedeclareAndWorkerStart(@TempDir Path dir) { + try (Taskito queue = open(dir)) { + queue.subscribe("orders", SEND_EMAIL); + assertTrue(queue.pauseSubscription("orders", SEND_EMAIL.name())); + + // Re-declaring must not resume the paused subscription. + queue.subscribe("orders", SEND_EMAIL); + assertTrue(queue.publish("orders", "o-1").isEmpty()); + + // Neither must a worker start re-registering the declaration. + try (Worker worker = + queue.worker().handle(SEND_EMAIL, payload -> payload).start()) { + assertTrue(queue.publish("orders", "o-2").isEmpty()); + } + } + } + + @Test + @Timeout(30) + void unsubscribeDropsTheLocalDeclarationSoWorkerStartDoesNotResurrectIt(@TempDir Path dir) { + try (Taskito queue = open(dir)) { + queue.subscribe("orders", SEND_EMAIL); + assertTrue(queue.unsubscribe("orders", SEND_EMAIL.name())); + try (Worker worker = + queue.worker().handle(SEND_EMAIL, payload -> payload).start()) { + assertTrue(queue.listSubscriptions("orders").isEmpty()); + assertTrue(queue.publish("orders", "o-1").isEmpty()); } } } diff --git a/sdks/java/test-support/src/main/java/org/byteveda/taskito/test/InMemoryQueueBackend.java b/sdks/java/test-support/src/main/java/org/byteveda/taskito/test/InMemoryQueueBackend.java index 1bf453b4..69d7036b 100644 --- a/sdks/java/test-support/src/main/java/org/byteveda/taskito/test/InMemoryQueueBackend.java +++ b/sdks/java/test-support/src/main/java/org/byteveda/taskito/test/InMemoryQueueBackend.java @@ -31,6 +31,14 @@ public final class InMemoryQueueBackend implements QueueBackend { private static final ObjectMapper JSON = new ObjectMapper(); private static final String DEFAULT_QUEUE = "default"; + /** + * Reap grace for freshly registered ephemeral rows, mirroring the core's + * {@code EPHEMERAL_SUBSCRIPTION_GRACE_MS}: a starting worker writes its + * subscriptions before its liveness is visible everywhere, and a concurrent + * reap must not race that gap. + */ + private static final long EPHEMERAL_SUBSCRIPTION_GRACE_MS = 60_000; + private final Map jobs = new ConcurrentHashMap<>(); private final Map settings = new ConcurrentHashMap<>(); private final Map> periodics = new ConcurrentHashMap<>(); @@ -93,6 +101,14 @@ private synchronized String enqueueOne(String taskName, byte[] payload, JsonNode long createdAt = now(); job.createdAt = createdAt; job.scheduledAt = createdAt + optLong(opts, "delayMs", 0); + JsonNode expiresMs = opts == null ? null : opts.get("expiresMs"); + if (expiresMs != null && !expiresMs.isNull()) { + job.expiresAt = createdAt + Math.max(0, expiresMs.asLong()); + } + JsonNode resultTtlMs = opts == null ? null : opts.get("resultTtlMs"); + if (resultTtlMs != null && !resultTtlMs.isNull()) { + job.resultTtlMs = resultTtlMs.asLong(); + } jobs.put(job.id, job); return job.id; } @@ -106,7 +122,15 @@ public Optional getJobJson(String jobId) { @Override public Optional getResult(String jobId) { JobRec job = jobs.get(jobId); - return job == null ? Optional.empty() : Optional.ofNullable(job.result); + if (job == null || resultExpired(job)) { + return Optional.empty(); + } + return Optional.ofNullable(job.result); + } + + /** Whether the job's result outlived its retention TTL (no TTL = kept). */ + private static boolean resultExpired(JobRec job) { + return job.resultTtlMs != null && job.completedAt != null && job.completedAt + job.resultTtlMs < now(); } // synchronized so a pending→cancelled transition can't race claimNext's @@ -448,12 +472,29 @@ public synchronized void registerSubscription( String queue, boolean durable, String ownerWorkerIdOrNull) { - // Full replacement mirrors the core's upsert: re-registering updates the - // routing target and clears any previously set owner. - subscriptions.put( - subscriptionKey(topic, subscriptionName), - new SubscriptionRec( - topic, subscriptionName, taskName, queue, durable, ownerWorkerIdOrNull, seq.incrementAndGet())); + // Mirrors the native guard: an ownerless ephemeral row would never be + // reaped yet keep receiving deliveries. + if (!durable && ownerWorkerIdOrNull == null) { + throw new TaskitoException("an ephemeral subscription (durable=false) requires ownerWorkerId"); + } + // Upsert mirrors the core: routing (task, queue, durable, owner) is + // replaced while active and creation time survive — re-declaring never + // resumes a paused subscription or refreshes the reap grace window. + subscriptions.compute(subscriptionKey(topic, subscriptionName), (key, existing) -> { + SubscriptionRec rec = new SubscriptionRec( + topic, + subscriptionName, + taskName, + queue, + durable, + ownerWorkerIdOrNull, + existing == null ? seq.incrementAndGet() : existing.createdSeq, + existing == null ? now() : existing.createdAt); + if (existing != null) { + rec.active = existing.active; + } + return rec; + }); } @Override @@ -485,16 +526,30 @@ public boolean setSubscriptionActive(String topic, String subscriptionName, bool @Override public long reapEphemeralSubscriptions() { + long cutoff = now() - EPHEMERAL_SUBSCRIPTION_GRACE_MS; long removed = 0; for (Map.Entry entry : subscriptions.entrySet()) { - String owner = entry.getValue().ownerWorkerId; - if (owner != null && !liveWorkers.contains(owner) && subscriptions.remove(entry.getKey()) != null) { + SubscriptionRec sub = entry.getValue(); + if (sub.ownerWorkerId == null || liveWorkers.contains(sub.ownerWorkerId) || sub.createdAt >= cutoff) { + continue; + } + // Conditional removal: a replacement registered under the same key + // after this scan must survive the reap. + if (subscriptions.remove(entry.getKey(), sub)) { removed++; } } return removed; } + /** Test seam: age a subscription past the reap grace window. */ + void backdateSubscription(String topic, String subscriptionName, long ageMs) { + SubscriptionRec sub = subscriptions.get(subscriptionKey(topic, subscriptionName)); + if (sub != null) { + sub.createdAt -= ageMs; + } + } + @Override public synchronized String publishJson(String topic, byte[] payload, String optionsJson) { JsonNode opts = readNode(optionsJson); @@ -549,6 +604,10 @@ private static ObjectNode deliveryOptions(JsonNode opts, SubscriptionRec sub) { delivery.put("maxRetries", resolveDeliveryInt(opts, taskDefaults, "maxRetries")); delivery.put("timeoutMs", resolveDeliveryLong(opts, taskDefaults, "timeoutMs")); delivery.put("delayMs", optLong(opts, "delayMs", 0)); + // Publish-level expiry and result retention pass straight through; the + // enqueue path resolves them (expiresMs → an absolute expiry). + copyIfSet(opts, delivery, "expiresMs"); + copyIfSet(opts, delivery, "resultTtlMs"); String idempotencyKey = text(opts, "idempotencyKey"); if (idempotencyKey != null) { delivery.put("uniqueKey", idempotencyKey + "::" + sub.name); @@ -561,6 +620,14 @@ private static ObjectNode deliveryOptions(JsonNode opts, SubscriptionRec sub) { return delivery; } + /** Copy a field from the publish options into the delivery, when present. */ + private static void copyIfSet(JsonNode opts, ObjectNode delivery, String field) { + JsonNode value = opts == null ? null : opts.get(field); + if (value != null && !value.isNull()) { + delivery.put(field, value.asLong()); + } + } + private static int resolveDeliveryInt(JsonNode opts, JsonNode taskDefaults, String field) { JsonNode override = opts == null ? null : opts.get(field); if (override != null && !override.isNull()) { @@ -597,8 +664,16 @@ private static String deliveryNotes(String callerNotes, SubscriptionRec sub) { public WorkerControl startWorker(WorkerBridge bridge, String optionsJson) { JsonNode options = readNode(optionsJson); String workerId = "im-worker-" + seq.incrementAndGet(); - registerWorkerSubscriptions(workerId, options); + // Mark the worker live before its ephemeral subscriptions exist (like + // the native path), so a concurrent reap never sees an owned row + // without a live owner. Roll back liveness if registration fails. liveWorkers.add(workerId); + try { + registerWorkerSubscriptions(workerId, options); + } catch (RuntimeException e) { + liveWorkers.remove(workerId); + throw e; + } InMemoryWorker worker = new InMemoryWorker(bridge, parseQueues(optionsJson), workerId); workers.add(worker); worker.start(); @@ -742,6 +817,14 @@ private synchronized JobRec claimNext(java.util.Set queues) { if (!"pending".equals(job.status) || job.scheduledAt > now || paused.contains(job.queue)) { continue; } + // Skip expired jobs, cancelling them — mirrors the core's dequeue, + // which archives an expired candidate as cancelled. + if (job.expiresAt != null && now > job.expiresAt) { + job.status = "cancelled"; + job.completedAt = now; + job.error = "expired before execution"; + continue; + } // Honor the worker's queue filter; empty filter = every queue. if (!queues.isEmpty() && !queues.contains(job.queue)) { continue; @@ -901,6 +984,8 @@ private static final class JobRec { long timeoutMs; Integer progress; String error; + Long expiresAt; + Long resultTtlMs; String uniqueKey; String namespace; String metadata; @@ -917,6 +1002,7 @@ private static final class SubscriptionRec { final boolean durable; final String ownerWorkerId; final long createdSeq; + volatile long createdAt; volatile boolean active = true; SubscriptionRec( @@ -926,7 +1012,8 @@ private static final class SubscriptionRec { String queue, boolean durable, String ownerWorkerId, - long createdSeq) { + long createdSeq, + long createdAt) { this.topic = topic; this.name = name; this.taskName = taskName; @@ -934,6 +1021,7 @@ private static final class SubscriptionRec { this.durable = durable; this.ownerWorkerId = ownerWorkerId; this.createdSeq = createdSeq; + this.createdAt = createdAt; } } @@ -1033,10 +1121,17 @@ public void stop() { if (loop != null) { loop.interrupt(); } - // This worker just went away: its ephemeral subscriptions are now - // dead-owned, so reap immediately (mirrors the native shutdown path). liveWorkers.remove(workerId); - reapEphemeralSubscriptions(); + // A graceful stop removes its own ephemeral subscriptions directly: + // unlike the generic reap, the owner is known-dead here, so the + // fresh-row registration grace does not apply. Conditional removal + // spares a row re-registered under a new owner meanwhile. + for (Map.Entry entry : subscriptions.entrySet()) { + SubscriptionRec sub = entry.getValue(); + if (workerId.equals(sub.ownerWorkerId)) { + subscriptions.remove(entry.getKey(), sub); + } + } } @Override diff --git a/sdks/java/test-support/src/test/java/org/byteveda/taskito/test/InMemoryPubSubTest.java b/sdks/java/test-support/src/test/java/org/byteveda/taskito/test/InMemoryPubSubTest.java index 4b0f440b..4175ce54 100644 --- a/sdks/java/test-support/src/test/java/org/byteveda/taskito/test/InMemoryPubSubTest.java +++ b/sdks/java/test-support/src/test/java/org/byteveda/taskito/test/InMemoryPubSubTest.java @@ -2,12 +2,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.time.Duration; import java.util.List; import java.util.Map; import org.byteveda.taskito.Taskito; +import org.byteveda.taskito.TaskitoException; import org.byteveda.taskito.model.Job; import org.byteveda.taskito.model.JobStatus; import org.byteveda.taskito.model.Subscription; @@ -55,6 +57,7 @@ void publishWithoutSubscribersIsANoOp() { } @Test + @Timeout(20) void redeclareUpsertsOnTopicAndName() { try (Taskito queue = InMemoryTaskito.open()) { queue.subscribe( @@ -64,6 +67,14 @@ void redeclareUpsertsOnTopicAndName() { List subs = queue.listSubscriptions("orders"); assertEquals(1, subs.size()); assertEquals(AUDIT.name(), subs.get(0).taskName); + + // The redeclare replaced (not appended to) the local declaration: + // a worker start re-registers exactly one subscription. + try (Worker worker = + queue.worker().handle(AUDIT, payload -> payload).start()) { + assertEquals(1, queue.listSubscriptions("orders").size()); + assertEquals(1, queue.publish("orders", "o-1").size()); + } } } @@ -76,6 +87,20 @@ void unsubscribeReportsWhetherAnythingWasRemoved() { } } + @Test + @Timeout(20) + void unsubscribeDropsTheLocalDeclarationSoWorkerStartDoesNotResurrectIt() { + try (Taskito queue = InMemoryTaskito.open()) { + queue.subscribe("orders", EMAIL); + assertTrue(queue.unsubscribe("orders", EMAIL.name())); + try (Worker worker = + queue.worker().handle(EMAIL, payload -> payload).start()) { + assertTrue(queue.listSubscriptions("orders").isEmpty()); + assertTrue(queue.publish("orders", "o-1").isEmpty()); + } + } + } + @Test void pauseBlocksDeliveriesAndResumeRestoresThem() { try (Taskito queue = InMemoryTaskito.open()) { @@ -94,6 +119,25 @@ void pauseBlocksDeliveriesAndResumeRestoresThem() { } } + @Test + @Timeout(20) + void pauseSurvivesRedeclareAndWorkerStart() { + try (Taskito queue = InMemoryTaskito.open()) { + queue.subscribe("orders", EMAIL); + assertTrue(queue.pauseSubscription("orders", EMAIL.name())); + + // Re-declaring must not resume the paused subscription. + queue.subscribe("orders", EMAIL); + assertTrue(queue.publish("orders", "o-1").isEmpty()); + + // Neither must a worker start re-registering the declaration. + try (Worker worker = + queue.worker().handle(EMAIL, payload -> payload).start()) { + assertTrue(queue.publish("orders", "o-2").isEmpty()); + } + } + } + @Test void idempotencyKeyDedupesPerSubscriberButNotAcrossThem() { try (Taskito queue = InMemoryTaskito.open()) { @@ -165,14 +209,81 @@ void ephemeralSubscriptionBindsToAWorkerAndIsReapedWhenItStops() { } @Test - void reapRemovesDeadOwnedSubscriptionsOnly() { + void reapRemovesDeadOwnedSubscriptionsOnlyAfterTheRegistrationGrace() { InMemoryQueueBackend backend = new InMemoryQueueBackend(); backend.registerSubscription("orders", "ghost", EMAIL.name(), "default", false, "gone-worker"); backend.registerSubscription("orders", "durable", EMAIL.name(), "default", true, null); + // A just-registered dead-owned row is inside the grace window: the + // owner's liveness may simply not be visible yet, so nothing is reaped. + assertEquals(0, backend.reapEphemeralSubscriptions()); + assertTrue(backend.listSubscriptionsJson("orders").contains("ghost")); + + backend.backdateSubscription("orders", "ghost", 61_000); assertEquals(1, backend.reapEphemeralSubscriptions()); assertEquals(0, backend.reapEphemeralSubscriptions()); assertTrue(backend.listSubscriptionsJson("orders").contains("durable")); assertFalse(backend.listSubscriptionsJson("orders").contains("ghost")); } + + @Test + void redeclareDoesNotRefreshTheReapGraceWindow() { + InMemoryQueueBackend backend = new InMemoryQueueBackend(); + backend.registerSubscription("orders", "ghost", EMAIL.name(), "default", false, "gone-worker"); + backend.backdateSubscription("orders", "ghost", 61_000); + + // The upsert preserves creation time, so re-declaring an aged + // dead-owned row leaves it eligible for the very next reap. + backend.registerSubscription("orders", "ghost", AUDIT.name(), "default", false, "gone-worker"); + assertEquals(1, backend.reapEphemeralSubscriptions()); + } + + @Test + void ephemeralRegistrationWithoutAnOwnerIsRejected() { + InMemoryQueueBackend backend = new InMemoryQueueBackend(); + TaskitoException rejected = assertThrows( + TaskitoException.class, + () -> backend.registerSubscription("orders", "sink", EMAIL.name(), "default", false, null)); + assertTrue(rejected.getMessage().contains("requires ownerWorkerId")); + } + + @Test + @Timeout(20) + void publishExpiryCancelsDeliveriesThatOutliveIt() throws Exception { + try (Taskito queue = InMemoryTaskito.open()) { + queue.subscribe("orders", EMAIL); + List deliveries = queue.publish( + "orders", "o-1", PublishOptions.builder().expiresMs(1L).build()); + assertEquals(1, deliveries.size()); + Thread.sleep(15); + + try (Worker worker = + queue.worker().handle(EMAIL, payload -> payload).start()) { + Job done = queue.awaitJob(deliveries.get(0).id, Duration.ofSeconds(10)) + .orElseThrow(); + assertEquals(JobStatus.CANCELLED, done.status); + assertEquals("expired before execution", done.error); + } + } + } + + @Test + @Timeout(20) + void publishResultTtlExpiresStoredResults() throws Exception { + try (Taskito queue = InMemoryTaskito.open()) { + queue.subscribe("orders", EMAIL); + List deliveries = queue.publish( + "orders", "o-1", PublishOptions.builder().resultTtlMs(1L).build()); + assertEquals(1, deliveries.size()); + + try (Worker worker = + queue.worker().handle(EMAIL, payload -> payload).start()) { + Job done = queue.awaitJob(deliveries.get(0).id, Duration.ofSeconds(10)) + .orElseThrow(); + assertEquals(JobStatus.COMPLETE, done.status); + Thread.sleep(15); + assertTrue(queue.getResult(done.id, String.class).isEmpty()); + } + } + } }