From 7404e27581fc84896233431f7aa49e861db98631 Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sat, 18 Jul 2026 13:08:09 +0530 Subject: [PATCH 1/2] feat(migrate): add raw_ddl escape hatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sea-query cannot express ALTER TABLE … SET (…); raw_ddl passes a code-defined literal through unrendered. Also covers the previously-untested add_column Postgres IF NOT EXISTS branch. --- crates/taskito-core/src/storage/migrate.rs | 45 ++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/crates/taskito-core/src/storage/migrate.rs b/crates/taskito-core/src/storage/migrate.rs index 1356d780..519fff08 100644 --- a/crates/taskito-core/src/storage/migrate.rs +++ b/crates/taskito-core/src/storage/migrate.rs @@ -44,6 +44,15 @@ pub struct Stmt { swallow_dup_column: bool, } +impl Stmt { + /// Rendered SQL. Test-only accessor so migration render tests co-located in a + /// migration's own module (a different module than `Stmt`) can inspect output. + #[cfg(test)] + pub(crate) fn sql(&self) -> &str { + &self.sql + } +} + /// A single schema version. `up` returns the rendered statements to apply. pub trait Migration { /// Stable, unique version key recorded in the ledger (e.g. `"0001_initial"`). @@ -78,6 +87,18 @@ pub fn add_column(backend: Backend, table: &str, column: &mut ColumnDef) -> Stmt } } +/// Escape hatch for DDL sea-query cannot model — currently only Postgres table +/// storage parameters (`ALTER TABLE … SET (…)`, which `TableAlterStatement` has +/// no method for). The SQL is a code-defined literal that never renders through +/// a dialect builder: no user input reaches here, and the caller owns dialect +/// gating (return no statements for the backends it does not apply to). +pub fn raw_ddl(sql: impl Into) -> Stmt { + Stmt { + sql: sql.into(), + swallow_dup_column: false, + } +} + /// Render a data statement (the `has_deps` backfill `UPDATE`). Literals are /// inlined by sea-query; only trusted, code-defined values reach here. pub fn dml(backend: Backend, stmt: &sea_query::UpdateStatement) -> Stmt { @@ -295,6 +316,15 @@ mod tests { .expect("read ledger") } + #[test] + fn raw_ddl_passes_sql_through_verbatim() { + // The escape hatch must not touch the SQL — it is a code-defined literal + // the caller has already made dialect-correct. + let stmt = raw_ddl("ALTER TABLE jobs SET (fillfactor = 85)"); + assert_eq!(stmt.sql, "ALTER TABLE jobs SET (fillfactor = 85)"); + assert!(!stmt.swallow_dup_column); + } + #[test] fn m0003_renders_partial_indexes_on_both_backends() { // The Postgres arm only renders under its feature — `render_schema` @@ -322,6 +352,21 @@ mod tests { } } + #[cfg(feature = "postgres")] + #[test] + fn add_column_renders_if_not_exists_on_postgres() { + // The Postgres `ADD COLUMN IF NOT EXISTS` branch had no coverage — SQLite + // renders a plain `ADD COLUMN` and swallows the dup-column error instead. + let mut col = ColumnDef::new(Alias::new("namespace")); + col.text(); + let stmt = add_column(Backend::Postgres, "jobs", &mut col); + assert!( + stmt.sql.contains("ADD COLUMN IF NOT EXISTS"), + "{}", + stmt.sql + ); + } + #[test] fn fresh_db_applies_all_migrations_and_is_idempotent() { let mut conn = mem(); From b0e36e7e17ec2a5eec2b1cb5d01b5d9bb0df0cdf Mon Sep 17 00:00:00 2001 From: Pratyush Sharma <56130065+pratyush618@users.noreply.github.com> Date: Sat, 18 Jul 2026 13:08:27 +0530 Subject: [PATCH 2/2] perf(postgres): tune fillfactor and autovacuum fillfactor on jobs/workers for HOT updates; aggressive autovacuum on the retention purge targets so bulk deletes are reclaimed at 2% not the 20% default. No-op on SQLite. --- .../migrations/m0004_postgres_tuning.rs | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 crates/taskito-core/migrations/m0004_postgres_tuning.rs diff --git a/crates/taskito-core/migrations/m0004_postgres_tuning.rs b/crates/taskito-core/migrations/m0004_postgres_tuning.rs new file mode 100644 index 00000000..1b545a1f --- /dev/null +++ b/crates/taskito-core/migrations/m0004_postgres_tuning.rs @@ -0,0 +1,101 @@ +//! Postgres storage tuning (`0004_postgres_tuning`). +//! +//! Two per-table storage-parameter overrides that `sea-query` cannot express +//! (`ALTER TABLE … SET (…)`), so they go through the `raw_ddl` escape hatch: +//! +//! - **fillfactor** leaves in-page room for HOT updates. `jobs` churns through +//! status/progress/retry updates and `workers` rewrites its heartbeat every +//! few seconds — both benefit from not having to migrate a row to a new page +//! on every update. It only affects *future* writes; existing pages turn over +//! as they are rewritten (we do not `VACUUM FULL` from a migration). +//! - **autovacuum** is made aggressive on the tables the retention purges churn. +//! The 20% default means a 10M-row table waits for 2M dead tuples before it is +//! vacuumed; the bulk deletes create dead tuples in exactly these tables. The +//! `threshold = 1000` is not optional — `scale_factor` alone still adds the +//! default `threshold = 50`, which fires constantly on a small table. +//! +//! Postgres-only: `up()` returns no statements on SQLite (which has neither +//! knob), so the migration still records as applied there but changes nothing. +//! `ALTER TABLE … SET` is catalog-only — instant, no table rewrite, only a brief +//! `ACCESS EXCLUSIVE` lock. These are per-table overrides that take precedence +//! over any an operator has tuned by hand. + +use crate::storage::migrate::{raw_ddl, Backend, Migration, Stmt}; + +pub struct M0004PostgresTuning; + +/// `(table, fillfactor)`. `jobs` carries a payload BLOB so rows are large and +/// few fit per page — 85 wastes less than a lower value would. `workers` is a +/// few hundred rows updated every heartbeat, so reserving space is pure upside. +const FILLFACTOR: &[(&str, u32)] = &[("jobs", 85), ("workers", 70)]; + +/// Tables the retention purges delete from in bulk, so their dead tuples must be +/// reclaimed promptly rather than at the 20% default. +const AUTOVACUUM_TABLES: &[&str] = &[ + "jobs", + "archived_jobs", + "dead_letter", + "task_logs", + "task_metrics", + "job_errors", +]; + +const AUTOVACUUM_PARAMS: &str = "autovacuum_vacuum_scale_factor = 0.02, \ + autovacuum_vacuum_threshold = 1000, \ + autovacuum_analyze_scale_factor = 0.02"; + +impl Migration for M0004PostgresTuning { + fn version(&self) -> &'static str { + "0004_postgres_tuning" + } + + fn up(&self, backend: Backend) -> Vec { + // SQLite has no per-table storage parameters; record as applied, do + // nothing. Bare identifiers are correct — `conn()` sets `search_path`. + if backend != Backend::Postgres { + return Vec::new(); + } + + let mut stmts = Vec::with_capacity(FILLFACTOR.len() + AUTOVACUUM_TABLES.len()); + for (table, factor) in FILLFACTOR { + stmts.push(raw_ddl(format!("ALTER TABLE {table} SET (fillfactor = {factor})"))); + } + for table in AUTOVACUUM_TABLES { + stmts.push(raw_ddl(format!("ALTER TABLE {table} SET ({AUTOVACUUM_PARAMS})"))); + } + stmts + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The Postgres arm renders through `raw_ddl` (a code literal, not a dialect + /// builder), so — unlike every other migration — it is testable without the + /// `postgres` feature and needs no database. + #[test] + fn tunes_postgres_and_is_a_noop_on_sqlite() { + let m = M0004PostgresTuning; + + // SQLite has no storage parameters: nothing to run, yet still recorded. + assert!(m.up(Backend::Sqlite).is_empty()); + + let pg = m + .up(Backend::Postgres) + .iter() + .map(|s| s.sql()) + .collect::>() + .join("\n"); + assert!(pg.contains("ALTER TABLE jobs SET (fillfactor = 85)"), "{pg}"); + assert!( + pg.contains("ALTER TABLE workers SET (fillfactor = 70)"), + "{pg}" + ); + // `threshold = 1000` must ride alongside `scale_factor` — scale alone + // keeps the default threshold of 50 and fires constantly on a small table. + assert!(pg.contains("autovacuum_vacuum_scale_factor = 0.02"), "{pg}"); + assert!(pg.contains("autovacuum_vacuum_threshold = 1000"), "{pg}"); + assert!(pg.contains("ALTER TABLE task_logs SET ("), "{pg}"); + } +}