diff --git a/.changeset/lazy-state-store-pool.md b/.changeset/lazy-state-store-pool.md new file mode 100644 index 0000000000..7dd64e673e --- /dev/null +++ b/.changeset/lazy-state-store-pool.md @@ -0,0 +1,20 @@ +--- +--- + +fix(training-agent): defer pool lookup in pickStateStore so first-attempt boot succeeds + +The state-store fix in #4071 unblocked deploys but exposed a boot-ordering +race: `mountTenantRoutes()` runs before `initializeDatabase()`, so the +eager `holder.get()` at boot calls `getPool()` and throws "Database not +initialized." The eager-mount catch resets `pendingInit`, the next +request retries, and by then the pool is up — so production heals +within ~5s. Visible in the post-#4071 deploy logs: + + T+0 init starting → "Database not initialized" (eager fail) + T+5 init starting → "Tenant registry initialized" totalMs=27 (heals) + +Wrap the pool in a lazy `PgQueryable` so `getPool()` runs at first query, +not at construction. Construction now always succeeds; by the time a +tool touches `ctx.store`, the pool is initialized regardless of mount +order. Eliminates one round-trip of boot retry and the noisy first +error. diff --git a/server/src/training-agent/tenants/registry.ts b/server/src/training-agent/tenants/registry.ts index 8340dcb337..30c4b586b3 100644 --- a/server/src/training-agent/tenants/registry.ts +++ b/server/src/training-agent/tenants/registry.ts @@ -157,9 +157,13 @@ function pickTaskRegistry(): TaskRegistry { * deployments would silently share state across resolved tenants. Each * tenant `register()` runs `createAdcpServer` for that tenant's platform * and trips this guard if `stateStore` is absent. Wire `PostgresStateStore` - * in production; fall back to a fresh `InMemoryStateStore` per - * registry-construction in dev/test (matches the legacy v5 default and - * keeps tests isolated). + * in production; use a fresh `InMemoryStateStore` in dev/test. + * + * The pool is resolved lazily through a `PgQueryable` adapter — calling + * `getPool()` at construction would fail because `mountTenantRoutes()` + * runs before `initializeDatabase()` in the boot order. Deferring the + * lookup to first query lets construction succeed; by the time a tool + * actually touches `ctx.store`, the pool is initialized. * * Migration: `server/src/db/migrations/466_adcp_state.sql`. */ @@ -168,16 +172,10 @@ function pickStateStore(): AdcpStateStore { if (!isProd) { return new InMemoryStateStore(); } - try { - return new PostgresStateStore(getPool()); - } catch (err) { - logger.error( - { err }, - 'Postgres state store init failed in production. Verify migration 466 ran and DATABASE_URL is set. ' + - 'Falling back to in-memory will trip the SDK production guard — re-throwing.', - ); - throw err; - } + const lazyPool = { + query: (text: string, values?: unknown[]) => getPool().query(text, values), + }; + return new PostgresStateStore(lazyPool); } function buildDefaultServerOptions(): CreateAdcpServerFromPlatformOptions {