From bbbf844e7be93e4f58d0549ba24a3d749635bc6e Mon Sep 17 00:00:00 2001
From: Brian O'Kelley <bokelley@scope3.com>
Date: Mon, 4 May 2026 07:53:15 -0400
Subject: [PATCH] fix(training-agent): defer pool lookup in pickStateStore so
 first-attempt boot succeeds
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The fix in #4071 unblocked deploys but exposed a boot-ordering race:
`mountTenantRoutes()` runs before `initializeDatabase()`, so the eager
`holder.get()` calls `getPool()` and throws "Database not initialized."
The eager-mount catch resets `pendingInit`, the next request retries,
and by then the pool is up — production heals within ~5s. Visible in
the post-#4071 deploy logs:

  T+0  init starting → "Database not initialized" (eager fail)
  T+5  init starting → "Tenant registry initialized" totalMs=27

Wrap the pool in a lazy `PgQueryable` so `getPool()` runs at first
query, not at construction. Construction always succeeds; by the time
a tool touches `ctx.store`, the pool is initialized regardless of
mount order. Eliminates one round-trip of boot retry and the noisy
first error.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 .changeset/lazy-state-store-pool.md           | 20 ++++++++++++++++
 server/src/training-agent/tenants/registry.ts | 24 +++++++++----------
 2 files changed, 31 insertions(+), 13 deletions(-)
 create mode 100644 .changeset/lazy-state-store-pool.md

diff --git a/.changeset/lazy-state-store-pool.md b/.changeset/lazy-state-store-pool.md
new file mode 100644
index 0000000000..7dd64e673e
--- /dev/null
+++ b/.changeset/lazy-state-store-pool.md
@@ -0,0 +1,20 @@
+---
+---
+
+fix(training-agent): defer pool lookup in pickStateStore so first-attempt boot succeeds
+
+The state-store fix in #4071 unblocked deploys but exposed a boot-ordering
+race: `mountTenantRoutes()` runs before `initializeDatabase()`, so the
+eager `holder.get()` at boot calls `getPool()` and throws "Database not
+initialized." The eager-mount catch resets `pendingInit`, the next
+request retries, and by then the pool is up — so production heals
+within ~5s. Visible in the post-#4071 deploy logs:
+
+  T+0  init starting → "Database not initialized" (eager fail)
+  T+5  init starting → "Tenant registry initialized" totalMs=27 (heals)
+
+Wrap the pool in a lazy `PgQueryable` so `getPool()` runs at first query,
+not at construction. Construction now always succeeds; by the time a
+tool touches `ctx.store`, the pool is initialized regardless of mount
+order. Eliminates one round-trip of boot retry and the noisy first
+error.
diff --git a/server/src/training-agent/tenants/registry.ts b/server/src/training-agent/tenants/registry.ts
index 8340dcb337..30c4b586b3 100644
--- a/server/src/training-agent/tenants/registry.ts
+++ b/server/src/training-agent/tenants/registry.ts
@@ -157,9 +157,13 @@ function pickTaskRegistry(): TaskRegistry {
  * deployments would silently share state across resolved tenants. Each
  * tenant `register()` runs `createAdcpServer` for that tenant's platform
  * and trips this guard if `stateStore` is absent. Wire `PostgresStateStore`
- * in production; fall back to a fresh `InMemoryStateStore` per
- * registry-construction in dev/test (matches the legacy v5 default and
- * keeps tests isolated).
+ * in production; use a fresh `InMemoryStateStore` in dev/test.
+ *
+ * The pool is resolved lazily through a `PgQueryable` adapter — calling
+ * `getPool()` at construction would fail because `mountTenantRoutes()`
+ * runs before `initializeDatabase()` in the boot order. Deferring the
+ * lookup to first query lets construction succeed; by the time a tool
+ * actually touches `ctx.store`, the pool is initialized.
  *
  * Migration: `server/src/db/migrations/466_adcp_state.sql`.
  */
@@ -168,16 +172,10 @@ function pickStateStore(): AdcpStateStore {
   if (!isProd) {
     return new InMemoryStateStore();
   }
-  try {
-    return new PostgresStateStore(getPool());
-  } catch (err) {
-    logger.error(
-      { err },
-      'Postgres state store init failed in production. Verify migration 466 ran and DATABASE_URL is set. ' +
-        'Falling back to in-memory will trip the SDK production guard — re-throwing.',
-    );
-    throw err;
-  }
+  const lazyPool = {
+    query: (text: string, values?: unknown[]) => getPool().query(text, values),
+  };
+  return new PostgresStateStore(lazyPool);
 }
 
 function buildDefaultServerOptions(): CreateAdcpServerFromPlatformOptions {