diff --git a/apps/cli/src/legacy/commands/db/diff/SIDE_EFFECTS.md b/apps/cli/src/legacy/commands/db/diff/SIDE_EFFECTS.md index 28836f4744..5aa9fed9bd 100644 --- a/apps/cli/src/legacy/commands/db/diff/SIDE_EFFECTS.md +++ b/apps/cli/src/legacy/commands/db/diff/SIDE_EFFECTS.md @@ -28,8 +28,12 @@ the native pg-delta or migra engine (both run inside Docker via edge-runtime). T ## Docker -- Edge-runtime container (pg-delta / migra diff scripts). -- Shadow Postgres container (provisioned + torn down via the Go `db __shadow` seam). +- Edge-runtime container (pg-delta / migra diff scripts; also runs the pg-delta + catalog-export script for explicit `--from/--to migrations` on a cache miss — + CLI-1959, native, no longer the hidden Go `__catalog` seam). +- Shadow Postgres container (provisioned + torn down via the Go `db __shadow` seam; + explicit `--from/--to migrations` reuses this same seam call — `mode: "diff"` — + on a cache miss, rather than a second, `__catalog`-specific shadow). - `supabase/migra` container — the migra OOM bash fallback only. ## API Routes (linked path, via the db-config resolver) @@ -82,6 +86,12 @@ Progress strings still go to stderr; stdout carries a single structured envelope binary (their side effects are Go's); the Go child's telemetry is disabled so the single `cli_command_executed` event comes from this TS command. - Explicit `--from`/`--to` mode always uses pg-delta and writes to `--output` (or stdout). +- The explicit `migrations` target resolves natively (CLI-1959): a bare + migrations-content hash cache lookup (`/supabase/.temp/pgdelta/catalog-local-migrations--.json`, + shared with `db push`'s post-apply cache write), and on a miss, the existing + `db __shadow --mode diff` seam call (unchanged — still Go, out of scope for + CLI-1959) plus a native pg-delta catalog export. No hidden Go + `db schema declarative __catalog` subprocess runs for this path any more. ### `--use-pg-schema` is deprecated (CLI-1960) — keep-in-Go exception diff --git a/apps/cli/src/legacy/commands/db/diff/diff.handler.ts b/apps/cli/src/legacy/commands/db/diff/diff.handler.ts index 6a2910aa87..957840ad12 100644 --- a/apps/cli/src/legacy/commands/db/diff/diff.handler.ts +++ b/apps/cli/src/legacy/commands/db/diff/diff.handler.ts @@ -26,6 +26,7 @@ import { legacyGetMigrationPath, } from "../../../shared/legacy-migration-file.ts"; import { legacyDiffMigra } from "../shared/legacy-migra.ts"; +import { legacyResolveMigrationsCatalogRef } from "../shared/legacy-pgdelta.cache.ts"; import { legacyWritePgDeltaMigrations } from "../shared/legacy-pgdelta-migrations.write.ts"; import { type LegacyPgDeltaContext, legacyDiffPgDelta } from "../shared/legacy-pgdelta.ts"; import { LegacyDeclarativeSeam } from "../shared/legacy-pgdelta.seam.service.ts"; @@ -229,16 +230,29 @@ export const legacyDbDiff = Effect.fn("legacy.db.diff")(function* (flags: Legacy } return legacyToPostgresURL(resolved.conn); } - case "migrations": - return yield* seam.exportCatalog({ - mode: "migrations", - noCache: false, - // Pass the linked ref only if one resolved earlier in the cascade, - // so the `__catalog` child merges the same remote override Go's - // in-process migrations catalog sees (`explicit.go:88-126`). Absent - // otherwise → base config, matching Go's resolution order. - ...(mergedLinkedRef !== undefined ? { projectRef: mergedLinkedRef } : {}), - }); + case "migrations": { + // Native (CLI-1959): mirrors Go's `resolveMigrationsCatalogRef` + // (`explicit.go:88-126`) exactly — see `legacyResolveMigrationsCatalogRef`'s + // doc comment. The pg-delta context is built from whatever `cfg` is + // current at this point in the cascade (possibly re-merged by an + // earlier "linked" ref above), matching Go's stateful pre-run. + const migrationsCtx: LegacyPgDeltaContext = { + projectId: Option.getOrElse(cliConfig.projectId, () => ""), + cwd: cliConfig.workdir, + npmVersion: Option.getOrUndefined(cfg.pgDelta.npmVersion), + denoVersion: cfg.denoVersion, + }; + // Pass the linked ref only if one resolved earlier in the cascade, so + // the shadow merges the same remote override Go's in-process + // migrations catalog sees (`explicit.go:88-126`). Absent otherwise → + // base config, matching Go's resolution order. + return yield* legacyResolveMigrationsCatalogRef( + fs, + path, + migrationsCtx, + mergedLinkedRef !== undefined ? { projectRef: mergedLinkedRef } : {}, + ); + } case "url": return ref; default: diff --git a/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts b/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts index e89962eafc..1b784983bc 100644 --- a/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; @@ -685,16 +686,55 @@ describe("legacy db diff", () => { }, ); - it.effect("explicit --from migrations resolves a shadow catalog via the seam", () => { + it.effect("explicit --from migrations resolves a shadow catalog natively", () => { + // CLI-1959: the migrations ref now resolves via `provisionShadow` (Go's + // unchanged `db __shadow --mode diff`) + a native pg-delta catalog export, + // instead of the retired `exportCatalog({mode:"migrations"})` seam call. const s = setup(tmp.current, { diffSql: "create table m ();\n" }); return Effect.gen(function* () { yield* legacyDbDiff(flags({ from: Option.some("migrations"), to: Option.some("local") })); - expect(s.exportCalls).toEqual(["migrations"]); + expect(s.exportCalls).toEqual([]); + expect(s.provisionCalls).toEqual([ + { mode: "diff", targetLocal: false, usePgDelta: false, projectRef: undefined }, + ]); + // `resolveMigrationsCatalogRef` (Go's `explicit.go:88-126`) calls the shadow + // primitives directly, without `DiffDatabase`'s own progress line — unlike + // `db schema declarative sync`'s `getMigrationsCatalogRef`, which DOES print + // it (`legacy-pgdelta.cache.ts`'s `legacyGetMigrationsCatalogRef`). This + // stderr asymmetry is the parity fix CLI-1959 makes; pin it here even though + // a shadow was actually provisioned on this cache miss. + expect(s.out.stderrText).not.toContain("Creating shadow database..."); }).pipe(Effect.provide(s.layer)); }); it.effect( - "explicit --from linked --to migrations exports the catalog with the linked ref", + "explicit --from migrations reuses an already-cached catalog without provisioning a shadow", + () => { + // A cache pre-warmed by a prior `db push` (`legacyTryCacheMigrationsCatalog`) + // or `db diff --from migrations` run keys off the BARE migrations hash + // (`pgcache.HashMigrations` — no setup-inputs token; see + // `legacyResolveMigrationsCatalogRef`'s doc comment), so it must be reused + // here without spinning up a new shadow database at all. + const noMigrationsHash = createHash("sha256").digest("hex"); + const tempDir = join(tmp.current, "supabase", ".temp", "pgdelta"); + mkdirSync(tempDir, { recursive: true }); + const cachedPath = join(tempDir, `catalog-local-migrations-${noMigrationsHash}-1000.json`); + writeFileSync(cachedPath, '{"cached":true}'); + const s = setup(tmp.current, { diffSql: "create table m ();\n" }); + return Effect.gen(function* () { + yield* legacyDbDiff(flags({ from: Option.some("migrations"), to: Option.some("local") })); + expect(s.provisionCalls).toEqual([]); + expect(s.exportCalls).toEqual([]); + const diffCall = s.edgeCalls.find((c) => c.script.includes("renderPlanFiles")); + expect(diffCall?.env["SOURCE"]).toBe( + `/workspace/${join("supabase", ".temp", "pgdelta", `catalog-local-migrations-${noMigrationsHash}-1000.json`)}`, + ); + }).pipe(Effect.provide(s.layer)); + }, + ); + + it.effect( + "explicit --from linked --to migrations provisions the shadow with the linked ref", () => { // Go resolves linked first (LoadConfig merges [remotes.]), so the later // migrations catalog is built from the remote-merged config (explicit.go). @@ -705,13 +745,13 @@ describe("legacy db diff", () => { }); return Effect.gen(function* () { yield* legacyDbDiff(flags({ from: Option.some("linked"), to: Option.some("migrations") })); - const migrations = s.exportCatalogCalls.find((c) => c.mode === "migrations"); + const migrations = s.provisionCalls.find((c) => c.mode === "diff" && !c.targetLocal); expect(migrations?.projectRef).toBe("abcdefghijklmnopqrst"); }).pipe(Effect.provide(s.layer)); }, ); - it.effect("explicit --from migrations --to linked exports the catalog with base config", () => { + it.effect("explicit --from migrations --to linked provisions the shadow with base config", () => { // Migrations is resolved BEFORE linked here, so Go's LoadConfig(ref) hasn't run // yet — the catalog must use base config (no ref forwarded), matching order. const s = setup(tmp.current, { @@ -721,7 +761,7 @@ describe("legacy db diff", () => { }); return Effect.gen(function* () { yield* legacyDbDiff(flags({ from: Option.some("migrations"), to: Option.some("linked") })); - const migrations = s.exportCatalogCalls.find((c) => c.mode === "migrations"); + const migrations = s.provisionCalls.find((c) => c.mode === "diff" && !c.targetLocal); expect(migrations?.projectRef).toBeUndefined(); }).pipe(Effect.provide(s.layer)); }); @@ -744,7 +784,7 @@ describe("legacy db diff", () => { linked: Option.some(true), }), ); - const migrations = s.exportCatalogCalls.find((c) => c.mode === "migrations"); + const migrations = s.provisionCalls.find((c) => c.mode === "diff" && !c.targetLocal); expect(migrations?.projectRef).toBe("abcdefghijklmnopqrst"); }).pipe(Effect.provide(s.layer)); }); diff --git a/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts b/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts index 2b581df3b0..d95be96a3f 100644 --- a/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts @@ -1,15 +1,25 @@ -import { mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; -import { Cause, Effect, Exit, Layer } from "effect"; +import { Cause, Effect, Exit, FileSystem, Layer, Path } from "effect"; +import { mockOutput } from "../../../../../../tests/helpers/mocks.ts"; import { type LegacyEdgeRuntimeRunOpts, LegacyEdgeRuntimeScript, } from "../../../../shared/legacy-edge-runtime-script.service.ts"; import { LegacyPgDeltaSslProbe } from "../../../../shared/legacy-pgdelta-ssl-probe.service.ts"; +import { + legacyBaselineCatalogFileName, + legacyBaselineCatalogKey, + legacyHashMigrations, + legacyMigrationCatalogFileName, + legacyMigrationsCatalogCacheKey, + legacySetupInputsToken, + type LegacySetupInputs, +} from "../../shared/legacy-pgdelta.cache.ts"; import { type LegacyCatalogMode, LegacyDeclarativeSeam, @@ -22,6 +32,13 @@ import { function mockSeam(paths: Record) { const calls: Array<{ mode: LegacyCatalogMode; noCache: boolean }> = []; + const provisionCalls: Array<{ + mode: string; + targetLocal: boolean; + usePgDelta: boolean; + projectRef?: string; + }> = []; + const removedContainers: string[] = []; const layer = Layer.succeed(LegacyDeclarativeSeam, { exportCatalog: ({ mode, noCache }) => { calls.push({ mode, noCache }); @@ -30,10 +47,24 @@ function mockSeam(paths: Record) { execInherit: () => Effect.succeed(0), ensureLocalDatabaseStarted: () => Effect.void, ensureLocalPostgresImageCurrent: () => Effect.void, - provisionShadow: () => Effect.die("provisionShadow not used in declarative tests"), - removeShadowContainer: () => Effect.void, + // The migrations-catalog source now resolves natively (CLI-1959) via + // `legacyGetMigrationsCatalogRef`, which provisions its shadow through this + // EXISTING `provisionShadow` (Go's unchanged `db __shadow --mode diff`) rather + // than the retired `exportCatalog({mode:"migrations"})` seam call. + provisionShadow: ({ mode, targetLocal, usePgDelta, projectRef }) => { + provisionCalls.push({ mode, targetLocal, usePgDelta, projectRef }); + return Effect.succeed({ + container: "shadow-1", + sourceUrl: "postgres://postgres:postgres@127.0.0.1:54320/postgres", + targetUrlOverride: undefined, + }); + }, + removeShadowContainer: (container) => + Effect.sync(() => { + removedContainers.push(container); + }), }); - return { layer, calls }; + return { layer, calls, provisionCalls, removedContainers }; } function mockEdge(stdout: string) { @@ -41,6 +72,13 @@ function mockEdge(stdout: string) { const layer = Layer.succeed(LegacyEdgeRuntimeScript, { run: (opts: LegacyEdgeRuntimeRunOpts) => { calls.push(opts); + // The catalog-export script (uniquely identified by its errPrefix) backs the + // native migrations-catalog resolution's shadow export — return a fixed, + // non-empty snapshot so it never trips `legacyExportCatalogPgDelta`'s + // empty-output check regardless of what `stdout` the diff/export scripts use. + if (opts.errPrefix === "error exporting pg-delta catalog") { + return Effect.succeed({ stdout: '{"schemas":[]}', stderr: "" }); + } // The pg-delta diff script (uniquely identified by `renderPlanFiles`) prints a // JSON envelope with one file per plan unit; wrap the test's raw SQL into a // single-unit envelope so `legacyDiffPgDelta` parses it. Other scripts @@ -67,48 +105,255 @@ const probe = Layer.succeed(LegacyPgDeltaSslProbe, { requireSslForHost: () => Effect.succeed(false), }); -const ctx = (declarativeDir: string): LegacyDeclarativeRunContext => ({ - pgDelta: { projectId: "cferry", cwd: "/proj", npmVersion: undefined, denoVersion: 2 }, +const ctx = (cwd: string, declarativeDir: string): LegacyDeclarativeRunContext => ({ + pgDelta: { projectId: "cferry", cwd, npmVersion: undefined, denoVersion: 2 }, formatOptions: "", declarativeDir, schema: [], noCache: false, }); +// A minimal, valid `LegacySetupInputs` — the exact field values don't matter to +// these tests (they only exercise the cache-miss/shadow-provision path), only +// that a real cache key can be derived from them. +const setupInputs: LegacySetupInputs = { + image: "supabase/postgres:17.6.1.135", + majorVersion: 17, + authEnabled: true, + storageEnabled: true, + realtimeEnabled: true, + autoExpose: false, + vaultNames: [], + rolesSql: "", +}; + describe("legacyDiffDeclarativeToMigrations", () => { - it.effect("provisions migrations + declarative catalogs via the seam and diffs them", () => { - const dir = mkdtempSync(join(tmpdir(), "legacy-decl-orch-")); - const declDir = join(dir, "supabase", "database"); - mkdirSync(declDir, { recursive: true }); - const seam = mockSeam({ - migrations: "supabase/.temp/pgdelta/mig.json", - declarative: "supabase/.temp/pgdelta/decl.json", - baseline: "supabase/.temp/pgdelta/base.json", - }); - const edge = mockEdge("ALTER TABLE x ADD COLUMN y int;\nDROP TABLE z;\n"); - return legacyDiffDeclarativeToMigrations(ctx(declDir)).pipe( - Effect.tap((result) => - Effect.sync(() => { - expect(seam.calls.map((c) => c.mode)).toEqual(["migrations", "declarative"]); - expect(result.sourceRef).toBe("supabase/.temp/pgdelta/mig.json"); - expect(result.targetRef).toBe("supabase/.temp/pgdelta/decl.json"); - expect(result.diffSQL).toContain("ALTER TABLE x"); - expect(result.dropWarnings).toEqual(["DROP TABLE z"]); - // The edge-runtime diff received the seam refs as SOURCE/TARGET. - expect(edge.calls[0]!.env["SOURCE"]).toBe("/workspace/supabase/.temp/pgdelta/mig.json"); - expect(edge.calls[0]!.env["TARGET"]).toBe("/workspace/supabase/.temp/pgdelta/decl.json"); - rmSync(dir, { recursive: true, force: true }); - }), - ), - Effect.provide(Layer.mergeAll(seam.layer, edge.layer, probe, BunServices.layer)), - ); - }); + it.effect( + "resolves the migrations catalog natively and diffs it against the seam-provisioned declarative catalog", + () => { + const dir = mkdtempSync(join(tmpdir(), "legacy-decl-orch-")); + const declDir = join(dir, "supabase", "database"); + mkdirSync(declDir, { recursive: true }); + const seam = mockSeam({ + declarative: "supabase/.temp/pgdelta/decl.json", + baseline: "supabase/.temp/pgdelta/base.json", + }); + const edge = mockEdge("ALTER TABLE x ADD COLUMN y int;\nDROP TABLE z;\n"); + const out = mockOutput(); + return legacyDiffDeclarativeToMigrations(ctx(dir, declDir), setupInputs).pipe( + Effect.tap((result) => + Effect.sync(() => { + // "declarative" still resolves via the seam; "migrations" no longer does + // (it resolves natively, provisioning through `provisionShadow` instead). + expect(seam.calls.map((c) => c.mode)).toEqual(["declarative"]); + expect(seam.provisionCalls).toEqual([ + { mode: "diff", targetLocal: false, usePgDelta: false, projectRef: undefined }, + ]); + expect(seam.removedContainers).toEqual(["shadow-1"]); + // No local migrations in the fresh temp dir → the zero-migrations branch + // writes (and returns) the platform-baseline catalog, workdir-relative. + expect(result.sourceRef).toMatch( + /^supabase[/\\]\.temp[/\\]pgdelta[/\\]catalog-baseline-.*\.json$/, + ); + expect(readFileSync(join(dir, result.sourceRef), "utf8")).toBe('{"schemas":[]}'); + expect(result.targetRef).toBe("supabase/.temp/pgdelta/decl.json"); + expect(result.diffSQL).toContain("ALTER TABLE x"); + expect(result.dropWarnings).toEqual(["DROP TABLE z"]); + // The edge-runtime diff received the migrations ref (workdir-relative, + // mapped to /workspace) and the seam's declarative ref as SOURCE/TARGET. + const diffCall = edge.calls.find((c) => c.script.includes("renderPlanFiles")); + expect(diffCall?.env["SOURCE"]).toBe(`/workspace/${result.sourceRef}`); + expect(diffCall?.env["TARGET"]).toBe("/workspace/supabase/.temp/pgdelta/decl.json"); + rmSync(dir, { recursive: true, force: true }); + }), + ), + Effect.provide(Layer.mergeAll(seam.layer, edge.layer, probe, out.layer, BunServices.layer)), + ); + }, + ); + + it.effect( + "reuses an already-warmed platform-baseline catalog without provisioning a shadow", + () => { + // A baseline catalog pre-warmed by a prior generate/sync run (same setup + // inputs, still zero local migrations) must be reused as-is — this is the + // whole point of the zero-migrations special case in + // `legacyGetMigrationsCatalogRef` (mirrors Go's `getMigrationsCatalogRef`, + // `declarative.go:380-392`). + const dir = mkdtempSync(join(tmpdir(), "legacy-decl-orch-")); + const declDir = join(dir, "supabase", "database"); + mkdirSync(declDir, { recursive: true }); + const tempDir = join(dir, "supabase", ".temp", "pgdelta"); + mkdirSync(tempDir, { recursive: true }); + const baselineKey = legacyBaselineCatalogKey(setupInputs); + const baselinePath = join(tempDir, legacyBaselineCatalogFileName(baselineKey)); + writeFileSync(baselinePath, '{"warmed":true}'); + const seam = mockSeam({ + declarative: "supabase/.temp/pgdelta/decl.json", + baseline: "supabase/.temp/pgdelta/base.json", + }); + const edge = mockEdge("ALTER TABLE x;\n"); + const out = mockOutput(); + return legacyDiffDeclarativeToMigrations(ctx(dir, declDir), setupInputs).pipe( + Effect.tap((result) => + Effect.sync(() => { + expect(seam.provisionCalls).toEqual([]); + expect(result.sourceRef).toBe( + join("supabase", ".temp", "pgdelta", `catalog-baseline-${baselineKey}.json`), + ); + expect(readFileSync(baselinePath, "utf8")).toBe('{"warmed":true}'); + rmSync(dir, { recursive: true, force: true }); + }), + ), + Effect.provide(Layer.mergeAll(seam.layer, edge.layer, probe, out.layer, BunServices.layer)), + ); + }, + ); + + it.effect( + "with local migrations present and cache enabled, provisions a shadow and caches the resulting catalog", + () => { + // The dominant real-world code path (a project WITH local migrations, cache + // enabled) — `legacyGetMigrationsCatalogRef`'s cache-miss/non-zero-migrations + // branch (declarative.go:393-430) — was previously never exercised by any + // test; every other test here uses a fresh temp dir with zero migrations. + const dir = mkdtempSync(join(tmpdir(), "legacy-decl-orch-")); + const declDir = join(dir, "supabase", "database"); + mkdirSync(declDir, { recursive: true }); + const migrationsDir = join(dir, "supabase", "migrations"); + mkdirSync(migrationsDir, { recursive: true }); + writeFileSync(join(migrationsDir, "20240101000000_init.sql"), "create table a();\n"); + const seam = mockSeam({ + declarative: "supabase/.temp/pgdelta/decl.json", + baseline: "supabase/.temp/pgdelta/base.json", + }); + const edge = mockEdge("ALTER TABLE x ADD COLUMN y int;\n"); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const migrationsHash = yield* legacyHashMigrations(fs, path, dir, migrationsDir); + const key = legacyMigrationsCatalogCacheKey( + legacySetupInputsToken(setupInputs), + migrationsHash, + ); + const result = yield* legacyDiffDeclarativeToMigrations(ctx(dir, declDir), setupInputs); + expect(result.sourceRef).toMatch( + new RegExp( + `^supabase[/\\\\]\\.temp[/\\\\]pgdelta[/\\\\]catalog-local-migrations-${key}-\\d+\\.json$`, + ), + ); + expect(readFileSync(join(dir, result.sourceRef), "utf8")).toBe('{"schemas":[]}'); + expect(out.stderrText).toContain("Creating shadow database...\n"); + expect(seam.provisionCalls).toEqual([ + { mode: "diff", targetLocal: false, usePgDelta: false, projectRef: undefined }, + ]); + expect(seam.removedContainers).toEqual(["shadow-1"]); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide(Layer.mergeAll(seam.layer, edge.layer, probe, out.layer, BunServices.layer)), + ); + }, + ); + + it.effect( + "reuses an already-cached migrations catalog for local migrations without provisioning a new shadow", + () => { + const dir = mkdtempSync(join(tmpdir(), "legacy-decl-orch-")); + const declDir = join(dir, "supabase", "database"); + mkdirSync(declDir, { recursive: true }); + const migrationsDir = join(dir, "supabase", "migrations"); + mkdirSync(migrationsDir, { recursive: true }); + writeFileSync(join(migrationsDir, "20240101000000_init.sql"), "create table a();\n"); + const tempDir = join(dir, "supabase", ".temp", "pgdelta"); + mkdirSync(tempDir, { recursive: true }); + const seam = mockSeam({ + declarative: "supabase/.temp/pgdelta/decl.json", + baseline: "supabase/.temp/pgdelta/base.json", + }); + const edge = mockEdge("ALTER TABLE x;\n"); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const migrationsHash = yield* legacyHashMigrations(fs, path, dir, migrationsDir); + const key = legacyMigrationsCatalogCacheKey( + legacySetupInputsToken(setupInputs), + migrationsHash, + ); + const cachedPath = join( + tempDir, + legacyMigrationCatalogFileName("local", key, 1_700_000_000_000), + ); + writeFileSync(cachedPath, '{"cached":true}'); + const result = yield* legacyDiffDeclarativeToMigrations(ctx(dir, declDir), setupInputs); + expect(result.sourceRef).toBe(path.relative(dir, cachedPath)); + expect(readFileSync(cachedPath, "utf8")).toBe('{"cached":true}'); + expect(seam.provisionCalls).toEqual([]); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide(Layer.mergeAll(seam.layer, edge.layer, probe, out.layer, BunServices.layer)), + ); + }, + ); + + it.effect( + "--no-cache ignores an already-cached migrations catalog, provisions a fresh shadow, and writes catalog-nocache-migrations.json", + () => { + const dir = mkdtempSync(join(tmpdir(), "legacy-decl-orch-")); + const declDir = join(dir, "supabase", "database"); + mkdirSync(declDir, { recursive: true }); + const migrationsDir = join(dir, "supabase", "migrations"); + mkdirSync(migrationsDir, { recursive: true }); + writeFileSync(join(migrationsDir, "20240101000000_init.sql"), "create table a();\n"); + const tempDir = join(dir, "supabase", ".temp", "pgdelta"); + mkdirSync(tempDir, { recursive: true }); + const seam = mockSeam({ + declarative: "supabase/.temp/pgdelta/decl.json", + baseline: "supabase/.temp/pgdelta/base.json", + }); + const edge = mockEdge("ALTER TABLE x;\n"); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + // Pre-warm the cache entry that a cache-enabled run would hit, proving + // --no-cache really skips the lookup rather than merely never having + // written that entry. + const migrationsHash = yield* legacyHashMigrations(fs, path, dir, migrationsDir); + const key = legacyMigrationsCatalogCacheKey( + legacySetupInputsToken(setupInputs), + migrationsHash, + ); + const cachedPath = join( + tempDir, + legacyMigrationCatalogFileName("local", key, 1_700_000_000_000), + ); + writeFileSync(cachedPath, '{"cached":true}'); + const result = yield* legacyDiffDeclarativeToMigrations( + { ...ctx(dir, declDir), noCache: true }, + setupInputs, + ); + expect(result.sourceRef).toBe( + join("supabase", ".temp", "pgdelta", "catalog-nocache-migrations.json"), + ); + expect(readFileSync(join(dir, result.sourceRef), "utf8")).toBe('{"schemas":[]}'); + expect(seam.provisionCalls).toEqual([ + { mode: "diff", targetLocal: false, usePgDelta: false, projectRef: undefined }, + ]); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide(Layer.mergeAll(seam.layer, edge.layer, probe, out.layer, BunServices.layer)), + ); + }, + ); it.effect("fails when the declarative dir is absent", () => { const dir = mkdtempSync(join(tmpdir(), "legacy-decl-orch-")); - const seam = mockSeam({ migrations: "m", declarative: "d", baseline: "b" }); + const seam = mockSeam({ declarative: "d", baseline: "b" }); const edge = mockEdge(""); - return legacyDiffDeclarativeToMigrations(ctx(join(dir, "missing"))).pipe( + const out = mockOutput(); + return legacyDiffDeclarativeToMigrations(ctx(dir, join(dir, "missing")), setupInputs).pipe( Effect.exit, Effect.tap((exit) => Effect.sync(() => { @@ -120,10 +365,11 @@ describe("legacyDiffDeclarativeToMigrations", () => { ); } expect(seam.calls).toEqual([]); + expect(seam.provisionCalls).toEqual([]); rmSync(dir, { recursive: true, force: true }); }), ), - Effect.provide(Layer.mergeAll(seam.layer, edge.layer, probe, BunServices.layer)), + Effect.provide(Layer.mergeAll(seam.layer, edge.layer, probe, out.layer, BunServices.layer)), ); }); }); @@ -131,7 +377,6 @@ describe("legacyDiffDeclarativeToMigrations", () => { describe("legacyGenerateDeclarativeOutput", () => { it.effect("diffs the baseline catalog against the live DB and returns files", () => { const seam = mockSeam({ - migrations: "m", declarative: "d", baseline: "supabase/.temp/pgdelta/base.json", }); @@ -142,7 +387,7 @@ describe("legacyGenerateDeclarativeOutput", () => { }; const edge = mockEdge(JSON.stringify(payload)); return legacyGenerateDeclarativeOutput( - ctx("/proj/supabase/database"), + ctx("/proj", "/proj/supabase/database"), "postgresql://postgres:postgres@127.0.0.1:54322/postgres?connect_timeout=10", ).pipe( Effect.tap((output) => diff --git a/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.ts b/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.ts index 954dc76ec6..57ba5901fb 100644 --- a/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.ts +++ b/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.ts @@ -1,10 +1,14 @@ -import { Effect, FileSystem } from "effect"; +import { Effect, FileSystem, Path } from "effect"; import { type LegacyPgDeltaContext, legacyDeclarativeExportPgDelta, legacyDiffPgDelta, } from "../../shared/legacy-pgdelta.ts"; +import { + type LegacySetupInputs, + legacyGetMigrationsCatalogRef, +} from "../../shared/legacy-pgdelta.cache.ts"; import { LegacyDeclarativeDiffError } from "./declarative.errors.ts"; import { LegacyDeclarativeSeam } from "../../shared/legacy-pgdelta.seam.service.ts"; import { legacyFindDropStatements } from "../../../../shared/legacy-sql-split.ts"; @@ -39,14 +43,18 @@ export interface LegacyDeclarativeSyncResult { /** * Computes the diff between local migrations state and the declarative schema. * Mirrors Go's `DiffDeclarativeToMigrations` (`declarative.go:170`): the - * migrations catalog (source) and declarative catalog (target) are provisioned - * via the Go seam (shadow DB + `SetupDatabase` + migrate / apply), then diffed - * natively with pg-delta. + * declarative catalog (target) is still provisioned via the Go seam (shadow DB + + * `SetupDatabase` + declarative apply); the migrations catalog (source) resolves + * natively (CLI-1959) via `legacyGetMigrationsCatalogRef`, which mirrors Go's + * `getMigrationsCatalogRef` (`declarative.go:368-430`) exactly. Both are then + * diffed natively with pg-delta, as before. */ export const legacyDiffDeclarativeToMigrations = Effect.fnUntraced(function* ( run: LegacyDeclarativeRunContext, + setupInputs: LegacySetupInputs, ) { const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const seam = yield* LegacyDeclarativeSeam; const exists = yield* fs.exists(run.declarativeDir).pipe(Effect.orElseSucceed(() => false)); @@ -59,7 +67,10 @@ export const legacyDiffDeclarativeToMigrations = Effect.fnUntraced(function* ( ); } - const sourceRef = yield* seam.exportCatalog({ mode: "migrations", noCache: run.noCache }); + const sourceRef = yield* legacyGetMigrationsCatalogRef(fs, path, run.pgDelta, setupInputs, { + noCache: run.noCache, + ...(run.linkedProjectRef !== undefined ? { projectRef: run.linkedProjectRef } : {}), + }); const targetRef = yield* seam.exportCatalog({ mode: "declarative", noCache: run.noCache }); const diff = yield* legacyDiffPgDelta(run.pgDelta, { sourceRef, diff --git a/apps/cli/src/legacy/commands/db/schema/declarative/sync/SIDE_EFFECTS.md b/apps/cli/src/legacy/commands/db/schema/declarative/sync/SIDE_EFFECTS.md index ba4ee2562b..4a17e22697 100644 --- a/apps/cli/src/legacy/commands/db/schema/declarative/sync/SIDE_EFFECTS.md +++ b/apps/cli/src/legacy/commands/db/schema/declarative/sync/SIDE_EFFECTS.md @@ -5,30 +5,32 @@ as a new timestamped migration. ## Files Read -| Path | Format | When | -| -------------------------------------------------------- | ---------- | -------------------------------------------------- | -| `/supabase/config.toml` | TOML | always — pg-delta gate, format options | -| `/supabase/.temp/pgdelta-version` | plain text | always — pins the `@supabase/pg-delta` npm version | -| `/supabase/.temp/edge-runtime-version` | plain text | always — pins the edge-runtime image tag | -| `/supabase/database/**/*.sql` (declarative dir) | SQL | always — must exist (else error) | -| `/supabase/migrations/*.sql` | SQL | shadow-DB migrations catalog (Go seam) | -| `/supabase/.temp/pgdelta/*.json` | JSON | catalog cache (read/written by the Go seam) | +| Path | Format | When | +| -------------------------------------------------------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/supabase/config.toml` | TOML | always — pg-delta gate, format options | +| `/supabase/.temp/pgdelta-version` | plain text | always — pins the `@supabase/pg-delta` npm version | +| `/supabase/.temp/edge-runtime-version` | plain text | always — pins the edge-runtime image tag | +| `/supabase/database/**/*.sql` (declarative dir) | SQL | always — must exist (else error) | +| `/supabase/migrations/*.sql` | SQL | migrations-catalog resolution (native, CLI-1959) — hashed for the cache key and, on a miss, replayed onto the shadow via `db __shadow --mode diff` | +| `/supabase/roles.sql` | SQL | native migrations-catalog cache key (setup-inputs token; empty when absent) | +| `/supabase/.temp/pgdelta/*.json` | JSON | migrations catalog cache (native, CLI-1959); declarative catalog cache (still the Go seam) | ## Files Written -| Path | Format | When | -| ------------------------------------------------------ | ------ | ----------------------------- | -| `/supabase/migrations/_.sql` | SQL | when schema changes are found | -| `/supabase/.temp/pgdelta/catalog-*.json` | JSON | catalog cache (Go seam) | +| Path | Format | When | +| --------------------------------------------------------------- | ------ | --------------------------------------------------- | +| `/supabase/migrations/_.sql` | SQL | when schema changes are found | +| `/supabase/.temp/pgdelta/catalog-*-migrations-*.json` | JSON | migrations catalog cache write (native, CLI-1959) | +| `/supabase/.temp/pgdelta/catalog-*-declarative-*.json` | JSON | declarative catalog cache write (still the Go seam) | ## Subprocesses / Containers -| What | When | -| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | -| `supabase-go db schema declarative __catalog --mode migrations --experimental` (seam) — shadow Postgres + `SetupDatabase` + apply migrations → catalog | always | -| `supabase-go db schema declarative __catalog --mode declarative --experimental` (seam) — shadow Postgres + `SetupDatabase` + apply declarative → catalog | always | -| Edge-runtime container running the pg-delta diff Deno script | always | -| `supabase-go db reset --local [--network-id ]` (seam) — only on the failed-apply recovery path; `db reset` is still Go-proxied (`wrapped`), so the reset itself shells out to the bundled binary | TTY only, apply failed, and the user confirms "reset and reapply" | +| What | When | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| `supabase-go db __shadow --mode diff` (seam, unchanged) — shadow Postgres + `SetupDatabase` + apply migrations; the catalog itself is exported natively via edge-runtime (CLI-1959 — no longer the hidden `db schema declarative __catalog --mode migrations` subprocess) | migrations-catalog cache miss only | +| `supabase-go db schema declarative __catalog --mode declarative --experimental` (seam) — shadow Postgres + `SetupDatabase` + apply declarative → catalog | always | +| Edge-runtime container running the pg-delta diff Deno script, and (on a migrations-catalog cache miss) the pg-delta catalog-export Deno script | always / cache miss | +| `supabase-go db reset --local [--network-id ]` (seam) — only on the failed-apply recovery path; `db reset` is still Go-proxied (`wrapped`), so the reset itself shells out to the bundled binary | TTY only, apply failed, and the user confirms "reset and reapply" | ## Environment Variables @@ -79,6 +81,14 @@ are mutually exclusive. `supabase/.temp/pgdelta/debug/` and, in a TTY, a reset-and-reapply is offered (the reset itself runs the bundled `supabase-go db reset --local`, since `db reset` is still `wrapped`). -- **Architecture:** the shadow-database platform baseline (migrations / declarative - catalogs) is provisioned by the bundled `supabase-go` via the hidden - `db schema declarative __catalog` seam; the diff is native pg-delta. +- **Architecture:** the migrations-catalog diff source resolves natively (CLI-1959): + the setup-inputs-folded cache key, the zero-local-migrations → platform-baseline + reuse, and the pg-delta catalog export are all native TS; only the shadow-database + platform-baseline provisioning + migrations apply still runs via the bundled + `supabase-go`, reusing the SAME `db __shadow --mode diff` seam call `db diff` + uses (not a `__catalog`-specific shadow). The declarative-catalog diff target + still provisions its shadow-database platform baseline (and applies declarative + files) via the hidden `db schema declarative __catalog --mode declarative` seam, + since neither a baseline-only shadow nor `pgdelta.ApplyDeclarative` has a native + TS port yet (tracked by CLI-1956/CLI-1823). The diff itself is native pg-delta + either way. diff --git a/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.handler.ts b/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.handler.ts index e7f946362b..520102cfc9 100644 --- a/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.handler.ts +++ b/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.handler.ts @@ -26,6 +26,7 @@ import { LegacyTelemetryState } from "../../../../../telemetry/legacy-telemetry- import { legacyListLocalMigrations, legacyPgDeltaTempPath, + legacyResolveSetupInputs, } from "../../../shared/legacy-pgdelta.cache.ts"; import { legacyResolveSmartTargetUrl } from "../declarative.smart-target.ts"; import { @@ -266,8 +267,20 @@ export const legacyDbSchemaDeclarativeSync = Effect.fn("legacy.db.schema.declara } // Step 2: diff migrations state vs declarative; on error, save a debug bundle. + // `setupInputs` is the cache-key/baseline-setup subset of `toml` that the now- + // native migrations-catalog resolution needs (CLI-1959) — see + // `legacyResolveSetupInputs`'s doc comment. + const setupInputs = yield* legacyResolveSetupInputs( + fs, + path, + cliConfig.workdir, + toml.majorVersion, + Option.getOrUndefined(toml.orioledbVersion), + toml.baseline, + ); const result: LegacyDeclarativeSyncResult = yield* legacyDiffDeclarativeToMigrations( run, + setupInputs, ).pipe( Effect.tapError((error) => Effect.gen(function* () { diff --git a/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts index a5acb0655e..72b4d43a6d 100644 --- a/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -75,6 +75,13 @@ function setup(workdir: string, opts: SetupOpts = {}) { // so tests can assert output ordering relative to the exports (e.g. the bootstrap's // written-to line lands after the declarative warm, before the diff's exports). const exportCatalogCalls: Array<{ mode: string; rawChunksAt: number }> = []; + // The migrations-catalog source now resolves natively (CLI-1959) via + // `legacyGetMigrationsCatalogRef`, which provisions its shadow through + // `provisionShadow` (Go's unchanged `db __shadow --mode diff`) instead of the + // retired `exportCatalog({mode:"migrations"})` seam call. "baseline"/ + // "declarative" still go through `exportCatalog`. + const provisionShadowCalls: Array<{ mode: string; targetLocal: boolean; rawChunksAt: number }> = + []; const seam = Layer.succeed(LegacyDeclarativeSeam, { exportCatalog: ({ mode }) => Effect.sync(() => { @@ -101,11 +108,25 @@ function setup(workdir: string, opts: SetupOpts = {}) { : Effect.void, ), ), - provisionShadow: () => Effect.die("provisionShadow not used in declarative tests"), + provisionShadow: ({ mode, targetLocal }) => + Effect.sync(() => { + provisionShadowCalls.push({ mode, targetLocal, rawChunksAt: out.rawChunks.length }); + return { + container: "shadow-1", + sourceUrl: "postgres://postgres:postgres@127.0.0.1:54320/postgres", + targetUrlOverride: undefined, + }; + }), removeShadowContainer: () => Effect.void, }); const edge = Layer.succeed(LegacyEdgeRuntimeScript, { run: (runOpts: LegacyEdgeRuntimeRunOpts) => { + // The native migrations-catalog resolution's shadow export — return a fixed, + // non-empty snapshot so it never trips `legacyExportCatalogPgDelta`'s + // empty-output check regardless of what `opts.diffSql` a given test sets. + if (runOpts.errPrefix === "error exporting pg-delta catalog") { + return Effect.succeed({ stdout: '{"schemas":[]}', stderr: "" }); + } if ( opts.exportJson !== undefined && runOpts.errPrefix === "error exporting declarative schema" @@ -203,6 +224,7 @@ function setup(workdir: string, opts: SetupOpts = {}) { cache, localPostgresImageChecks, exportCatalogCalls, + provisionShadowCalls, }; } @@ -490,9 +512,11 @@ describe("legacy db schema declarative sync integration", () => { // The warm (first declarative-mode export) fires before the line is printed… const warm = s.exportCatalogCalls.find((c) => c.mode === "declarative"); expect(warm?.rawChunksAt).toBeLessThanOrEqual(lineAt); - // …and the diff's first export (migrations catalog) fires after it, so the - // line sits at the end of the bootstrap, matching Go's ordering. - const diffStart = s.exportCatalogCalls.find((c) => c.mode === "migrations"); + // …and the diff's migrations-catalog resolution (now native, CLI-1959 — + // provisions its shadow via `provisionShadow` instead of a seam `exportCatalog` + // call) fires after it, so the line sits at the end of the bootstrap, matching + // Go's ordering. + const diffStart = s.provisionShadowCalls.find((c) => c.mode === "diff" && !c.targetLocal); expect(diffStart?.rawChunksAt).toBeGreaterThan(lineAt); // The generated files actually landed in the printed (resolved) dir. expect( diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.ts index f06b687000..a9c8ada348 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.ts @@ -1,9 +1,12 @@ import { createHash } from "node:crypto"; -import { Effect, type FileSystem, Option, type Path } from "effect"; +import { Clock, Effect, type FileSystem, Option, type Path } from "effect"; import { Output } from "../../../../shared/output/output.service.ts"; +import type { LegacyBaselineTomlConfig } from "../../../shared/legacy-db-config.toml-read.ts"; +import { legacyResolveDbImage } from "../../../shared/legacy-db-image.ts"; import { LegacyMigrationsReadError } from "../../../shared/legacy-migration.errors.ts"; import { type LegacyPgDeltaContext, legacyExportCatalogPgDelta } from "./legacy-pgdelta.ts"; +import { LegacyDeclarativeSeam } from "./legacy-pgdelta.seam.service.ts"; /** * Declarative catalog-cache key builders + on-disk catalog resolution, ported @@ -11,6 +14,14 @@ import { type LegacyPgDeltaContext, legacyExportCatalogPgDelta } from "./legacy- * `internal/db/pgcache/cache.go`). Byte-stable parity matters: caches under * `supabase/.temp/pgdelta/` are shared with the Go binary, so a drifting key * would silently miss (re-provision) or over-hit (reuse a stale snapshot). + * + * Beyond the pure key/path builders, this file also owns the migrations-catalog + * RESOLUTION path for both `db diff --from/--to migrations` and `db schema + * declarative sync` ({@link legacyResolveMigrationsCatalogRef}, + * {@link legacyGetMigrationsCatalogRef}) — including shadow-database provisioning/ + * removal via `LegacyDeclarativeSeam` (Docker orchestration, unchanged from the Go + * seam) and the "Creating shadow database..." stderr side effect the latter prints + * on a cache miss. It is not a pure module. */ const CATALOG_PREFIX_PATTERN = /[^a-zA-Z0-9._-]+/g; @@ -103,11 +114,66 @@ export function legacyBaselineCatalogKey(inputs: LegacySetupInputs): string { )}`; } +/** + * Resolves {@link LegacySetupInputs} from the caller's already-loaded db config: + * the resolved Postgres image, and `supabase/roles.sql`'s content (empty when + * absent, mirroring Go's `errors.Is(err, os.ErrNotExist)` tolerance in + * `setupInputsToken`, `apps/cli-go/internal/db/declarative/declarative.go:711-714`). + * Callers pass `toml.baseline` (`legacy-db-config.toml-read.ts`'s + * `LegacyBaselineTomlConfig`, already exactly this cache-key subset) verbatim. + */ +export const legacyResolveSetupInputs = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + workdir: string, + majorVersion: number, + orioledbVersion: string | undefined, + baseline: LegacyBaselineTomlConfig, +) { + const image = yield* legacyResolveDbImage(fs, path, workdir, majorVersion, orioledbVersion); + const rolesPath = path.join(workdir, "supabase", "roles.sql"); + const rolesSql = yield* fs + .readFileString(rolesPath) + .pipe( + Effect.catchTag("PlatformError", (error) => + error.reason._tag === "NotFound" ? Effect.succeed("") : Effect.fail(error), + ), + ); + return { + image, + majorVersion, + authEnabled: baseline.authEnabled, + storageEnabled: baseline.storageEnabled, + realtimeEnabled: baseline.realtimeEnabled, + autoExpose: + Option.isSome(baseline.apiAutoExposeNewTables) && baseline.apiAutoExposeNewTables.value, + vaultNames: baseline.vaultNames, + rolesSql, + } satisfies LegacySetupInputs; +}); + /** Mirrors Go's `declarativeCatalogCacheKey` (`declarative.go:753`): `-`. */ export function legacyDeclarativeCatalogCacheKey(setupToken: string, schemaHash: string): string { return `${setupToken}-${schemaHash}`; } +/** + * Mirrors Go's `migrationsCatalogCacheKey` (`declarative.go:765`): `- + * `. Used ONLY by {@link legacyGetMigrationsCatalogRef} (the + * `db schema declarative sync` migrations source) — `db diff`'s explicit + * `--from/--to migrations` uses a bare, setup-token-less hash instead (Go's + * `resolveMigrationsCatalogRef`, `internal/db/diff/explicit.go:88`; see + * {@link legacyResolveMigrationsCatalogRef}). These are deliberately two different + * cache-key schemes over the same `catalog-local-migrations-*.json` filename + * family, matching Go exactly (CLI-1959). + */ +export function legacyMigrationsCatalogCacheKey( + setupToken: string, + migrationsHash: string, +): string { + return `${setupToken}-${migrationsHash}`; +} + /** `catalog-baseline-.json` (`declarative.go:44`). */ export function legacyBaselineCatalogFileName(key: string): string { return `catalog-baseline-${key}.json`; @@ -275,19 +341,21 @@ const listJsonEntries = Effect.fnUntraced(function* (fs: FileSystem.FileSystem, }); /** - * Resolves the newest cached declarative catalog for `(prefix, hash)`. Mirrors - * Go's `resolveDeclarativeCatalogPath` (`declarative.go:578`): of all - * `catalog--declarative--.json`, returns the highest `ts`. + * Shared "highest suffixed timestamp wins" scan behind both + * {@link legacyResolveDeclarativeCatalogPath} and {@link legacyResolveMigrationCatalogPath}: + * of every `.json` entry in `tempDir`, returns the path with the + * highest `ts`. Mirrors both Go's `resolveDeclarativeCatalogPath` + * (`declarative.go:578`) and `pgcache.ResolveMigrationCatalogPath` + * (`internal/db/pgcache/cache.go:112-149`), which share this exact scan over their + * own filename family — only the family prefix differs between callers. */ -export const legacyResolveDeclarativeCatalogPath = Effect.fnUntraced(function* ( +const resolveLatestByFamily = Effect.fnUntraced(function* ( fs: FileSystem.FileSystem, path: Path.Path, tempDir: string, - prefix: string, - hash: string, + familyPrefix: string, ) { const entries = yield* listJsonEntries(fs, tempDir); - const familyPrefix = `catalog-${legacySanitizedCatalogPrefix(prefix)}-declarative-${hash}-`; let latestPath = Option.none(); let latest = -1; for (const name of entries) { @@ -301,6 +369,26 @@ export const legacyResolveDeclarativeCatalogPath = Effect.fnUntraced(function* ( return latestPath; }); +/** + * Resolves the newest cached declarative catalog for `(hash, prefix)`. Mirrors + * Go's `resolveDeclarativeCatalogPath` (`declarative.go:578`): of all + * `catalog--declarative--.json`, returns the highest `ts`. + */ +export const legacyResolveDeclarativeCatalogPath = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + tempDir: string, + hash: string, + prefix: string, +) { + return yield* resolveLatestByFamily( + fs, + path, + tempDir, + `catalog-${legacySanitizedCatalogPrefix(prefix)}-declarative-${hash}-`, + ); +}); + const cleanupOldCatalogsByFamily = Effect.fnUntraced(function* ( fs: FileSystem.FileSystem, path: Path.Path, @@ -366,6 +454,31 @@ export function legacyMigrationCatalogFileName( return `catalog-${legacySanitizedCatalogPrefix(prefix)}-migrations-${hash}-${timestampMillis}.json`; } +/** + * Resolves the newest cached migrations catalog for `(hash, prefix)`. Mirrors + * Go's `pgcache.ResolveMigrationCatalogPath` (`internal/db/pgcache/cache.go:112-149`). + * Go's fallback to a pre-timestamp legacy filename (`catalog--migrations- + * .json`, no `-` suffix) is intentionally NOT replicated: nothing in the + * Go tree writes that name any more — `pgcache.MigrationCatalogPath` has always + * produced the timestamped form since the fallback was added in the same commit + * (CLI-1959 go-parity-auditor finding) — so it is unreachable dead code on both + * sides. + */ +export const legacyResolveMigrationCatalogPath = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + tempDir: string, + hash: string, + prefix: string, +) { + return yield* resolveLatestByFamily( + fs, + path, + tempDir, + `catalog-${legacySanitizedCatalogPrefix(prefix)}-migrations-${hash}-`, + ); +}); + /** * Writes a migrations-catalog snapshot to `/catalog--migrations--.json` * and prunes older snapshots for the same `(prefix)` family. Mirrors Go's @@ -434,3 +547,184 @@ export const legacyTryCacheMigrationsCatalog = Effect.fnUntraced(function* ( params.nowMillis, ); }); + +/** + * Shared shadow-provision → pg-delta export → persist → cleanup mechanics behind + * both {@link legacyResolveMigrationsCatalogRef} and {@link legacyGetMigrationsCatalogRef} + * on a cache miss: provisions the shadow via the EXISTING + * `LegacyDeclarativeSeam.provisionShadow` (Go's `db __shadow --mode diff`, unchanged + * / out of scope for CLI-1959 — `CreateShadowDatabase` + `MigrateShadowDatabase` are + * the exact same Go primitives both callers' Go counterparts call directly, + * `internal/db/diff/shadow.go:37-53` with `targetLocal=false` skipping its only + * extra branch), exports its catalog via the already-native + * {@link legacyExportCatalogPgDelta} (the same edge-runtime script Go's own + * `ExportCatalogPgDelta` runs), hands the snapshot to `persist` to decide where it + * lands on disk, then ALWAYS removes the shadow container (`Effect.ensuring`, + * success or failure) before returning. The persisted path is made relative to + * `ctx.cwd` before returning: every caller feeds this ref into pg-delta's + * edge-runtime scripts as SOURCE/TARGET, which prefix a bare (non-postgres://) ref + * with `/workspace/` — matching the container bind `${ctx.cwd}:/workspace` + * (`legacyPgDeltaContainerRef`, `legacy-pgdelta.ts:100-103`). Go's equivalent + * (`pgcache.WriteMigrationCatalogSnapshot`) is only ever built from `utils.TempDir`, + * a workdir-RELATIVE constant (Go chdirs into the workdir first), so the ref it + * returns is relative too; return the same shape here rather than the absolute host + * path `persist` builds internally. The two public functions differ only in their + * cache-decision and `persist`'s cache-write logic, not in this mechanics. + */ +const exportViaShadowCatalog = ( + path: Path.Path, + ctx: LegacyPgDeltaContext, + provisionParams: { readonly projectRef?: string }, + persist: (snapshot: string) => Effect.Effect, +) => + Effect.gen(function* () { + const seam = yield* LegacyDeclarativeSeam; + const shadow = yield* seam.provisionShadow({ + mode: "diff", + targetLocal: false, + usePgDelta: false, + schema: [], + ...(provisionParams.projectRef !== undefined + ? { projectRef: provisionParams.projectRef } + : {}), + }); + const written = yield* Effect.gen(function* () { + const snapshot = yield* legacyExportCatalogPgDelta(ctx, { + targetRef: shadow.sourceUrl, + role: "postgres", + }); + return yield* persist(snapshot); + }).pipe(Effect.ensuring(seam.removeShadowContainer(shadow.container))); + return path.relative(ctx.cwd, written); + }); + +/** + * Resolves the pg-delta migrations-catalog ref for `db diff`'s explicit + * `--from migrations` / `--to migrations` target — the native replacement for + * the hidden Go seam `db schema declarative __catalog --mode migrations` this + * call site used to shell out to (CLI-1959). Mirrors Go's + * `resolveMigrationsCatalogRef` (`apps/cli-go/internal/db/diff/explicit.go:88-126`) + * EXACTLY — not {@link legacyGetMigrationsCatalogRef} below, which backs a + * different Go function (`declarative.go`'s `getMigrationsCatalogRef`, used by + * `db schema declarative sync`). The two diverge on purpose: this one uses a + * BARE migrations-content hash (no setup-inputs token — `explicit.go:89`'s + * `pgcache.HashMigrations`), always consults the cache (`db diff` has no + * `--no-cache` flag on this path), has no zero-migrations/baseline special case, + * and prints no "Creating shadow database..." line (Go calls the shadow + * primitives directly, without `DiffDatabase`'s own progress line). + * + * On a cache miss, the shadow-provision/export/persist/cleanup mechanics are + * shared with {@link legacyGetMigrationsCatalogRef} via {@link exportViaShadowCatalog} + * — see its doc comment. The catalog is cached with + * {@link legacyWriteMigrationCatalogSnapshot}. + */ +export const legacyResolveMigrationsCatalogRef = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + ctx: LegacyPgDeltaContext, + params: { readonly projectRef?: string }, +) { + const tempDir = legacyPgDeltaTempPath(path, ctx.cwd); + const migrationsDir = path.join(ctx.cwd, "supabase", "migrations"); + const hash = yield* legacyHashMigrations(fs, path, ctx.cwd, migrationsDir); + const cached = yield* legacyResolveMigrationCatalogPath(fs, path, tempDir, hash, "local"); + if (Option.isSome(cached)) return path.relative(ctx.cwd, cached.value); + + return yield* exportViaShadowCatalog(path, ctx, params, (snapshot) => + Effect.gen(function* () { + const timestamp = yield* Clock.currentTimeMillis; + return yield* legacyWriteMigrationCatalogSnapshot( + fs, + path, + tempDir, + "local", + hash, + snapshot, + timestamp, + ); + }), + ); +}); + +/** `catalog-nocache-migrations.json` — Go's `noCacheMigrationsCatalogPath` (`declarative.go:51`). */ +const NO_CACHE_MIGRATIONS_CATALOG_NAME = "catalog-nocache-migrations.json"; + +/** + * Resolves (and caches under `supabase/.temp/pgdelta/`) the pg-delta migrations + * catalog — platform baseline + local migrations applied — for `db schema + * declarative sync`'s diff SOURCE. The native replacement for the hidden Go seam + * `db schema declarative __catalog --mode migrations` this call site used to + * shell out to (CLI-1959). Mirrors Go's `getMigrationsCatalogRef` + * (`apps/cli-go/internal/db/declarative/declarative.go:368-430`) — see + * {@link legacyResolveMigrationsCatalogRef}'s doc comment for exactly how this + * diverges from `db diff`'s bare-hash version: this one folds the setup-inputs + * token into the cache key, special-cases zero local migrations by reusing/ + * writing the platform-baseline catalog, honors `--no-cache`, and prints + * "Creating shadow database..." to stderr on a cache miss + * (`declarative.go:490`, reached only when `createShadow` actually runs). + * + * On a cache miss, the shadow-provision/export/persist/cleanup mechanics are + * shared with {@link legacyResolveMigrationsCatalogRef} via + * {@link exportViaShadowCatalog} — see its doc comment. + */ +export const legacyGetMigrationsCatalogRef = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + ctx: LegacyPgDeltaContext, + setupInputs: LegacySetupInputs, + params: { readonly noCache: boolean; readonly projectRef?: string }, +) { + const output = yield* Output; + const tempDir = legacyPgDeltaTempPath(path, ctx.cwd); + const migrationsDir = path.join(ctx.cwd, "supabase", "migrations"); + const migrations = yield* legacyListLocalMigrations(fs, path, migrationsDir); + const zeroMigrations = migrations.length === 0; + + const baselinePath = path.join( + tempDir, + legacyBaselineCatalogFileName(legacyBaselineCatalogKey(setupInputs)), + ); + if (zeroMigrations && !params.noCache) { + const exists = yield* fs.exists(baselinePath).pipe(Effect.orElseSucceed(() => false)); + if (exists) return path.relative(ctx.cwd, baselinePath); + } + + // Mirrors Go's unconditional `migrationsCatalogCacheKey` call (`declarative.go:393`), + // which always runs — even on the zeroMigrations/noCache paths — since it is pure + // and only unused there, not because it needs to run early for a side effect. + const setupToken = legacySetupInputsToken(setupInputs); + const migrationsHash = yield* legacyHashMigrations(fs, path, ctx.cwd, migrationsDir); + const hash = legacyMigrationsCatalogCacheKey(setupToken, migrationsHash); + + if (!params.noCache && !zeroMigrations) { + const cached = yield* legacyResolveMigrationCatalogPath(fs, path, tempDir, hash, "local"); + if (Option.isSome(cached)) return path.relative(ctx.cwd, cached.value); + } + + yield* output.raw("Creating shadow database...\n", "stderr"); + return yield* exportViaShadowCatalog(path, ctx, params, (snapshot) => + Effect.gen(function* () { + if (params.noCache) { + yield* fs.makeDirectory(tempDir, { recursive: true }).pipe(Effect.ignore); + const noCachePath = path.join(tempDir, NO_CACHE_MIGRATIONS_CATALOG_NAME); + yield* fs.writeFileString(noCachePath, snapshot); + return noCachePath; + } + if (zeroMigrations) { + yield* fs.makeDirectory(tempDir, { recursive: true }).pipe(Effect.ignore); + yield* fs.writeFileString(baselinePath, snapshot); + return baselinePath; + } + const timestamp = yield* Clock.currentTimeMillis; + return yield* legacyWriteMigrationCatalogSnapshot( + fs, + path, + tempDir, + "local", + hash, + snapshot, + timestamp, + ); + }), + ); +}); diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.unit.test.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.unit.test.ts index 83535b91c0..36df7041a3 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.unit.test.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.cache.unit.test.ts @@ -22,7 +22,10 @@ import { legacyHashMigrations, legacyListLocalMigrations, legacyMigrationCatalogFileName, + legacyMigrationsCatalogCacheKey, legacyResolveDeclarativeCatalogPath, + legacyResolveMigrationCatalogPath, + legacyResolveSetupInputs, legacySanitizedCatalogPrefix, legacySetupInputsToken, legacyWriteMigrationCatalogSnapshot, @@ -104,6 +107,16 @@ describe("catalog keys + file names", () => { ); }); + it("composes the migrations cache key used by `db schema declarative sync` (setup-token-folded)", () => { + // Mirrors Go's `migrationsCatalogCacheKey` (`declarative.go:765`) — deliberately + // different from `db diff`'s bare `pgcache.HashMigrations` key (CLI-1959): this + // one folds the setup-inputs token in so a baseline/config change self- + // invalidates the sync migrations catalog too. + expect(legacyMigrationsCatalogCacheKey("setup12chars", "migrationshash")).toBe( + "setup12chars-migrationshash", + ); + }); + it("formats catalog file names", () => { expect(legacyBaselineCatalogFileName("17.6.1.135-abc")).toBe( "catalog-baseline-17.6.1.135-abc.json", @@ -309,7 +322,7 @@ describe("legacyResolveDeclarativeCatalogPath + cleanup", () => { writeFileSync(join(tempDir, "catalog-local-declarative-other-50.json"), "{}"); return withServices((fs, path) => Effect.gen(function* () { - const latest = yield* legacyResolveDeclarativeCatalogPath(fs, path, tempDir, "local", "h"); + const latest = yield* legacyResolveDeclarativeCatalogPath(fs, path, tempDir, "h", "local"); expect(Option.getOrNull(latest)?.endsWith("catalog-local-declarative-h-300.json")).toBe( true, ); @@ -364,6 +377,92 @@ describe("legacyCatalogPrefixFromConfig", () => { }); }); +describe("legacyResolveMigrationCatalogPath", () => { + it.effect("resolves the newest snapshot for the (hash, prefix) family", () => { + const dir = withTemp(); + const tempDir = join(dir, "pgdelta"); + mkdirSync(tempDir, { recursive: true }); + for (const ts of [100, 300, 200]) { + writeFileSync(join(tempDir, `catalog-local-migrations-h-${ts}.json`), "{}"); + } + // A different hash in the same prefix family must not be picked up. + writeFileSync(join(tempDir, "catalog-local-migrations-other-500.json"), "{}"); + return withServices((fs, path) => + Effect.gen(function* () { + const latest = yield* legacyResolveMigrationCatalogPath(fs, path, tempDir, "h", "local"); + expect(Option.getOrNull(latest)?.endsWith("catalog-local-migrations-h-300.json")).toBe( + true, + ); + }), + ).pipe(Effect.tap(() => Effect.sync(() => rmSync(dir, { recursive: true, force: true })))); + }); + + it.effect("returns None on a cache miss (no matching family member)", () => { + const dir = withTemp(); + const tempDir = join(dir, "pgdelta"); + return withServices((fs, path) => + Effect.gen(function* () { + const resolved = yield* legacyResolveMigrationCatalogPath(fs, path, tempDir, "h", "local"); + expect(Option.isNone(resolved)).toBe(true); + }), + ).pipe(Effect.tap(() => Effect.sync(() => rmSync(dir, { recursive: true, force: true })))); + }); +}); + +describe("legacyResolveSetupInputs", () => { + it.effect("resolves the image and tolerates a missing roles.sql", () => { + const dir = withTemp(); + return withServices((fs, path) => + legacyResolveSetupInputs(fs, path, dir, 17, undefined, { + authEnabled: true, + storageEnabled: false, + realtimeEnabled: true, + apiAutoExposeNewTables: Option.none(), + vaultNames: ["a_secret"], + }), + ).pipe( + Effect.tap((inputs) => + Effect.sync(() => { + expect(inputs).toMatchObject({ + majorVersion: 17, + authEnabled: true, + storageEnabled: false, + realtimeEnabled: true, + autoExpose: false, + vaultNames: ["a_secret"], + rolesSql: "", + }); + expect(inputs.image.length).toBeGreaterThan(0); + rmSync(dir, { recursive: true, force: true }); + }), + ), + ); + }); + + it.effect("reads roles.sql content and resolves the effective auto-expose bool", () => { + const dir = withTemp(); + mkdirSync(join(dir, "supabase"), { recursive: true }); + writeFileSync(join(dir, "supabase", "roles.sql"), "create role app;"); + return withServices((fs, path) => + legacyResolveSetupInputs(fs, path, dir, 17, undefined, { + authEnabled: true, + storageEnabled: true, + realtimeEnabled: true, + apiAutoExposeNewTables: Option.some(true), + vaultNames: [], + }), + ).pipe( + Effect.tap((inputs) => + Effect.sync(() => { + expect(inputs.rolesSql).toBe("create role app;"); + expect(inputs.autoExpose).toBe(true); + rmSync(dir, { recursive: true, force: true }); + }), + ), + ); + }); +}); + describe("legacyMigrationCatalogFileName", () => { it("formats catalog--migrations--.json", () => { expect(legacyMigrationCatalogFileName("local", "h", 1700)).toBe( diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.layer.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.layer.ts index 6a52434c89..9c3923ed0a 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.layer.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.layer.ts @@ -22,6 +22,11 @@ import { legacyInjectPostgresPassword } from "./legacy-pgdelta.seam.url.ts"; * `db schema declarative __catalog --mode --experimental` with stdout piped * (the catalog path) and stderr inherited (shadow-DB progress / image pulls). * The Go binary is resolved exactly like `LegacyGoProxy` (`resolveBinary`). + * + * `exportCatalog`'s `mode` is now restricted to `"baseline" | "declarative"` + * (CLI-1959 removed `"migrations"` from `LegacyCatalogMode` — see that type's + * doc comment in `legacy-pgdelta.seam.service.ts` for why those two modes still + * need this hidden Go command while `"migrations"` no longer does). */ export const legacyDeclarativeSeamLayer = Layer.effect( LegacyDeclarativeSeam, diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.service.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.service.ts index 11f0501bcd..c7354e8504 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.service.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.service.ts @@ -2,8 +2,23 @@ import { Context, type Effect } from "effect"; import type { LegacyDeclarativeShadowDbError } from "./legacy-pgdelta.errors.ts"; -/** Which shadow-database catalog the Go seam should produce. */ -export type LegacyCatalogMode = "baseline" | "migrations" | "declarative"; +/** + * Which shadow-database catalog the Go seam should produce. + * + * `"migrations"` was removed from this union under CLI-1959: both call sites + * that used it (`db diff`'s explicit `--from/--to migrations`, and + * `db schema declarative sync`'s migrations-catalog diff source) now resolve + * natively — see `legacy-pgdelta.cache.ts`'s `legacyResolveMigrationsCatalogRef` + * and `legacyGetMigrationsCatalogRef` respectively. `"baseline"` and + * `"declarative"` remain seam-backed because they need a shadow provisioned with + * ONLY the platform baseline (no migrations) or with declarative files applied — + * neither has a native TS equivalent yet (`start.SetupDatabase` against an + * arbitrary shadow, and `pgdelta.ApplyDeclarative`), and porting either + * overlaps with CLI-1956's in-progress native shadow-provisioning work. CLI-1823 + * (native pg-delta lib) and CLI-1956 are the tracked follow-ups for retiring the + * rest of this seam. + */ +export type LegacyCatalogMode = "baseline" | "declarative"; /** * Which live shadow database the Go seam should provision and leave running: @@ -30,15 +45,16 @@ export interface LegacyShadowSource { interface LegacyDeclarativeSeamShape { /** - * Provisions the shadow-database platform baseline (and, for - * `migrations`/`declarative`, applies migrations / declarative files) via the - * bundled Go binary's hidden `db schema declarative __catalog` command, and - * returns the workdir-relative path of the exported pg-delta catalog (cached - * under `supabase/.temp/pgdelta/`). Go's progress is teed to stderr; only the - * catalog path is captured from stdout. + * Provisions the shadow-database platform baseline (and, for `declarative`, + * applies declarative files) via the bundled Go binary's hidden + * `db schema declarative __catalog` command, and returns the workdir-relative + * path of the exported pg-delta catalog (cached under `supabase/.temp/pgdelta/`). + * Go's progress is teed to stderr; only the catalog path is captured from stdout. * * This is the seam for `start.SetupDatabase` (the auth/storage/realtime service - * migrations), which is not yet ported to TypeScript. + * migrations) run against an arbitrary shadow, and for `pgdelta.ApplyDeclarative` + * (the `declarative` mode), neither of which is yet ported to TypeScript + * (CLI-1959/CLI-1956/CLI-1823 — see {@link LegacyCatalogMode}'s doc comment). */ readonly exportCatalog: (opts: { readonly mode: LegacyCatalogMode; diff --git a/apps/cli/src/legacy/shared/legacy-db-config.toml-read.ts b/apps/cli/src/legacy/shared/legacy-db-config.toml-read.ts index 74582272eb..33c3dfe15c 100644 --- a/apps/cli/src/legacy/shared/legacy-db-config.toml-read.ts +++ b/apps/cli/src/legacy/shared/legacy-db-config.toml-read.ts @@ -130,8 +130,14 @@ interface LegacyDbVaultSecretToml { readonly resolved: boolean; } -/** Cache-key inputs from `[auth]`/`[storage]`/`[realtime]`/`[api]`/`[db.vault]`. */ -interface LegacyBaselineTomlConfig { +/** + * Cache-key inputs from `[auth]`/`[storage]`/`[realtime]`/`[api]`/`[db.vault]`. + * Exported so callers that build this cache-key subset directly (e.g. + * `legacyResolveSetupInputs` in `legacy-pgdelta.cache.ts`) reference this shape + * instead of re-declaring it inline, making field drift a compile error rather + * than a silent cache-key gap. + */ +export interface LegacyBaselineTomlConfig { /** `[auth] enabled`, default true. Gates `initSchema`'s auth service migration. */ readonly authEnabled: boolean; /** `[storage] enabled`, default true. */