diff --git a/apps/cli-go/cmd/db.go b/apps/cli-go/cmd/db.go index df04364e44..b6f68e5ccc 100644 --- a/apps/cli-go/cmd/db.go +++ b/apps/cli-go/cmd/db.go @@ -200,76 +200,6 @@ var ( }, } - shadowMode string - shadowTargetLocal bool - shadowUsePgDelta bool - shadowSchema []string - shadowProjectRef string - - // dbShadowCmd is a hidden seam used by the native-TypeScript db diff/pull - // commands to provision the throwaway shadow database that the diff "source" - // runs against, then leave it running so the TS caller can run the differ - // (migra or pg-delta) itself and remove the container afterwards. It prints - // three newline-separated lines to stdout: the container id, the source - // Postgres URL, and an optional target-override URL (empty unless the - // local-target declarative branch redirects the diff target to a second - // shadow database). The URLs are emitted WITHOUT the password - // (ToPostgresURLWithoutPassword) so we never log a credential to stdout - // (CWE-312); the TS caller re-injects the local Postgres password it already - // resolves from config.toml, which is the same value the shadow uses. Shadow - // provisioning (start.SetupDatabase) is not yet ported, which is why this - // stays in Go. - dbShadowCmd = &cobra.Command{ - Use: "__shadow", - Hidden: true, - Short: "Internal: provision a shadow database for the native db diff/pull commands", - RunE: func(cmd *cobra.Command, args []string) error { - // The hidden __shadow command carries none of the db-url/local/linked - // target flags, so the root PersistentPreRunE's ParseDatabaseConfig - // never loads supabase/config.toml (it only loads when a target flag - // is set, internal/utils/flags/db_url.go:46-90). Load it explicitly so - // the shadow is provisioned from the project's [db] settings — shadow - // port, Postgres version, service baseline, and especially the - // password: the native-TS caller injects the config.toml password into - // the seam URLs, so the shadow must be created with that same password. - fsys := afero.NewOsFs() - // On the linked path the native-TS caller passes the resolved project - // ref via --project-ref so the shadow is built from the same - // remote-merged config the Go monolith uses: LoadConfig seeds - // utils.Config.ProjectId from flags.ProjectRef and merges the matching - // [remotes.] block (pkg/config/config.go). Omitted on local/db-url - // shadows, which the monolith never remote-merges, so the base config is - // used exactly as before. - if len(shadowProjectRef) > 0 { - flags.ProjectRef = shadowProjectRef - } - if err := flags.LoadConfig(fsys); err != nil { - return err - } - var src diff.ShadowSource - var err error - switch shadowMode { - case "declarative": - src, err = diff.PrepareRawShadow(cmd.Context()) - case "diff", "": - src, err = diff.PrepareShadowSource(cmd.Context(), shadowSchema, shadowTargetLocal, shadowUsePgDelta, fsys) - default: - return fmt.Errorf("unknown shadow mode: %s", shadowMode) - } - if err != nil { - return err - } - fmt.Println(src.Container) - fmt.Println(utils.ToPostgresURLWithoutPassword(src.Source)) - if src.TargetOverride != nil { - fmt.Println(utils.ToPostgresURLWithoutPassword(*src.TargetOverride)) - } else { - fmt.Println("") - } - return nil - }, - } - dbRemoteCmd = &cobra.Command{ Hidden: true, Use: "remote", @@ -612,14 +542,6 @@ func init() { pullFlags.StringVarP(&dbPassword, "password", "p", "", "Password to your remote Postgres database.") cobra.CheckErr(viper.BindPFlag("DB_PASSWORD", pullFlags.Lookup("password"))) dbCmd.AddCommand(dbPullCmd) - // Build hidden shadow-provisioning seam command - shadowFlags := dbShadowCmd.Flags() - shadowFlags.StringVar(&shadowMode, "mode", "diff", "Shadow mode: diff (baseline + migrations) or declarative (bare shadow).") - shadowFlags.BoolVar(&shadowTargetLocal, "target-local", false, "Whether the diff target is the local database (enables the declarative-schema branch).") - shadowFlags.BoolVar(&shadowUsePgDelta, "use-pg-delta", false, "Whether pg-delta is the active diff engine (selects the declarative-apply path).") - shadowFlags.StringSliceVarP(&shadowSchema, "schema", "s", []string{}, "Comma separated list of schema to include.") - shadowFlags.StringVar(&shadowProjectRef, "project-ref", "", "Linked project ref, so the shadow merges the matching [remotes.] config override.") - dbCmd.AddCommand(dbShadowCmd) // Build remote command remoteFlags := dbRemoteCmd.PersistentFlags() remoteFlags.StringSliceVarP(&schema, "schema", "s", []string{}, "Comma separated list of schema to include.") diff --git a/apps/cli-go/internal/utils/connect.go b/apps/cli-go/internal/utils/connect.go index 406e515370..6dad6c5c4a 100644 --- a/apps/cli-go/internal/utils/connect.go +++ b/apps/cli-go/internal/utils/connect.go @@ -26,17 +26,6 @@ func ToPostgresURL(config pgconn.Config) string { return toPostgresURL(config, url.UserPassword(config.User, config.Password)) } -// ToPostgresURLWithoutPassword renders the connection URL exactly like -// ToPostgresURL but omits the password from the userinfo. Use it for callers that -// print the URL to stdout (the hidden `db __shadow` seam): embedding the password -// there is clear-text logging of a credential (CWE-312, flagged by CodeQL). The -// password is never the seam's to share — the TS caller that consumes the seam -// output re-injects the local Postgres password it already resolves from -// config.toml (`utils.Config.Db.Password`). -func ToPostgresURLWithoutPassword(config pgconn.Config) string { - return toPostgresURL(config, url.User(config.User)) -} - func toPostgresURL(config pgconn.Config, userinfo *url.Userinfo) string { timeoutSecond := int64(config.ConnectTimeout.Seconds()) if timeoutSecond == 0 { diff --git a/apps/cli-go/internal/utils/connect_test.go b/apps/cli-go/internal/utils/connect_test.go index 80684df8d1..876d7ea7c7 100644 --- a/apps/cli-go/internal/utils/connect_test.go +++ b/apps/cli-go/internal/utils/connect_test.go @@ -398,23 +398,6 @@ func TestPostgresURL(t *testing.T) { assert.Equal(t, `postgresql://postgres:%21%40%23$%25%5E&%2A%28%29@[2406:da18:4fd:9b0d:80ec:9812:3e65:450b]:5432/?connect_timeout=10&options=test`, url) } -func TestPostgresURLWithoutPassword(t *testing.T) { - config := pgconn.Config{ - Host: "2406:da18:4fd:9b0d:80ec:9812:3e65:450b", - Port: 5432, - User: "postgres", - Password: "!@#$%^&*()", - RuntimeParams: map[string]string{ - "options": "test", - }, - } - url := ToPostgresURLWithoutPassword(config) - // Same as ToPostgresURL but with the password omitted from the userinfo, so a - // credential is never written to stdout by the db __shadow seam. - assert.Equal(t, `postgresql://postgres@[2406:da18:4fd:9b0d:80ec:9812:3e65:450b]:5432/?connect_timeout=10&options=test`, url) - assert.NotContains(t, url, "%21%40%23") -} - func TestPreserveTLSConfig(t *testing.T) { const dsn = "postgresql://postgres:pw@example.com:5432/postgres" diff --git a/apps/cli/docs/go-cli-porting-status.md b/apps/cli/docs/go-cli-porting-status.md index ac13f8c62c..a87044acaa 100644 --- a/apps/cli/docs/go-cli-porting-status.md +++ b/apps/cli/docs/go-cli-porting-status.md @@ -82,7 +82,7 @@ These commands exist in the TS CLI today but have no direct top-level equivalent | Old command | TS status | TS command path or `missing` | Missing flags/params | Extra TS flags/params | Notes | | --------------------------------- | --------- | -------------------------------------------------- | -------------------- | --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `db diff` | `ported` | `legacy/commands/db/diff/` | `n/a` | `n/a` | Native TS port. Native pg-delta / migra diff via edge-runtime against a Go-seam-provisioned live shadow (`db __shadow`); `--use-pgadmin` / `--use-pg-schema` delegate to the Go binary. | +| `db diff` | `ported` | `legacy/commands/db/diff/` | `n/a` | `n/a` | Native TS port. Native pg-delta / migra diff via edge-runtime against a natively-provisioned live shadow (CLI-1956 removed the last Go delegation on shadow-database provisioning — the hidden `db __shadow` seam no longer exists); `--use-pgadmin` / `--use-pg-schema` still delegate to the Go binary. | | `db dump` | `ported` | `legacy/commands/db/dump/` | `n/a` | `n/a` | Native TS port. Streams `pg_dump`/`pg_dumpall` via a Docker container (`LegacyDockerRun`); schema/data/role modes, `--dry-run` script print, IPv4 transaction-pooler fallback. | | `db lint` | `ported` | `legacy/commands/db/lint/` | `n/a` | `n/a` | Native TS port. Runs `plpgsql_check` in a rolled-back transaction via LegacyDbConnection; emits Go-parity pretty JSON. | | `db pull` | `ported` | `legacy/commands/db/pull/` | `n/a` | `n/a` | Native TS port. Native pg-delta / migra migration + `--declarative` pg-delta export; reconciles `schema_migrations`. The initial-migra pull dumps the remote schema natively (`pg_dump`) then appends the migra diff. Only `--experimental` (structured dump) still delegates to Go, pending a TS PostgreSQL DDL parser for `format.WriteStructuredSchemas`. | diff --git a/apps/cli/src/legacy/commands/db/diff/SIDE_EFFECTS.md b/apps/cli/src/legacy/commands/db/diff/SIDE_EFFECTS.md index a019c95594..fed8029b06 100644 --- a/apps/cli/src/legacy/commands/db/diff/SIDE_EFFECTS.md +++ b/apps/cli/src/legacy/commands/db/diff/SIDE_EFFECTS.md @@ -7,29 +7,33 @@ the native pg-delta or migra engine (both run inside Docker via edge-runtime). T ## Files Read -| Path | Format | When | -| -------------------------------------------------- | ---------- | ----------------------------------------------------------------- | -| `/supabase/config.toml` | TOML | always (db port/password, `[experimental.pgdelta]`, deno_version) | -| `/supabase/migrations/*.sql` | SQL | shadow provisioning (applied to the shadow source) | -| `/supabase/database/**` (declarative dir) | SQL | local target when declarative schemas exist | -| `~/.supabase/access-token` | plain text | `--linked` / `--db-url` with no `SUPABASE_ACCESS_TOKEN` | -| `/supabase/.temp/project-ref` | plain text | `--linked` ref resolution | -| `/supabase/.temp/pgdelta/*.json` | JSON | explicit `--from/--to migrations` catalog (cache) | +| Path | Format | When | +| ------------------------------------------------------------------------------------------------------------------------------------ | ---------- | -------------------------------------------------------------------------------------- | +| `/supabase/config.toml` | TOML | always (db port/password, `[experimental.pgdelta]`, deno_version) | +| `/supabase/migrations/*.sql` | SQL | shadow provisioning (applied to the shadow source) | +| `[db.migrations].schema_paths` globs / `/supabase/database/**` (pg-delta declarative dir) / `/supabase/schemas/**` | SQL | local target: 3-source declarative-schema fallback ladder, first non-empty source wins | +| `~/.supabase/access-token` | plain text | `--linked` / `--db-url` with no `SUPABASE_ACCESS_TOKEN` | +| `/supabase/.temp/project-ref` | plain text | `--linked` ref resolution | +| `/supabase/.temp/pgdelta/*.json` | JSON | explicit `--from/--to migrations` catalog (cache) | ## Files Written -| Path | Format | When | -| ----------------------------------------------------------- | ------ | ----------------------------------------------- | -| `/supabase/migrations/_.sql` | SQL | `--file ` and the diff is non-empty | -| `` (from `--output` / `-o`) | SQL | explicit `--from/--to` mode with `--output` | -| `/supabase/.temp/pgdelta/*.json` | JSON | explicit `--from/--to migrations` catalog cache | -| `~/.supabase//linked-project.json` | JSON | `--linked` (post-run cache) | -| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | +| Path | Format | When | +| ----------------------------------------------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/supabase/migrations/_.sql` | SQL | `--file ` and the diff is non-empty | +| `` (from `--output` / `-o`) | SQL | explicit `--from/--to` mode with `--output` | +| `/supabase/.temp/pgdelta/*.json` | JSON | explicit `--from/--to migrations` catalog cache | +| `~/.supabase//linked-project.json` | JSON | `--linked` (post-run cache) | +| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | +| `/supabase/.temp/start-secrets/shadow-/secret-0` | binary | PG >= 15 only: the shadow container's pgsodium root key, staged as a host bind-mount source. Randomized per invocation, reclaimed (`rm -rf`) once the shadow container is torn down — see `legacyRemoveShadowDatabase`'s own doc comment. | ## Docker -- Edge-runtime container (pg-delta / migra diff scripts). -- Shadow Postgres container (provisioned + torn down via the Go `db __shadow` seam). +- Edge-runtime container (pg-delta / migra diff scripts; also the declarative + pg-delta apply script for the local-target branch). +- Shadow Postgres container — provisioned and torn down natively (`legacyPrepareShadowSource` + in `legacy/commands/db/shared/legacy-shadow-source.ts`, over the lower-level primitives in + `legacy/shared/db-bootstrap/shadow-database.ts`), no longer via a Go seam. - `supabase/migra` container — the migra OOM bash fallback only. ## API Routes (linked path, via the db-config resolver) diff --git a/apps/cli/src/legacy/commands/db/diff/diff.handler.ts b/apps/cli/src/legacy/commands/db/diff/diff.handler.ts index 61fb818dbc..56ac45521e 100644 --- a/apps/cli/src/legacy/commands/db/diff/diff.handler.ts +++ b/apps/cli/src/legacy/commands/db/diff/diff.handler.ts @@ -1,9 +1,15 @@ import { Clock, Effect, FileSystem, Option, Path } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; -import { LegacyDnsResolverFlag } from "../../../../shared/legacy/global-flags.ts"; +import { + LegacyDebugFlag, + LegacyDnsResolverFlag, + LegacyNetworkIdFlag, +} from "../../../../shared/legacy/global-flags.ts"; import { LegacyGoProxy } from "../../../../shared/legacy/go-proxy.service.ts"; import { detectGitBranch } from "../../../../shared/git/git-branch.ts"; import { Output } from "../../../../shared/output/output.service.ts"; +import { RuntimeInfo } from "../../../../shared/runtime/runtime-info.service.ts"; import { LegacyCliConfig } from "../../../config/legacy-cli-config.service.ts"; import { legacyAqua, legacyYellow } from "../../../shared/legacy-colors.ts"; import { legacyReadDbToml } from "../../../shared/legacy-db-config.toml-read.ts"; @@ -14,6 +20,8 @@ import { legacyMakeDir } from "../../../shared/legacy-make-dir.ts"; import { legacyToPostgresURL } from "../../../shared/legacy-postgres-url.ts"; import { legacySchemaToCsvField } from "../../../shared/legacy-schema-flags.ts"; import { legacyFindDropStatements } from "../../../shared/legacy-sql-split.ts"; +import { legacyBuildLocalDbContainerInputs } from "../../../shared/db-bootstrap/local-container-inputs.ts"; +import { legacyRemoveShadowDatabase } from "../../../shared/db-bootstrap/shadow-database.ts"; import { LegacyLinkedProjectCache } from "../../../telemetry/legacy-linked-project-cache.service.ts"; import { LegacyTelemetryState } from "../../../telemetry/legacy-telemetry-state.service.ts"; import { @@ -29,6 +37,10 @@ import { legacyDiffMigra } from "../shared/legacy-migra.ts"; import { legacyWritePgDeltaMigrations } from "../shared/legacy-pgdelta-migrations.write.ts"; import { type LegacyPgDeltaContext, legacyDiffPgDelta } from "../shared/legacy-pgdelta.ts"; import { LegacyDeclarativeSeam } from "../shared/legacy-pgdelta.seam.service.ts"; +import { + legacyPrepareShadowSource, + legacyShadowRunInputFromLocalContainerInputs, +} from "../shared/legacy-shadow-source.ts"; import type { LegacyDbDiffFlags } from "./diff.command.ts"; import { legacyClassifyExplicitRef, legacyUnknownTargetMessage } from "./diff.explicit.ts"; import { @@ -92,6 +104,7 @@ export const legacyDbDiff = Effect.fn("legacy.db.diff")(function* (flags: Legacy const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const dnsResolver = yield* LegacyDnsResolverFlag; + const debug = yield* LegacyDebugFlag; // Resolved linked ref, captured so the post-run finalizer caches the project // (GET /v1/projects/{ref}) — Go's `ensureProjectGroupsCached` (cmd/root.go:214). @@ -378,15 +391,36 @@ export const legacyDbDiff = Effect.fn("legacy.db.diff")(function* (flags: Legacy }); yield* output.raw("Creating shadow database...\n", "stderr"); - const shadow = yield* seam.provisionShadow({ - mode: "diff", + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const runtimeInfo = yield* RuntimeInfo; + const networkIdFlag = yield* LegacyNetworkIdFlag; + const localInputs = yield* legacyBuildLocalDbContainerInputs( + spawner, + cliConfig.workdir, + networkIdFlag, + runtimeInfo.platform, + debug, + // So the shadow's own container spec (image/JWT secret/root key/db.settings/service + // enabled-for-setup flags) reflects the matching `[remotes.]` override too, same + // as `cfg` above (`legacyReadDbToml(..., linkedRef)`) — Go remote-merges the WHOLE + // config uniformly on the linked path (`LoadConfig` seeds `flags.ProjectRef` before + // every field read). + connType === "linked" ? linkedRef : undefined, + ); + const resolvedShadowImage = yield* localInputs.resolvePostgresImage; + const shadow = yield* legacyPrepareShadowSource(spawner, { + ...legacyShadowRunInputFromLocalContainerInputs( + localInputs, + resolvedShadowImage, + cfg, + fs, + path, + ), targetLocal: resolved.isLocal, usePgDelta: useDelta, - schema: flags.schema, - // Linked path only: the shadow merges the same `[remotes.]` override - // the engine/format read above (Go builds the shadow from the remote-merged - // config). Default `db diff` is local, which never merges a remote block. - projectRef: connType === "linked" ? linkedRef : undefined, + schemaPaths: localInputs.context.config.db.migrations.schema_paths, + pgDelta: cfg.pgDelta, + ctx, }); const diffResult = yield* Effect.gen(function* () { @@ -418,7 +452,15 @@ export const legacyDbDiff = Effect.fn("legacy.db.diff")(function* (flags: Legacy // The migra engine has no execution-aware plan units, so it always writes a // single migration file (Go's `SaveDiff` single-file path). return { sql, files: undefined }; - }).pipe(Effect.ensuring(seam.removeShadowContainer(shadow.container))); + }).pipe( + Effect.ensuring( + legacyRemoveShadowDatabase(spawner, { + containerId: shadow.container, + secretDirId: shadow.secretDirId, + workdir: cliConfig.workdir, + }), + ), + ); const out = diffResult.sql; // Detect the branch from the resolved workdir, not the caller's CWD: Go diff --git a/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts b/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts index f1a531c7a9..52dc0d0a40 100644 --- a/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/diff/diff.integration.test.ts @@ -3,24 +3,35 @@ import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit, Layer, Option } from "effect"; +import * as HttpClient from "effect/unstable/http/HttpClient"; +import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; import { stripAnsi } from "../../../../../tests/helpers/ansi.ts"; import { + LEGACY_FAKE_SHADOW_CONTAINER_ID, legacyFailWriteStringOnNthCallFsLayer, mockLegacyCliConfig, mockLegacyLinkedProjectCacheTracked, + mockLegacyShadowContainerCliSpawner, mockLegacyTelemetryStateTracked, useLegacyTempWorkdir, } from "../../../../../tests/helpers/legacy-mocks.ts"; import { mockOutput, mockRuntimeInfo } from "../../../../../tests/helpers/mocks.ts"; +import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; import { + LegacyDebugFlag, LegacyDnsResolverFlag, + LegacyExperimentalFlag, LegacyNetworkIdFlag, } from "../../../../shared/legacy/global-flags.ts"; import { LegacyGoProxy } from "../../../../shared/legacy/go-proxy.service.ts"; import type { OutputFormat } from "../../../../shared/output/types.ts"; import { LegacyDbConfigResolver } from "../../../shared/legacy-db-config.service.ts"; -import { LegacyDbConnection } from "../../../shared/legacy-db-connection.service.ts"; +import { + LegacyDbConnection, + type LegacyDbSession, + type LegacyPgConnInput, +} from "../../../shared/legacy-db-connection.service.ts"; import { LegacyDockerRun } from "../../../shared/legacy-docker-run.service.ts"; import { LegacyEdgeRuntimeScriptError } from "../../../shared/legacy-edge-runtime-script.errors.ts"; import { @@ -40,26 +51,52 @@ interface SetupOpts { // When set, the pg-delta edge mock emits a multi-unit plan envelope (one file // per entry) instead of the single-unit wrap of `diffSql`. readonly diffFiles?: ReadonlyArray<{ readonly name: string; readonly sql: string }>; - readonly targetOverride?: string; readonly oom?: boolean; // edge-runtime OOMs; the bash fallback returns `diffSql` readonly delegateStdout?: string; // stdout returned by a captured Go-delegate run + // When set, the shadow's own PG15+ one-shot platform-baseline job(s) exit + // non-zero, exercising cleanup-on-partial-failure (the shadow is still removed). + readonly failShadowSetupJob?: boolean; readonly networkId?: string; // --network-id value forwarded to docker runs // When set, the Nth `writeFileString` fails, exercising cleanup-on-failure. readonly failWriteOnCall?: number; } +const alwaysReadyHttpClientLayer = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, new Response(null, { status: 200 }))), + ), +); + +/** Records every `LegacyDbConnection.connect` target's database name, and every `exec`/`query` SQL run against it. */ +function fakeShadowDbConnection() { + const connectedDatabases: Array = []; + const execCalls: Array = []; + const layer = Layer.succeed(LegacyDbConnection, { + connect: (cfg: LegacyPgConnInput) => + Effect.sync(() => { + connectedDatabases.push(cfg.database); + const session: LegacyDbSession = { + exec: (sql) => + Effect.sync(() => { + execCalls.push(sql); + }), + query: () => Effect.succeed([]), + extensionExists: () => Effect.succeed(false), + copyToCsv: () => Effect.succeed(new Uint8Array()), + queryRaw: () => Effect.succeed({ fields: [], rows: [], commandTag: "" }), + }; + return session; + }), + }); + return { layer, connectedDatabases, execCalls }; +} + function setup(workdir: string, opts: SetupOpts = {}) { const out = mockOutput({ format: opts.format ?? "text" }); const telemetry = mockLegacyTelemetryStateTracked(); const cache = mockLegacyLinkedProjectCacheTracked(); - const provisionCalls: Array<{ - mode: string; - targetLocal: boolean; - usePgDelta: boolean; - projectRef?: string; - }> = []; - const removedContainers: string[] = []; const exportCalls: string[] = []; const exportCatalogCalls: Array<{ mode: string; projectRef?: string }> = []; const seam = Layer.succeed(LegacyDeclarativeSeam, { @@ -71,20 +108,14 @@ function setup(workdir: string, opts: SetupOpts = {}) { execInherit: () => Effect.succeed(0), ensureLocalDatabaseStarted: () => Effect.void, ensureLocalPostgresImageCurrent: () => Effect.void, - provisionShadow: ({ mode, targetLocal, usePgDelta, projectRef }) => { - provisionCalls.push({ mode, targetLocal, usePgDelta, projectRef }); - return Effect.succeed({ - container: "shadow-1", - sourceUrl: "postgres://postgres:postgres@127.0.0.1:54320/postgres", - targetUrlOverride: opts.targetOverride, - }); - }, - removeShadowContainer: (container) => - Effect.sync(() => { - removedContainers.push(container); - }), }); + // Shadow provisioning is native (CLI-1956): a real docker-spawner fake backs + // container create/start/health-inspect/cleanup, and a real (fake) Postgres + // session backs the shadow's own platform-baseline/migration/declarative setup. + const shadowSpawner = mockLegacyShadowContainerCliSpawner(); + const shadowDbConnection = fakeShadowDbConnection(); + const edgeCalls: LegacyEdgeRuntimeRunOpts[] = []; const edge = Layer.succeed(LegacyEdgeRuntimeScript, { run: (runOpts: LegacyEdgeRuntimeRunOpts) => { @@ -119,8 +150,14 @@ function setup(workdir: string, opts: SetupOpts = {}) { }, }); - // Exercised only by the migra OOM bash fallback. + // `dockerCalls` tracks the migra OOM bash fallback's own `runCapture` calls — the + // native shadow's PG15+ one-shot setup jobs (`legacyRunStartMigrateJob`) go through + // `runStream` instead (constant-memory stdout discard, matching Go's `io.Discard` + // writer for these jobs), so they're tracked separately in `shadowSetupJobCalls` + // (their `env`, notably `DB_HOST`, is the one shadow-specific parameterization + // CLI-1956 exists to get right). const dockerCalls: unknown[] = []; + const shadowSetupJobCalls: Array<{ readonly env: Readonly> }> = []; const docker = Layer.succeed(LegacyDockerRun, { run: () => Effect.die("run unused"), runCapture: (dockerOpts) => { @@ -131,11 +168,14 @@ function setup(workdir: string, opts: SetupOpts = {}) { stderr: "", }); }, - runStream: () => Effect.die("runStream unused"), - }); - - const dbConnection = Layer.succeed(LegacyDbConnection, { - connect: () => Effect.die("connect unused"), + // The shadow's own PG15+ one-shot platform-baseline job(s). + runStream: (dockerOpts) => { + shadowSetupJobCalls.push(dockerOpts); + return Effect.succeed({ + exitCode: opts.failShadowSetupJob === true ? 1 : 0, + stderr: "", + }); + }, }); const resolverCalls: unknown[] = []; @@ -170,13 +210,20 @@ function setup(workdir: string, opts: SetupOpts = {}) { }); const baseLayer = Layer.mergeAll( + // `BunServices.layer` is listed FIRST so every fake service layer below (most + // importantly `shadowSpawner.layer`'s fake `ChildProcessSpawner`) OVERRIDES its + // real implementation — `Layer.mergeAll` is last-wins on a shared service, + // matching `start.integration.test.ts`'s own established ordering. + BunServices.layer, out.layer, telemetry.layer, cache.layer, seam, edge, docker, - dbConnection, + shadowDbConnection.layer, + shadowSpawner.layer, + alwaysReadyHttpClientLayer, resolver, proxy, mockLegacyCliConfig({ workdir, projectId: Option.some("test") }), @@ -189,11 +236,12 @@ function setup(workdir: string, opts: SetupOpts = {}) { requireSsl: () => Effect.succeed(false), requireSslForHost: () => Effect.succeed(false), }), + Layer.succeed(LegacyExperimentalFlag, false), + Layer.succeed(LegacyDebugFlag, false), + Layer.succeed(CliArgs, { args: [] }), mockRuntimeInfo(), - BunServices.layer, ); - // Merged last so its `FileSystem` overrides `BunServices` (last-wins); `Path` - // still resolves from `BunServices`. + // Merged last so its `FileSystem` overrides everything above (last-wins). const layer = opts.failWriteOnCall === undefined ? baseLayer @@ -204,8 +252,6 @@ function setup(workdir: string, opts: SetupOpts = {}) { out, cache, telemetry, - provisionCalls, - removedContainers, exportCalls, exportCatalogCalls, edgeCalls, @@ -213,6 +259,10 @@ function setup(workdir: string, opts: SetupOpts = {}) { proxyCalls, proxyCaptureCalls, dockerCalls, + shadowSetupJobCalls, + shadowSpawned: shadowSpawner.spawned, + shadowConnectedDatabases: shadowDbConnection.connectedDatabases, + shadowExecCalls: shadowDbConnection.execCalls, }; } @@ -253,13 +303,40 @@ describe("legacy db diff", () => { const s = setup(tmp.current, { diffSql: "create table players ();\n" }); return Effect.gen(function* () { yield* legacyDbDiff(flags()); - expect(s.provisionCalls).toEqual([{ mode: "diff", targetLocal: true, usePgDelta: false }]); + // The native shadow was created once (one `docker create`) and removed once + // (one `docker rm -f -v`) — see `mockLegacyShadowContainerCliSpawner`. + expect(s.shadowSpawned.filter((c) => c.args[0] === "create")).toHaveLength(1); + expect(s.shadowSpawned.filter((c) => c.args[0] === "rm")).toHaveLength(1); expect(stdout(s.out)).toBe("create table players ();\n\n"); expect(stderr(s.out)).toContain("Creating shadow database..."); expect(stderr(s.out)).toContain("Diffing schemas..."); expect(stderr(s.out)).toContain("Finished supabase db diff on branch"); - expect(s.removedContainers).toEqual(["shadow-1"]); expect(s.telemetry.flushed).toBe(true); + // The shadow's PG15+ one-shot platform-baseline job(s) connect to the shadow over + // Docker's embedded DNS using the shadow container's OWN 12-char short id as `DB_HOST` + // (Go's `container[:12]`, `diff.go:172`) — NOT the real `db` container's name, and not + // some other slice length (a mutation from `.slice(0, 12)` to `.slice(0, 8)` must fail + // this). This is the one shadow-specific parameterization this port exists to get right + // (`legacyBuildShadowSetupDatabaseInput`'s `dbHost`). The default config enables realtime + // (and PG >= 15 by default), so this always exercises at least one one-shot job — + // Realtime's own env sets `DB_HOST` directly; Storage/Auth embed the same host inside a + // `DATABASE_URL`-style connection string instead. + const expectedHost = LEGACY_FAKE_SHADOW_CONTAINER_ID.slice(0, 12); + expect(s.shadowSetupJobCalls.length).toBeGreaterThan(0); + let sawHost = false; + for (const call of s.shadowSetupJobCalls) { + if (call.env["DB_HOST"] !== undefined) { + expect(call.env["DB_HOST"]).toBe(expectedHost); + sawHost = true; + } + for (const value of Object.values(call.env)) { + if (value.includes("@") && value.includes(":")) { + expect(value).toContain(`@${expectedHost}:`); + sawHost = true; + } + } + } + expect(sawHost).toBe(true); }).pipe(Effect.provide(s.layer)); }); @@ -267,12 +344,50 @@ describe("legacy db diff", () => { const s = setup(tmp.current, { diffSql: "create table p ();\n" }); return Effect.gen(function* () { yield* legacyDbDiff(flags({ usePgDelta: Option.some(true), schema: ["public"] })); - expect(s.provisionCalls).toEqual([{ mode: "diff", targetLocal: true, usePgDelta: true }]); + // pg-delta selection is observable via the edge-runtime script it runs. + expect(s.edgeCalls[0]?.script).toContain("renderPlanFiles"); expect(stderr(s.out)).toContain("Diffing schemas: public"); expect(stdout(s.out)).toBe("create table p ();\n\n"); }).pipe(Effect.provide(s.layer)); }); + it.effect("PG14: provisions a shadow via the SQL-exec init path (no PG15+ one-shot jobs)", () => { + // Go's own shadow test coverage hardcodes PG14 (`diff_test.go`); the PG15+ short-id + // DNS resolution path was verified separately (empirical Docker probe, see the + // task's own header) — this covers the OTHER major-version branch of the SAME + // `legacySetupDatabase` pipeline, which execs SQL directly via the session + // instead of the three one-shot `LegacyDockerRun` jobs. + mkdirSync(join(tmp.current, "supabase"), { recursive: true }); + writeFileSync(join(tmp.current, "supabase", "config.toml"), "[db]\nmajor_version = 14\n"); + const s = setup(tmp.current, { diffSql: "create table pg14 ();\n" }); + return Effect.gen(function* () { + yield* legacyDbDiff(flags()); + expect(stdout(s.out)).toBe("create table pg14 ();\n\n"); + expect(s.shadowSpawned.filter((c) => c.args[0] === "create")).toHaveLength(1); + expect(s.shadowSpawned.filter((c) => c.args[0] === "rm")).toHaveLength(1); + // PG14's `legacyStartInitSchemaPre15` execs SQL over the session directly — + // no one-shot `LegacyDockerRun` jobs (Go's `initSchema15` never runs). + expect(s.dockerCalls).toEqual([]); + expect(s.shadowExecCalls.length).toBeGreaterThan(0); + }).pipe(Effect.provide(s.layer)); + }); + + it.effect( + "removes the shadow even when its own platform-baseline setup fails midway (ok-sentinel cleanup)", + () => { + // Mirrors Go's `ok`-sentinel + `defer` pattern (`shadow.go:42-47`): once the + // shadow container is created, ANY later failure (here, a PG15+ one-shot + // platform-baseline job exiting non-zero) still removes it. + const s = setup(tmp.current, { diffSql: "create table x ();\n", failShadowSetupJob: true }); + return Effect.gen(function* () { + const exit = yield* legacyDbDiff(flags()).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(s.shadowSpawned.filter((c) => c.args[0] === "create")).toHaveLength(1); + expect(s.shadowSpawned.filter((c) => c.args[0] === "rm")).toHaveLength(1); + }).pipe(Effect.provide(s.layer)); + }, + ); + it.effect("a linked [remotes.] block enabling pg-delta selects the pg-delta engine", () => { // Go loads the project ref before LoadConfig on the linked path, merging the // matching [remotes.] block before experimental.pgdelta.enabled is read @@ -301,13 +416,54 @@ describe("legacy db diff", () => { }); return Effect.gen(function* () { yield* legacyDbDiff(flags({ linked: Option.some(true) })); - expect(s.provisionCalls[0]?.usePgDelta).toBe(true); - // The shadow is provisioned with the resolved ref so the `db __shadow` child - // merges the same `[remotes.]` override into the shadow baseline. - expect(s.provisionCalls[0]?.projectRef).toBe("abcdefghijklmnopqrst"); + // pg-delta selection (ref-aware: read from the remote-merged `cfg.pgDelta`) is + // observable via the edge-runtime script the diff runs. + expect(s.edgeCalls[0]?.script).toContain("renderPlanFiles"); }).pipe(Effect.provide(s.layer)); }); + it.effect( + "a linked [remotes.] db.major_version override reaches the shadow's OWN container spec, not just cfg", + () => { + // Go remote-merges the WHOLE config uniformly on the linked path (`LoadConfig` seeds + // `flags.ProjectRef` before every field read) — the shadow's container spec (image, + // JWT secret, root key, db.settings, service enabled-for-setup flags) must reflect the + // matched `[remotes.]` override too, not just the `cfg`/`toml` read used for + // pg-delta/schema_paths. `major_version` is a clean, directly-observable probe: PG <= 14 + // is the ONLY branch that emits a `--tmpfs` flag on the shadow's `docker create` argv + // (`legacyBuildShadowPostgresContainerSpec`) — a base config of 17 (>= 15, no tmpfs) + // overridden by a remote block's `major_version = 14` must flip that flag on. + mkdirSync(join(tmp.current, "supabase"), { recursive: true }); + writeFileSync( + join(tmp.current, "supabase", "config.toml"), + [ + "[db]", + "major_version = 17", + "", + "[remotes.staging]", + 'project_id = "abcdefghijklmnopqrst"', + "", + "[remotes.staging.db]", + "major_version = 14", + "", + ].join("\n"), + ); + const s = setup(tmp.current, { + isLocal: false, + linkedRef: "abcdefghijklmnopqrst", + diffSql: "alter table x;\n", + }); + return Effect.gen(function* () { + yield* legacyDbDiff(flags({ linked: Option.some(true) })); + const createArgs = s.shadowSpawned.find((c) => c.args[0] === "create")?.args ?? []; + expect(createArgs).toContain("--tmpfs"); + // The PG15+ one-shot platform-baseline jobs (`initSchema15`) never run for PG14 — + // it execs SQL directly over the session instead — corroborating the same override. + expect(s.dockerCalls).toEqual([]); + }).pipe(Effect.provide(s.layer)); + }, + ); + it.effect("the base config (default local target) does not merge a remote block", () => { // The default db diff target is local; Go never calls LoadProjectRef for local, // so a [remotes.] override must be ignored and the base engine (migra) wins. @@ -329,9 +485,8 @@ describe("legacy db diff", () => { const s = setup(tmp.current, { diffSql: "create table players ();\n" }); return Effect.gen(function* () { yield* legacyDbDiff(flags()); - expect(s.provisionCalls[0]?.usePgDelta).toBe(false); - // The local default never passes a ref, so the shadow uses base config. - expect(s.provisionCalls[0]?.projectRef).toBeUndefined(); + // The local default never merges a remote block, so the base (migra) engine wins. + expect(s.edgeCalls[0]?.script).not.toContain("renderPlanFiles"); }).pipe(Effect.provide(s.layer)); }); @@ -343,22 +498,28 @@ describe("legacy db diff", () => { }); return Effect.gen(function* () { yield* legacyDbDiff(flags({ linked: Option.some(true) })); - expect(s.provisionCalls[0]?.targetLocal).toBe(false); expect(s.cache.cached).toBe(true); }).pipe(Effect.provide(s.layer)); }); - it.effect("uses the seam's target override for the local declarative branch", () => { - const s = setup(tmp.current, { - targetOverride: "postgres://postgres:postgres@127.0.0.1:54320/contrib_regression", - diffSql: "create table o ();\n", - }); - return Effect.gen(function* () { - yield* legacyDbDiff(flags()); - expect(stdout(s.out)).toBe("create table o ();\n\n"); - expect(s.removedContainers).toEqual(["shadow-1"]); - }).pipe(Effect.provide(s.layer)); - }); + it.effect( + "provisions a local-target declarative shadow and diffs against the override database", + () => { + // A declarative schema file under supabase/schemas makes `loadDeclaredSchemas` + // non-empty, so the native `--target-local` branch redirects the diff target to + // a second (contrib_regression) database on the SAME shadow container. + mkdirSync(join(tmp.current, "supabase", "schemas"), { recursive: true }); + writeFileSync(join(tmp.current, "supabase", "schemas", "public.sql"), "select 1;\n"); + const s = setup(tmp.current, { diffSql: "create table o ();\n" }); + return Effect.gen(function* () { + yield* legacyDbDiff(flags()); + expect(stdout(s.out)).toBe("create table o ();\n\n"); + // The declarative-schema file was migrated into the contrib_regression override. + expect(s.shadowConnectedDatabases).toContain("contrib_regression"); + expect(s.shadowSpawned.filter((c) => c.args[0] === "rm")).toHaveLength(1); + }).pipe(Effect.provide(s.layer)); + }, + ); it.effect("delegates --use-pgadmin to the Go binary (telemetry disabled on the child)", () => { const s = setup(tmp.current); @@ -367,7 +528,8 @@ describe("legacy db diff", () => { expect(s.proxyCalls).toHaveLength(1); expect(s.proxyCalls[0]?.args).toEqual(["db", "diff", "--use-pgadmin"]); expect(s.proxyCalls[0]?.env).toEqual({ SUPABASE_TELEMETRY_DISABLED: "1" }); - expect(s.provisionCalls).toEqual([]); + // The pgadmin/pg-schema delegate short-circuits before ever creating a shadow. + expect(s.shadowSpawned.filter((c) => c.args[0] === "create")).toEqual([]); }).pipe(Effect.provide(s.layer)); }); @@ -589,7 +751,7 @@ describe("legacy db diff", () => { return Effect.gen(function* () { yield* legacyDbDiff(flags({ from: Option.some("local"), to: Option.some("linked") })); // Explicit mode is pg-delta and never provisions a shadow. - expect(s.provisionCalls).toEqual([]); + expect(s.shadowSpawned.filter((c) => c.args[0] === "create")).toEqual([]); expect(stdout(s.out)).toBe("create table e ();\n"); }).pipe(Effect.provide(s.layer)); }); @@ -760,7 +922,7 @@ describe("legacy db diff", () => { return Effect.gen(function* () { yield* legacyDbDiff(flags({ from: Option.some(""), to: Option.some("") })); // Reaching the native path proves it didn't enter explicit mode and error. - expect(s.provisionCalls).toHaveLength(1); + expect(s.shadowSpawned.filter((c) => c.args[0] === "create")).toHaveLength(1); expect(stdout(s.out)).toBe("create table e ();\n\n"); }).pipe(Effect.provide(s.layer)); }); diff --git a/apps/cli/src/legacy/commands/db/diff/diff.layers.ts b/apps/cli/src/legacy/commands/db/diff/diff.layers.ts index 8c2ab09380..b9a8f8a771 100644 --- a/apps/cli/src/legacy/commands/db/diff/diff.layers.ts +++ b/apps/cli/src/legacy/commands/db/diff/diff.layers.ts @@ -1,6 +1,7 @@ import { Layer } from "effect"; import { commandRuntimeLayer } from "../../../../shared/runtime/command-runtime.layer.ts"; +import { legacyHttpClientLayer } from "../../../auth/legacy-http-debug.layer.ts"; import { legacyCliConfigLayer } from "../../../config/legacy-cli-config.layer.ts"; import { legacyDbConfigLayer } from "../../../shared/legacy-db-config.layer.ts"; import { legacyDbConnectionLayer } from "../../../shared/legacy-db-connection.layer.ts"; @@ -18,9 +19,12 @@ import { legacyDeclarativeSeamLayer } from "../shared/legacy-pgdelta.seam.layer. * * Mirrors `db schema declarative generate` (`generate.layers.ts`): the db-config * resolver plus the native pg-delta / migra stack — the edge-runtime runner, the - * SSL probe, and the Go shadow-database seam (`provisionShadow`). `LegacyDockerRun` - * is exposed in the merge (not just provided to the edge-runtime layer) because the - * migra OOM bash fallback runs the `supabase/migra` container directly. + * SSL probe, `HttpClient` (the native shadow's health-check wait), and the Go + * seam (`exportCatalog`, still used by the explicit `--from migrations`/`--to + * migrations` path; shadow provisioning itself is native — see + * `commands/db/shared/legacy-shadow-source.ts`). `LegacyDockerRun` is exposed in + * the merge (not just provided to the edge-runtime layer) because the migra OOM + * bash fallback runs the `supabase/migra` container directly. * Per the "provide doesn't share to siblings" rule, `LegacyCliConfig` is provided * to every layer that needs it. */ @@ -43,6 +47,8 @@ const edgeRuntime = legacyEdgeRuntimeScriptLayer.pipe( const seam = legacyDeclarativeSeamLayer.pipe(Layer.provide(cliConfig)); +const httpClient = legacyHttpClientLayer.pipe(Layer.provide(legacyDebugLoggerLayer)); + export const legacyDbDiffRuntimeLayer = Layer.mergeAll( dbConfig, legacyDbConnectionLayer, @@ -50,6 +56,7 @@ export const legacyDbDiffRuntimeLayer = Layer.mergeAll( edgeRuntime, legacyPgDeltaSslProbeLayer, seam, + httpClient, cliConfig, legacyIdentityStitchLayer, legacyTelemetryStateLayer, diff --git a/apps/cli/src/legacy/commands/db/pull/SIDE_EFFECTS.md b/apps/cli/src/legacy/commands/db/pull/SIDE_EFFECTS.md index 6ca74fd5f5..267ebf57ae 100644 --- a/apps/cli/src/legacy/commands/db/pull/SIDE_EFFECTS.md +++ b/apps/cli/src/legacy/commands/db/pull/SIDE_EFFECTS.md @@ -11,26 +11,31 @@ binary (it needs `format.WriteStructuredSchemas`, which has no TS port yet). ## Files Read -| Path | Format | When | -| -------------------------------------- | ---------- | --------------------------------------------------- | -| `/supabase/config.toml` | TOML | always (db port/password, `[experimental.pgdelta]`) | -| `/supabase/migrations/*.sql` | SQL | history reconciliation + shadow provisioning | -| `~/.supabase/access-token` | plain text | linked target with no `SUPABASE_ACCESS_TOKEN` | -| `/supabase/.temp/project-ref` | plain text | linked ref resolution | +| Path | Format | When | +| ------------------------------------------------------------------------------------------------------------------------------------ | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `/supabase/config.toml` | TOML | always (db port/password, `[experimental.pgdelta]`) | +| `/supabase/migrations/*.sql` | SQL | history reconciliation + shadow provisioning | +| `~/.supabase/access-token` | plain text | linked target with no `SUPABASE_ACCESS_TOKEN` | +| `/supabase/.temp/project-ref` | plain text | linked ref resolution | +| `[db.migrations].schema_paths` globs / `/supabase/database/**` (pg-delta declarative dir) / `/supabase/schemas/**` | SQL | migration-style pull against the local target only: 3-source declarative-schema fallback ladder, first non-empty source wins (same as `db diff`) | ## Files Written -| Path | Format | When | -| ----------------------------------------------------------- | ------ | -------------------------------------------------------------------------- | -| `/supabase/migrations/_.sql` | SQL | migration-style pull (non-empty diff, or the initial-migra `pg_dump` seed) | -| `/supabase/database/**` | SQL | `--declarative` | -| `~/.supabase//linked-project.json` | JSON | linked (post-run cache) | -| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | +| Path | Format | When | +| ----------------------------------------------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/supabase/migrations/_.sql` | SQL | migration-style pull (non-empty diff, or the initial-migra `pg_dump` seed) | +| `/supabase/database/**` | SQL | `--declarative` | +| `~/.supabase//linked-project.json` | JSON | linked (post-run cache) | +| `~/.supabase/telemetry.json` | JSON | every invocation (post-run) | +| `/supabase/.temp/start-secrets/shadow-/secret-0` | binary | PG >= 15 only: the shadow container's pgsodium root key, staged as a host bind-mount source. Randomized per invocation, reclaimed (`rm -rf`) once the shadow container is torn down — see `legacyRemoveShadowDatabase`'s own doc comment. | ## Docker - Edge-runtime container (pg-delta export / pg-delta or migra diff). -- Shadow Postgres container (provisioned + torn down via the Go `db __shadow` seam). +- Shadow Postgres container — provisioned and torn down natively (`legacyPrepareShadowSource` in + `legacy/commands/db/shared/legacy-shadow-source.ts` / `legacyPrepareRawShadow` in + `legacy/shared/db-bootstrap/shadow-database.ts`, which also owns the lower-level primitives + both build on), no longer via a Go seam. - `supabase/migra` container — the migra OOM bash fallback only. - `pg_dump` container — the initial-migra pull's native remote-schema dump (`legacyStreamPgDump`, shared with `db dump`). diff --git a/apps/cli/src/legacy/commands/db/pull/pull.handler.ts b/apps/cli/src/legacy/commands/db/pull/pull.handler.ts index fdb2452543..9644f4b480 100644 --- a/apps/cli/src/legacy/commands/db/pull/pull.handler.ts +++ b/apps/cli/src/legacy/commands/db/pull/pull.handler.ts @@ -1,13 +1,17 @@ import { Clock, Effect, FileSystem, Option, Path } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; import { + LegacyDebugFlag, LegacyDnsResolverFlag, LegacyExperimentalFlag, + LegacyNetworkIdFlag, legacyResolveYesWithProjectEnv, } from "../../../../shared/legacy/global-flags.ts"; import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; import { LegacyGoProxy } from "../../../../shared/legacy/go-proxy.service.ts"; import { Output } from "../../../../shared/output/output.service.ts"; +import { RuntimeInfo } from "../../../../shared/runtime/runtime-info.service.ts"; import { LegacyCliConfig } from "../../../config/legacy-cli-config.service.ts"; import { legacyAqua, legacyBold } from "../../../shared/legacy-colors.ts"; import { legacyPromptYesNo } from "../../../../shared/legacy/legacy-prompt-yes-no.ts"; @@ -31,6 +35,11 @@ import type { LegacyDbConnType } from "../../../shared/legacy-db-target-flags.ts import { legacyMakeDir } from "../../../shared/legacy-make-dir.ts"; import { legacyToPostgresURL } from "../../../shared/legacy-postgres-url.ts"; import { legacySchemaToCsvField } from "../../../shared/legacy-schema-flags.ts"; +import { legacyBuildLocalDbContainerInputs } from "../../../shared/db-bootstrap/local-container-inputs.ts"; +import { + legacyPrepareRawShadow, + legacyRemoveShadowDatabase, +} from "../../../shared/db-bootstrap/shadow-database.ts"; import { LegacyLinkedProjectCache } from "../../../telemetry/legacy-linked-project-cache.service.ts"; import { LegacyTelemetryState } from "../../../telemetry/legacy-telemetry-state.service.ts"; import { @@ -66,7 +75,10 @@ import { legacyIsPgDeltaDebugEnabled, } from "../shared/legacy-pgdelta.ts"; import { legacySaveEmptyPgDeltaPullDebug } from "./pull.debug.ts"; -import { LegacyDeclarativeSeam } from "../shared/legacy-pgdelta.seam.service.ts"; +import { + legacyPrepareShadowSource, + legacyShadowRunInputFromLocalContainerInputs, +} from "../shared/legacy-shadow-source.ts"; import type { LegacyDbPullFlags } from "./pull.command.ts"; import { LegacyDbPullDumpError, @@ -133,7 +145,6 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy const output = yield* Output; const resolver = yield* LegacyDbConfigResolver; const connection = yield* LegacyDbConnection; - const seam = yield* LegacyDeclarativeSeam; const proxy = yield* LegacyGoProxy; const cliConfig = yield* LegacyCliConfig; const telemetryState = yield* LegacyTelemetryState; @@ -142,6 +153,7 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const dnsResolver = yield* LegacyDnsResolverFlag; + const debug = yield* LegacyDebugFlag; const cliArgs = yield* CliArgs; // `--yes` OR `SUPABASE_YES` (Go's `viper.GetBool("YES")`, root.go:318-320). Go @@ -361,15 +373,34 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy yield* output.raw("Preparing declarative schema export using pg-delta...\n", "stderr"); const declarativeDirRel = legacyResolveDeclarativeDir(path, toml.pgDelta); const declarativeDir = path.resolve(cliConfig.workdir, declarativeDirRel); - const shadow = yield* seam.provisionShadow({ - mode: "declarative", - targetLocal: false, - usePgDelta: true, - schema: flags.schema, - // Linked path only: merge the same `[remotes.]` override into the - // shadow baseline (Go builds the shadow from the remote-merged config). - projectRef: connType === "linked" ? linkedRef : undefined, - }); + const declSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const declRuntimeInfo = yield* RuntimeInfo; + const declNetworkIdFlag = yield* LegacyNetworkIdFlag; + const declLocalInputs = yield* legacyBuildLocalDbContainerInputs( + declSpawner, + cliConfig.workdir, + declNetworkIdFlag, + declRuntimeInfo.platform, + debug, + // So the shadow's own container spec reflects the matching `[remotes.]` + // override, same as `toml` above — see `diff.handler.ts`'s identical call site. + connType === "linked" ? linkedRef : undefined, + ); + const resolvedDeclShadowImage = yield* declLocalInputs.resolvePostgresImage; + // `legacyPrepareRawShadow` needs none of the `setup`/declarative-branch fields the + // adapter also returns (a bare shadow never runs `MigrateShadowDatabase`) — its own + // input type (`LegacyShadowConnectionInput`) is structurally narrower, so the extra + // fields are simply never read. + const shadow = yield* legacyPrepareRawShadow( + declSpawner, + legacyShadowRunInputFromLocalContainerInputs( + declLocalInputs, + resolvedDeclShadowImage, + toml, + fs, + path, + ), + ); const exported = yield* withPoolerFallback(targetUrl, (targetRef) => legacyDeclarativeExportPgDelta(ctx, { sourceRef: shadow.sourceUrl, @@ -377,7 +408,15 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy schema: flags.schema, formatOptions, }), - ).pipe(Effect.ensuring(seam.removeShadowContainer(shadow.container))); + ).pipe( + Effect.ensuring( + legacyRemoveShadowDatabase(declSpawner, { + containerId: shadow.container, + secretDirId: shadow.secretDirId, + workdir: cliConfig.workdir, + }), + ), + ); yield* legacyWriteDeclarativeSchemas(fs, path, declarativeDir, exported).pipe( Effect.mapError((cause) => new LegacyDbPullWriteError({ message: cause.message })), ); @@ -570,38 +609,74 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy // For the initial pull (no local migrations) the schema filter is ignored, // matching Go's `diffRemoteSchema(ctx, nil, …)`. const diffSchema = sync.kind === "missing" ? [] : flags.schema; - // Go's `DiffDatabase` emits these to stderr before provisioning + diffing - // (`internal/db/diff/diff.go:189,234-237`); the shadow seam doesn't, so the - // pull handler emits them itself to match the migration-style `db pull` output. - yield* output.raw("Creating shadow database...\n", "stderr"); - const shadow = yield* seam.provisionShadow({ - mode: "diff", - // Mirror Go's `DiffDatabase` → `PrepareShadowSource(ctx, schema, - // utils.IsLocalDatabase(config), …)` (`internal/db/diff/diff.go:190`): - // a local target with declarative schema files gets a second - // `contrib_regression` shadow returned as the target override. - targetLocal: resolved.isLocal, - usePgDelta: usePgDeltaDiff, - schema: diffSchema, - // Linked path only: merge the same `[remotes.]` override into the - // shadow baseline (Go builds the shadow from the remote-merged config). - projectRef: connType === "linked" ? linkedRef : undefined, - }); - const diffOutcome = yield* Effect.gen(function* () { - // Use the declarative target override when present (Go substitutes it - // for the diff target, `diff.go:196-197`); for remote pulls it's - // undefined, so this is the direct target URL as before. - const target = shadow.targetUrlOverride ?? targetUrl; - yield* output.raw( - diffSchema.length > 0 - ? `Diffing schemas: ${diffSchema.join(",")}\n` - : "Diffing schemas...\n", - "stderr", - ); - return yield* withPoolerFallback(target, (targetRef) => - // Wrap the engine choice in a gen so both branches' error/requirement - // channels unify into one `Effect` the helper can retry generically. - Effect.gen(function* () { + const pullSpawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const pullRuntimeInfo = yield* RuntimeInfo; + const pullNetworkIdFlag = yield* LegacyNetworkIdFlag; + const pullLocalInputs = yield* legacyBuildLocalDbContainerInputs( + pullSpawner, + cliConfig.workdir, + pullNetworkIdFlag, + pullRuntimeInfo.platform, + debug, + // So the shadow's own container spec reflects the matching `[remotes.]` + // override, same as `toml` above — see `diff.handler.ts`'s identical call site. + connType === "linked" ? linkedRef : undefined, + ); + // Go's `diffRemoteSchema` retries the ENTIRE `diff.DiffDatabase` call — shadow + // provisioning included — against the pooler config on an IPv6 failure, not + // just the diff step (`internal/db/pull/pull.go:176-190`): `DiffDatabase` + // prints "Creating shadow database..." and runs `PrepareShadowSource` before + // ever touching the remote/target connection (`internal/db/diff/diff.go:211- + // 217`), so a pooler retry re-prints the creation/diff banners and provisions + // + tears down a second, fresh shadow. Mirror that observable behavior by + // wrapping the full prepare-shadow-then-diff operation in the retried + // closure — each attempt gets its own shadow and its own teardown — instead + // of provisioning one shadow and only retrying the diff engine against it. + const runShadowDiff = (targetRef: string) => + Effect.gen(function* () { + // Go's `DiffDatabase` emits these to stderr before provisioning + diffing + // (`internal/db/diff/diff.go:212,223-226`); `legacyPrepareShadowSource` + // doesn't print its own banner, so the pull handler emits it itself to + // match the migration-style `db pull` output. + yield* output.raw("Creating shadow database...\n", "stderr"); + // Resolved AFTER the banner, inside the retried closure — Go's + // `CreateShadowDatabase` → `utils.DockerStart` (where the postgres image is + // resolved/pulled) runs inside `PrepareShadowSource`, which is itself called + // after `DiffDatabase` prints "Creating shadow database..." above, and is + // re-run fresh on every pooler-retry attempt (see the comment above). Resolving + // it earlier, outside this closure (as `diff.handler.ts`'s sibling call site does + // NOT do — it also resolves after its own banner), would both print nothing on an + // image-resolution failure before the banner and skip re-resolving it on retry. + const resolvedPullShadowImage = yield* pullLocalInputs.resolvePostgresImage; + // Mirror Go's `DiffDatabase` → `PrepareShadowSource(ctx, schema, + // utils.IsLocalDatabase(config), …)` (`internal/db/diff/diff.go:213`): a + // local target with declarative schema files gets a second + // `contrib_regression` shadow returned as the target override. + const shadow = yield* legacyPrepareShadowSource(pullSpawner, { + ...legacyShadowRunInputFromLocalContainerInputs( + pullLocalInputs, + resolvedPullShadowImage, + toml, + fs, + path, + ), + targetLocal: resolved.isLocal, + usePgDelta: usePgDeltaDiff, + schemaPaths: pullLocalInputs.context.config.db.migrations.schema_paths, + pgDelta: toml.pgDelta, + ctx, + }); + return yield* Effect.gen(function* () { + // Use the declarative target override when present (Go substitutes it + // for the diff target, `diff.go:196-197`); for remote pulls it's + // undefined, so this is this attempt's resolved target URL. + const target = shadow.targetUrlOverride ?? targetRef; + yield* output.raw( + diffSchema.length > 0 + ? `Diffing schemas: ${diffSchema.join(",")}\n` + : "Diffing schemas...\n", + "stderr", + ); if (usePgDeltaDiff) { // With PGDELTA_DEBUG set, capture the shadow baseline catalog so an // empty diff can be inspected later (Go's DiffDatabase, @@ -624,7 +699,7 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy : undefined; const result = yield* legacyDiffPgDelta(ctx, { sourceRef: shadow.sourceUrl, - targetRef, + targetRef: target, schema: diffSchema, formatOptions, }); @@ -636,14 +711,22 @@ export const legacyDbPull = Effect.fn("legacy.db.pull")(function* (flags: Legacy } const sql = yield* legacyDiffMigra(ctx, { source: shadow.sourceUrl, - target: targetRef, + target, schema: diffSchema, connectOptions: { isLocal: resolved.isLocal, dnsResolver }, }); return { sql, files: undefined, capture: undefined }; - }), - ); - }).pipe(Effect.ensuring(seam.removeShadowContainer(shadow.container))); + }).pipe( + Effect.ensuring( + legacyRemoveShadowDatabase(pullSpawner, { + containerId: shadow.container, + secretDirId: shadow.secretDirId, + workdir: cliConfig.workdir, + }), + ), + ); + }); + const diffOutcome = yield* withPoolerFallback(targetUrl, runShadowDiff); const out = diffOutcome.sql; const diffEmpty = out.trim().length === 0; diff --git a/apps/cli/src/legacy/commands/db/pull/pull.integration.test.ts b/apps/cli/src/legacy/commands/db/pull/pull.integration.test.ts index aaac55ec20..c2a4b96b77 100644 --- a/apps/cli/src/legacy/commands/db/pull/pull.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/pull/pull.integration.test.ts @@ -3,12 +3,15 @@ import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Exit, Layer, Option } from "effect"; +import * as HttpClient from "effect/unstable/http/HttpClient"; +import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; import { stripAnsi } from "../../../../../tests/helpers/ansi.ts"; import { legacyFailWriteStringOnNthCallFsLayer, mockLegacyCliConfig, mockLegacyLinkedProjectCacheTracked, + mockLegacyShadowContainerCliSpawner, mockLegacyTelemetryStateTracked, useLegacyTempWorkdir, } from "../../../../../tests/helpers/legacy-mocks.ts"; @@ -19,6 +22,7 @@ import { mockTty, } from "../../../../../tests/helpers/mocks.ts"; import { + LegacyDebugFlag, LegacyDnsResolverFlag, LegacyExperimentalFlag, LegacyNetworkIdFlag, @@ -40,6 +44,13 @@ import { LegacyDeclarativeSeam } from "../shared/legacy-pgdelta.seam.service.ts" import type { LegacyDbPullFlags } from "./pull.command.ts"; import { legacyDbPull } from "./pull.handler.ts"; +const alwaysReadyHttpClientLayer = Layer.succeed( + HttpClient.HttpClient, + HttpClient.make((request) => + Effect.succeed(HttpClientResponse.fromWeb(request, new Response(null, { status: 200 }))), + ), +); + const EXPORT_JSON = JSON.stringify({ version: 1, mode: "declarative", @@ -70,7 +81,6 @@ interface SetupOpts { readonly pipedAnswers?: ReadonlyArray; readonly yes?: boolean; readonly experimental?: boolean; - readonly shadowTargetOverride?: string; readonly promptConfirmResponses?: ReadonlyArray; readonly resolvedRef?: string; // Fail the first edge-runtime run with this message (the second succeeds with @@ -107,32 +117,17 @@ function setup(workdir: string, opts: SetupOpts = {}) { const telemetry = mockLegacyTelemetryStateTracked(); const cache = mockLegacyLinkedProjectCacheTracked(); - const provisionCalls: Array<{ - mode: string; - usePgDelta: boolean; - targetLocal: boolean; - projectRef?: string; - }> = []; - const removedContainers: string[] = []; const seam = Layer.succeed(LegacyDeclarativeSeam, { exportCatalog: () => Effect.succeed("supabase/.temp/pgdelta/x.json"), execInherit: () => Effect.succeed(0), ensureLocalDatabaseStarted: () => Effect.void, ensureLocalPostgresImageCurrent: () => Effect.void, - provisionShadow: ({ mode, usePgDelta, targetLocal, projectRef }) => { - provisionCalls.push({ mode, usePgDelta, targetLocal, projectRef }); - return Effect.succeed({ - container: "shadow-1", - sourceUrl: "postgres://postgres:postgres@127.0.0.1:54320/postgres", - targetUrlOverride: opts.shadowTargetOverride, - }); - }, - removeShadowContainer: (container) => - Effect.sync(() => { - removedContainers.push(container); - }), }); + // Shadow provisioning is native (CLI-1956): a real docker-spawner fake backs + // container create/start/health-inspect/cleanup. + const shadowSpawner = mockLegacyShadowContainerCliSpawner(); + let edgeRunCount = 0; const edge = Layer.succeed(LegacyEdgeRuntimeScript, { run: (runOpts: LegacyEdgeRuntimeRunOpts) => { @@ -160,6 +155,15 @@ function setup(workdir: string, opts: SetupOpts = {}) { runCapture: () => Effect.die("runCapture unused"), runStream: (runOpts, streamOpts) => Effect.gen(function* () { + // The native shadow's PG15+ one-shot platform-baseline jobs + // (`legacyRunStartMigrateJob`) go through this same `runStream`, always + // `skipImageResolve: true` (the real `pg_dump` `runStream` call never sets + // it) — succeed unconditionally so shadow setup itself never fails; this + // suite has no assertions over the one-shot jobs' own output, and they must + // not be counted alongside the real `dumpCalls` this suite DOES assert on. + if (runOpts.skipImageResolve === true) { + return { exitCode: 0, stderr: "" }; + } dumpRunCount += 1; dumpCalls.push({ env: runOpts.env, image: runOpts.image }); if (opts.dumpFailFirstWith !== undefined && dumpRunCount === 1) { @@ -177,21 +181,37 @@ function setup(workdir: string, opts: SetupOpts = {}) { const execLog: string[] = []; const historyUpserts: ReadonlyArray[] = []; - const session = { + const connectedDatabases: Array = []; + // The resolver mock's own target connection always dials port 5432; the native + // shadow (platform baseline, `CREATE_TEMPLATE`, migrations, and — on the + // declarative branch — the `contrib_regression` override) always dials the + // schema-default shadow port (54320) instead — a reliable way to tell "the + // REAL remote/local target's own history upsert" (which `historyUpserts` is + // meant to count) apart from the shadow's OWN internal migration replay (which + // ALSO issues a parameterized `INSERT_MIGRATION_VERSION` query, into its own + // separate in-shadow history table). + const TARGET_PORT = 5432; + const makeSession = (isShadow: boolean) => ({ exec: (sql: string) => Effect.sync(() => void execLog.push(sql)), query: (sql: string, params?: ReadonlyArray) => { if (/SELECT version/u.test(sql)) { return Effect.succeed((opts.remoteVersions ?? []).map((v) => ({ version: v }))); } - if (params !== undefined) historyUpserts.push(params); + if (!isShadow && params !== undefined) historyUpserts.push(params); return Effect.succeed([] as ReadonlyArray>); }, extensionExists: () => Effect.die("extensionExists unused"), copyToCsv: () => Effect.die("copyToCsv unused"), queryRaw: () => Effect.die("queryRaw unused"), - }; + }); + const targetSession = makeSession(false); + const shadowSession = makeSession(true); const dbConnection = Layer.succeed(LegacyDbConnection, { - connect: () => Effect.succeed(session), + connect: (cfg: { readonly database: string; readonly port: number }) => + Effect.sync(() => { + connectedDatabases.push(cfg.database); + return cfg.port === TARGET_PORT ? targetSession : shadowSession; + }), }); const poolerFallbackCalls: unknown[] = []; @@ -242,6 +262,11 @@ function setup(workdir: string, opts: SetupOpts = {}) { }); const baseLayer = Layer.mergeAll( + // `BunServices.layer` is listed FIRST so every fake service layer below (most + // importantly `shadowSpawner.layer`'s fake `ChildProcessSpawner`) OVERRIDES its + // real implementation — `Layer.mergeAll` is last-wins on a shared service, + // matching `start.integration.test.ts`'s own established ordering. + BunServices.layer, out.layer, telemetry.layer, cache.layer, @@ -249,6 +274,8 @@ function setup(workdir: string, opts: SetupOpts = {}) { edge, docker, dbConnection, + shadowSpawner.layer, + alwaysReadyHttpClientLayer, resolver, proxy, mockLegacyCliConfig({ workdir, projectId: Option.some("test") }), @@ -259,6 +286,7 @@ function setup(workdir: string, opts: SetupOpts = {}) { ), Layer.succeed(LegacyYesFlag, opts.yes ?? false), Layer.succeed(LegacyExperimentalFlag, opts.experimental ?? false), + Layer.succeed(LegacyDebugFlag, false), Layer.succeed(LegacyDnsResolverFlag, "native"), Layer.succeed(LegacyNetworkIdFlag, Option.none()), Layer.succeed(LegacyPgDeltaSslProbe, { @@ -267,10 +295,8 @@ function setup(workdir: string, opts: SetupOpts = {}) { }), Layer.succeed(CliArgs, { args: opts.args ?? [] }), mockRuntimeInfo(), - BunServices.layer, ); - // Merged last so its `FileSystem` overrides `BunServices` (last-wins); `Path` - // still resolves from `BunServices`. + // Merged last so its `FileSystem` overrides everything above (last-wins). const layer = opts.failWriteOnCall === undefined ? baseLayer @@ -279,14 +305,14 @@ function setup(workdir: string, opts: SetupOpts = {}) { return { layer, out, - provisionCalls, - removedContainers, proxyCalls, proxyCaptureCalls, historyUpserts, execLog, + connectedDatabases, poolerFallbackCalls, dumpCalls, + shadowSpawned: shadowSpawner.spawned, get edgeRunCount() { return edgeRunCount; }, @@ -496,7 +522,8 @@ describe("legacy db pull", () => { }); return Effect.gen(function* () { yield* legacyDbPull(flags()); - expect(s.provisionCalls[0]?.usePgDelta).toBe(false); + // Migra engine selection is proven by `edgeStdout` parsing as raw SQL below + // (a pg-delta selection would instead try — and fail — to `JSON.parse` it). const err = streamText(s.out, "stderr"); // Go's `ConnectByConfig` prints the Connecting line to stderr before dialing // (`internal/utils/connect.go:348`), ahead of any other pull output. @@ -529,7 +556,10 @@ describe("legacy db pull", () => { expect( existsSync(join(tmp.current, "supabase", "database", "schemas", "public", "t.sql")), ).toBe(true); - expect(s.provisionCalls[0]?.mode).toBe("declarative"); + // Declarative mode's bare shadow (`legacyPrepareRawShadow`) never connects to set + // up a platform baseline or `contrib_regression` template — the only connect is + // the top-level target connect (`resolved.conn`, database "postgres"). + expect(s.connectedDatabases).toEqual(["postgres"]); }).pipe(Effect.provide(s.layer)); }); @@ -613,7 +643,6 @@ describe("legacy db pull", () => { yield* legacyDbPull( flags({ declarative: Option.some(true), usePgDelta: Option.some(false) }), ); - expect(s.provisionCalls[0]?.mode).toBe("diff"); expect(s.historyUpserts.length).toBe(1); }).pipe(Effect.provide(s.layer)); }, @@ -633,7 +662,6 @@ describe("legacy db pull", () => { yield* legacyDbPull( flags({ declarative: Option.some(false), usePgDelta: Option.some(true) }), ); - expect(s.provisionCalls[0]?.mode).toBe("diff"); expect(s.historyUpserts.length).toBe(1); }).pipe(Effect.provide(s.layer)); }, @@ -646,7 +674,12 @@ describe("legacy db pull", () => { }); return Effect.gen(function* () { yield* legacyDbPull(flags({ declarative: Option.some(true), usePgDelta: Option.some(true) })); - expect(s.provisionCalls[0]?.mode).toBe("declarative"); + // Reaching the declarative write (rather than a migration file / history + // upsert) proves the declarative export path ran. + expect( + existsSync(join(tmp.current, "supabase", "database", "schemas", "public", "t.sql")), + ).toBe(true); + expect(s.historyUpserts.length).toBe(0); }).pipe(Effect.provide(s.layer)); }); @@ -678,8 +711,6 @@ describe("legacy db pull", () => { expect(s.dumpCalls).toHaveLength(1); expect(s.dumpCalls[0]?.env["EXTRA_SED"]).toBe("/^--/d"); expect(s.dumpCalls[0]?.env["EXCLUDED_SCHEMAS"]).toContain("auth"); - // The diff ran against the shadow with the migra engine (no schema filter). - expect(s.provisionCalls[0]?.usePgDelta).toBe(false); // The migration file holds the dump output followed by the appended diff. const dir = join(tmp.current, "supabase", "migrations"); const file = readdirSync(dir).find((f) => f.endsWith("_remote_schema.sql")); @@ -803,7 +834,7 @@ describe("legacy db pull", () => { const error = yield* legacyDbPull(flags()).pipe(Effect.flip); expect(error.message).toContain("error running container: exit 1"); // The diff pass never ran — the dump failure aborts before provisioning a shadow. - expect(s.provisionCalls).toHaveLength(0); + expect(s.shadowSpawned.filter((c) => c.args[0] === "create")).toEqual([]); }).pipe(Effect.provide(s.layer)); }); @@ -1263,24 +1294,28 @@ describe("legacy db pull", () => { }); return Effect.gen(function* () { yield* legacyDbPull(flags()); - expect(s.provisionCalls[0]?.usePgDelta).toBe(true); + // pg-delta selection is proven by `edgeStdout`'s envelope shape parsing + // successfully below (a migra selection would instead treat it as raw SQL). }).pipe(Effect.provide(s.layer)); }); it.effect("db pull --local provisions a local-target shadow and uses the target override", () => { // Go derives the shadow targetLocal from utils.IsLocalDatabase and substitutes - // the declarative contrib_regression target override (diff.go:190,196-197); - // the native handler must pass targetLocal and honor shadow.targetUrlOverride. + // the declarative contrib_regression target override (diff.go:190,196-197); a + // real declarative schema file makes the native `loadDeclaredSchemas` branch + // non-empty, so `legacyPrepareShadowSource` redirects the diff target to the + // shadow's own `contrib_regression` override database. seedMigration(tmp.current, "20240101000000"); + mkdirSync(join(tmp.current, "supabase", "schemas"), { recursive: true }); + writeFileSync(join(tmp.current, "supabase", "schemas", "public.sql"), "select 1;\n"); const s = setup(tmp.current, { remoteVersions: ["20240101000000"], edgeStdout: "create table remote ();\n", yes: true, - shadowTargetOverride: "postgres://postgres:postgres@127.0.0.1:54320/contrib_regression", }); return Effect.gen(function* () { yield* legacyDbPull(flags({ local: Option.some(true) })); - expect(s.provisionCalls[0]?.targetLocal).toBe(true); + expect(s.connectedDatabases).toContain("contrib_regression"); // A local target prints the local wording (Go's `IsLocalDatabase` branch in // `ConnectByConfigStream`, `internal/utils/connect.go:344-346`). expect(streamText(s.out, "stderr")).toContain("Connecting to local database...\n"); @@ -1390,18 +1425,69 @@ describe("legacy db pull", () => { }); return Effect.gen(function* () { yield* legacyDbPull(flags({ linked: Option.some(true) })); - expect(s.provisionCalls[0]?.usePgDelta).toBe(true); - // The resolved ref is forwarded to the shadow so the `db __shadow` child - // merges the same `[remotes.]` override into the shadow baseline. - expect(s.provisionCalls[0]?.projectRef).toBe("abcdefghijklmnopqrst"); + // pg-delta selection is ref-aware (read from the remote-merged `toml.pgDelta`) + // and is proven by `edgeStdout`'s envelope shape parsing successfully below. + expect(streamText(s.out, "stderr")).toMatch( + /Schema written to supabase[/\\]migrations[/\\]\d{14}_remote_schema\.sql\n/u, + ); }).pipe(Effect.provide(s.layer)); }); + it.effect( + "a linked [remotes.] db.major_version override reaches the shadow's OWN container spec, not just toml", + () => { + // Go remote-merges the WHOLE config uniformly on the linked path (`LoadConfig` seeds + // `flags.ProjectRef` before every field read) — the shadow's container spec (image, JWT + // secret, root key, db.settings, service enabled-for-setup flags) must reflect the + // matched `[remotes.]` override too, not just the `toml` read used for + // pg-delta/schema_paths (mirrors `diff.integration.test.ts`'s identically-named test). + // `major_version` is a clean, directly-observable probe: PG <= 14 is the ONLY branch + // that emits a `--tmpfs` flag on the shadow's `docker create` argv + // (`legacyBuildShadowPostgresContainerSpec`) — a base config of 17 (>= 15, no tmpfs) + // overridden by a remote block's `major_version = 14` must flip that flag on. + seedMigration(tmp.current, "20240101000000"); + mkdirSync(join(tmp.current, "supabase"), { recursive: true }); + writeFileSync( + join(tmp.current, "supabase", "config.toml"), + [ + "[db]", + "major_version = 17", + "", + "[remotes.staging]", + 'project_id = "abcdefghijklmnopqrst"', + "", + "[remotes.staging.db]", + "major_version = 14", + "", + ].join("\n"), + ); + const s = setup(tmp.current, { + remoteVersions: ["20240101000000"], + edgeStdout: "alter table x;\n", + yes: true, + resolvedRef: "abcdefghijklmnopqrst", + }); + return Effect.gen(function* () { + yield* legacyDbPull(flags({ linked: Option.some(true) })); + const createArgs = s.shadowSpawned.find((c) => c.args[0] === "create")?.args ?? []; + expect(createArgs).toContain("--tmpfs"); + }).pipe(Effect.provide(s.layer)); + }, + ); + it.effect("retries the migration-style diff through the IPv4 pooler on an IPv6 error", () => { // Go wraps the linked diff with PoolerFallbackConfig and retries against the // IPv4 pooler when the direct host is unreachable over IPv6 from the container // (internal/db/pull/pull.go, diffRemoteSchema). The first edge run fails with // an IPv6 connectivity error; the retry succeeds and the migration is written. + // + // Go's `diffRemoteSchema` retries the WHOLE `diff.DiffDatabase` call on this + // path, not just the diff engine (`internal/db/diff/diff.go:211-217` runs + // `PrepareShadowSource` and prints "Creating shadow database..."/"Diffing + // schemas..." before ever touching the target connection) — so the pooler + // retry re-provisions and tears down a FRESH shadow and re-prints both + // banners, rather than reusing the first attempt's shadow. Assert that shape + // directly, not just that the migration eventually gets written. seedMigration(tmp.current, "20240101000000"); const s = setup(tmp.current, { remoteVersions: ["20240101000000"], @@ -1414,18 +1500,31 @@ describe("legacy db pull", () => { yield* legacyDbPull( flags({ linked: Option.some(true), diffEngine: Option.some("pg-delta") }), ); - expect(streamText(s.out, "stderr")).toContain("does not support IPv6"); - expect(streamText(s.out, "stderr")).toContain("Retrying via the IPv4 connection pooler"); + const err = streamText(s.out, "stderr"); + expect(err).toContain("does not support IPv6"); + expect(err).toContain("Retrying via the IPv4 connection pooler"); expect(s.edgeRunCount).toBe(2); - expect(streamText(s.out, "stderr")).toMatch( + expect(err).toMatch( /Schema written to supabase[/\\]migrations[/\\]\d{14}_remote_schema\.sql\n/u, ); + expect(s.shadowSpawned.filter((c) => c.args[0] === "create")).toHaveLength(2); + expect( + s.shadowSpawned.filter((c) => c.args[0] === "rm" && c.args.includes("-f")), + ).toHaveLength(2); + expect(err.split("Creating shadow database...")).toHaveLength(3); + expect(err.split("Diffing schemas...")).toHaveLength(3); }).pipe(Effect.provide(s.layer)); }); it.effect("retries the declarative export through the IPv4 pooler on an IPv6 error", () => { // Go's pullDeclarativePgDelta retries DeclarativeExportPgDelta through the - // pooler in the same IPv6 scenario (internal/db/pull/pull.go). + // pooler in the same IPv6 scenario (internal/db/pull/pull.go), but unlike + // diffRemoteSchema/DiffDatabase it calls `diff.PrepareRawShadow` ONCE before + // the retry and only re-runs the export against the same shadow + // (`pull.go:92-115`) — a deliberate asymmetry in Go's own code, not a gap to + // close. Assert the single-shadow-reuse shape so a future change doesn't + // accidentally "fix" this path to double-provision like the migration-style + // diff path correctly does. const s = setup(tmp.current, { edgeFailFirstWith: "error exporting declarative schema:\nnetwork is unreachable", edgeStdout: EXPORT_JSON, @@ -1438,6 +1537,7 @@ describe("legacy db pull", () => { expect(streamText(s.out, "stderr")).toContain( `Declarative schema written to ${join("supabase", "database")}\n`, ); + expect(s.shadowSpawned.filter((c) => c.args[0] === "create")).toHaveLength(1); }).pipe(Effect.provide(s.layer)); }); diff --git a/apps/cli/src/legacy/commands/db/pull/pull.layers.ts b/apps/cli/src/legacy/commands/db/pull/pull.layers.ts index 821fd07acd..ba3689949f 100644 --- a/apps/cli/src/legacy/commands/db/pull/pull.layers.ts +++ b/apps/cli/src/legacy/commands/db/pull/pull.layers.ts @@ -1,6 +1,7 @@ import { Layer } from "effect"; import { commandRuntimeLayer } from "../../../../shared/runtime/command-runtime.layer.ts"; +import { legacyHttpClientLayer } from "../../../auth/legacy-http-debug.layer.ts"; import { legacyCliConfigLayer } from "../../../config/legacy-cli-config.layer.ts"; import { legacyDbConfigLayer } from "../../../shared/legacy-db-config.layer.ts"; import { legacyDbConnectionLayer } from "../../../shared/legacy-db-connection.layer.ts"; @@ -12,13 +13,16 @@ import { legacyLinkedDbResolverRuntimeLayer } from "../../../shared/legacy-manag import { legacyPgDeltaSslProbeLayer } from "../../../shared/legacy-pgdelta-ssl-probe.layer.ts"; import { legacyTelemetryStateLayer } from "../../../telemetry/legacy-telemetry-state.layer.ts"; import { stdinLayer } from "../../../../shared/runtime/stdin.layer.ts"; -import { legacyDeclarativeSeamLayer } from "../shared/legacy-pgdelta.seam.layer.ts"; /** - * Runtime layer for `supabase db pull`. Same composition as `db diff`: the - * db-config resolver, the native pg-delta / migra stack (edge-runtime, SSL probe, - * the Go shadow seam), `LegacyDbConnection` (remote connect + `schema_migrations` - * reconciliation / history update), and `LegacyDockerRun` for the migra fallback. + * Runtime layer for `supabase db pull`. The db-config resolver, the native pg-delta / migra + * stack (edge-runtime, SSL probe, `HttpClient` for the native shadow's health-check wait — + * shadow provisioning itself is native, see `commands/db/shared/legacy-shadow-source.ts` / + * `shared/db-bootstrap/shadow-database.ts`), `LegacyDbConnection` (remote connect + + * `schema_migrations` reconciliation / history update), and `LegacyDockerRun` for the migra + * fallback. Unlike `db diff`, no `LegacyDeclarativeSeam` — `db pull` has no Go-delegate branch + * left that needs it (native shadow provisioning replaced the Go seam here entirely; `db diff` + * still delegates `--use-pgadmin`/`--use-pg-schema` to Go, so it still wires the seam layer). */ const cliConfig = legacyCliConfigLayer.pipe(Layer.provide(legacyDebugLoggerLayer)); @@ -34,7 +38,7 @@ const edgeRuntime = legacyEdgeRuntimeScriptLayer.pipe( Layer.provide(cliConfig), ); -const seam = legacyDeclarativeSeamLayer.pipe(Layer.provide(cliConfig)); +const httpClient = legacyHttpClientLayer.pipe(Layer.provide(legacyDebugLoggerLayer)); export const legacyDbPullRuntimeLayer = Layer.mergeAll( dbConfig, @@ -42,7 +46,7 @@ export const legacyDbPullRuntimeLayer = Layer.mergeAll( legacyDockerRunLayer, edgeRuntime, legacyPgDeltaSslProbeLayer, - seam, + httpClient, cliConfig, legacyIdentityStitchLayer, legacyTelemetryStateLayer, diff --git a/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts b/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts index 2b581df3b0..4dd0cc8e51 100644 --- a/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/schema/declarative/declarative.orchestrate.integration.test.ts @@ -30,8 +30,6 @@ function mockSeam(paths: Record) { execInherit: () => Effect.succeed(0), ensureLocalDatabaseStarted: () => Effect.void, ensureLocalPostgresImageCurrent: () => Effect.void, - provisionShadow: () => Effect.die("provisionShadow not used in declarative tests"), - removeShadowContainer: () => Effect.void, }); return { layer, calls }; } diff --git a/apps/cli/src/legacy/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/legacy/commands/db/schema/declarative/generate/generate.integration.test.ts index b8c4e2733d..d272941c48 100644 --- a/apps/cli/src/legacy/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -106,8 +106,6 @@ function setup(workdir: string, opts: SetupOpts = {}) { : Effect.void, ), ), - provisionShadow: () => Effect.die("provisionShadow not used in declarative tests"), - removeShadowContainer: () => Effect.void, }); const edgeCalls: LegacyEdgeRuntimeRunOpts[] = []; const edge = Layer.succeed(LegacyEdgeRuntimeScript, { diff --git a/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts index a5acb0655e..5213266983 100644 --- a/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -101,8 +101,6 @@ function setup(workdir: string, opts: SetupOpts = {}) { : Effect.void, ), ), - provisionShadow: () => Effect.die("provisionShadow not used in declarative tests"), - removeShadowContainer: () => Effect.void, }); const edge = Layer.succeed(LegacyEdgeRuntimeScript, { run: (runOpts: LegacyEdgeRuntimeRunOpts) => { diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.apply.integration.test.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.apply.integration.test.ts new file mode 100644 index 0000000000..b277a91b4b --- /dev/null +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.apply.integration.test.ts @@ -0,0 +1,747 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Cause, Effect, Exit, FileSystem, Layer } from "effect"; + +import { LegacyDebugFlag } from "../../../../shared/legacy/global-flags.ts"; +import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; +import { + type LegacyEdgeRuntimeRunOpts, + type LegacyEdgeRuntimeRunResult, + LegacyEdgeRuntimeScript, +} from "../../../shared/legacy-edge-runtime-script.service.ts"; +import { LegacyEdgeRuntimeScriptError } from "../../../shared/legacy-edge-runtime-script.errors.ts"; +import { legacyApplyDeclarativePgDelta } from "./legacy-pgdelta.apply.ts"; +import type { LegacyPgDeltaContext } from "./legacy-pgdelta.ts"; + +const CTX: LegacyPgDeltaContext = { + projectId: "ref", + cwd: "/proj", + npmVersion: undefined, + denoVersion: 2, +}; + +function fakeEdgeRuntime(outcome: { stdout?: string; stderr?: string; fail?: string } = {}) { + const calls: Array = []; + const layer = Layer.succeed(LegacyEdgeRuntimeScript, { + run: (opts: LegacyEdgeRuntimeRunOpts) => { + calls.push(opts); + if (outcome.fail !== undefined) { + return Effect.fail(new LegacyEdgeRuntimeScriptError({ message: outcome.fail })); + } + return Effect.succeed({ + stdout: outcome.stdout ?? "", + stderr: outcome.stderr ?? "", + } satisfies LegacyEdgeRuntimeRunResult); + }, + }); + return { layer, calls }; +} + +function makeDeclarativeDir(): string { + const dir = mkdtempSync(join(tmpdir(), "legacy-pgdelta-apply-")); + mkdirSync(join(dir, "declarative"), { recursive: true }); + writeFileSync(join(dir, "declarative", "public.sql"), "create table t ();"); + return join(dir, "declarative"); +} + +const failError = (exit: Exit.Exit) => + Exit.isFailure(exit) ? exit.cause.reasons.find(Cause.isFailReason)?.error : undefined; + +describe("legacyApplyDeclarativePgDelta", () => { + it.effect("fails with LegacyDeclarativeApplyError when the declarative dir doesn't exist", () => { + const edge = fakeEdgeRuntime(); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: "/does/not/exist", + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect((failError(exit) as { message: string }).message).toContain( + "declarative schema directory not found", + ); + // Never even reaches the edge-runtime — the exists() check runs first. + expect(edge.calls).toHaveLength(0); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }); + + it.effect("maps an edge-runtime failure to LegacyDeclarativeApplyError", () => { + const dir = makeDeclarativeDir(); + const edge = fakeEdgeRuntime({ fail: "error running pg-delta script: boom" }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect((failError(exit) as { message: string }).message).toBe( + "error running pg-delta script: boom", + ); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }); + + it.effect("fails with a parse error WITHOUT the raw stdout when --debug is unset", () => { + const dir = makeDeclarativeDir(); + const edge = fakeEdgeRuntime({ stdout: "not json{" }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + const message = (failError(exit) as { message: string }).message; + expect(message).toContain("failed to parse pg-delta apply output"); + expect(message).not.toContain("stdout:"); + expect(message).not.toContain("not json{"); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }); + + it.effect("fails with a parse error INCLUDING the raw stdout when --debug is set", () => { + const dir = makeDeclarativeDir(); + const edge = fakeEdgeRuntime({ stdout: "not json{" }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + const message = (failError(exit) as { message: string }).message; + expect(message).toContain("failed to parse pg-delta apply output"); + expect(message).toContain("stdout: not json{"); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, true), + ), + ), + ); + }); + + it.effect( + "fails with LegacyDeclarativeApplyError (not an unhandled defect) when stdout is syntactically valid but non-object JSON", + () => { + // A configured or future pg-delta version emitting `null`/an array is valid JSON, so a + // bare `JSON.parse(...) as LegacyPgDeltaApplyResult` cast would let `parsed.status` throw + // an unhandled TypeError instead of failing typed. + const dir = makeDeclarativeDir(); + const edge = fakeEdgeRuntime({ stdout: "null" }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect((failError(exit) as { message: string }).message).toContain( + "failed to parse pg-delta apply output", + ); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); + + it.effect("fails with LegacyDeclarativeApplyError when stdout is a JSON array", () => { + const dir = makeDeclarativeDir(); + const edge = fakeEdgeRuntime({ stdout: "[1,2,3]" }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect((failError(exit) as { message: string }).message).toContain( + "failed to parse pg-delta apply output", + ); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }); + + it.effect( + "fails with LegacyDeclarativeApplyError (not an unhandled defect) when a field typed as an array arrives as an object", + () => { + // A configured or future pg-delta emitting `{"status":"error","errors":{"length":1}}` must + // not reach `legacyFormatApplyFailure`'s `for (const issue of errors)`, which would throw an + // unhandled TypeError on a non-iterable object — Go's `json.Unmarshal` rejects this the same + // way, since `Errors` is declared `[]ApplyIssue` (`apps/cli-go/internal/pgdelta/apply.go:33`). + const dir = makeDeclarativeDir(); + const edge = fakeEdgeRuntime({ + stdout: JSON.stringify({ status: "error", errors: { length: 1 } }), + }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect((failError(exit) as { message: string }).message).toContain( + "failed to parse pg-delta apply output", + ); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); + + it.effect( + "fails with LegacyDeclarativeApplyError (not treated as a false success) when an errors array element is a number", + () => { + // A configured or future pg-delta emitting `{"status":"success","errors":[123]}` must not + // be accepted as a successful apply. Verified against Go's real `ApplyIssue.UnmarshalJSON` + // (`apps/cli-go/internal/pgdelta/apply.go:124-142`): a numeric element fails BOTH its + // string-arm and its object-arm unmarshal, which fails the WHOLE `ApplyResult` decode — + // Go never reaches a "success" status in this case, so the TS guard must reject it too. + const dir = makeDeclarativeDir(); + const edge = fakeEdgeRuntime({ + stdout: JSON.stringify({ status: "success", errors: [123] }), + }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect((failError(exit) as { message: string }).message).toContain( + "failed to parse pg-delta apply output", + ); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); + + it.effect( + "fails with LegacyDeclarativeApplyError (not treated as a false success) when a diagnostics array element is a bare string", + () => { + // Unlike `ApplyIssue`, Go's `ApplyDiagnosis.UnmarshalJSON` (`apply.go:79-116`) has no + // bare-string acceptance branch, so `{"diagnostics":["boom"]}` fails Go's whole decode too + // (verified: unmarshaling a JSON string into `ApplyDiagnosis`'s shadow struct errors). + const dir = makeDeclarativeDir(); + const edge = fakeEdgeRuntime({ + stdout: JSON.stringify({ status: "success", diagnostics: ["boom"] }), + }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect((failError(exit) as { message: string }).message).toContain( + "failed to parse pg-delta apply output", + ); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); + + it.effect( + "accepts a diagnostics element whose statementId is a mistyped, non-object/non-string value (Go degrades it silently)", + () => { + // Unlike a top-level array-element shape mismatch, Go's `ApplyDiagnosis.UnmarshalJSON` + // decodes `statementId` into a `json.RawMessage` first (accepts ANY valid JSON value), then + // tries `ApplyStatementLocation`, then a bare string, and silently leaves `StatementID` nil + // if BOTH fail — never propagating an error. A mistyped `statementId` must NOT fail the + // whole parse. + const dir = makeDeclarativeDir(); + const payload = { + status: "success", + diagnostics: [{ message: "note", statementId: 42 }], + }; + const edge = fakeEdgeRuntime({ stdout: JSON.stringify(payload) }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(Exit.isSuccess(exit)).toBe(true); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); + + it.effect( + "drops a diagnostics element's statementId when a nested field is mistyped, instead of rendering a bogus location (Go's nil fallback)", + () => { + // Unlike the mistyped-non-object/non-string `statementId` case above, this reproduces a + // mistyped FIELD INSIDE an otherwise object-shaped `statementId` + // (`{"filePath":123,...}`). Go's `(d *ApplyDiagnosis) UnmarshalJSON` (`apply.go:100-115`) + // tries the `ApplyStatementLocation` object shape first — the mistyped `filePath` fails + // that decode — then falls back to a bare string, which ALSO fails (it's an object, not a + // string) — so Go silently leaves `StatementID` nil rather than erroring the whole parse, + // verified empirically. Rendering the raw object anyway would show a bogus `(123#1)` + // location Go never emits. + const dir = makeDeclarativeDir(); + const payload = { + status: "success", + diagnostics: [{ message: "note", statementId: { filePath: 123, statementIndex: 1 } }], + }; + const edge = fakeEdgeRuntime({ stdout: JSON.stringify(payload) }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(Exit.isSuccess(exit)).toBe(true); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); + + it.effect( + "accepts a null scalar field on an errors/diagnostics element and formats it as absent (Go's encoding/json leaves the zero value)", + () => { + // `ApplyIssue`'s non-`Statement` fields (`Code`/`Message`/`IsDependencyError`/`Position`/ + // `Detail`/`Hint`) and `ApplyDiagnosis`'s (`Code`/`Message`/`SuggestedFix`) are all plain, + // non-pointer Go types decoded via the default `encoding/json` — verified empirically that + // a JSON `null` for a non-pointer struct field produces NO error and leaves the zero value, + // so `{"errors":[{"message":null}]}` is a valid, Go-accepted payload, not a parse failure. + // The formatter's existing `String(issue.message ?? "")` already renders a zero-value + // message as "unknown pg-delta issue" once the guard lets the `null` through. + const dir = makeDeclarativeDir(); + const payload = { + status: "error", + totalApplied: 0, + totalRounds: 1, + totalSkipped: 0, + errors: [{ message: null, code: null, isDependencyError: null, position: null }], + diagnostics: [{ message: null, code: null, suggestedFix: null }], + }; + const edge = fakeEdgeRuntime({ stdout: JSON.stringify(payload) }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect(out.stderrText).toContain("- unknown pg-delta issue"); + expect(out.stderrText).toContain("- unknown pg-delta diagnostic"); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, true), + ), + ), + ); + }, + ); + + it.effect( + "accepts a null top-level counter and formats it as zero (Go's encoding/json leaves the zero value)", + () => { + // `ApplyResult` has no custom `UnmarshalJSON` of its own, so its plain, non-pointer `int` + // counters (`TotalStatements`/`TotalRounds`/`TotalApplied`/`TotalSkipped`) decode via the + // default `encoding/json` — verified empirically that a JSON `null` for a non-pointer `int` + // field produces NO error and leaves the zero value, so + // `{"status":"success","totalApplied":null}` is a valid, Go-accepted payload, not a parse + // failure — same "null means absent" rule already applied to nested issue/diagnostic + // scalar fields above. + const dir = makeDeclarativeDir(); + const payload = { status: "success", totalApplied: null, totalRounds: null }; + const edge = fakeEdgeRuntime({ stdout: JSON.stringify(payload) }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(Exit.isSuccess(exit)).toBe(true); + expect(out.stderrText).toContain("Applied 0 statements in 0 round(s)."); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); + + it.effect( + "accepts a null errors/stuckStatements/validationErrors/diagnostics array and treats it as empty (Go's encoding/json leaves a nil slice)", + () => { + // `ApplyResult`'s array fields have no custom `UnmarshalJSON` of their own, so Go's + // `encoding/json` accepts a JSON `null` for a `[]T` slice field with no error, leaving a + // nil (zero-length) slice — verified empirically: + // `json.Unmarshal([]byte(\`{"status":"error","errors":null}\`), &r)` returns `err == nil` + // with `len(r.Errors) == 0`. A payload reporting all four as `null` must format as if none + // were reported at all, not fail the parse. + const dir = makeDeclarativeDir(); + const payload = { + status: "error", + totalApplied: 0, + totalRounds: 1, + totalSkipped: 0, + errors: null, + stuckStatements: null, + validationErrors: null, + diagnostics: null, + }; + const edge = fakeEdgeRuntime({ stdout: JSON.stringify(payload) }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect(out.stderrText).toContain("No per-statement diagnostics were reported by pg-delta."); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); + + it.effect( + "fails with LegacyDeclarativeApplyError (not an unhandled defect) when a field typed as a number arrives as a string", + () => { + // Same reasoning as the array-typed-field test above, for `ApplyResult`'s numeric fields + // (`TotalApplied int`, etc.) — a malformed counter must fail the parse, not be silently + // treated as a genuine successful-apply summary. + const dir = makeDeclarativeDir(); + const edge = fakeEdgeRuntime({ + stdout: JSON.stringify({ status: "success", totalApplied: "5" }), + }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect((failError(exit) as { message: string }).message).toContain( + "failed to parse pg-delta apply output", + ); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); + + it.effect( + "fails with LegacyDeclarativeApplyError (not an unhandled defect) when a field typed as an int arrives as a fractional number", + () => { + // Go's `TotalApplied int` (and its `int`-typed siblings) reject any JSON number literal + // with a decimal point via `strconv.ParseInt` on the raw literal text — verified + // empirically that `json.Unmarshal` on `{"totalApplied":1.5}` errors identically to a + // string-typed field mismatch, so `1.5` must fail the parse here too, not be treated as a + // truncated/rounded successful-apply count. + const dir = makeDeclarativeDir(); + const edge = fakeEdgeRuntime({ + stdout: JSON.stringify({ status: "success", totalApplied: 1.5 }), + }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect((failError(exit) as { message: string }).message).toContain( + "failed to parse pg-delta apply output", + ); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); + + it.effect( + "on a non-success status, prints the formatted failure to stderr but not the raw payload when --debug is unset", + () => { + const dir = makeDeclarativeDir(); + const payload = { + status: "error", + totalApplied: 0, + totalRounds: 1, + totalSkipped: 0, + errors: ["boom"], + }; + const edge = fakeEdgeRuntime({ stdout: JSON.stringify(payload) }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const exit = yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(failError(exit)?.constructor.name).toBe("LegacyDeclarativeApplyError"); + expect((failError(exit) as { message: string }).message).toBe( + "pg-delta declarative apply failed with status: error", + ); + expect(out.stderrText).toContain('pg-delta apply returned status "error".'); + expect(out.stderrText).toContain("- boom"); + expect(out.stderrText).not.toContain("pg-delta apply result:"); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); + + it.effect( + "on a non-success status with --debug set, additionally dumps the pretty-printed raw payload", + () => { + const dir = makeDeclarativeDir(); + const payload = { + status: "error", + totalApplied: 0, + totalRounds: 1, + totalSkipped: 0, + errors: ["boom"], + }; + const edge = fakeEdgeRuntime({ stdout: JSON.stringify(payload) }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }).pipe(Effect.exit); + expect(out.stderrText).toContain("pg-delta apply result:"); + expect(out.stderrText).toContain(JSON.stringify(payload, null, 2)); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, true), + ), + ), + ); + }, + ); + + it.effect( + "on success, prints the applied-statements summary and forwards SCHEMA_PATH/TARGET/binds", + () => { + const dir = makeDeclarativeDir(); + const payload = { + status: "success", + totalStatements: 3, + totalApplied: 3, + totalRounds: 2, + totalSkipped: 0, + }; + const edge = fakeEdgeRuntime({ stdout: JSON.stringify(payload) }); + const out = mockOutput(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + yield* legacyApplyDeclarativePgDelta(CTX, { + fs, + declarativeDirAbs: dir, + target: "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + }); + expect(out.stderrText).toContain("Applying declarative schemas via pg-delta..."); + expect(out.stderrText).toContain("Applied 3 statements in 2 round(s)."); + const opts = edge.calls[0]!; + expect(opts.env["SCHEMA_PATH"]).toBe("/declarative"); + expect(opts.env["TARGET"]).toBe( + "postgresql://postgres:postgres@127.0.0.1:54320/contrib_regression", + ); + expect(opts.binds).toEqual([ + "supabase_edge_runtime_ref:/root/.cache/deno:rw", + `${dir}:/declarative:ro`, + ]); + expect(opts.errPrefix).toBe("error running pg-delta script"); + rmSync(dir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + edge.layer, + out.layer, + Layer.succeed(LegacyDebugFlag, false), + ), + ), + ); + }, + ); +}); diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.apply.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.apply.ts new file mode 100644 index 0000000000..d9421bf720 --- /dev/null +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.apply.ts @@ -0,0 +1,855 @@ +/** + * Port of Go's `pgdelta.ApplyDeclarative` (`apps/cli-go/internal/pgdelta/apply.go:299-360`) — + * CLI-1956's declarative-apply runner: applies `supabase/database` (or the configured + * declarative dir) to the shadow's `contrib_regression` override database via pg-delta's + * declarative apply engine, run inside the edge-runtime container. + * + * This is genuinely NEW work, not a seam removal: the Deno script template itself + * (`legacyPgDeltaDeclarativeApplyScript`) already existed (ported for a different, now-dead + * seam), but nothing in TS ever invoked it — every declarative apply ran through the bundled + * Go binary until now. + */ + +import { Data, Effect, type FileSystem } from "effect"; + +import { LegacyDebugFlag } from "../../../../shared/legacy/global-flags.ts"; +import { Output } from "../../../../shared/output/output.service.ts"; +import { LegacyEdgeRuntimeScript } from "../../../shared/legacy-edge-runtime-script.service.ts"; +import { + legacyInterpolatePgDeltaScript, + legacyPgDeltaDeclarativeApplyScript, +} from "./legacy-pgdelta.deno-templates.ts"; +import { + legacyEdgeRuntimeId, + legacyPgDeltaNpmRegistryOption, + type LegacyPgDeltaContext, +} from "./legacy-pgdelta.ts"; + +const errMessage = (e: unknown): string => + typeof e === "object" && e !== null && "message" in e && typeof e.message === "string" + ? e.message + : String(e); + +/** `pgdelta.ApplyDeclarative` failed — Go's own error messages at each step (see call sites below). */ +export class LegacyDeclarativeApplyError extends Data.TaggedError("LegacyDeclarativeApplyError")<{ + readonly message: string; +}> {} + +/** Go's `containerSchemaPath` (`apply.go:311`). */ +const LEGACY_PG_DELTA_APPLY_CONTAINER_SCHEMA_PATH = "/declarative"; + +/** One statement/error entry — Go's `ApplyIssue`, which may arrive as a bare string or an object. */ +export interface LegacyPgDeltaApplyIssue { + readonly statement?: { + // Optional (not required): `legacyIsValidApplyIssueElement` only checks the TYPE of each + // present field (matching Go's per-field `json.Unmarshal` type check), not that every + // field is present — so a partially-populated `statement` object (e.g. a future pg-delta + // release that only reports `id`) must still render, not throw — see + // `legacyFormatApplyIssue`'s defensive `?? ""` handling below. Go's own `(i + // *ApplyIssue) UnmarshalJSON` is deliberately just as permissive about ABSENT fields, + // while still rejecting a MISTYPED one for the whole payload — see + // `legacyIsValidApplyIssueElement`'s own doc comment. + // + // `| null` on each of `id`/`sql`/`statementClass` (not just `?`): these are plain, + // non-pointer `string` fields on Go's `ApplyStatement`, which has no custom + // `UnmarshalJSON` of its own — so they decode via the default `encoding/json`, which + // (verified empirically) accepts a JSON `null` for a non-pointer field with NO error and + // leaves the zero value (`""`), the same "null means absent" rule as every other scalar + // on this interface — see {@link LegacyPgDeltaApplyIssue.code}'s doc comment. + readonly id?: string | null; + readonly sql?: string | null; + readonly statementClass?: string | null; + // `| null` (not just `?`): Go's `Statement *ApplyStatement` is a pointer, so a JSON + // `"statement":null` entry (e.g. `{"statement":null,"message":"failed"}`) unmarshals to a + // nil pointer — `formatApplyIssue`'s `issue.Statement == nil` (`apply.go:202`) treats that + // identically to a missing field. `legacyFormatApplyIssue`'s guard below must check for + // `null` as well as `undefined`, or a `JSON.parse`'d `null` reaches `issue.statement.*` and + // throws a `TypeError` instead of rendering the message. + } | null; + // `| null` on every scalar below (not just `?`): `ApplyIssue`'s non-`Statement` fields + // (`Code`/`Message`/`IsDependencyError`/`Position`/`Detail`/`Hint`) are all plain, + // non-pointer Go types (`string`/`bool`/`int`) decoded via the default `encoding/json` + // inside `(i *ApplyIssue) UnmarshalJSON`'s `json.Unmarshal(trimmed, &parsed)` call + // (`apply.go:133-138`) — verified empirically that unmarshaling a JSON `null` into a + // non-pointer struct field produces NO error and leaves the zero value untouched (Go's + // documented "null means absent" rule applies to any Go type, not just pointers/maps/ + // slices/interfaces). So `{"message":null}` is a valid, Go-accepted `ApplyIssue` element — + // rejecting it here would turn an otherwise-parseable pg-delta payload into a spurious + // "failed to parse pg-delta apply output" instead of rendering `unknown pg-delta issue` + // the way `legacyFormatApplyIssueMessage`'s existing `String(issue.message ?? "")` already + // does once this type (and `legacyIsValidApplyIssueElement`) let a null through. + readonly code?: string | null; + readonly message?: string | null; + readonly isDependencyError?: boolean | null; + readonly position?: number | null; + readonly detail?: string | null; + readonly hint?: string | null; +} + +/** + * Go's `ApplyStatementLocation` (pg-topo's `StatementId` shape). `ApplyStatementLocation` + * has no custom `UnmarshalJSON` of its own, so `filePath`/`statementIndex` are plain, + * non-pointer Go types decoded via the default `encoding/json` — same "null means absent" + * rule as every other scalar in this file (verified empirically, see {@link + * LegacyPgDeltaApplyIssue.code}'s doc comment), hence `| null` on both. + */ +export interface LegacyPgDeltaApplyStatementLocation { + readonly filePath?: string | null; + readonly statementIndex?: number | null; +} + +/** Go's `ApplyDiagnosis` — a pg-topo static-analysis diagnostic. */ +export interface LegacyPgDeltaApplyDiagnosis { + // `| null` on `code`/`message`/`suggestedFix` (not just `?`): `(d *ApplyDiagnosis) + // UnmarshalJSON`'s shadow `raw` struct (`apply.go:87-92`) declares these as plain, + // non-pointer `string` fields with no custom unmarshaler of their own, so — same + // empirically-verified "null means absent" `encoding/json` rule as + // {@link LegacyPgDeltaApplyIssue.code} — a JSON `null` for any of them decodes with no + // error and leaves `""`, not a rejected payload. + readonly code?: string | null; + readonly message?: string | null; + // `| null` (not just `?`): Go's `(d *ApplyDiagnosis) UnmarshalJSON` (`apply.go:79-108`) + // explicitly maps a JSON `"statementId":null` to a nil `*ApplyStatementLocation`, and + // `formatStatementLocation` (`apply.go:263-274`) returns `""` for a nil pointer — so the TS + // path must accept `null` here as absent too, or a `JSON.parse`'d `null` reaches + // `legacyFormatStatementLocation`'s `resolved.filePath` and throws a `TypeError` instead of + // rendering the rest of the diagnostic. + readonly statementId?: LegacyPgDeltaApplyStatementLocation | string | null; + readonly suggestedFix?: string | null; +} + +/** + * The JSON payload `pgdelta_declarative_apply.ts` prints on stdout. Go's `ApplyResult`. + * + * `| null` on each `total*` counter (not just `?`): `ApplyResult` has no custom + * `UnmarshalJSON` of its own, so these plain, non-pointer `int` fields decode via the + * default `encoding/json`, which — verified empirically, same rule as {@link + * LegacyPgDeltaApplyIssue.code} — accepts a JSON `null` for a non-pointer `int` field with + * NO error and leaves the zero value. So `{"status":"success","totalApplied":null}` is a + * valid, Go-accepted `ApplyResult`, not a parse failure. + * + * `| null` on each array field too (`errors`/`stuckStatements`/`validationErrors`/ + * `diagnostics`): these are plain, non-pointer Go `[]T` slice fields with no custom + * unmarshaler on `ApplyResult` itself, and `encoding/json` accepts a JSON `null` for a + * slice field with NO error, leaving a nil (zero-length) slice — verified empirically: + * `json.Unmarshal([]byte(\`{"status":"error","errors":null}\`), &r)` returns `err == nil` + * with `r.Errors == nil` (`len(r.Errors) == 0`). `formatApplyFailure`'s `len(result.Errors) + * > 0` guards treat a nil slice identically to an empty one, so `{"status":"error", + * "errors":null}` must be accepted here too, not rejected as a parse failure. + */ +export interface LegacyPgDeltaApplyResult { + readonly status: string; + readonly totalStatements?: number | null; + readonly totalRounds?: number | null; + readonly totalApplied?: number | null; + readonly totalSkipped?: number | null; + readonly errors?: ReadonlyArray | null; + readonly stuckStatements?: ReadonlyArray | null; + readonly validationErrors?: ReadonlyArray | null; + readonly diagnostics?: ReadonlyArray | null; +} + +/** + * Go's `int`-typed fields (`TotalStatements`/`TotalRounds`/`TotalApplied`/`TotalSkipped` on + * `ApplyResult`, `Position` on `ApplyIssue`) reject any JSON number literal containing a decimal + * point or exponent — Go's `json.Unmarshal` parses the literal text via `strconv.ParseInt` + * rather than decoding a `float64` and truncating it, so even a "whole" float like `1.0` fails + * identically to `1.5` (verified empirically: `json.Unmarshal([]byte(\`{"totalApplied":1.0}\`), + * &r)` and the `1.5` variant both return `cannot unmarshal number ... into ... type int`). A + * `JSON.parse`'d `1.0` is already indistinguishable from the integer `1` by the time it reaches + * this guard — `JSON.parse` itself collapses that distinction, so that exact literal-text + * sub-case can't be reproduced post-parse — but `Number.isInteger` still correctly rejects any + * genuinely fractional value like `1.5`, which is the reachable and observable part of this + * parity gap. + */ +function legacyIsGoIntNumber(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value); +} + +/** + * Go's `(i *ApplyIssue) UnmarshalJSON` (`apply.go:124-142`) accepts `null`, a bare string, or + * an object whose PRESENT fields each match `ApplyIssue`'s declared JSON types — anything else + * (a number, boolean, array, or an object with a mistyped field) fails Go's `json.Unmarshal` + * for the WHOLE `ApplyResult`, not just that element. Verified empirically against Go's real + * struct definitions: `{"errors":[123]}` returns `cannot unmarshal number into Go struct field + * ApplyResult.errors of type main.alias`, and `{"errors":[{"message":123}]}` returns `cannot + * unmarshal number into Go struct field ApplyResult.errors.message of type string` — both abort + * the ENTIRE parse rather than degrading that one element, so a payload like + * `{"status":"success","errors":[123]}` must be rejected here too, not accepted as a (false) + * success. Nested `statement` is checked the same way, one level deep — Go's `ApplyStatement` + * has no custom `UnmarshalJSON`, so a mistyped `id`/`sql`/`statementClass` fails identically. + * + * A JSON `null` for any INDIVIDUAL scalar field, though — top-level (`code`/`message`/ + * `isDependencyError`/`position`/`detail`/`hint`) or nested under `statement` + * (`id`/`sql`/`statementClass`) — is NOT a mistyped field: every one of these is a plain, + * non-pointer Go type with no custom unmarshaler, and `encoding/json` accepts `null` for those + * with no error, leaving the zero value (verified empirically — see + * {@link LegacyPgDeltaApplyIssue.code}'s doc comment). So `null` is tolerated alongside each + * field's declared type below, matching Go exactly instead of rejecting an otherwise + * Go-compatible payload like `{"message":null}`. + */ +function legacyIsValidApplyIssueElement(value: unknown): boolean { + if (value === null || typeof value === "string") return true; + if (typeof value !== "object" || Array.isArray(value)) return false; + if ("statement" in value) { + const statement = value.statement; + if (statement !== null && statement !== undefined) { + if (typeof statement !== "object" || Array.isArray(statement)) return false; + if ("id" in statement && statement.id !== null && typeof statement.id !== "string") { + return false; + } + if ("sql" in statement && statement.sql !== null && typeof statement.sql !== "string") { + return false; + } + if ( + "statementClass" in statement && + statement.statementClass !== null && + typeof statement.statementClass !== "string" + ) { + return false; + } + } + } + if ("code" in value && value.code !== null && typeof value.code !== "string") return false; + if ("message" in value && value.message !== null && typeof value.message !== "string") { + return false; + } + if ( + "isDependencyError" in value && + value.isDependencyError !== null && + typeof value.isDependencyError !== "boolean" + ) { + return false; + } + if ("position" in value && value.position !== null && !legacyIsGoIntNumber(value.position)) { + return false; + } + if ("detail" in value && value.detail !== null && typeof value.detail !== "string") return false; + if ("hint" in value && value.hint !== null && typeof value.hint !== "string") return false; + return true; +} + +/** + * Go's `(d *ApplyDiagnosis) UnmarshalJSON` (`apply.go:79-116`) — unlike `ApplyIssue`, there is + * NO bare-string acceptance branch, so only `null` or an object is valid; a bare + * string/number/boolean/array element fails the whole `ApplyResult` unmarshal. Verified + * empirically: `{"diagnostics":["boom"]}` returns `cannot unmarshal string into Go struct field + * ApplyResult.diagnostics of type struct {...}`. `statementId` is deliberately NOT type-checked + * here: Go decodes it into a `json.RawMessage` first (accepts any valid JSON value), then tries + * `ApplyStatementLocation`, then a bare string, and silently leaves `StatementID` nil if BOTH + * fail — it never propagates an error for a mistyped `statementId` (verified empirically: + * `{"statementId":42}` and `{"statementId":{"filePath":123}}` both unmarshal with `err: `), + * so `legacyNormalizeApplyDiagnosis`/`legacyFormatStatementLocation`'s existing defensive + * handling is the correct (and only) place that degrades gracefully. + * + * Same "null tolerated on a scalar field" rule as {@link legacyIsValidApplyIssueElement} + * applies to `code`/`message`/`suggestedFix` here too: `UnmarshalJSON`'s shadow `raw` struct + * (`apply.go:87-92`) decodes them via the default `encoding/json`, which accepts a JSON + * `null` for a plain `string` field with no error (verified empirically). + */ +function legacyIsValidApplyDiagnosisElement(value: unknown): boolean { + if (value === null) return true; + if (typeof value !== "object" || Array.isArray(value)) return false; + if ("code" in value && value.code !== null && typeof value.code !== "string") return false; + if ("message" in value && value.message !== null && typeof value.message !== "string") { + return false; + } + if ( + "suggestedFix" in value && + value.suggestedFix !== null && + typeof value.suggestedFix !== "string" + ) { + return false; + } + return true; +} + +/** + * Structural guard for Go's `ApplyResult` JSON shape, applied to an untrusted + * `JSON.parse` of the pg-delta subprocess's stdout. A syntactically valid but non-object + * payload (`null`, an array, a bare string/number — e.g. a future pg-delta release that + * changes its output shape) must fail typed as {@link LegacyDeclarativeApplyError}, not + * crash `parsed.status` with an unhandled `TypeError`. + * + * Every field `ApplyResult` itself declares a type for is checked when present — Go's + * `json.Unmarshal` rejects the whole payload with an `UnmarshalTypeError` the moment any of + * these doesn't match its struct field's declared type (`Errors []ApplyIssue`, `TotalApplied + * int`, etc., `apps/cli-go/internal/pgdelta/apply.go:27-44`), so e.g. an `errors` field that + * arrives as an object (`{"length":1}`) instead of an array must fail here too, not reach + * `legacyFormatApplyFailure`'s `for (const issue of errors)` and throw an unhandled + * `TypeError` defect. Each ARRAY field's elements are also validated ({@link + * legacyIsValidApplyIssueElement}/{@link legacyIsValidApplyDiagnosisElement}) since Go's own + * per-element `UnmarshalJSON` implementations reject a malformed element by failing the WHOLE + * `ApplyResult` decode, not by skipping just that element — see those functions' own doc + * comments for the empirical verification. This is also the AGENTS.md-mandated way to narrow + * `unknown` without an `as` cast. + * + * Each array field also tolerates a JSON `null` (not just an absent key): `ApplyResult`'s + * `[]ApplyIssue`/`[]ApplyDiagnosis` fields have no custom unmarshaler of their own, and + * Go's `encoding/json` accepts `null` for a slice field with no error, leaving a nil + * (zero-length) slice — verified empirically, see {@link LegacyPgDeltaApplyResult}'s own + * doc comment. So `{"status":"error","errors":null}` is a valid, Go-accepted payload, not + * a rejected one. + */ +function legacyIsPgDeltaApplyResult(value: unknown): value is LegacyPgDeltaApplyResult { + if ( + typeof value !== "object" || + value === null || + Array.isArray(value) || + !("status" in value) || + typeof value.status !== "string" + ) { + return false; + } + if ( + "totalStatements" in value && + value.totalStatements !== null && + !legacyIsGoIntNumber(value.totalStatements) + ) { + return false; + } + if ( + "totalRounds" in value && + value.totalRounds !== null && + !legacyIsGoIntNumber(value.totalRounds) + ) { + return false; + } + if ( + "totalApplied" in value && + value.totalApplied !== null && + !legacyIsGoIntNumber(value.totalApplied) + ) { + return false; + } + if ( + "totalSkipped" in value && + value.totalSkipped !== null && + !legacyIsGoIntNumber(value.totalSkipped) + ) { + return false; + } + if ("errors" in value && value.errors !== null) { + if (!Array.isArray(value.errors) || !value.errors.every(legacyIsValidApplyIssueElement)) { + return false; + } + } + if ("stuckStatements" in value && value.stuckStatements !== null) { + if ( + !Array.isArray(value.stuckStatements) || + !value.stuckStatements.every(legacyIsValidApplyIssueElement) + ) { + return false; + } + } + if ("validationErrors" in value && value.validationErrors !== null) { + if ( + !Array.isArray(value.validationErrors) || + !value.validationErrors.every(legacyIsValidApplyIssueElement) + ) { + return false; + } + } + if ("diagnostics" in value && value.diagnostics !== null) { + if ( + !Array.isArray(value.diagnostics) || + !value.diagnostics.every(legacyIsValidApplyDiagnosisElement) + ) { + return false; + } + } + return true; +} + +/** Go's `(i *ApplyIssue) UnmarshalJSON` string/object dual shape, applied post-`JSON.parse`. */ +function legacyNormalizeApplyIssue( + raw: LegacyPgDeltaApplyIssue | string | null | undefined, +): LegacyPgDeltaApplyIssue { + if (raw === null || raw === undefined) return {}; + if (typeof raw === "string") return { message: raw }; + return raw; +} + +/** + * Go's `(d *ApplyDiagnosis) UnmarshalJSON` three-way `statementId` fallback + * (`apply.go:100-115`): decode into `ApplyStatementLocation` first — an object whose + * PRESENT `filePath`/`statementIndex` fields each match the declared type (`null` + * tolerated per field, same rule as {@link legacyIsValidApplyIssueElement}) — and if + * that fails (a non-object, or an object with a mistyped field), fall back to a bare + * string; if BOTH fail, Go silently leaves `StatementID` nil rather than erroring the + * whole `ApplyResult` parse. Verified empirically: `{"statementId":{"filePath":123, + * "statementIndex":1}}` decodes with `StatementID == nil` in Go — the object-shape + * unmarshal fails on the mistyped `filePath`, and the string fallback also fails since + * the value is an object, not a string. `legacyIsValidApplyDiagnosisElement` deliberately + * does NOT check `statementId`'s shape (see its own doc comment — Go defers this into a + * `json.RawMessage` that never fails the outer parse), so this is the only place that can + * drop a malformed location instead of `legacyFormatStatementLocation`'s `String(...)` + * coercion rendering a bogus location (e.g. `123#1`) Go would never have shown. + */ +function legacyNormalizeApplyStatementId( + raw: LegacyPgDeltaApplyStatementLocation | string | null | undefined, +): LegacyPgDeltaApplyStatementLocation | undefined { + if (raw === null || raw === undefined) return undefined; + if (typeof raw === "string") return { filePath: raw }; + if (typeof raw !== "object" || Array.isArray(raw)) return undefined; + const filePathOk = + !("filePath" in raw) || raw.filePath === null || typeof raw.filePath === "string"; + const indexOk = + !("statementIndex" in raw) || + raw.statementIndex === null || + legacyIsGoIntNumber(raw.statementIndex); + if (filePathOk && indexOk) return raw; + return undefined; +} + +/** Go's `(d *ApplyDiagnosis) UnmarshalJSON` defensive `statementId` handling. */ +function legacyNormalizeApplyDiagnosis( + raw: LegacyPgDeltaApplyDiagnosis | null | undefined, +): LegacyPgDeltaApplyDiagnosis { + if (raw === null || raw === undefined) return {}; + return { ...raw, statementId: legacyNormalizeApplyStatementId(raw.statementId) }; +} + +/** + * Go's `formatStatementLocation` (`apply.go:262-275`). `String(... ?? "")` rather than a bare + * `?? ""` before `.trim()`: `filePath` is typed as `string | undefined`, but this whole module + * types an untrusted `JSON.parse` of subprocess output, so a malformed payload can hand this a + * non-string value (e.g. a number) at runtime — `?? ""` alone only substitutes `null`/ + * `undefined`, so a non-string, non-nullish value would still reach `.trim()` and throw. The + * `resolved === null` check (not just `undefined`) is the same shape: Go's `StatementID + * *ApplyStatementLocation` is a pointer, so `"statementId":null` unmarshals to `nil` and + * `formatStatementLocation`'s own `loc == nil` (`apply.go:264`) treats it as absent — checking + * only `undefined` here would fall through to `resolved.filePath` on a `null` and throw a + * `TypeError` instead of rendering the rest of the diagnostic. + */ +function legacyFormatStatementLocation( + loc: LegacyPgDeltaApplyStatementLocation | string | null | undefined, +): string { + const resolved = typeof loc === "string" ? { filePath: loc } : loc; + if (resolved === null || resolved === undefined) return ""; + const path = String(resolved.filePath ?? "").trim(); + if (path.length === 0) return ""; + if ((resolved.statementIndex ?? 0) > 0) return `${path}#${resolved.statementIndex}`; + return path; +} + +/** + * Go's `formatStatementSQL` (`apply.go:277-283`): collapse whitespace, then truncate at 120 + * UTF-8 bytes — not JS UTF-16 code units. Go's `len(normalized)` and `normalized[:maxLen-3]` + * both count/slice raw bytes, so a statement with multibyte (e.g. non-ASCII identifier) + * characters can be far longer in bytes than in UTF-16 units — a `.length`/`.slice()` guard + * would under-truncate (or not truncate at all) relative to Go's 120-byte limit, changing the + * legacy stderr contract for an already-failed apply. + * + * Returns a `Buffer`, not a `string`: Go's `[:maxLen-3]` is a raw byte slice with no regard + * for codepoint boundaries, so a multibyte (e.g. non-ASCII identifier) character straddling + * byte 117 is cut mid-sequence, leaving an intentionally INVALID trailing UTF-8 fragment — + * exactly what Go writes to stderr, unvalidated. `Buffer#toString("utf-8")` on that same + * fragment does NOT reproduce it: Node's UTF-8 decoder substitutes U+FFFD for the incomplete + * sequence, and re-encoding that string back to bytes for output yields a DIFFERENT (and + * differently-sized) byte sequence than Go's raw slice — verified empirically: slicing Go's + * own `formatStatementSQL` at a non-boundary-aligned cut produces a 120-byte, deliberately + * invalid-UTF-8 result (`utf8.ValidString` reports `false`), while + * `Buffer.from(sql,"utf-8").subarray(...).toString("utf-8")` on that exact byte range + * decodes+re-encodes to a 121-byte result containing U+FFFD instead. Keeping this a `Buffer` + * all the way to `output.rawBytes` (see {@link legacyFormatApplyFailure}) avoids that + * lossy string round-trip and reproduces Go's bytes exactly, valid or not. + */ +function legacyFormatStatementSql(sql: string): Buffer { + const normalized = sql + .split(/\s+/u) + .filter((part) => part.length > 0) + .join(" "); + const maxLen = 120; + const normalizedBytes = Buffer.from(normalized, "utf-8"); + if (normalizedBytes.byteLength <= maxLen) return normalizedBytes; + return Buffer.concat([normalizedBytes.subarray(0, maxLen - 3), Buffer.from("...", "utf-8")]); +} + +/** + * Joins Buffer "lines" with `\n` — a Buffer-safe equivalent of `Array#join("\n")`, used so + * {@link legacyFormatApplyIssue}/{@link legacyFormatApplyFailure} can embed + * {@link legacyFormatStatementSql}'s raw (possibly invalid-UTF-8) bytes without ever + * decoding them back into a JS string. + */ +function legacyJoinLines(lines: ReadonlyArray): Buffer { + const newline = Buffer.from("\n", "utf-8"); + const parts: Array = []; + lines.forEach((line, index) => { + if (index > 0) parts.push(newline); + parts.push(line); + }); + return Buffer.concat(parts); +} + +/** + * Go's `json.Indent` (`encoding/json/indent.go`): re-flows compact/pretty JSON by inserting + * whitespace between tokens ONLY — every token (string, number, `true`/`false`/`null`) is + * copied byte-for-byte from `src`, never decoded into a value and re-encoded. This is NOT the + * same as `JSON.parse` + `JSON.stringify`: parsing a number decodes it into a JS `float64`, + * which silently loses precision for an integer literal beyond + * `Number.MAX_SAFE_INTEGER` (e.g. a snowflake-style id), and re-stringifying a string + * re-escapes it using `JSON.stringify`'s own rules, which can change an existing escape's + * representation (e.g. `\/` becomes a literal `/`) — both would corrupt the exact debug + * payload users are asked to attach to bug reports. `legacyGoJsonIndentTokens` instead scans + * `src` as a token stream (only tracking string boundaries, via backslash-escape skipping, to + * avoid misreading punctuation inside a string as structural) and reproduces Go's exact + * spacing rules: verified empirically against `encoding/json.Indent` for nested objects/ + * arrays, empty `{}`/`[]` (no inserted newline), a `\/`-escaped string, an emoji (multi-UTF-16 + * code point) string, and an integer literal beyond `Number.MAX_SAFE_INTEGER` — all byte- + * identical to Go's own output. Caller ({@link legacyFormatDebugJson}) is responsible for + * validating `src` is well-formed JSON first; this function assumes it and does not itself + * detect malformed input. + */ +function legacyGoJsonIndentTokens(src: string): string { + let out = ""; + let depth = 0; + let needIndent = false; + let i = 0; + const n = src.length; + const newline = (): void => { + out += `\n${" ".repeat(depth)}`; + }; + const openIndentIfNeeded = (): void => { + if (!needIndent) return; + needIndent = false; + depth++; + newline(); + }; + while (i < n) { + const c = src[i]; + if (c === " " || c === "\t" || c === "\r" || c === "\n") { + i++; + continue; + } + if (c === '"') { + const start = i; + i++; + while (i < n) { + if (src[i] === "\\") { + i += 2; + continue; + } + if (src[i] === '"') { + i++; + break; + } + i++; + } + openIndentIfNeeded(); + out += src.slice(start, i); + continue; + } + if (c === "{" || c === "[") { + openIndentIfNeeded(); + out += c; + needIndent = true; + i++; + continue; + } + if (c === "}" || c === "]") { + if (needIndent) { + needIndent = false; + } else { + depth--; + newline(); + } + out += c; + i++; + continue; + } + if (c === ",") { + openIndentIfNeeded(); + out += c; + newline(); + i++; + continue; + } + if (c === ":") { + openIndentIfNeeded(); + out += ": "; + i++; + continue; + } + openIndentIfNeeded(); + out += c; + i++; + } + return out; +} + +/** + * Go's `formatDebugJSON` (`apply.go:285-294`): pretty-print if parseable, else the trimmed raw + * bytes. `JSON.parse` here is used ONLY as a well-formedness check (its result is discarded); + * the actual reformatting goes through {@link legacyGoJsonIndentTokens} so token values are + * never decoded and re-encoded — see that function's own doc comment for why + * `JSON.stringify(JSON.parse(...))` would corrupt the payload Go's `json.Indent` preserves. + */ +export function legacyFormatDebugJson(raw: string): string { + const trimmed = raw.trim(); + if (trimmed.length === 0) return ""; + try { + JSON.parse(trimmed); + } catch { + return trimmed; + } + return legacyGoJsonIndentTokens(trimmed); +} + +/** Go's `formatApplyIssueMessage` (`apply.go:222-238`). `String(x ?? "")` throughout — see {@link legacyFormatApplyIssue}'s own doc comment for why. */ +function legacyFormatApplyIssueMessage(issue: LegacyPgDeltaApplyIssue): string { + const trimmed = String(issue.message ?? "").trim(); + const message = trimmed.length > 0 ? trimmed : "unknown pg-delta issue"; + const metadata: Array = []; + const code = String(issue.code ?? ""); + if (code.length > 0) metadata.push(`SQLSTATE ${code}`); + if ((issue.position ?? 0) > 0) metadata.push(`position ${issue.position}`); + if (issue.isDependencyError === true) metadata.push("dependency error"); + if (metadata.length === 0) return message; + return `${message} (${metadata.join(", ")})`; +} + +/** + * Go's `formatApplyIssue` (`apply.go:202-221`). Every `issue.statement.*`/`issue.*` field is + * defaulted with `String(x ?? "")` before use — not a bare `?? ""`: a malformed subprocess + * payload (e.g. a pg-delta release that reports `detail`/`hint`/`sql` as a number) can hand any + * of these a non-string value, which `?? ""` alone does not catch (it only substitutes + * `null`/`undefined`), and the very next call on several of these fields is a string-only + * method (`.trim()`, `legacyFormatStatementSql`'s `.split()`) that throws a `TypeError` on + * anything else — turning an actionable SQL error into an unhandled defect, the worst place for + * a rendering bug to exist, since this only ever runs on an ALREADY-FAILED apply. + * + * The no-statement guard checks both `undefined` and `null`: Go's `Statement *ApplyStatement` + * is a pointer, so `{"statement":null,...}` unmarshals to `nil` and `issue.Statement == nil` + * (`apply.go:202`) treats it exactly like a missing field. A `JSON.parse`'d `null` is not + * `=== undefined`, so checking only `undefined` would fall through to `issue.statement.*` and + * throw a `TypeError` instead of rendering the message. + * + * Returns a `Buffer`, not a `string`: the `SQL: ` line embeds {@link legacyFormatStatementSql}'s + * raw bytes directly (via {@link legacyJoinLines}) rather than interpolating them into a + * template string, so a truncation that lands mid-codepoint reaches `output.rawBytes` + * unmodified instead of being silently corrupted by a UTF-8 decode/re-encode round-trip. + */ +function legacyFormatApplyIssue(rawIssue: LegacyPgDeltaApplyIssue | string | null): Buffer { + const issue = legacyNormalizeApplyIssue(rawIssue); + if (issue.statement === undefined || issue.statement === null) { + return Buffer.from(`- ${legacyFormatApplyIssueMessage(issue)}`, "utf-8"); + } + const statementClass = String(issue.statement.statementClass ?? ""); + const classSuffix = statementClass.length > 0 ? ` [${statementClass}]` : ""; + const lines: Array = [ + Buffer.from(`- ${String(issue.statement.id ?? "")}${classSuffix}`, "utf-8"), + Buffer.from(` ${legacyFormatApplyIssueMessage(issue)}`, "utf-8"), + ]; + const detail = String(issue.detail ?? "").trim(); + if (detail.length > 0) lines.push(Buffer.from(` Detail: ${detail}`, "utf-8")); + const hint = String(issue.hint ?? "").trim(); + if (hint.length > 0) lines.push(Buffer.from(` Hint: ${hint}`, "utf-8")); + const sql = legacyFormatStatementSql(String(issue.statement.sql ?? "")); + if (sql.byteLength > 0) { + lines.push(Buffer.concat([Buffer.from(" SQL: ", "utf-8"), sql])); + } + return legacyJoinLines(lines); +} + +/** Go's `formatApplyDiagnosis` (`apply.go:240-258`). `String(x ?? "")` throughout — see {@link legacyFormatApplyIssue}'s own doc comment for why. */ +function legacyFormatApplyDiagnosis(rawDiagnosis: LegacyPgDeltaApplyDiagnosis | null): string { + const diagnosis = legacyNormalizeApplyDiagnosis(rawDiagnosis); + const trimmed = String(diagnosis.message ?? "").trim(); + const message = trimmed.length > 0 ? trimmed : "unknown pg-delta diagnostic"; + let out = "- "; + const code = String(diagnosis.code ?? "").trim(); + if (code.length > 0) out += `[${code}] `; + out += message; + const loc = legacyFormatStatementLocation(diagnosis.statementId); + if (loc.length > 0) out += ` (${loc})`; + const fix = String(diagnosis.suggestedFix ?? "").trim(); + if (fix.length > 0) out += `\n Suggested fix: ${fix}`; + return out; +} + +/** + * Port of Go's `formatApplyFailure` (`apply.go:145-183`): a human-readable summary of an + * unsuccessful pg-delta apply, rendered on failure regardless of `--debug`. `verbose` + * (Go's `viper.GetBool("DEBUG")`) only expands pg-topo diagnostics inline — collapsed to a + * one-line count by default since a large schema can produce hundreds of them. + * + * Returns a `Buffer`, not a `string` — see {@link legacyFormatStatementSql}'s doc comment: + * an embedded truncated SQL statement can be intentionally invalid UTF-8 (matching Go's raw + * byte slice), and only a `Buffer` carried through to `output.rawBytes` reproduces those + * exact bytes instead of a lossy decode/re-encode round-trip. Callers that only need the + * text for display/assertions (this module's own unit tests) can `.toString("utf-8")` it — + * safe for every case except the one pathological truncation this return type exists to + * preserve exactly. + */ +export function legacyFormatApplyFailure( + result: LegacyPgDeltaApplyResult, + verbose: boolean, +): Buffer { + const errors = result.errors ?? []; + const stuckStatements = result.stuckStatements ?? []; + const validationErrors = result.validationErrors ?? []; + const diagnostics = result.diagnostics ?? []; + + let totalStatements = result.totalStatements ?? 0; + if (totalStatements === 0) { + totalStatements = + (result.totalApplied ?? 0) + (result.totalSkipped ?? 0) + stuckStatements.length; + } + + const lines: Array = [ + Buffer.from(`pg-delta apply returned status "${result.status}".`, "utf-8"), + Buffer.from( + `${result.totalApplied ?? 0}/${totalStatements} statements applied in ${ + result.totalRounds ?? 0 + } round(s); ${result.totalSkipped ?? 0} skipped.`, + "utf-8", + ), + ]; + if (errors.length > 0) { + lines.push(Buffer.from("Errors:", "utf-8")); + for (const issue of errors) lines.push(legacyFormatApplyIssue(issue)); + } + if (stuckStatements.length > 0) { + lines.push(Buffer.from("Stuck statements:", "utf-8")); + for (const issue of stuckStatements) lines.push(legacyFormatApplyIssue(issue)); + } + if (validationErrors.length > 0) { + lines.push(Buffer.from("Validation errors (from check_function_bodies=on pass):", "utf-8")); + for (const issue of validationErrors) lines.push(legacyFormatApplyIssue(issue)); + } + if (diagnostics.length > 0) { + if (verbose) { + lines.push(Buffer.from("Diagnostics:", "utf-8")); + for (const diagnosis of diagnostics) { + lines.push(Buffer.from(legacyFormatApplyDiagnosis(diagnosis), "utf-8")); + } + } else { + lines.push( + Buffer.from( + `${diagnostics.length} pg-topo diagnostic(s) omitted (re-run with --debug to view).`, + "utf-8", + ), + ); + } + } + // pg-delta may report status "error" without populating any issue arrays (e.g. an internal + // assertion in a future pg-delta release) — point the user at how to get more information + // rather than leaving them with just the bare status line. + if (errors.length === 0 && stuckStatements.length === 0 && validationErrors.length === 0) { + lines.push( + Buffer.from( + [ + "No per-statement diagnostics were reported by pg-delta.", + "Re-run with --debug to print the raw pg-delta payload, or open an issue at", + "https://github.com/supabase/pg-toolbelt/issues with the debug bundle attached.", + ].join("\n"), + "utf-8", + ), + ); + } + return legacyJoinLines(lines); +} + +/** + * Port of Go's `pgdelta.ApplyDeclarative` (`apps/cli-go/internal/pgdelta/apply.go:299-360`): + * applies `declarativeDirAbs` to `target` (the shadow's `contrib_regression` override + * database) via pg-delta's declarative apply engine. Unlike the diff/export/catalog scripts + * (`legacy-pgdelta.ts`), this binds the declarative directory itself read-only at + * `/declarative` rather than mounting the whole project at `/workspace` — Go's own + * `ApplyDeclarative` never needs the wider project tree, only the schema files. `target` is + * always a LOCAL shadow connection (never a remote/Supabase-hosted endpoint), so — unlike + * `legacyDiffPgDelta`'s SOURCE/TARGET — no SSL/CA-bundle preparation applies here, matching + * Go's own plain `"TARGET="+utils.ToPostgresURL(config)` (no TLS handling at all). + */ +export const legacyApplyDeclarativePgDelta = Effect.fnUntraced(function* ( + ctx: LegacyPgDeltaContext, + params: { + readonly fs: FileSystem.FileSystem; + /** Absolute host path to the declarative schema directory. */ + readonly declarativeDirAbs: string; + /** The shadow override database's Postgres URL. */ + readonly target: string; + }, +) { + const exists = yield* params.fs + .exists(params.declarativeDirAbs) + .pipe(Effect.orElseSucceed(() => false)); + if (!exists) { + return yield* Effect.fail( + new LegacyDeclarativeApplyError({ + message: `declarative schema directory not found: ${params.declarativeDirAbs}`, + }), + ); + } + + const output = yield* Output; + const edgeRuntime = yield* LegacyEdgeRuntimeScript; + const debug = yield* LegacyDebugFlag; + + yield* output.raw("Applying declarative schemas via pg-delta...\n", "stderr"); + + const env: Record = { + SCHEMA_PATH: LEGACY_PG_DELTA_APPLY_CONTAINER_SCHEMA_PATH, + TARGET: params.target, + }; + const binds = [ + `${legacyEdgeRuntimeId(ctx.projectId)}:/root/.cache/deno:rw`, + `${params.declarativeDirAbs}:${LEGACY_PG_DELTA_APPLY_CONTAINER_SCHEMA_PATH}:ro`, + ]; + const npm = legacyPgDeltaNpmRegistryOption(); + const result = yield* edgeRuntime + .run({ + script: legacyInterpolatePgDeltaScript(legacyPgDeltaDeclarativeApplyScript, ctx.npmVersion), + env, + binds, + errPrefix: "error running pg-delta script", + extraFiles: npm.extraFiles, + extraEnv: npm.extraEnv, + denoVersion: ctx.denoVersion, + }) + .pipe(Effect.mapError((cause) => new LegacyDeclarativeApplyError({ message: cause.message }))); + + const parsed = yield* Effect.try({ + try: () => { + const raw: unknown = JSON.parse(result.stdout); + if (!legacyIsPgDeltaApplyResult(raw)) { + throw new Error("pg-delta apply output was not a JSON object"); + } + return raw; + }, + catch: (cause) => + new LegacyDeclarativeApplyError({ + message: debug + ? `failed to parse pg-delta apply output: ${errMessage(cause)}\nstdout: ${result.stdout}` + : `failed to parse pg-delta apply output: ${errMessage(cause)}`, + }), + }); + + if (parsed.status !== "success") { + // `output.rawBytes`, not `output.raw`: `legacyFormatApplyFailure` returns a `Buffer` that + // may contain intentionally-invalid trailing UTF-8 bytes (a truncated SQL statement cut + // mid-codepoint, matching Go's raw byte slice) — decoding it into a string here would + // corrupt exactly the bytes that Buffer exists to preserve. See its own doc comment. + yield* output.rawBytes( + Buffer.concat([legacyFormatApplyFailure(parsed, debug), Buffer.from("\n", "utf-8")]), + "stderr", + ); + if (debug) { + const debugJson = legacyFormatDebugJson(result.stdout); + if (debugJson.length > 0) { + yield* output.raw("pg-delta apply result:\n", "stderr"); + yield* output.raw(`${debugJson}\n`, "stderr"); + } + } + return yield* Effect.fail( + new LegacyDeclarativeApplyError({ + message: `pg-delta declarative apply failed with status: ${parsed.status}`, + }), + ); + } + yield* output.raw( + `Applied ${parsed.totalApplied ?? 0} statements in ${parsed.totalRounds ?? 0} round(s).\n`, + "stderr", + ); +}); diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.apply.unit.test.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.apply.unit.test.ts new file mode 100644 index 0000000000..4b58a1986a --- /dev/null +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.apply.unit.test.ts @@ -0,0 +1,395 @@ +import { describe, expect, test } from "vitest"; + +import { + legacyFormatApplyFailure, + legacyFormatDebugJson, + type LegacyPgDeltaApplyDiagnosis, + type LegacyPgDeltaApplyIssue, + type LegacyPgDeltaApplyResult, + type LegacyPgDeltaApplyStatementLocation, +} from "./legacy-pgdelta.apply.ts"; + +describe("legacyFormatApplyFailure", () => { + test("renders the status + counts summary line, with no per-statement sections when there are no issues", () => { + const result: LegacyPgDeltaApplyResult = { + status: "error", + totalStatements: 4, + totalRounds: 2, + totalApplied: 3, + totalSkipped: 1, + }; + const message = legacyFormatApplyFailure(result, false).toString("utf-8"); + expect(message).toContain('pg-delta apply returned status "error".'); + expect(message).toContain("3/4 statements applied in 2 round(s); 1 skipped."); + expect(message).toContain("No per-statement diagnostics were reported by pg-delta."); + expect(message).toContain("https://github.com/supabase/pg-toolbelt/issues"); + }); + + test("derives totalStatements from applied + skipped + stuck when omitted", () => { + const result: LegacyPgDeltaApplyResult = { + status: "error", + totalRounds: 1, + totalApplied: 2, + totalSkipped: 1, + stuckStatements: ["stuck one"], + }; + const message = legacyFormatApplyFailure(result, false).toString("utf-8"); + expect(message).toContain("2/4 statements applied in 1 round(s); 1 skipped."); + }); + + test("renders a structured issue with no `statement` field as its message, with SQLSTATE/position/dependency metadata appended", () => { + const issue: LegacyPgDeltaApplyIssue = { + message: "relation already exists", + code: "42P07", + position: 15, + isDependencyError: true, + }; + const result: LegacyPgDeltaApplyResult = { + status: "error", + totalApplied: 0, + totalRounds: 1, + totalSkipped: 0, + errors: [issue], + }; + const message = legacyFormatApplyFailure(result, false).toString("utf-8"); + expect(message).toContain("Errors:"); + expect(message).toContain( + "- relation already exists (SQLSTATE 42P07, position 15, dependency error)", + ); + }); + + test("renders a genuine bare string issue (Go's ApplyIssue string-arm) as its own message", () => { + const result: LegacyPgDeltaApplyResult = { + status: "error", + totalApplied: 0, + totalRounds: 1, + totalSkipped: 0, + errors: ["relation already exists"], + }; + const message = legacyFormatApplyFailure(result, false).toString("utf-8"); + expect(message).toContain("Errors:\n- relation already exists"); + }); + + test("renders a structured issue with its statement id/class, detail, hint, and truncated SQL", () => { + const issue: LegacyPgDeltaApplyIssue = { + message: "column does not exist", + statement: { + id: "001_add_column", + statementClass: "alter_table", + sql: "alter table t add column c int;", + }, + detail: "Column c was dropped earlier in this plan.", + hint: "Check the plan ordering.", + }; + const result: LegacyPgDeltaApplyResult = { + status: "error", + totalApplied: 0, + totalRounds: 1, + totalSkipped: 0, + errors: [issue], + }; + const message = legacyFormatApplyFailure(result, false).toString("utf-8"); + expect(message).toContain("- 001_add_column [alter_table]"); + expect(message).toContain(" column does not exist"); + expect(message).toContain(" Detail: Column c was dropped earlier in this plan."); + expect(message).toContain(" Hint: Check the plan ordering."); + expect(message).toContain(" SQL: alter table t add column c int;"); + }); + + test("truncates a multibyte SQL statement by UTF-8 bytes, not UTF-16 code units", () => { + // Go's `formatStatementSQL` (`apply.go:277-283`) truncates via `len(normalized)` and + // `normalized[:maxLen-3]`, both of which count/slice raw UTF-8 bytes. 70 repetitions of a + // single 3-byte CJK character is only 70 JS UTF-16 code units (well under the 120-char + // threshold a naive `.length`/`.slice()` guard would use — it would never truncate at all), + // but 210 UTF-8 bytes — well over Go's 120-byte limit. `117 / 3 === 39` lands the byte cut + // exactly on a codepoint boundary, so the expected output is unambiguous. + const sql = "字".repeat(70); + const issue: LegacyPgDeltaApplyIssue = { + message: "boom", + statement: { id: "001_a", sql }, + }; + const result: LegacyPgDeltaApplyResult = { + status: "error", + totalApplied: 0, + totalRounds: 1, + totalSkipped: 0, + errors: [issue], + }; + const message = legacyFormatApplyFailure(result, false).toString("utf-8"); + expect(sql.length).toBeLessThanOrEqual(120); + expect(Buffer.byteLength(sql, "utf-8")).toBe(210); + expect(message).toContain(` SQL: ${"字".repeat(39)}...`); + expect(message).not.toContain(sql); + }); + + test("preserves Go's exact (possibly invalid-UTF-8) truncated bytes when the byte cut lands mid-codepoint", () => { + // Unlike the boundary-aligned CJK-repeat case above, a single leading ASCII byte shifts + // every subsequent 3-byte CJK character by one, so the byte-117 cut now lands ONE byte + // into a character instead of exactly on a boundary — reproducing the pathological case + // where Go's raw `normalized[:117]` slice is intentionally invalid UTF-8. Verified against + // Go's own `formatStatementSQL` (`apply.go:277-283`): slicing this exact byte range + // produces a 120-byte result that `unicode/utf8.ValidString` reports as `false`. A naive + // `Buffer#toString("utf-8")` truncation would instead substitute U+FFFD for the incomplete + // trailing sequence, corrupting the byte-exact stderr contract. + const sql = `a${"字".repeat(60)}`; + const issue: LegacyPgDeltaApplyIssue = { + message: "boom", + statement: { id: "001_a", sql }, + }; + const result: LegacyPgDeltaApplyResult = { + status: "error", + totalApplied: 0, + totalRounds: 1, + totalSkipped: 0, + errors: [issue], + }; + const message = legacyFormatApplyFailure(result, false); + const normalizedBytes = Buffer.from(sql, "utf-8"); + const expectedTruncatedTail = Buffer.concat([ + normalizedBytes.subarray(0, 117), + Buffer.from("...", "utf-8"), + ]); + expect(expectedTruncatedTail.byteLength).toBe(120); + expect( + message.includes(Buffer.concat([Buffer.from(" SQL: ", "utf-8"), expectedTruncatedTail])), + ).toBe(true); + // No replacement character (the tell-tale sign of a lossy UTF-8 decode/re-encode + // round-trip) should ever appear in the output. + expect(message.includes(Buffer.from("�", "utf-8"))).toBe(false); + }); + + test("treats a null errors/stuckStatements/validationErrors/diagnostics array as empty, matching Go's nil-slice decode", () => { + // Go's `encoding/json` accepts a JSON `null` for a `[]T` slice field with no error, + // leaving a nil (zero-length) slice — verified empirically: + // `json.Unmarshal([]byte(\`{"status":"error","errors":null}\`), &r)` returns `err == nil` + // with `len(r.Errors) == 0`. `legacyFormatApplyFailure` itself already treats a JS `null`/ + // `undefined` array as empty via `?? []`; this exercises that the TYPE also tolerates it + // (the earlier structural-guard bug — `legacyIsPgDeltaApplyResult` — is covered by the + // integration test in `legacy-pgdelta.apply.integration.test.ts`, since it isn't exported). + const result: LegacyPgDeltaApplyResult = { + status: "error", + totalApplied: 0, + totalRounds: 1, + totalSkipped: 0, + errors: null, + stuckStatements: null, + validationErrors: null, + diagnostics: null, + }; + const message = legacyFormatApplyFailure(result, false).toString("utf-8"); + expect(message).toContain("No per-statement diagnostics were reported by pg-delta."); + expect(message).not.toContain("Errors:"); + expect(message).not.toContain("Stuck statements:"); + }); + + test("stuck statements and validation errors get their own labeled sections", () => { + const result: LegacyPgDeltaApplyResult = { + status: "error", + totalApplied: 0, + totalRounds: 1, + totalSkipped: 0, + stuckStatements: ["still stuck"], + validationErrors: ["bad function body"], + }; + const message = legacyFormatApplyFailure(result, false).toString("utf-8"); + expect(message).toContain("Stuck statements:\n- still stuck"); + expect(message).toContain( + "Validation errors (from check_function_bodies=on pass):\n- bad function body", + ); + }); + + test("diagnostics collapse to a one-line count unless verbose", () => { + const result: LegacyPgDeltaApplyResult = { + status: "error", + totalApplied: 1, + totalRounds: 1, + totalSkipped: 0, + errors: ["some error"], + diagnostics: [{ message: "unused index" }, { message: "missing default" }], + }; + const collapsed = legacyFormatApplyFailure(result, false).toString("utf-8"); + expect(collapsed).toContain("2 pg-topo diagnostic(s) omitted (re-run with --debug to view)."); + expect(collapsed).not.toContain("unused index"); + + const verbose = legacyFormatApplyFailure(result, true).toString("utf-8"); + expect(verbose).toContain("Diagnostics:"); + expect(verbose).toContain("- unused index"); + expect(verbose).toContain("- missing default"); + }); + + test("renders a partially-populated statement (missing sql/statementClass) without throwing", () => { + // Reproduces feeding a real pg-delta subprocess's malformed stdout + // (`{"errors":[{"message":"boom","statement":{"id":"s1"}}]}`) through + // `legacyApplyDeclarativePgDelta` — that function only validates the top-level shape + // (`{status: string}`), not nested fields, and this only ever runs on an + // ALREADY-FAILED apply, so a formatter crash here would turn an actionable SQL error + // into an unhandled defect. + const parsed = JSON.parse( + '{"status":"error","totalApplied":0,"totalRounds":1,"totalSkipped":0,"errors":[{"message":"boom","statement":{"id":"s1"}}]}', + ) as LegacyPgDeltaApplyResult; + expect(() => legacyFormatApplyFailure(parsed, false).toString("utf-8")).not.toThrow(); + const message = legacyFormatApplyFailure(parsed, false).toString("utf-8"); + expect(message).toContain("- s1"); + expect(message).toContain(" boom"); + expect(message).not.toContain("undefined"); + }); + + test("renders an issue with a null `statement` field as its message, without throwing", () => { + // Reproduces feeding a real pg-delta subprocess's stdout + // (`{"errors":[{"statement":null,"message":"failed"}]}`) through + // `legacyApplyDeclarativePgDelta` — Go's `Statement *ApplyStatement` is a pointer, so + // `"statement":null` unmarshals to `nil` and `formatApplyIssue`'s `issue.Statement == nil` + // (`apply.go:202`) treats it identically to a missing field. A no-statement guard that only + // checks `=== undefined` would fall through to `issue.statement.statementClass` on `null` + // and throw a `TypeError` instead of rendering the message. + const parsed = JSON.parse( + '{"status":"error","totalApplied":0,"totalRounds":1,"totalSkipped":0,"errors":[{"statement":null,"message":"failed"}]}', + ) as LegacyPgDeltaApplyResult; + expect(() => legacyFormatApplyFailure(parsed, false).toString("utf-8")).not.toThrow(); + const message = legacyFormatApplyFailure(parsed, false).toString("utf-8"); + expect(message).toContain("Errors:\n- failed"); + }); + + test("renders an issue whose detail/hint/sql/statementClass arrived as non-strings without throwing", () => { + // A malformed pg-delta payload can hand any of these fields a non-string value (e.g. a + // future release that reports a numeric `detail`) — a bare `?? ""` guard (rather than + // `String(x ?? "")`) would still pass the number straight to `.trim()`/`.split()` and throw. + const parsed = JSON.parse( + '{"status":"error","totalApplied":0,"totalRounds":1,"totalSkipped":0,"errors":[{"message":"boom","statement":{"id":"s1","statementClass":42,"sql":7},"detail":123,"hint":456}]}', + ) as LegacyPgDeltaApplyResult; + expect(() => legacyFormatApplyFailure(parsed, false).toString("utf-8")).not.toThrow(); + const message = legacyFormatApplyFailure(parsed, false).toString("utf-8"); + expect(message).toContain("- s1 [42]"); + expect(message).toContain(" Detail: 123"); + expect(message).toContain(" Hint: 456"); + expect(message).toContain(" SQL: 7"); + }); + + test("renders a diagnosis whose message/code/suggestedFix arrived as non-strings without throwing", () => { + const parsed = JSON.parse( + '{"status":"error","totalApplied":0,"totalRounds":1,"totalSkipped":0,"errors":["e"],"diagnostics":[{"message":123,"code":456,"suggestedFix":789}]}', + ) as LegacyPgDeltaApplyResult; + expect(() => legacyFormatApplyFailure(parsed, true).toString("utf-8")).not.toThrow(); + const message = legacyFormatApplyFailure(parsed, true).toString("utf-8"); + expect(message).toContain("[456] 123"); + expect(message).toContain("Suggested fix: 789"); + }); + + test("drops a diagnosis's statementId when a nested field is mistyped, matching Go's nil fallback", () => { + // Go's `(d *ApplyDiagnosis) UnmarshalJSON` (`apply.go:79-108`) tries decoding `statementId` + // as an `ApplyStatementLocation` object first; a mistyped `filePath` (a number, not a + // string) fails that decode, and its bare-string fallback ALSO fails since the value is an + // object, not a string — so Go silently leaves `StatementID` nil, never erroring the whole + // `ApplyResult` parse. Verified empirically against Go's real struct + fallback chain: + // `{"statementId":{"filePath":123,"statementIndex":1}}` decodes with `StatementID == nil`. + // Rendering the raw object anyway (coercing `filePath` via `String(123)`) would show a + // bogus `(123#1)` location Go never emits. + const parsed = JSON.parse( + '{"status":"error","totalApplied":0,"totalRounds":1,"totalSkipped":0,"errors":["e"],"diagnostics":[{"message":"d","statementId":{"filePath":123,"statementIndex":1}}]}', + ) as LegacyPgDeltaApplyResult; + expect(() => legacyFormatApplyFailure(parsed, true).toString("utf-8")).not.toThrow(); + const message = legacyFormatApplyFailure(parsed, true).toString("utf-8"); + expect(message).toContain("- d"); + expect(message).not.toContain("123#1"); + expect(message).not.toContain("(123"); + }); + + test("renders a diagnosis with a null statementId as having no location, without throwing", () => { + // Reproduces a real pg-delta subprocess emitting + // `{"diagnostics":[{"message":"failed","statementId":null}]}` — Go's + // `(d *ApplyDiagnosis) UnmarshalJSON` (`apply.go:79-108`) explicitly maps a JSON + // `"statementId":null` to a nil `*ApplyStatementLocation`, and `formatStatementLocation` + // (`apply.go:263-274`) returns `""` for a nil pointer. A guard that only checked + // `resolved === undefined` (not `null`) would fall through to + // `legacyFormatStatementLocation`'s `resolved.filePath` and dereference a `null`, throwing a + // `TypeError` instead of rendering the rest of the diagnostic. + const parsed = JSON.parse( + '{"status":"error","totalApplied":0,"totalRounds":1,"totalSkipped":0,"errors":["e"],"diagnostics":[{"message":"failed","statementId":null}]}', + ) as LegacyPgDeltaApplyResult; + expect(() => legacyFormatApplyFailure(parsed, true).toString("utf-8")).not.toThrow(); + const message = legacyFormatApplyFailure(parsed, true).toString("utf-8"); + expect(message).toContain("- failed"); + expect(message).not.toContain("undefined"); + }); + + test("a diagnosis with a statementId location and suggestedFix renders both", () => { + const statementId: LegacyPgDeltaApplyStatementLocation = { + filePath: "001_a.sql", + statementIndex: 2, + }; + const diagnosis: LegacyPgDeltaApplyDiagnosis = { + code: "PGT001", + message: "circular dependency", + statementId, + suggestedFix: "Split the statement across two files.", + }; + const result: LegacyPgDeltaApplyResult = { + status: "error", + totalApplied: 1, + totalRounds: 1, + totalSkipped: 0, + errors: ["some error"], + diagnostics: [diagnosis], + }; + const message = legacyFormatApplyFailure(result, true).toString("utf-8"); + expect(message).toContain("- [PGT001] circular dependency (001_a.sql#2)"); + expect(message).toContain("Suggested fix: Split the statement across two files."); + }); +}); + +describe("legacyFormatDebugJson", () => { + test("pretty-prints valid JSON", () => { + expect(legacyFormatDebugJson('{"status":"error","totalApplied":1}')).toBe( + JSON.stringify({ status: "error", totalApplied: 1 }, null, 2), + ); + }); + + test("returns the trimmed raw string when it isn't valid JSON", () => { + expect(legacyFormatDebugJson(" not json ")).toBe("not json"); + }); + + test("returns empty for blank input", () => { + expect(legacyFormatDebugJson(" ")).toBe(""); + }); + + test("preserves an integer literal beyond Number.MAX_SAFE_INTEGER byte-for-byte", () => { + // Go's `json.Indent` (`encoding/json/indent.go`) only inserts whitespace between existing + // tokens — it never decodes a number into a value and re-encodes it. `JSON.parse` would + // decode this literal into a `float64`-backed JS number, silently rounding it (verified: + // `JSON.parse("9007199254740993").toString()` is `"9007199254740992"`), and + // `JSON.stringify` would then re-emit the ROUNDED value — corrupting the exact debug + // payload users are asked to attach to bug reports. + const raw = '{"id":9007199254740993}'; + expect(legacyFormatDebugJson(raw)).toBe('{\n "id": 9007199254740993\n}'); + }); + + test("preserves an existing string escape's exact representation (e.g. an escaped forward slash)", () => { + // Go's `json.Indent` copies string tokens byte-for-byte, so an existing `\/` escape stays + // `\/`. `JSON.stringify(JSON.parse(...))` would instead re-escape the decoded `/` using its + // own (unescaped) convention, changing the payload's exact bytes. + const raw = '{"path":"a\\/b"}'; + expect(legacyFormatDebugJson(raw)).toBe('{\n "path": "a\\/b"\n}'); + }); + + test("matches Go's json.Indent shape for nested objects/arrays, including empty ones", () => { + const raw = '{"a":1,"b":{"c":2,"d":[1,{"e":3}]},"empty":{},"emptyArr":[]}'; + expect(legacyFormatDebugJson(raw)).toBe( + [ + "{", + ' "a": 1,', + ' "b": {', + ' "c": 2,', + ' "d": [', + " 1,", + " {", + ' "e": 3', + " }", + " ]", + " },", + ' "empty": {},', + ' "emptyArr": []', + "}", + ].join("\n"), + ); + }); +}); diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.layer.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.layer.ts index 5c453dabb4..78e3f26f4c 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.layer.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.layer.ts @@ -5,7 +5,7 @@ import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner import { LegacyNetworkIdFlag, LegacyProfileFlag } from "../../../../shared/legacy/global-flags.ts"; import { resolveBinary } from "../../../../shared/legacy/go-proxy.layer.ts"; import { LegacyCliConfig } from "../../../config/legacy-cli-config.service.ts"; -import { containerCliExitCode, spawnContainerCli } from "../../../shared/legacy-container-cli.ts"; +import { spawnContainerCli } from "../../../shared/legacy-container-cli.ts"; import { legacyResolveDbImage } from "../../../shared/legacy-db-image.ts"; import { legacyReadDbToml } from "../../../shared/legacy-db-config.toml-read.ts"; import { legacyGetRegistryImageUrl } from "../../../shared/legacy-docker-registry.ts"; @@ -14,8 +14,7 @@ import { localDbContainerId, } from "../../../shared/legacy-docker-ids.ts"; import { LegacyDeclarativeShadowDbError } from "./legacy-pgdelta.errors.ts"; -import { LegacyDeclarativeSeam, type LegacyShadowSource } from "./legacy-pgdelta.seam.service.ts"; -import { legacyInjectPostgresPassword } from "./legacy-pgdelta.seam.url.ts"; +import { LegacyDeclarativeSeam } from "./legacy-pgdelta.seam.service.ts"; /** * Real `LegacyDeclarativeSeam`: runs the bundled `supabase-go`'s hidden @@ -75,7 +74,9 @@ export const legacyDeclarativeSeamLayer = Layer.effect( // calls `flags.LoadConfig` directly without `LoadProjectRef`, so the // env (read only by LoadProjectRef) never reaches the merge — the Go // command seeds `flags.ProjectRef` from `--project-ref` before - // LoadConfig instead (mirrors `db __shadow`). + // LoadConfig instead (the same trick the Go `db __shadow` hidden + // command used to use, before CLI-1956 removed it in favor of a + // native shadow-provisioning port). ...(projectRef !== undefined ? ["--project-ref", projectRef] : []), ...profileArgs, ]; @@ -379,128 +380,6 @@ export const legacyDeclarativeSeamLayer = Layer.effect( ); }), ), - provisionShadow: ({ mode, targetLocal, usePgDelta, schema, projectRef }) => - Effect.scoped( - Effect.gen(function* () { - if (!("found" in resolved)) { - return yield* Effect.fail( - new LegacyDeclarativeShadowDbError({ - message: - "Could not find the supabase-go binary required to provision the shadow database.", - }), - ); - } - const args = [ - "db", - "__shadow", - "--mode", - mode, - ...(targetLocal ? ["--target-local"] : []), - ...(usePgDelta ? ["--use-pg-delta"] : []), - ...(schema.length > 0 ? ["--schema", schema.join(",")] : []), - ...(Option.isSome(networkId) ? ["--network-id", networkId.value] : []), - // Linked path only: pass the resolved ref so the hidden `db __shadow` - // child's LoadConfig merges the matching `[remotes.]` override - // into the shadow baseline (db.major_version, service enables, vault), - // matching the Go monolith which builds the shadow from the - // remote-merged config. A flag (not env) keeps the Go-proxy channel - // parity and avoids over-merging on local/db-url shadows. - ...(projectRef !== undefined ? ["--project-ref", projectRef] : []), - ...profileArgs, - ]; - const command = ChildProcess.make(resolved.found, args, { - cwd: cliConfig.workdir, - stdin: "inherit", - stdout: "pipe", - stderr: "inherit", - extendEnv: true, - // Disable the child's telemetry so the hidden `db __shadow` seam - // doesn't record its own `cli_command_executed` (and run Go post-run - // work) on top of the user's TS command, matching the explicit - // LegacyGoProxy delegates which set the same env. - env: { SUPABASE_TELEMETRY_DISABLED: "1" }, - detached: false, - }); - const handle = yield* spawner.spawn(command).pipe( - Effect.mapError( - () => - new LegacyDeclarativeShadowDbError({ - message: "failed to run the shadow-database provisioner (supabase-go).", - }), - ), - ); - const chunks: Array = []; - yield* Stream.runForEach(handle.stdout, (chunk) => - Effect.sync(() => { - chunks.push(chunk); - }), - ).pipe(Effect.mapError(() => failure())); - const exitCode = yield* handle.exitCode.pipe(Effect.mapError(() => failure())); - if (exitCode !== 0) { - return yield* Effect.fail(failure(exitCode)); - } - const total = chunks.reduce((size, chunk) => size + chunk.length, 0); - const bytes = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - bytes.set(chunk, offset); - offset += chunk.length; - } - // stdout is three newline-separated lines: container id, source URL, - // and an optional target-override URL (empty unless the local-target - // declarative branch redirected the target to a second shadow db). - // The URLs arrive WITHOUT a password — the Go seam prints them via - // ToPostgresURLWithoutPassword so it never logs a credential to stdout - // (CWE-312). The shadow uses the local Postgres password, so we re-inject - // the password resolved from config.toml before handing the URLs to the - // differ / sql-pg connection. On the linked path the child built the - // shadow from the remote-merged config (via --project-ref), so re-read - // with the same ref to pick up a `[remotes.].db.password` override — - // otherwise the injected password wouldn't match the shadow's and the - // connection would fail auth. Absent (local/db-url) → base config. - const lines = new TextDecoder().decode(bytes).split(/\r?\n/u); - const container = (lines[0] ?? "").trim(); - const sourceUrl = (lines[1] ?? "").trim(); - const targetOverride = (lines[2] ?? "").trim(); - if (container.length === 0 || sourceUrl.length === 0) { - return yield* Effect.fail(failure()); - } - const password = yield* legacyReadDbToml(fs, path, cliConfig.workdir, projectRef).pipe( - Effect.map((toml) => toml.password), - Effect.mapError( - () => - new LegacyDeclarativeShadowDbError({ - message: - "failed to read the local database password from config.toml to connect to the shadow database.", - }), - ), - ); - return { - container, - sourceUrl: legacyInjectPostgresPassword(sourceUrl, password), - targetUrlOverride: - targetOverride.length > 0 - ? legacyInjectPostgresPassword(targetOverride, password) - : undefined, - } satisfies LegacyShadowSource; - }), - ), - removeShadowContainer: (container) => - Effect.gen(function* () { - if (container.length === 0) return; - // Remove the shadow left running by provisionShadow. Best-effort — a - // failure here must never mask the diff result. `-v` removes the - // Postgres anonymous data volume too, matching Go's `DockerRemove` - // (`RemoveOptions{RemoveVolumes: true, Force: true}`, - // `internal/utils/docker.go:330`); without it every shadow leaves a - // dangling volume behind. - yield* containerCliExitCode(spawner, ["rm", "-f", "-v", container], { - stdin: "ignore", - stdout: "ignore", - stderr: "ignore", - extendEnv: true, - }).pipe(Effect.ignore); - }), }); }), ); diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.service.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.service.ts index 16593f5f75..6b26e75384 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.service.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.service.ts @@ -5,29 +5,6 @@ import type { LegacyDeclarativeShadowDbError } from "./legacy-pgdelta.errors.ts" /** Which shadow-database catalog the Go seam should produce. */ export type LegacyCatalogMode = "baseline" | "migrations" | "declarative"; -/** - * Which live shadow database the Go seam should provision and leave running: - * - `diff`: platform baseline + local migrations (the `db diff` / migration-style - * `db pull` diff source), plus the local-target declarative branch. - * - `declarative`: a bare shadow with no baseline/migrations (the `db pull - * --declarative` empty export source). - */ -type LegacyShadowMode = "diff" | "declarative"; - -/** A live shadow database left running for the caller to diff against and remove. */ -export interface LegacyShadowSource { - /** Container id; the caller removes it via `removeShadowContainer` when done. */ - readonly container: string; - /** The diff source Postgres URL (the provisioned shadow). */ - readonly sourceUrl: string; - /** - * When set, replaces the diff target with a second shadow database - * (`contrib_regression` with declarative schemas applied). Mirrors Go's - * local-target declarative branch, where the user's local DB is not diffed. - */ - readonly targetUrlOverride: string | undefined; -} - interface LegacyDeclarativeSeamShape { /** * Provisions the shadow-database platform baseline (and, for @@ -37,8 +14,13 @@ interface LegacyDeclarativeSeamShape { * under `supabase/.temp/pgdelta/`). Go's progress is teed to stderr; only the * catalog path is captured from stdout. * - * This is the seam for `start.SetupDatabase` (the auth/storage/realtime service - * migrations), which is not yet ported to TypeScript. + * The shadow-database provisioning this needs (`start.SetupDatabase`, the + * auth/storage/realtime service migrations) IS now natively ported + * (`legacySetupDatabase`, `shared/db-bootstrap/db-setup.ts`, CLI-1956) — `db diff`/ + * `db pull` no longer go through this Go seam for their own shadow at all (see + * `commands/db/shared/legacy-shadow-source.ts`). This method stays Go-delegated + * only because `db schema declarative generate`/`sync` haven't been natively + * ported yet, not because the underlying shadow primitive is missing. */ readonly exportCatalog: (opts: { readonly mode: LegacyCatalogMode; @@ -82,33 +64,6 @@ interface LegacyDeclarativeSeamShape { void, LegacyDeclarativeShadowDbError >; - /** - * Provisions a live shadow database via the bundled Go binary's hidden - * `db __shadow` command and returns it running (the container is NOT removed — - * the caller must call `removeShadowContainer` when the diff completes). This - * is the diff "source" that both the migra and pg-delta engines run against in - * `db diff` / `db pull`, mirroring Go's `DiffDatabase` (`differ(shadow, target)`). - * Go's shadow-provisioning progress is teed to stderr. - */ - readonly provisionShadow: (opts: { - readonly mode: LegacyShadowMode; - readonly targetLocal: boolean; - readonly usePgDelta: boolean; - readonly schema: ReadonlyArray; - /** - * Resolved linked project ref, passed ONLY on the `--linked` path so the - * shadow merges the matching `[remotes.]` config override (Go builds the - * shadow from the already-remote-merged global config on the linked path). - * Omitted for local/db-url shadows, which Go never remote-merges. - */ - readonly projectRef?: string; - }) => Effect.Effect; - /** - * Removes a shadow database container left running by `provisionShadow` - * (`docker rm -f `). Best-effort: a failure to remove is swallowed so it - * never masks the underlying diff result. - */ - readonly removeShadowContainer: (container: string) => Effect.Effect; } export class LegacyDeclarativeSeam extends Context.Service< diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.url.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.url.ts deleted file mode 100644 index 644586df5d..0000000000 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.url.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Injects the Postgres password into a connection URL that the Go `db __shadow` - * seam emitted WITHOUT one. - * - * The Go seam prints the shadow source/target URLs via - * `ToPostgresURLWithoutPassword` so it never writes a credential to stdout - * (CWE-312). The shadow database always uses the local Postgres password - * (`utils.Config.Db.Password`), which the TS caller resolves independently from - * `config.toml` (`legacyReadDbToml().password`) — so we re-attach it here before - * the URL is handed to the differ (migra / pg-delta) or a sql-pg connection. - * - * The host, port, database, and query params are left exactly as the Go seam - * produced them (Go remains the authority for IPv6 bracketing, `connect_timeout`, - * and runtime params); only the userinfo password is set. The `URL` setter - * percent-encodes the password, matching Go's `url.UserPassword` encoding, and - * the pg driver decodes it back to the same secret. - */ -export function legacyInjectPostgresPassword(connectionUrl: string, password: string): string { - const url = new URL(connectionUrl); - url.password = password; - return url.toString(); -} diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.url.unit.test.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.url.unit.test.ts deleted file mode 100644 index f8298aa30d..0000000000 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.seam.url.unit.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { legacyInjectPostgresPassword } from "./legacy-pgdelta.seam.url.ts"; - -describe("legacyInjectPostgresPassword", () => { - it("injects the password into a password-less IPv4 shadow URL", () => { - expect( - legacyInjectPostgresPassword( - "postgresql://postgres@127.0.0.1:54320/postgres?connect_timeout=10", - "postgres", - ), - ).toBe("postgresql://postgres:postgres@127.0.0.1:54320/postgres?connect_timeout=10"); - }); - - it("preserves IPv6 bracketing, the database name, and query params", () => { - expect( - legacyInjectPostgresPassword( - "postgresql://postgres@[::1]:54320/contrib_regression?connect_timeout=10&options=test", - "postgres", - ), - ).toBe( - "postgresql://postgres:postgres@[::1]:54320/contrib_regression?connect_timeout=10&options=test", - ); - }); - - it("percent-encodes a password with special characters so it round-trips", () => { - const injected = legacyInjectPostgresPassword( - "postgresql://postgres@127.0.0.1:54320/postgres?connect_timeout=10", - "p@ss:w/rd", - ); - expect(injected).toBe( - "postgresql://postgres:p%40ss%3Aw%2Frd@127.0.0.1:54320/postgres?connect_timeout=10", - ); - // The pg driver decodes the userinfo back to the original secret. - expect(decodeURIComponent(new URL(injected).password)).toBe("p@ss:w/rd"); - }); - - it("overwrites any existing userinfo password", () => { - expect( - legacyInjectPostgresPassword( - "postgresql://postgres:stale@127.0.0.1:54320/postgres?connect_timeout=10", - "fresh", - ), - ).toBe("postgresql://postgres:fresh@127.0.0.1:54320/postgres?connect_timeout=10"); - }); -}); diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.ts b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.ts index 4dc3de042f..1e7a8637e7 100644 --- a/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.ts +++ b/apps/cli/src/legacy/commands/db/shared/legacy-pgdelta.ts @@ -127,9 +127,11 @@ export function legacyIsPgDeltaDebugEnabled(): boolean { * Mirrors Go's `PgDeltaNpmRegistryOption` (`internal/utils/pgdelta_local.go:30`): * when `PGDELTA_NPM_REGISTRY` is set, drop a project-local `.npmrc` scoping the * `@supabase` registry and forward both `PGDELTA_NPM_REGISTRY` and the universal - * `NPM_CONFIG_REGISTRY` into the container. + * `NPM_CONFIG_REGISTRY` into the container. Exported so `legacy-pgdelta.apply.ts`'s + * declarative-apply runner (CLI-1956) can reuse the same option, matching every other + * pg-delta edge-runtime invocation in this file. */ -function legacyPgDeltaNpmRegistryOption(): { +export function legacyPgDeltaNpmRegistryOption(): { readonly extraFiles?: ReadonlyArray; readonly extraEnv?: Readonly>; } { diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-shadow-source.ts b/apps/cli/src/legacy/commands/db/shared/legacy-shadow-source.ts new file mode 100644 index 0000000000..f3f69f7458 --- /dev/null +++ b/apps/cli/src/legacy/commands/db/shared/legacy-shadow-source.ts @@ -0,0 +1,739 @@ +/** + * The composed shadow-database shapes `db diff`/`db pull` actually call — Go's + * `PrepareShadowSource`/`PrepareRawShadow` (`apps/cli-go/internal/db/diff/shadow.go`), built + * on top of `shared/db-bootstrap/shadow-database.ts`'s lower-level primitives plus the + * `--target-local` declarative-schema branch (Go's `loadDeclaredSchemas`/ + * `shouldApplyDeclarativeWithPgDelta`/`migrateBaseDatabase`, `internal/db/diff/diff.go:52-115, + * 261-274`) and pg-delta's declarative apply engine (`legacy-pgdelta.apply.ts`). + * + * Go's `PrepareShadowSource(ctx, schema []string, targetLocal, usePgDelta bool, fsys, + * options...)` takes a `schema` parameter that is NEVER referenced anywhere in the function + * body (verified by reading the whole function) — dead code in Go itself, making `db __shadow + * --schema` a no-op. Deliberately NOT ported here: there is nothing to port. + */ + +import { Effect, Option, Result, type FileSystem, type Path } from "effect"; +import type { PlatformError } from "effect/PlatformError"; +import type { GlobalFlag } from "effect/unstable/cli"; +import type * as HttpClient from "effect/unstable/http/HttpClient"; +import type { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"; + +import { Output } from "../../../../shared/output/output.service.ts"; +import type { RuntimeInfo } from "../../../../shared/runtime/runtime-info.service.ts"; +import { legacyBold } from "../../../shared/legacy-colors.ts"; +import type { LegacyEdgeRuntimeScript } from "../../../shared/legacy-edge-runtime-script.service.ts"; +import { + LegacyDbConnection, + type LegacyPgConnInput, +} from "../../../shared/legacy-db-connection.service.ts"; +import { + legacyResolveDeclarativeDir, + legacyResolveSeedSqlPath, + type LegacyPgDeltaTomlConfig, +} from "../../../shared/legacy-db-config.toml-read.ts"; +import type { LegacyDbConfigLoadError } from "../../../shared/legacy-db-config.errors.ts"; +import { legacyResolveUnderWorkdir, legacyGlobPattern } from "../../../shared/legacy-glob.ts"; +import type { LegacyDockerRun } from "../../../shared/legacy-docker-run.service.ts"; +import type { LegacyImagePrepullError } from "../../../shared/containers/image-prepull.ts"; +import type { LegacyHealthCheckTimeoutError } from "../../../shared/containers/health-check.ts"; +import { legacyWaitForHealthyServices } from "../../../shared/containers/health-check.ts"; +import { legacySeedGlobals } from "../../../shared/legacy-migration-apply.ts"; +import { LEGACY_BAD_PATTERN_MESSAGE, legacyPathMatch } from "../../../shared/legacy-path-match.ts"; +import { legacyToPostgresURL } from "../../../shared/legacy-postgres-url.ts"; +import type { LegacyLocalDbContainerInputs } from "../../../shared/db-bootstrap/local-container-inputs.ts"; +import type { LegacyVaultSecret } from "../../../shared/legacy-vault.ts"; +import { + legacyCreateShadowDatabase, + legacyMigrateShadowDatabase, + legacyRemoveShadowDatabase, + LegacyShadowDbError, + type LegacyShadowConnectionInput, + type LegacyShadowDbSetupInput, + type LegacyShadowSourceResult, +} from "../../../shared/db-bootstrap/shadow-database.ts"; +import type { LegacyStartSetupLocalDatabaseError } from "../../../shared/db-bootstrap/db-setup.ts"; +import { + LegacyDeclarativeApplyError, + legacyApplyDeclarativePgDelta, +} from "./legacy-pgdelta.apply.ts"; +import { LegacyDeclarativeShadowDbError } from "./legacy-pgdelta.errors.ts"; +import type { LegacyPgDeltaContext } from "./legacy-pgdelta.ts"; + +type Spawner = ChildProcessSpawner["Service"]; + +export type { LegacyShadowSourceResult }; + +/** + * Adapts {@link LegacyLocalDbContainerInputs} (`local-container-inputs.ts`, the SAME + * config/image/JWKS resolution prelude `db start`/`db reset` share) plus the caller's own + * already-loaded `config.toml` slice into {@link LegacyShadowConnectionInput} + * (`shadow-database.ts`) — every field {@link legacyPrepareShadowSource}/ + * `legacyPrepareRawShadow` (`shadow-database.ts`) need EXCEPT the diff/pull-specific ones + * (`targetLocal`/`usePgDelta`/`schemaPaths`/`pgDelta`/`ctx`/`setup`, left to each call site). + * Hoisted here so `db diff`/`db pull` don't each declare an identical ~20-field object literal. + * + * On `db diff --linked`/`db pull` (linked), the caller passes its own resolved ref straight + * through to {@link legacyBuildLocalDbContainerInputs} (its own `projectRef` parameter — see + * that function's doc comment), which threads it into `legacyLoadLocalProjectContext` -> + * `loadProjectConfig({ projectRef })`. So the shadow's OWN container config (image, JWT + * secret, root key, `db.settings`, service enabled-for-setup flags, sourced from + * `localInputs.context.config`/`postgresSpecBase`) reflects the matching `[remotes.]` + * override, same as `toml` (the caller's own `legacyReadDbToml(..., linkedRef)` result, + * which feeds `pgDelta`/vault/`apiAutoExposeNewTables` below) — matching Go's own uniform + * remote-merge on the linked path (`LoadConfig` seeds `flags.ProjectRef` before every field + * read). The two config reads still go through independent remote-merge implementations + * (`@supabase/config`'s `applyRemoteOverride` for `localInputs.context.config`; + * `legacy-db-config.toml-read.ts`'s own TOML-based merge for `toml`) rather than a single + * shared decode — unifying those is a larger, out-of-scope refactor, not a per-command gap. + */ +export function legacyShadowRunInputFromLocalContainerInputs( + localInputs: LegacyLocalDbContainerInputs, + resolvedImage: string, + toml: { + readonly shadowPort: number; + readonly password: string; + readonly baseline: { readonly apiAutoExposeNewTables: Option.Option }; + readonly vault: ReadonlyArray; + }, + fs: FileSystem.FileSystem, + path: Path.Path, +): Omit< + LegacyPrepareShadowSourceInput, + "targetLocal" | "usePgDelta" | "schemaPaths" | "pgDelta" | "ctx" +> { + const { postgresSpecBase } = localInputs; + return { + db: { + major_version: postgresSpecBase.db.major_version, + settings: postgresSpecBase.db.settings, + }, + experimental: postgresSpecBase.experimental, + jwtSecret: postgresSpecBase.jwtSecret, + jwtExpiry: postgresSpecBase.jwtExpiry, + networkId: localInputs.networkId, + image: resolvedImage, + configImage: postgresSpecBase.configImage, + rootKey: postgresSpecBase.rootKey, + shadowPort: toml.shadowPort, + projectId: localInputs.context.projectId, + isBitbucketPipeline: localInputs.containerOpts.isBitbucketPipeline, + workdir: localInputs.containerOpts.workdir, + extraHosts: localInputs.containerOpts.extraHosts, + fs, + path, + hostname: localInputs.context.hostname, + password: toml.password, + healthTimeoutSeconds: localInputs.dbHealthTimeoutSeconds, + setup: { + majorVersion: localInputs.setup.majorVersion, + config: localInputs.setup.config, + // NOT `localInputs.setup.dbUrl` — that carries the REGULAR local container's own + // hardcoded-"postgres" password (`legacy-local-config-values.ts`'s `DEFAULT_DB_PASSWORD`), + // for a DIFFERENT container. The shadow's own one-shot setup jobs + // (`legacyBuildShadowSetupDatabaseInput`) only ever consume this `dbUrl` to extract a + // password (`legacyStartInternalDbPassword`) for the SHADOW they actually run against, so + // it must carry the SAME resolved `toml.password` the shadow container itself is + // initialized with (see `legacyBuildShadowPostgresContainerSpec`) — otherwise a non-default + // `[db] password` authenticates against the wrong secret and every setup job fails. + dbUrl: legacyToPostgresURL({ + host: localInputs.context.hostname, + port: toml.shadowPort, + user: "postgres", + password: toml.password, + database: "postgres", + }), + jwtSecret: localInputs.setup.jwtSecret, + jwks: localInputs.setup.jwks, + apiUrl: localInputs.setup.apiUrl, + authExternalUrl: localInputs.setup.authExternalUrl, + siteUrl: localInputs.setup.siteUrl, + anonKey: localInputs.setup.anonKey, + serviceRoleKey: localInputs.setup.serviceRoleKey, + storageTargetMigration: localInputs.setup.storageTargetMigration, + realtimeEnabledForSetup: localInputs.setup.realtimeEnabledForSetup, + storageEnabledForSetup: localInputs.setup.storageEnabledForSetup, + authEnabledForSetup: localInputs.setup.authEnabledForSetup, + serviceVersionOverrides: localInputs.setup.serviceVersionOverrides, + projectEnvValues: localInputs.setup.projectEnvValues, + debug: localInputs.setup.debug, + apiAutoExposeNewTables: toml.baseline.apiAutoExposeNewTables, + vault: toml.vault, + }, + }; +} + +export interface LegacyPrepareShadowSourceInput extends LegacyShadowConnectionInput { + readonly setup: LegacyShadowDbSetupInput; + /** Go's `utils.IsLocalDatabase(config)` — the only target-derived input the shadow prep needs. */ + readonly targetLocal: boolean; + /** Selects the declarative-apply engine for the local-declared branch, matching `DiffDatabase`. */ + readonly usePgDelta: boolean; + /** `db.migrations.schema_paths`, RAW (unresolved) — Go's `Config.Db.Migrations.SchemaPaths` pre-`config.go:976-979`-resolution form. */ + readonly schemaPaths: ReadonlyArray; + readonly pgDelta: LegacyPgDeltaTomlConfig; + /** Ambient pg-delta edge-runtime context, only read on the pg-delta declarative-apply sub-branch. */ + readonly ctx: LegacyPgDeltaContext; +} + +/** Every failure {@link legacyPrepareShadowSource} can produce, beyond its own `E` (JWKS resolution). */ +export type LegacyPrepareShadowSourceError = + | LegacyShadowDbError + | LegacyDeclarativeShadowDbError + | LegacyHealthCheckTimeoutError + | LegacyStartSetupLocalDatabaseError + | LegacyImagePrepullError + | LegacyDeclarativeApplyError; + +/** + * Port of Go's `PrepareShadowSource` (`apps/cli-go/internal/db/diff/shadow.go:37-91`): + * create -> health-wait -> `MigrateShadowDatabase` (platform baseline + local migrations + + * the `contrib_regression` template database) -> build the diff-source config -> when + * `targetLocal`, the declarative-schema override branch. On ANY failure after creation the + * shadow container is removed (Go's `ok`-sentinel + `defer` pattern); `Effect.onError` mirrors + * this exactly (fires on a typed failure OR an interrupt, matching Go's defer running + * regardless of *how* the function returns early) rather than `Effect.tapError` (which never + * sees a pure interrupt). + */ +export const legacyPrepareShadowSource = ( + spawner: Spawner, + input: LegacyPrepareShadowSourceInput, +): Effect.Effect< + LegacyShadowSourceResult, + LegacyPrepareShadowSourceError | E, + | Output + | LegacyDockerRun + | RuntimeInfo + | HttpClient.HttpClient + | LegacyDbConnection + | LegacyEdgeRuntimeScript + | GlobalFlag.Setting.Identifier<"debug"> +> => + Effect.gen(function* () { + const { containerId, secretDirId } = yield* legacyCreateShadowDatabase(spawner, input); + + return yield* Effect.gen(function* () { + yield* legacyWaitForHealthyServices(spawner, [containerId], { + timeoutSeconds: input.healthTimeoutSeconds, + }); + + const connConfig: LegacyPgConnInput = { + host: input.hostname, + port: input.shadowPort, + user: "postgres", + password: input.password, + database: "postgres", + }; + yield* legacyMigrateShadowDatabase(spawner, { + fs: input.fs, + path: input.path, + workdir: input.workdir, + projectId: input.projectId, + container: containerId, + networkId: input.networkId, + connConfig, + setup: input.setup, + }); + + const sourceUrl = legacyToPostgresURL(connConfig); + + let targetUrlOverride: string | undefined; + if (input.targetLocal) { + const declared = yield* legacyLoadDeclaredSchemas( + input.fs, + input.path, + input.workdir, + input.schemaPaths, + input.pgDelta, + ); + if (declared.length > 0) { + const overrideConn: LegacyPgConnInput = { ...connConfig, database: "contrib_regression" }; + const useDeclarativePgDelta = legacyShouldApplyDeclarativeWithPgDelta( + input.path, + input.usePgDelta, + input.schemaPaths, + input.pgDelta, + ); + let appliedViaPgDelta = false; + if (useDeclarativePgDelta) { + const declDirRel = legacyResolveDeclarativeDir(input.path, input.pgDelta); + const declDirAbs = legacyResolveUnderWorkdir(input.path, input.workdir, declDirRel); + // Go's `afero.DirExists` (`shadow.go:72`) — a non-directory path is treated as + // absent here too, same reasoning as `legacyLoadDeclaredSchemas` below. + const declDirExists = yield* input.fs.stat(declDirAbs).pipe( + Effect.map((info) => info.type === "Directory"), + Effect.orElseSucceed(() => false), + ); + if (declDirExists) { + yield* legacyApplyDeclarativePgDelta(input.ctx, { + fs: input.fs, + declarativeDirAbs: declDirAbs, + target: legacyToPostgresURL(overrideConn), + }); + appliedViaPgDelta = true; + } + } + if (!appliedViaPgDelta) { + yield* legacyMigrateBaseDatabase( + input.fs, + input.path, + input.workdir, + overrideConn, + declared, + ); + } + targetUrlOverride = legacyToPostgresURL(overrideConn); + } + } + + return { + container: containerId, + secretDirId, + sourceUrl, + targetUrlOverride, + } satisfies LegacyShadowSourceResult; + }).pipe( + Effect.onError(() => + legacyRemoveShadowDatabase(spawner, { + containerId, + secretDirId, + workdir: input.workdir, + }), + ), + ); + }); + +/** Go's `pkg/config.hasGlobMeta` (`config.go:211-213`) — `*?[` only, NOT `io/fs.hasMeta`'s broader set (which also counts `\`). */ +function legacyHasConfigGlobMeta(pattern: string): boolean { + return /[*?[]/u.test(pattern); +} + +/** + * Go's `sort.Strings` compares byte-wise over each string's UTF-8 encoding; JS's default + * `Array.prototype.sort()` instead compares UTF-16 CODE UNITS, which diverges from byte/codepoint + * order for a supplementary-plane character (encoded as a surrogate pair, code units + * `0xD800`-`0xDBFF` + `0xDC00`-`0xDFFF`) alongside a BMP private-use character (`0xE000`- + * `0xFFFF`): JS ranks the surrogate pair BEFORE the private-use character (`0xD800 < 0xE000`), + * while Go's UTF-8 byte order — which preserves Unicode codepoint order — ranks the + * supplementary-plane codepoint (`>= U+10000 > U+FFFF`) AFTER it. Verified empirically: + * `["a\u{1F600}.sql","a.sql"].sort()` (default) disagrees with `Buffer.compare` on the + * same two strings' UTF-8 bytes. Used for every `sort.Strings` this module ports so a schema + * directory with such filenames applies in the same order Go would. + */ +function legacyCompareUtf8Bytes(a: string, b: string): number { + return Buffer.compare(Buffer.from(a, "utf8"), Buffer.from(b, "utf8")); +} + +/** + * Manual, no-follow-symlink directory walk shared by `legacyGlobDeclaredSchemaPaths` (Go's + * `walkMatchedDir`/`fs.WalkDir`) and `legacyWalkSqlFilesSorted` (Go's `afero.Walk`). Both Go + * walkers are `Lstat`-based and therefore never descend into a symlinked directory — + * `io/fs.WalkDir`'s doc comment: "WalkDir does not follow symbolic links found in directories, + * but if root itself is a symbolic link, its target will be walked"; `afero.walk` confirms the + * same via its own `lstatIfPossible` call (`github.com/spf13/afero/path.go`), which reports a + * symlinked subdirectory's `IsDir()` as false so the recursive `walk` call returns without + * descending. Effect's `FileSystem.readDirectory(dir, { recursive: true })` is instead backed by + * Node's recursive `fs.readdir` (`NodeFileSystem.ts`'s `readDirectory` passes `options` straight + * to `fs.promises.readdir`), which DOES follow symlinked subdirectories — verified empirically: a + * directory containing a symlink to an external directory has the external directory's files + * appear in the recursive listing. Left uncorrected, a schema directory symlinking outside the + * configured schema tree would leak external `.sql` files into a local-target diff/pull that Go + * would never have picked up. Walking manually here, one `fs.readDirectory(dir)` (non-recursive) + * per level, and testing each entry with `readLink` BEFORE `stat` (the same no-follow-detector + * idiom as `cp.handler.ts`'s `walkUploadDir`) — skipping a symlinked directory entirely, exactly + * like Lstat-based Go — reproduces that behavior. Both Go walkers also finish with a plain + * `sort.Strings` over the complete set of collected paths (`config.go:186`, + * `internal/db/diff/diff.go:75,95`), which is a full lexicographic sort over full relative paths, + * NOT merely a per-directory-level sort — so the final `.sort()` below is required even though + * entries are already read in sorted order at each level; it uses {@link legacyCompareUtf8Bytes}, + * not JS's default comparator, to match Go's byte order — see that function's own doc comment. + * A per-entry `fs.stat` failure (permission denied, I/O error, a concurrent filesystem change + * between `readDirectory` and `stat`) is NOT swallowed: both Go walkers pass the entry's error to + * their callback, which returns it and aborts the whole walk — silently treating it as "file + * absent" here could build an incomplete declarative target instead. + */ +function legacyWalkRegularSqlFilesNoFollow( + fs: FileSystem.FileSystem, + path: Path.Path, + rootAbs: string, +): Effect.Effect, PlatformError> { + return Effect.gen(function* () { + const result: Array = []; + + const visit = (dirAbs: string, dirRel: string): Effect.Effect => + Effect.gen(function* () { + const names = [...(yield* fs.readDirectory(dirAbs))].sort(); + for (const name of names) { + const entryAbs = path.join(dirAbs, name); + const entryRel = dirRel === "" ? name : `${dirRel}/${name}`; + const isSymlink = yield* fs.readLink(entryAbs).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + if (isSymlink) continue; + const entryStat = yield* fs.stat(entryAbs); + if (entryStat.type === "Directory") { + yield* visit(entryAbs, entryRel); + } else if (entryStat.type === "File" && entryRel.endsWith(".sql")) { + result.push(entryRel); + } + } + }); + + yield* visit(rootAbs, ""); + return result.sort(legacyCompareUtf8Bytes); + }); +} + +/** + * Port of Go's `Glob.SQLFiles(fsys, WithSkipEmptyGlobs(), WithErrorOnAllSkippedGlobs())` + * (`apps/cli-go/pkg/config/config.go:119-192`), the exact option combination + * `loadDeclaredSchemas`'s `schema_paths` branch uses. Deliberately separate from + * `legacy-migrate-and-seed.ts`'s `legacyResolveSchemaPathFiles` (Go's SAME `Glob.SQLFiles` + * with ZERO options, `applySchemaFiles`) — the two option sets are genuinely different: a + * per-pattern "no files matched" is unconditionally an error here UNLESS the pattern + * contains a glob metacharacter (`skipEmptyGlobs`), in which case it's only converted back + * into an error when EVERY pattern ended up skipped and the combined result is still empty + * (`errorOnAllSkippedGlobs`) — and, unlike `applySchemaFiles`'s caller (which swallows any + * collected errors once `len(declared) > 0`), `loadDeclaredSchemas`'s caller propagates + * ANY error unconditionally, regardless of whether other patterns matched. + */ +function legacyGlobDeclaredSchemaPaths( + fs: FileSystem.FileSystem, + path: Path.Path, + workdir: string, + patterns: ReadonlyArray, +): Effect.Effect, LegacyDeclarativeShadowDbError> { + return Effect.gen(function* () { + const seen = new Set(); + const result: Array = []; + const problems: Array = []; + const skipped: Array = []; + + for (const rawPattern of patterns) { + // Go's `config.go:976-979`: a non-empty, non-absolute `schema_paths` entry is resolved + // under `supabase/` (via `path.Join`, which also cleans the result) at config-load + // time — `legacyResolveSeedSqlPath` already implements the identical resolution Go + // applies to `[db.seed] sql_paths`, the same shape. Go's `Glob.files` then normalizes + // to forward slashes immediately before globbing (`fs.Glob(fsys, + // filepath.ToSlash(pattern))`, `config.go:145`) — an absolute Windows entry such as + // `C:\repo\schema.sql` must become `C:/repo/schema.sql` before `legacyPathMatch`/ + // `legacyGlobPattern` (which only recognize `/` as a segment separator) ever see it. + // Mirrors `legacy-seed-ops.ts`'s identical `toSlash` step for `[db.seed] sql_paths`. + // + // NOT gated on `platform === "win32"` the way `legacyCleanSchemaPath` below gates its + // OWN `\`->`/` conversion: verified empirically against the real + // `config.Glob.SQLFiles` (`apps/cli-go/pkg/config/config.go:119-192`) on darwin, fed a + // pattern containing a literal `\` (`supabase/foo\bar/x.sql`) — Go's `path.Match` + // (which `fs.Glob` compiles down to) treats `\` as an ESCAPE metacharacter on every + // platform, not a literal filename byte, so it matched `supabase/foobar/x.sql` (the + // backslash consumed, "b" required literally), never a real directory named `foo\bar`. + // A POSIX build of Go can therefore NEVER glob-match a literal backslash in a + // `schema_paths` entry either way — "preserving" it here wouldn't reproduce that + // escape semantics (this port's own `legacyPathMatch`/`legacyGlobPattern` don't + // segment-join an unslashed `\` the same way Go's chunked matcher does), it would just + // swap one non-parity POSIX result for a different, no-more-correct one. Left + // unconditional pending a real fix to `legacyPathMatch`/`legacyGlobPattern`'s own + // cross-segment escape handling, which is a pre-existing gap in that shared module + // (also used by `[db.seed] sql_paths`), not something specific to this PR's shadow + // provisioning. + const pattern = legacyResolveSeedSqlPath(path, rawPattern).replaceAll("\\", "/"); + if (legacyPathMatch(pattern, "").badPattern) { + problems.push(`failed to glob files: ${LEGACY_BAD_PATTERN_MESSAGE}`); + continue; + } + // Go's `io/fs.Glob` never matches an empty pattern: its literal (no-metacharacter) + // branch calls `Stat(fsys, "")`, which fails on a real OS filesystem (there is no file + // whose path is the empty string), so `Glob` returns zero matches — verified empirically + // against the real `config.Glob.SQLFiles` (`apps/cli-go/pkg/config/config.go:119-133`) + // fed pattern `""` against an `afero.NewOsFs()`: it reports `no files matched pattern: `, + // the same as any other non-matching literal pattern. `legacyGlobPattern`'s own + // literal-pattern branch, however, resolves an empty pattern to the WORKDIR itself + // (`legacyResolveUnderWorkdir(path, workdir, "")` is the workdir, which always exists), + // so without this guard an empty `schema_paths` entry would recurse into and collect + // every `.sql` file in the entire project instead of matching nothing. Short-circuit + // before calling it, rather than fixing `legacyGlobPattern` itself, since that shared + // helper (`legacy-glob.ts`) also backs `[db.seed] sql_paths` (`legacy-seed.ts`) and + // `legacy-migrate-and-seed.ts`, both out of scope for this PR. + // Go's `sort.Strings(matches)` (`config.go:154`) — byte order, not JS's default UTF-16 + // code-unit order; see `legacyCompareUtf8Bytes`'s own doc comment. + const matches = + pattern.length === 0 + ? [] + : [...(yield* legacyGlobPattern(fs, path, workdir, pattern))].sort( + legacyCompareUtf8Bytes, + ); + if (matches.length === 0) { + if (legacyHasConfigGlobMeta(pattern)) { + skipped.push(pattern); + continue; + } + // Go always resolves `SchemaPaths` (`config.go:976-979`) before this error can fire + // (resolution happens at config-load time, ahead of any glob), so the error must show + // the RESOLVED, `supabase/`-prefixed pattern, matching the all-skipped-globs branch + // below — not the raw, caller-supplied one. + problems.push(`no files matched pattern: ${pattern}`); + continue; + } + for (const match of matches) { + const absMatch = legacyResolveUnderWorkdir(path, workdir, match); + const statResult = yield* fs.stat(absMatch).pipe(Effect.result); + if (Result.isFailure(statResult)) { + problems.push(`failed to stat matched file: ${statResult.failure.message}`); + continue; + } + if (statResult.success.type !== "Directory") { + if (!seen.has(match)) { + seen.add(match); + result.push(match); + } + continue; + } + // Go's `walkMatchedDir` (`pkg/config/config.go:194-211`) propagates ANY `fs.WalkDir` + // error (e.g. a permission-denied or I/O-erroring subdirectory) as `failed to walk + // matched directory: ` — it does NOT treat an unreadable directory as an empty + // match set, since silently doing so can omit declared schemas and compare a + // local-target diff against the wrong target. `legacyWalkRegularSqlFilesNoFollow` also + // matches Go's no-follow-symlink walk semantics — see its doc comment. + const sqlRelativeResult = yield* legacyWalkRegularSqlFilesNoFollow(fs, path, absMatch).pipe( + Effect.result, + ); + if (Result.isFailure(sqlRelativeResult)) { + problems.push(`failed to walk matched directory: ${sqlRelativeResult.failure.message}`); + continue; + } + for (const relative of sqlRelativeResult.success) { + const relativeToWorkdir = `${match}/${relative}`; + if (!seen.has(relativeToWorkdir)) { + seen.add(relativeToWorkdir); + result.push(relativeToWorkdir); + } + } + } + } + + if (result.length === 0 && skipped.length > 0) { + for (const pattern of skipped) problems.push(`no files matched pattern: ${pattern}`); + } + if (problems.length > 0) { + return yield* Effect.fail( + new LegacyDeclarativeShadowDbError({ message: problems.join("\n") }), + ); + } + return result; + }); +} + +/** + * Port of Go's `afero.Walk` + regular-`.sql`-file filter + `sort.Strings` (the shared tail of + * both `loadDeclaredSchemas`'s pg-delta-declarative-dir and `SchemasDir` branches, + * `apps/cli-go/internal/db/diff/diff.go:65-76,86-96`). `legacyWalkRegularSqlFilesNoFollow` also + * matches Go's no-follow-symlink walk semantics — see its doc comment. + * + * The walk ROOT itself is checked for being a symlink here, unlike `legacyGlobDeclaredSchemaPaths`'s + * directory branch (Go's `fs.WalkDir`, whose own doc comment says "if root itself is a symbolic + * link, its target will be walked" — so a symlinked `schema_paths` match is deliberately followed, + * matching `legacyWalkRegularSqlFilesNoFollow`'s existing never-checks-its-own-root behavior). + * `afero.Walk` is the opposite: its `Walk(fs, root, walkFn)` entry point `Lstat`s the root BEFORE + * ever calling `walkFn`, so a symlinked root is treated as a non-directory and produces zero files + * silently, never descending into the target — verified against `afero`'s own source + * (`path.go`'s `Walk`/`lstatIfPossible`). The PRECEDING `fs.stat`-based existence check in + * `legacyLoadDeclaredSchemas` (which follows symlinks, matching Go's `afero.DirExists` — also + * `fs.Stat`-based) can't substitute for this: existence and walkability are different checks in + * Go, and only the latter uses `Lstat`. + * + * Paths are joined with the injected `Path` service (not a literal `/` template) so a symlink-free + * result matches Go's own `filepath.Join`-built path on every platform — on Windows this yields + * native backslashes (Go's `afero.Walk` never calls `filepath.ToSlash` on this branch, unlike + * `walkMatchedDir`'s `schema_paths` branch, which does), and `path.join` normalizes ANY `/` + * `legacyWalkRegularSqlFilesNoFollow`'s own relative-path construction produced internally, not + * just the outer `dirRel`/`relative` join (verified: `path.win32.join("supabase/database", + * "sub/dir/file.sql")` returns `"supabase\\database\\sub\\dir\\file.sql"`, not a mixed-separator + * string) — on POSIX this is a no-op (`path.posix.join` is byte-identical to the old template). + * + * `errorPrefix` lets the two callers preserve Go's own DIFFERENT wrapping messages for the same + * walk failure: the pg-delta declarative-dir branch reports `"failed to walk declarative dir: + * %w"` while the `supabase/schemas` fallback reports `"failed to walk dir: %w"` + * (`apps/cli-go/internal/db/diff/diff.go:65-76,86-96` — same walk, genuinely different prefix + * per source), so stderr still identifies which configured source failed. + */ +function legacyWalkSqlFilesSorted( + fs: FileSystem.FileSystem, + path: Path.Path, + workdir: string, + dirRel: string, + errorPrefix: string, +): Effect.Effect, LegacyDeclarativeShadowDbError> { + return Effect.gen(function* () { + const dirAbs = legacyResolveUnderWorkdir(path, workdir, dirRel); + const isSymlinkRoot = yield* fs.readLink(dirAbs).pipe( + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + if (isSymlinkRoot) return []; + const sqlRelative = yield* legacyWalkRegularSqlFilesNoFollow(fs, path, dirAbs).pipe( + Effect.mapError( + (cause) => + new LegacyDeclarativeShadowDbError({ message: `${errorPrefix}: ${cause.message}` }), + ), + ); + return sqlRelative.map((relative) => path.join(dirRel, relative)); + }); +} + +/** + * Port of Go's `loadDeclaredSchemas` (`apps/cli-go/internal/db/diff/diff.go:52-101`): a + * three-source priority ladder — `db.migrations.schema_paths` (when non-empty) -> + * pg-delta's declarative dir (when `[experimental.pgdelta] enabled` AND the dir exists) -> + * `supabase/schemas` (when it exists) -> `[]`. Each source is `sort.Strings`-ordered. + */ +export function legacyLoadDeclaredSchemas( + fs: FileSystem.FileSystem, + path: Path.Path, + workdir: string, + schemaPaths: ReadonlyArray, + pgDelta: LegacyPgDeltaTomlConfig, +): Effect.Effect, LegacyDeclarativeShadowDbError> { + return Effect.gen(function* () { + if (schemaPaths.length > 0) { + return yield* legacyGlobDeclaredSchemaPaths(fs, path, workdir, schemaPaths); + } + if (pgDelta.enabled) { + const declDirRel = legacyResolveDeclarativeDir(path, pgDelta); + const declDirAbs = legacyResolveUnderWorkdir(path, workdir, declDirRel); + // Go's `afero.DirExists` (`diff.go:63`) — a path that exists but is a regular file is + // "not a directory" (`err == nil && exists` is false), not an error, so it falls through + // to the `supabase/schemas` source below rather than being walked as a directory. + const isDeclDir = yield* fs.stat(declDirAbs).pipe( + Effect.map((info) => info.type === "Directory"), + Effect.orElseSucceed(() => false), + ); + if (isDeclDir) { + return yield* legacyWalkSqlFilesSorted( + fs, + path, + workdir, + declDirRel, + "failed to walk declarative dir", + ); + } + } + const schemasDirRel = "supabase/schemas"; + const schemasDirAbs = legacyResolveUnderWorkdir(path, workdir, schemasDirRel); + // Same `afero.DirExists` semantics as above (`diff.go:80`): a missing path or a path that + // exists but isn't a directory both resolve to "no declared schemas" (`[]`), not an error — + // only a genuine stat failure (permission denied, I/O error) propagates. + const isSchemasDir = yield* fs.stat(schemasDirAbs).pipe( + Effect.matchEffect({ + onFailure: (cause) => + cause.reason._tag === "NotFound" + ? Effect.succeed(false) + : Effect.fail( + new LegacyDeclarativeShadowDbError({ + message: `failed to check schemas: ${cause.message}`, + }), + ), + onSuccess: (info) => Effect.succeed(info.type === "Directory"), + }), + ); + if (!isSchemasDir) return []; + return yield* legacyWalkSqlFilesSorted(fs, path, workdir, schemasDirRel, "failed to walk dir"); + }); +} + +/** + * Go's `cleanSchemaPath` (`apps/cli-go/internal/db/diff/diff.go:117-119`): + * `filepath.ToSlash(filepath.Clean(path))`. `filepath.Clean`/`ToSlash` only treat `\` as a path + * separator on the Windows build of the Go CLI (`filepath.Separator == '\\'` there) — on every + * POSIX build (darwin/linux, what this TS binary stands in for on those hosts) a backslash is + * just a literal filename character that survives untouched. Verified empirically: + * `filepath.ToSlash(filepath.Clean(\`supabase/foo\bar\`))` compiled for `GOOS=darwin` returns + * `supabase/foo\bar`, not `supabase/foo/bar`. Gate the separator-normalization on the host + * platform so this matches whichever Go build this TS binary is standing in for. + */ +function legacyCleanSchemaPath( + rawPath: string, + platform: NodeJS.Platform = process.platform, +): string { + const normalized = platform === "win32" ? rawPath.replaceAll("\\", "/") : rawPath; + const isAbsolute = normalized.startsWith("/"); + const out: Array = []; + for (const segment of normalized.split("/")) { + if (segment === "" || segment === ".") continue; + if (segment === "..") { + if (out.length > 0 && out[out.length - 1] !== "..") out.pop(); + else if (!isAbsolute) out.push(".."); + } else { + out.push(segment); + } + } + const joined = out.join("/"); + if (joined.length === 0) return isAbsolute ? "/" : "."; + return isAbsolute ? `/${joined}` : joined; +} + +/** + * Port of Go's `shouldApplyDeclarativeWithPgDelta` (`apps/cli-go/internal/db/diff/diff.go: + * 103-115`): `usePgDelta` false -> false; zero `schema_paths` -> true; more than one + * `schema_paths` entry -> false; exactly one entry -> true only when it resolves (Go's + * `config.go:976-979` resolution, matching `legacyResolveSeedSqlPath`) to the SAME cleaned + * path as the effective declarative dir. + */ +export function legacyShouldApplyDeclarativeWithPgDelta( + path: Path.Path, + usePgDelta: boolean, + schemaPaths: ReadonlyArray, + pgDelta: LegacyPgDeltaTomlConfig, + platform: NodeJS.Platform = process.platform, +): boolean { + if (!usePgDelta) return false; + if (schemaPaths.length === 0) return true; + if (schemaPaths.length !== 1) return false; + const resolvedSchema = legacyCleanSchemaPath( + legacyResolveSeedSqlPath(path, schemaPaths[0]!), + platform, + ); + const declDir = legacyCleanSchemaPath(legacyResolveDeclarativeDir(path, pgDelta), platform); + return resolvedSchema === declDir; +} + +/** + * Port of Go's `migrateBaseDatabase` (`apps/cli-go/internal/db/diff/diff.go:261-274`): prints + * the declarative-schema file list, connects to `config` (the shadow's `contrib_regression` + * override), then seeds `migrations` as globals (Go's `migration.SeedGlobals` — no history + * row, no history table, WITHOUT the migra-engine schema files' own transactional/seed + * distinctions {@link legacySeedGlobals} already reproduces for every other caller of it). + */ +function legacyMigrateBaseDatabase( + fs: FileSystem.FileSystem, + path: Path.Path, + workdir: string, + config: LegacyPgConnInput, + migrations: ReadonlyArray, +): Effect.Effect { + return Effect.scoped( + Effect.gen(function* () { + const output = yield* Output; + yield* output.raw("Creating local database from declarative schemas:\n", "stderr"); + const msg = migrations.map((m) => ` • ${legacyBold(m)}`).join("\n"); + yield* output.raw(`${msg}\n`, "stderr"); + + const dbConnection = yield* LegacyDbConnection; + const session = yield* dbConnection + .connect(config, { isLocal: true, dnsResolver: "native" }) + .pipe( + Effect.mapError( + (cause) => new LegacyDeclarativeShadowDbError({ message: cause.message }), + ), + ); + + const absolutePaths = migrations.map((m) => legacyResolveUnderWorkdir(path, workdir, m)); + yield* legacySeedGlobals( + session, + fs, + path, + absolutePaths, + (message) => new LegacyDeclarativeShadowDbError({ message }), + ); + }), + ); +} diff --git a/apps/cli/src/legacy/commands/db/shared/legacy-shadow-source.unit.test.ts b/apps/cli/src/legacy/commands/db/shared/legacy-shadow-source.unit.test.ts new file mode 100644 index 0000000000..e19bcf1c38 --- /dev/null +++ b/apps/cli/src/legacy/commands/db/shared/legacy-shadow-source.unit.test.ts @@ -0,0 +1,628 @@ +import { chmodSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Exit, FileSystem, Layer, Option, Path, PlatformError } from "effect"; + +import { + legacyLoadDeclaredSchemas, + legacyShouldApplyDeclarativeWithPgDelta, +} from "./legacy-shadow-source.ts"; +import type { LegacyPgDeltaTomlConfig } from "../../../shared/legacy-db-config.toml-read.ts"; + +function pgDelta(overrides: Partial = {}): LegacyPgDeltaTomlConfig { + return { + enabled: false, + declarativeSchemaPath: Option.none(), + formatOptions: Option.none(), + npmVersion: Option.none(), + ...overrides, + }; +} + +function makeWorkdir(): string { + return mkdtempSync(join(tmpdir(), "legacy-shadow-source-")); +} + +// Root bypasses POSIX permission bits, so chmod 000 wouldn't block readdir() there. +const isRoot = typeof process.getuid === "function" && process.getuid() === 0; + +describe("legacyShouldApplyDeclarativeWithPgDelta", () => { + it.effect("is false whenever usePgDelta is false, regardless of schema_paths", () => + Effect.gen(function* () { + const path = yield* Path.Path; + expect(legacyShouldApplyDeclarativeWithPgDelta(path, false, [], pgDelta())).toBe(false); + expect( + legacyShouldApplyDeclarativeWithPgDelta(path, false, ["schemas/x.sql"], pgDelta()), + ).toBe(false); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("is true when usePgDelta and zero schema_paths are configured", () => + Effect.gen(function* () { + const path = yield* Path.Path; + expect(legacyShouldApplyDeclarativeWithPgDelta(path, true, [], pgDelta())).toBe(true); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("is false when more than one schema_paths entry is configured", () => + Effect.gen(function* () { + const path = yield* Path.Path; + expect( + legacyShouldApplyDeclarativeWithPgDelta(path, true, ["a.sql", "b.sql"], pgDelta()), + ).toBe(false); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect( + "is true when exactly one schema_paths entry resolves to the effective declarative dir", + () => + Effect.gen(function* () { + const path = yield* Path.Path; + expect(legacyShouldApplyDeclarativeWithPgDelta(path, true, ["database"], pgDelta())).toBe( + true, + ); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("is false when the single schema_paths entry does not match the declarative dir", () => + Effect.gen(function* () { + const path = yield* Path.Path; + expect(legacyShouldApplyDeclarativeWithPgDelta(path, true, ["schemas"], pgDelta())).toBe( + false, + ); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("matches a configured (non-default) declarative_schema_path the same way", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const configured = pgDelta({ declarativeSchemaPath: Option.some("supabase/custom-decl") }); + expect(legacyShouldApplyDeclarativeWithPgDelta(path, true, ["custom-decl"], configured)).toBe( + true, + ); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect( + "on POSIX, a backslash in schema_paths is a literal character, not a path separator", + () => + Effect.gen(function* () { + const path = yield* Path.Path; + // Go's `filepath.Clean`/`ToSlash` only treat `\` as a separator on a Windows build — + // on darwin/linux it's untouched, so a `foo\bar` schema_paths entry (which + // `legacyResolveSeedSqlPath` joins under `supabase/` unresolved) must NOT be treated + // as equivalent to the slash-separated declarative dir `supabase/foo/bar`. + const configured = pgDelta({ declarativeSchemaPath: Option.some("supabase/foo/bar") }); + expect( + legacyShouldApplyDeclarativeWithPgDelta(path, true, ["foo\\bar"], configured, "darwin"), + ).toBe(false); + }).pipe(Effect.provide(BunServices.layer)), + ); + + it.effect("on win32, a backslash in schema_paths normalizes as a path separator", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const configured = pgDelta({ declarativeSchemaPath: Option.some("supabase/foo/bar") }); + expect( + legacyShouldApplyDeclarativeWithPgDelta(path, true, ["foo\\bar"], configured, "win32"), + ).toBe(true); + }).pipe(Effect.provide(BunServices.layer)), + ); +}); + +describe("legacyLoadDeclaredSchemas", () => { + it.effect( + "returns [] when neither schema_paths, an enabled pg-delta dir, nor supabase/schemas exist", + () => { + const workdir = makeWorkdir(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas(fs, path, workdir, [], pgDelta()); + expect(result).toEqual([]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect( + "falls back to sorted supabase/schemas/*.sql when no schema_paths/pg-delta dir apply", + () => { + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "schemas"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "schemas", "b.sql"), "select 2;\n"); + writeFileSync(join(workdir, "supabase", "schemas", "a.sql"), "select 1;\n"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas(fs, path, workdir, [], pgDelta()); + expect(result).toEqual(["supabase/schemas/a.sql", "supabase/schemas/b.sql"]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect( + "prefers the pg-delta declarative dir over supabase/schemas when pg-delta is enabled and the dir exists", + () => { + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "database"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "database", "t.sql"), "select 1;\n"); + mkdirSync(join(workdir, "supabase", "schemas"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "schemas", "unused.sql"), "select 2;\n"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas( + fs, + path, + workdir, + [], + pgDelta({ enabled: true }), + ); + expect(result).toEqual(["supabase/database/t.sql"]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect( + "prefers db.migrations.schema_paths over both the pg-delta dir and supabase/schemas", + () => { + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "custom"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "custom", "x.sql"), "select 1;\n"); + mkdirSync(join(workdir, "supabase", "database"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "database", "unused.sql"), "select 2;\n"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas( + fs, + path, + workdir, + ["custom/*.sql"], + pgDelta({ enabled: true }), + ); + expect(result).toEqual(["supabase/custom/x.sql"]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect("fails when a literal (non-glob) schema_paths entry matches nothing", () => { + const workdir = makeWorkdir(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const exit = yield* legacyLoadDeclaredSchemas( + fs, + path, + workdir, + ["missing.sql"], + pgDelta(), + ).pipe(Effect.exit); + expect(exit._tag).toBe("Failure"); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }); + + it.effect( + 'an empty schema_paths entry matches nothing, not the entire project (Go\'s fs.Glob(""))', + () => { + // Go's `io/fs.Glob` never matches an empty pattern — its literal-pattern branch calls + // `Stat(fsys, "")`, which fails on a real OS filesystem, so `Glob.SQLFiles` reports + // `no files matched pattern: ` for it (verified empirically against the real + // `config.Glob.SQLFiles` fed `""` over an `afero.NewOsFs()`). Without this guard, + // `legacyGlobPattern`'s literal-pattern branch resolves `""` to the workdir itself + // (which always exists) and recursively collects every `.sql` file in the project, + // including files well outside any declared schema path. + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "migrations"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "migrations", "001_init.sql"), "select 1;\n"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const exit = yield* legacyLoadDeclaredSchemas(fs, path, workdir, [""], pgDelta()).pipe( + Effect.exit, + ); + expect(exit._tag).toBe("Failure"); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect("a glob schema_paths entry matching nothing is silently skipped, not an error", () => { + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "custom"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "custom", "x.sql"), "select 1;\n"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas( + fs, + path, + workdir, + ["custom/*.sql", "empty-glob/*.sql"], + pgDelta(), + ); + expect(result).toEqual(["supabase/custom/x.sql"]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }); + + it.effect( + "normalizes a backslash-separated schema_paths entry before globbing (Go's filepath.ToSlash)", + () => { + // Go calls `fs.Glob(fsys, filepath.ToSlash(pattern))` immediately before globbing + // (`pkg/config/config.go:145`) — a pattern containing `\` (as every absolute Windows + // `schema_paths` entry does) must be forward-slashed first, or `legacyPathMatch`/ + // `legacyGlobPattern` (which only recognize `/` as a segment separator, and treat `\` + // as a glob escape) mis-parse it entirely. + // + // NOT gated on platform, unlike `legacyCleanSchemaPath`'s own win32-only conversion: + // verified empirically against the real `config.Glob.SQLFiles` + // (`apps/cli-go/pkg/config/config.go:119-192`) on darwin that Go's own `path.Match` + // treats `\` as an escape metacharacter on every platform, not a literal filename byte — + // a `foo\bar/x.sql` pattern matched `foobar/x.sql`, never a directory actually named + // `foo\bar`. A POSIX Go build can therefore never glob-match a literal backslash in a + // `schema_paths` entry either way, so "preserving" it here wouldn't reproduce Go's real + // behavior; it would just swap one non-parity result for a different one. + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "custom"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "custom", "x.sql"), "select 1;\n"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas( + fs, + path, + workdir, + ["custom\\x.sql"], + pgDelta(), + ); + expect(result).toEqual(["supabase/custom/x.sql"]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect( + "excludes a symlinked .sql file from a recursively-matched schema_paths directory", + () => { + // Go's `entry.Type().IsRegular()` (`config.go:127`) is a no-follow check — a symlink + // is never "regular", so `walkMatchedDir` excludes it even when it resolves to a real + // `.sql` file. + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "custom"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "custom", "real.sql"), "select 1;\n"); + const secretTarget = join(workdir, "outside.sql"); + writeFileSync(secretTarget, "select 2;\n"); + symlinkSync(secretTarget, join(workdir, "supabase", "custom", "linked.sql")); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas(fs, path, workdir, ["custom"], pgDelta()); + expect(result).toEqual(["supabase/custom/real.sql"]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect("excludes a symlinked .sql file from the supabase/schemas fallback walk", () => { + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "schemas"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "schemas", "real.sql"), "select 1;\n"); + const secretTarget = join(workdir, "outside.sql"); + writeFileSync(secretTarget, "select 2;\n"); + symlinkSync(secretTarget, join(workdir, "supabase", "schemas", "linked.sql")); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas(fs, path, workdir, [], pgDelta()); + expect(result).toEqual(["supabase/schemas/real.sql"]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }); + + it.effect( + "does not follow a symlinked subdirectory in a recursively-matched schema_paths directory", + () => { + // Go's `fs.WalkDir` (`walkMatchedDir`, `config.go:194-211`) is `Lstat`-based and never + // descends into a symlinked directory (`io/fs.WalkDir` doc: "WalkDir does not follow + // symbolic links found in directories") — a schema dir symlinking OUT of the configured + // schema tree must not leak the linked directory's files into the diff/pull target. + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "custom"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "custom", "real.sql"), "select 1;\n"); + const outsideDir = join(workdir, "outside"); + mkdirSync(outsideDir, { recursive: true }); + writeFileSync(join(outsideDir, "secret.sql"), "select 2;\n"); + symlinkSync(outsideDir, join(workdir, "supabase", "custom", "linked-dir"), "dir"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas(fs, path, workdir, ["custom"], pgDelta()); + expect(result).toEqual(["supabase/custom/real.sql"]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect( + "does not follow a symlinked subdirectory in the supabase/schemas fallback walk", + () => { + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "schemas"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "schemas", "real.sql"), "select 1;\n"); + const outsideDir = join(workdir, "outside"); + mkdirSync(outsideDir, { recursive: true }); + writeFileSync(join(outsideDir, "secret.sql"), "select 2;\n"); + symlinkSync(outsideDir, join(workdir, "supabase", "schemas", "linked-dir"), "dir"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas(fs, path, workdir, [], pgDelta()); + expect(result).toEqual(["supabase/schemas/real.sql"]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect( + "falls back to supabase/schemas when the pg-delta declarative path exists but is a regular file", + () => { + // Go's `afero.DirExists` (`apps/cli-go/internal/db/diff/diff.go:63`) treats a non-directory + // path as absent, not present-but-unwalkable — a stray `supabase/database` FILE (e.g. left + // over from a previous config) must fall through to `supabase/schemas`, not make + // `legacyWalkSqlFilesSorted` try (and fail) to read a file as a directory. + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "database"), "not a directory"); + mkdirSync(join(workdir, "supabase", "schemas"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "schemas", "a.sql"), "select 1;\n"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas( + fs, + path, + workdir, + [], + pgDelta({ enabled: true }), + ); + expect(result).toEqual(["supabase/schemas/a.sql"]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect( + "returns [] when supabase/schemas exists but is a regular file, not a directory", + () => { + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "schemas"), "not a directory"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas(fs, path, workdir, [], pgDelta()); + expect(result).toEqual([]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect( + "returns [] (does not follow) when the pg-delta declarative dir itself is a symlink", + () => { + // Go's `afero.Walk(fsys, declDir, ...)` Lstat's the ROOT before ever calling `walkFn` + // (`afero`'s own `Walk`/`lstatIfPossible`) — a symlinked root is treated as a + // non-directory and produces zero files, silently, never descending into the target. + // The PRECEDING `afero.DirExists`-equivalent existence check (which follows symlinks, + // matching Go's own `fs.Stat`-based `DirExists`) reports the symlinked dir as present, so + // only the WALK itself (not the existence check) must reject it. + const workdir = makeWorkdir(); + const realDir = join(workdir, "real-database"); + mkdirSync(realDir, { recursive: true }); + writeFileSync(join(realDir, "t.sql"), "select 1;\n"); + mkdirSync(join(workdir, "supabase"), { recursive: true }); + symlinkSync(realDir, join(workdir, "supabase", "database"), "dir"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas( + fs, + path, + workdir, + [], + pgDelta({ enabled: true }), + ); + expect(result).toEqual([]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect("returns [] (does not follow) when supabase/schemas itself is a symlink", () => { + const workdir = makeWorkdir(); + const realDir = join(workdir, "real-schemas"); + mkdirSync(realDir, { recursive: true }); + writeFileSync(join(realDir, "t.sql"), "select 1;\n"); + mkdirSync(join(workdir, "supabase"), { recursive: true }); + symlinkSync(realDir, join(workdir, "supabase", "schemas"), "dir"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas(fs, path, workdir, [], pgDelta()); + expect(result).toEqual([]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }); + + it.effect( + "sorts declared schema paths by UTF-8 byte order, not JS's default UTF-16 code-unit order", + () => { + // A supplementary-plane character (U+1F600, a surrogate pair in UTF-16) alongside a BMP + // private-use character (U+E000) is the textbook case where JS's default `.sort()` + // (UTF-16 code units) disagrees with Go's `sort.Strings` (UTF-8 bytes, which preserves + // codepoint order): JS ranks the surrogate pair first (0xD800 < 0xE000), Go ranks the + // supplementary-plane codepoint last (it's numerically > U+FFFF). Verified empirically + // against `Buffer.compare` on the two filenames' UTF-8 encodings. + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "schemas"), { recursive: true }); + const supplementary = "a\u{1F600}.sql"; + const privateUse = "a.sql"; + writeFileSync(join(workdir, "supabase", "schemas", supplementary), "select 1;\n"); + writeFileSync(join(workdir, "supabase", "schemas", privateUse), "select 2;\n"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const result = yield* legacyLoadDeclaredSchemas(fs, path, workdir, [], pgDelta()); + expect(result).toEqual([ + `supabase/schemas/${privateUse}`, + `supabase/schemas/${supplementary}`, + ]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect( + "propagates (rather than silently drops) a per-entry stat failure during the pg-delta/schemas walk", + () => { + // Both Go walkers (`afero.Walk`, `fs.WalkDir`) pass a per-entry stat/lstat error to their + // callback, which returns it and aborts the whole walk — an entry that can't be statted + // after its parent was listed (permissions, I/O error, a concurrent filesystem change) + // must not be silently omitted, which could build an incomplete declarative target. + const workdir = makeWorkdir(); + mkdirSync(join(workdir, "supabase", "schemas"), { recursive: true }); + writeFileSync(join(workdir, "supabase", "schemas", "a.sql"), "select 1;\n"); + const brokenAbs = join(workdir, "supabase", "schemas", "broken.sql"); + writeFileSync(brokenAbs, "select 2;\n"); + const statFs = Layer.effect( + FileSystem.FileSystem, + Effect.map(FileSystem.FileSystem, (real) => ({ + ...real, + stat: (statPath: string) => + statPath === brokenAbs + ? Effect.fail( + PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "stat", + description: "simulated stat failure", + pathOrDescriptor: statPath, + }), + ) + : real.stat(statPath), + })), + ).pipe(Layer.provideMerge(BunServices.layer)); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const exit = yield* legacyLoadDeclaredSchemas(fs, path, workdir, [], pgDelta()).pipe( + Effect.exit, + ); + expect(exit._tag).toBe("Failure"); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(statFs)); + }, + ); + + it.effect.skipIf(isRoot)( + "fails (rather than silently treating as empty) when a matched schema directory can't be read, and keeps the underlying cause in the message", + () => { + // Go's `walkMatchedDir` (`pkg/config/config.go:194-211`) propagates ANY `fs.WalkDir` + // error as `failed to walk matched directory: ` — an unreadable directory must + // surface as a failure, not silently contribute zero files (which could compare a + // local-target diff against the wrong target or generate an incomplete migration), and + // the reported message must carry the real underlying error (permission denied, here), + // not just the directory name — otherwise a user can't tell WHY the walk failed. + const workdir = makeWorkdir(); + const locked = join(workdir, "supabase", "locked"); + mkdirSync(locked, { recursive: true }); + chmodSync(locked, 0o000); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const exit = yield* legacyLoadDeclaredSchemas( + fs, + path, + workdir, + ["locked"], + pgDelta(), + ).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + const errorJson = JSON.stringify(exit.cause); + expect(errorJson).toContain("failed to walk matched directory:"); + expect(errorJson).not.toContain("failed to walk matched directory: locked"); + } + chmodSync(locked, 0o755); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect.skipIf(isRoot)( + "reports the pg-delta declarative dir walk failure as 'failed to walk declarative dir', not the generic 'failed to walk dir'", + () => { + // Go's `loadDeclaredSchemas` (`apps/cli-go/internal/db/diff/diff.go:52-101`) wraps the + // SAME `afero.Walk` failure with a DIFFERENT prefix per source: the pg-delta declarative + // dir branch reports `failed to walk declarative dir: %w`, while the `supabase/schemas` + // fallback (covered by the sibling test below) reports `failed to walk dir: %w` — both + // walks share `legacyWalkSqlFilesSorted`, which must be told which source it's walking. + const workdir = makeWorkdir(); + const declDir = join(workdir, "supabase", "database"); + mkdirSync(declDir, { recursive: true }); + chmodSync(declDir, 0o000); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const exit = yield* legacyLoadDeclaredSchemas( + fs, + path, + workdir, + [], + pgDelta({ enabled: true }), + ).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + const errorJson = JSON.stringify(exit.cause); + expect(errorJson).toContain("failed to walk declarative dir:"); + expect(errorJson).not.toContain("failed to walk dir:"); + } + chmodSync(declDir, 0o755); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); + + it.effect.skipIf(isRoot)( + "reports the supabase/schemas fallback walk failure as 'failed to walk dir', not the declarative-dir prefix", + () => { + const workdir = makeWorkdir(); + const schemasDir = join(workdir, "supabase", "schemas"); + mkdirSync(schemasDir, { recursive: true }); + chmodSync(schemasDir, 0o000); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const exit = yield* legacyLoadDeclaredSchemas(fs, path, workdir, [], pgDelta()).pipe( + Effect.exit, + ); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) { + const errorJson = JSON.stringify(exit.cause); + expect(errorJson).toContain("failed to walk dir:"); + expect(errorJson).not.toContain("failed to walk declarative dir:"); + } + chmodSync(schemasDir, 0o755); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); +}); diff --git a/apps/cli/src/legacy/commands/db/start/start.integration.test.ts b/apps/cli/src/legacy/commands/db/start/start.integration.test.ts index 9963d01eb3..c3dca80b60 100644 --- a/apps/cli/src/legacy/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/legacy/commands/db/start/start.integration.test.ts @@ -304,7 +304,6 @@ function setup(opts: SetupOpts = {}) { opts.networkId === undefined ? Option.none() : Option.some(opts.networkId), ), Layer.succeed(CliArgs, { args: ["db", "start"] }), - Layer.succeed(LegacyDebugFlag, false), Layer.succeed(LegacyExperimentalFlag, opts.experimental ?? false), Layer.succeed(LegacyDebugFlag, opts.debug ?? false), ); @@ -652,6 +651,31 @@ describe("legacy db start", () => { }); }); + it.live( + "an explicitly empty --network-id falls back to the generated network name, not a literal empty override", + () => { + // Go's gate is `len(viper.GetString("network-id")) > 0` (docker.go:379-383), not merely + // "the flag was passed" — an empty override (e.g. a shell expanding an unset var to "") + // must fall through to the generated `supabase_network_` name, not produce a + // literal `--network ""` on the `docker create` call. + const { layer, child } = setup({ + route: freshVolumeRoute(defaultRoute()), + networkId: "", + }); + return Effect.gen(function* () { + yield* legacyDbStart(DEFAULT_FLAGS).pipe(Effect.provide(layer)); + expect( + child.spawned.some( + (s) => s.args[0] === "network" && s.args.at(-1) === "supabase_network_test", + ), + ).toBe(true); + const args = createArgs(child.spawned); + const networkIndex = args?.indexOf("--network") ?? -1; + expect(args?.[networkIndex + 1]).toBe("supabase_network_test"); + }); + }, + ); + it.live( "fails with a typed config error on a malformed SUPABASE_DB_HEALTH_TIMEOUT, before any container is created", () => { diff --git a/apps/cli/src/legacy/shared/containers/container-lifecycle.ts b/apps/cli/src/legacy/shared/containers/container-lifecycle.ts index d69e00924e..07908ea94d 100644 --- a/apps/cli/src/legacy/shared/containers/container-lifecycle.ts +++ b/apps/cli/src/legacy/shared/containers/container-lifecycle.ts @@ -125,6 +125,25 @@ export interface LegacyContainerOpts { * Go applies it identically to every container this orchestrator creates. */ readonly extraHosts: ReadonlyArray; + /** + * Fallback identifier for {@link legacyStageStartSecretFiles}'s per-container secret + * directory, used ONLY when `spec.containerName` is empty (Docker auto-generates the + * real name — see that field's own doc comment): the deterministic, name-keyed + * directory this mechanism otherwise relies on isn't knowable until AFTER the + * container is created. Every real service container has a non-empty `containerName` + * and never reaches this fallback. + * + * REQUIRED (not merely "the caller's responsibility to supply") whenever + * `spec.containerName === ""`: {@link legacyStageStartSecretFiles} unconditionally + * `rm -rf`s its target directory FIRST on every call, even when `spec.secretFiles` is + * empty. Falling through to an empty identifier would resolve to the shared + * `/supabase/.temp/start-secrets/` ROOT directory itself (an empty trailing + * path segment collapses away), wiping every OTHER container's staged secrets sharing + * that root — {@link legacyCreateContainer} fails loudly with a + * {@link LegacyContainerCreateError} instead of ever falling through to that empty + * default. + */ + readonly secretDirId?: string; } /** @@ -742,9 +761,28 @@ export function legacyCreateContainer( opts.isBitbucketPipeline, ); + // `finalSpec.containerName` is empty only for a shadow database (Docker auto-generates + // the real name) — `opts.secretDirId` is the caller-REQUIRED fallback identifier for that + // case; see both fields' own doc comments. Fail loudly rather than silently falling + // through to an empty identifier, which would resolve to the shared `start-secrets/` ROOT + // directory itself and let `legacyStageStartSecretFiles`'s unconditional `rm -rf` wipe + // every OTHER container's staged secrets sharing that root. + let secretDirId: string; + if (finalSpec.containerName.length > 0) { + secretDirId = finalSpec.containerName; + } else if (opts.secretDirId !== undefined && opts.secretDirId.length > 0) { + secretDirId = opts.secretDirId; + } else { + return yield* Effect.fail( + new LegacyContainerCreateError({ + message: + "failed to create docker container: an unnamed container spec requires opts.secretDirId", + }), + ); + } const { binds: secretBinds, cleanup: cleanupSecretFiles } = yield* legacyStageStartSecretFiles( finalSpec.secretFiles ?? [], - finalSpec.containerName, + secretDirId, opts.workdir, ); const specWithSecretBinds: LegacyStartContainerSpec = diff --git a/apps/cli/src/legacy/shared/containers/container-lifecycle.unit.test.ts b/apps/cli/src/legacy/shared/containers/container-lifecycle.unit.test.ts index dd9a9dec52..622a7e1549 100644 --- a/apps/cli/src/legacy/shared/containers/container-lifecycle.unit.test.ts +++ b/apps/cli/src/legacy/shared/containers/container-lifecycle.unit.test.ts @@ -944,3 +944,111 @@ describe("legacyRemoveVolume", () => { ); }); }); + +describe("legacyCreateContainer with an empty containerName (the shadow database)", () => { + it.live( + "omits --name from the create argv, stages secretFiles under `opts.secretDirId` instead of the (empty) containerName, and leaves them in place after a successful start", + () => { + let hostPath: string | undefined; + const mock = mockSpawner((args) => { + if (args[0] === "create") { + expect(args).not.toContain("--name"); + const bind = args.find((a) => + a.endsWith(":/etc/postgresql-custom/pgsodium_root.key:ro,Z"), + ); + if (bind !== undefined) { + hostPath = bind.slice( + 0, + bind.length - ":/etc/postgresql-custom/pgsodium_root.key:ro,Z".length, + ); + } + return { exitCode: 0, stdout: "shadow-container-id\n" }; + } + return { exitCode: 0 }; + }); + + const spec: LegacyStartContainerSpec = { + ...baseSpec, + containerName: "", + binds: [], + networkAliases: undefined, + autoRemove: true, + secretFiles: [ + { containerPath: "/etc/postgresql-custom/pgsodium_root.key", content: "root-key" }, + ], + }; + + return legacyCreateContainer(mock.spawner, spec, { + projectId: "proj", + isBitbucketPipeline: false, + workdir, + extraHosts: [], + secretDirId: "shadow", + }).pipe( + Effect.map((containerId) => { + expect(containerId).toBe("shadow-container-id"); + expect(hostPath).toBeDefined(); + expect(hostPath).toBe( + join(workdir, "supabase", ".temp", "start-secrets", "shadow", "secret-0"), + ); + // Not reclaimed here: eagerly deleting right after `docker start` returns isn't + // safe on every Docker backend (see `legacyCreateShadowDatabase`'s own doc + // comment), so the staged file stays in place immediately after create/start — + // it is reclaimed later, at container teardown, by `legacyRemoveShadowDatabase` + // (keyed off the same `secretDirId` this test passed in). + expect(existsSync(hostPath ?? "")).toBe(true); + }), + ); + }, + ); + + it.live( + "still stages secretFiles under the real name when containerName is non-empty, ignoring an (irrelevant) secretDirId", + () => { + const mock = alwaysSucceed("real-name-container-id\n"); + const spec: LegacyStartContainerSpec = { + ...baseSpec, + secretFiles: [{ containerPath: "/etc/kong/kong.yml", content: "secret" }], + }; + return legacyCreateContainer(mock.spawner, spec, { + projectId: "proj", + isBitbucketPipeline: false, + workdir, + extraHosts: [], + secretDirId: "should-be-ignored", + }).pipe( + Effect.map(() => { + expect( + existsSync(join(workdir, "supabase", ".temp", "start-secrets", spec.containerName)), + ).toBe(true); + expect( + existsSync(join(workdir, "supabase", ".temp", "start-secrets", "should-be-ignored")), + ).toBe(false); + }), + ); + }, + ); + + it.live( + "fails loudly instead of silently defaulting to the shared start-secrets root when containerName is empty and no secretDirId is supplied", + () => { + const mock = alwaysSucceed("shadow-container-id\n"); + const spec: LegacyStartContainerSpec = { ...baseSpec, containerName: "", binds: [] }; + return legacyCreateContainer(mock.spawner, spec, { + projectId: "proj", + isBitbucketPipeline: false, + workdir, + extraHosts: [], + }).pipe( + Effect.flip, + Effect.map((error) => { + expect(error).toBeInstanceOf(LegacyContainerCreateError); + expect(error.message).toContain("requires opts.secretDirId"); + // Never even reaches `docker create` — this is a caller-programming-error + // check, not a Docker-level failure. + expect(mock.spawned).toEqual([]); + }), + ); + }, + ); +}); diff --git a/apps/cli/src/legacy/shared/containers/docker-create-args.ts b/apps/cli/src/legacy/shared/containers/docker-create-args.ts index c729321e88..0c3a5017d2 100644 --- a/apps/cli/src/legacy/shared/containers/docker-create-args.ts +++ b/apps/cli/src/legacy/shared/containers/docker-create-args.ts @@ -125,7 +125,14 @@ interface LegacyStartSecretFileSpec { export interface LegacyStartContainerSpec { /** `container.Config.Image` (already resolved/pulled — resolution is out of scope here). */ readonly image: string; - /** The 4th `DockerStart` positional argument — `--name`. */ + /** + * The 4th `DockerStart` positional argument — `--name`. An empty string mirrors Go + * passing `""` (e.g. `CreateShadowDatabase`, `apps/cli-go/internal/db/diff/diff.go:150`) + * and lets Docker auto-generate one — {@link legacyBuildStartContainerCreateArgs} omits + * `--name` entirely in that case (docker rejects an explicit empty `--name` value, unlike + * the Engine API's empty `containerName` positional, which it happily treats as "generate + * one"). Every real service container still passes a non-empty name, unchanged. + */ readonly containerName: string; /** * `container.Config.Hostname`. Only Logflare sets this (`start.go:353`, @@ -257,6 +264,15 @@ export interface LegacyStartContainerSpec { * supported for completeness/future callers, per the task brief. */ readonly restartPolicy?: "unless-stopped" | "no" | "always" | "on-failure"; + /** + * `container.HostConfig.AutoRemove` — only the shadow-database container sets this + * (`CreateShadowDatabase`, `apps/cli-go/internal/db/diff/diff.go:144`), via `--rm`. + * `AutoRemove` only fires once the container's own main process exits on its own; it + * does NOT make an explicit remove redundant for a still-running container (verified + * empirically), so callers still remove the shadow explicitly once they are done with + * it — see `shadow-database.ts`'s `legacyRemoveShadowDatabase`. + */ + readonly autoRemove?: boolean; /** * `container.HostConfig.SecurityOpt`. Only Vector sets this * (`start.go:441`, `"label:disable"`, when mounting a non-root Docker @@ -435,8 +451,8 @@ export function legacyBuildStartContainerCreateArgs( ): ReadonlyArray { return [ "create", - "--name", - spec.containerName, + ...(spec.containerName.length === 0 ? [] : ["--name", spec.containerName]), + ...(spec.autoRemove === true ? ["--rm"] : []), ...(spec.hostname === undefined ? [] : ["--hostname", spec.hostname]), ...Object.entries(spec.env).flatMap(([key, value]) => legacyIsDockerClientEnvKey(key) ? ["-e", `${key}=${value}`] : ["-e", key], diff --git a/apps/cli/src/legacy/shared/containers/docker-create-args.unit.test.ts b/apps/cli/src/legacy/shared/containers/docker-create-args.unit.test.ts index e55386f87a..391b77ca09 100644 --- a/apps/cli/src/legacy/shared/containers/docker-create-args.unit.test.ts +++ b/apps/cli/src/legacy/shared/containers/docker-create-args.unit.test.ts @@ -115,6 +115,36 @@ describe("legacyBuildStartContainerCreateArgs", () => { ]); }); + test("omits --name entirely when containerName is empty (Docker auto-generates one, e.g. the shadow database)", () => { + const spec: LegacyStartContainerSpec = { + image: "supabase/postgres:17.4.1.030", + containerName: "", + env: {}, + binds: [], + networkId: "supabase_network_proj", + labels: {}, + }; + const args = legacyBuildStartContainerCreateArgs(spec); + expect(args).not.toContain("--name"); + expect(args).toEqual(["create", "--network", "supabase_network_proj", spec.image]); + }); + + test("emits --rm when autoRemove is true, omits it otherwise", () => { + const base: LegacyStartContainerSpec = { + image: "supabase/postgres:17.4.1.030", + containerName: "", + env: {}, + binds: [], + networkId: "supabase_network_proj", + labels: {}, + }; + expect(legacyBuildStartContainerCreateArgs(base)).not.toContain("--rm"); + expect(legacyBuildStartContainerCreateArgs({ ...base, autoRemove: true })).toContain("--rm"); + expect(legacyBuildStartContainerCreateArgs({ ...base, autoRemove: false })).not.toContain( + "--rm", + ); + }); + test("never serializes env values into argv (CWE-214: secrets must not leak to ps)", () => { const args = legacyBuildStartContainerCreateArgs(full); expect(args).toContain("DB_PASSWORD"); diff --git a/apps/cli/src/legacy/shared/db-bootstrap/db-setup.ts b/apps/cli/src/legacy/shared/db-bootstrap/db-setup.ts index 225a5fc034..31fc593f20 100644 --- a/apps/cli/src/legacy/shared/db-bootstrap/db-setup.ts +++ b/apps/cli/src/legacy/shared/db-bootstrap/db-setup.ts @@ -103,13 +103,17 @@ import { LegacyDbConnection, type LegacyDbSession } from "../legacy-db-connectio import type { LegacyDbConnectError } from "../legacy-db-connection.errors.ts"; import { LegacyDbConfigLoadError } from "../legacy-db-config.errors.ts"; import { legacyCheckDbToml, legacyResolveSeedSqlPath } from "../legacy-db-config.toml-read.ts"; -import { LEGACY_CLI_PROJECT_LABEL, legacyServiceContainerName } from "../legacy-docker-ids.ts"; +import { LEGACY_CLI_PROJECT_LABEL, localDbContainerId } from "../legacy-docker-ids.ts"; import { LegacyDockerRun, type LegacyDockerRunOpts } from "../legacy-docker-run.service.ts"; import { legacyMigrateAndSeed } from "../legacy-migrate-and-seed.ts"; import { LegacyMigrationApplyError, legacyExecSqlFile } from "../legacy-migration-apply.ts"; import type { LegacyMigrationSeedError, LegacySeedConfig } from "../legacy-seed.ts"; import { ramInBytes } from "../legacy-size-units.ts"; -import { LegacyMigrationVaultError, legacyUpsertVaultSecrets } from "../legacy-vault.ts"; +import { + LegacyMigrationVaultError, + type LegacyVaultSecret, + legacyUpsertVaultSecrets, +} from "../legacy-vault.ts"; import { legacyEnsureImagesCached, type LegacyImagePrepullError, @@ -163,7 +167,7 @@ export type LegacyStartSetupLocalDatabaseError = | LegacyImagePrepullError; /** Already-resolved Docker images for the three PG15+ one-shot migrate jobs (`initSchema15`'s `initJobs`). */ -interface LegacyStartDbSetupImages { +export interface LegacyStartDbSetupImages { /** `utils.Config.Realtime.Image`, resolved by the caller (not part of the decoded `ProjectConfig` schema — `toml:"-"`). */ readonly realtime: string; /** `utils.Config.Storage.Image`, ditto. */ @@ -174,16 +178,17 @@ interface LegacyStartDbSetupImages { /** * Computes the three PG15+ one-shot setup jobs' PINNED image names (`initSchema15`'s - * `initRealtimeJob`/`initStorageJob`/`initAuthJob`) for {@link legacyRunFreshDbSetup} — the - * ONE place both real Go callers (`db start`'s fresh-volume branch and `db reset`'s PG15 - * recreate) reach this from. Mirrors Go's `initSchema15`, which uses the SAME - * already-pin-rewritten `utils.Config.{Realtime,Storage,Auth}.Image` fields the - * long-running containers would use, regardless of `--exclude` — resolved via - * `legacyResolvePinnedImage`, not the raw Dockerfile default, so a linked project's - * version pins apply here too. Deliberately does NOT resolve these against the registry - * (`legacyEnsureImagesCached`) as a batch: Go resolves (and pulls) each one-shot job's - * own image individually, sequentially, right before THAT job runs (`DockerRunJob` -> - * `DockerStart` -> `DockerResolveImageIfNotCached`, `start.go:334-355`, + * `initRealtimeJob`/`initStorageJob`/`initAuthJob`) for {@link legacyResolveDbSetupPrelude}, + * the ONE place every real caller (`db start`'s fresh-volume branch, `db reset`'s PG15 + * recreate, and the shadow-database variant's `legacySetupShadowDatabase`/ + * `legacyMigrateShadowDatabase`) reaches this resolution from — see that function's own doc + * comment. Mirrors Go's `initSchema15`, which uses the SAME already-pin-rewritten + * `utils.Config.{Realtime,Storage,Auth}.Image` fields the long-running containers would use, + * regardless of `--exclude` — resolved via `legacyResolvePinnedImage`, not the raw Dockerfile + * default, so a linked project's version pins apply here too. Deliberately does NOT resolve + * these against the registry (`legacyEnsureImagesCached`) as a batch: Go resolves (and pulls) + * each one-shot job's own image individually, sequentially, right before THAT job runs + * (`DockerRunJob` -> `DockerStart` -> `DockerResolveImageIfNotCached`, `start.go:334-355`, * `docker.go:363-365`) — {@link legacyRunStartMigrateJob} does that lazily itself, right * before running each job (see its own doc comment): a batch resolve here would let one * unreachable image fail the WHOLE setup before an earlier job Go would already have run @@ -199,8 +204,54 @@ function legacyResolveDbSetupImages( }; } -/** Input to {@link legacyStartSetupLocalDatabase}. */ -export interface LegacyStartSetupLocalDatabaseInput { +/** + * Resolves JWKS (lazily, only when `majorVersion >= 15` AND `realtimeEnabledForSetup`) + the + * PG15+ one-shot job images' PINNED names (via {@link legacyResolveDbSetupImages}) — the + * exact prelude BOTH {@link legacyRunFreshDbSetup} (the real local `db` container) and + * `shadow-database.ts`'s `legacySetupShadowDatabase`/`legacyMigrateShadowDatabase` need + * before calling {@link legacySetupDatabase}. Hoisted here (CLI-1956 review follow-up) so + * the shadow path shares this exact resolution instead of keeping its own copy, which had + * silently drifted (a dead, never-forwarded `jwtExpiry` field on the shadow's own setup-input + * shape). Structurally typed against just the fields this needs (not the full {@link + * LegacyFreshDbSetupInput}) so both that type and `shadow-database.ts`'s + * `LegacyShadowDbSetupInput` — which is itself derived from it — satisfy this signature + * without an explicit cast. Synchronous other than the caller-supplied `jwks` effect: neither + * step here talks to Docker at all (see {@link legacyResolveDbSetupImages}'s own doc comment + * for why the images are only PINNED, not registry-resolved, at this stage). + * + * The `majorVersion >= 15` gate matters, not just an optimization: Go's `initSchema` + * (`apps/cli-go/internal/db/start/start.go:243-253`) returns via `InitSchema14` for + * `MajorVersion <= 14` WITHOUT ever calling `initSchema15`, so `Config.Auth.ResolveJWKS` + * (`start.go:338`, only reached from `initSchema15`) never runs at all on PG13/14 — even + * with realtime enabled. `ResolveJWKS` can perform live discovery/JWKS HTTP requests for + * configured `auth.third_party` providers, so resolving it unconditionally on PG14 is not + * just wasted work: it can fail (or hang) when Go's own shadow/setup never would. + */ +export const legacyResolveDbSetupPrelude = (setup: { + readonly majorVersion: number; + readonly realtimeEnabledForSetup: boolean; + readonly serviceVersionOverrides: LocalServiceVersionOverrides; + readonly jwks: Effect.Effect; +}): Effect.Effect<{ readonly jwks: string; readonly images: LegacyStartDbSetupImages }, E> => + Effect.gen(function* () { + const jwks = setup.majorVersion >= 15 && setup.realtimeEnabledForSetup ? yield* setup.jwks : ""; + const images = legacyResolveDbSetupImages(setup.serviceVersionOverrides); + return { jwks, images }; + }); + +/** + * Input to {@link legacySetupDatabase} — Go's EXPORTED `SetupDatabase(ctx, conn, host, w, + * fsys)` (`start.go:383-399`): `initSchema -> ApplyApiPrivileges -> vault upsert -> + * SeedGlobals(roles.sql)`, deliberately WITHOUT `apply.MigrateAndSeed` (that extra step is + * what makes {@link LegacyStartSetupLocalDatabaseInput}/{@link legacyStartSetupLocalDatabase} + * bigger — see that interface's own doc comment). Extracted as its own exported shape + * (CLI-1956) so shadow-database provisioning (`shadow-database.ts`) can reach the exact same + * platform-baseline pipeline the real local `db` container's fresh-volume setup does, without + * also replaying migrations a second time or reaching `legacyMigrateAndSeed`'s + * declarative-schema-files branch, neither of which Go's own shadow provisioning + * (`setupShadowConn`) ever does either. + */ +export interface LegacySetupDatabaseInput { /** * An already-open session to the local Postgres database, dialed the same way * Go's `ConnectLocalPostgres(ctx, pgconn.Config{})` does (`internal/utils/ @@ -209,7 +260,7 @@ export interface LegacyStartSetupLocalDatabaseInput { * config.layer.ts`'s own `--local` branch already dials (`legacy-db-config. * layer.ts:518-529`). This is deliberately NOT the internal Docker-network `db` * container address the PG15+ one-shot jobs below connect through (see - * `networkId`/`projectId`) — the two addressing schemes are independent, exactly + * `networkId`/`dbHost`) — the two addressing schemes are independent, exactly * like Go's `conn` (host-facing) vs. `host` parameter (`utils.DbId`) in * `SetupDatabase(ctx, conn, utils.DbId, w, fsys)`. */ @@ -222,15 +273,30 @@ export interface LegacyStartSetupLocalDatabaseInput { readonly config: ProjectConfig; /** `db.major_version` (13-17) — Go's `utils.Config.Db.MajorVersion`, resolved by the caller once, ahead of the `db` container's own image tag selection. */ readonly majorVersion: number; - /** Go's `Config.ProjectId`, already sanitized (`legacySanitizeProjectId`) — derives the `db` container's internal Docker name for the PG15+ one-shot jobs (`legacyServiceContainerName("db", projectId)`, Go's `utils.DbId`). */ - readonly projectId: string; /** - * `--experimental`/`SUPABASE_EXPERIMENTAL`, resolved by the caller (Go's - * `viper.GetBool("EXPERIMENTAL")`) — threaded straight into - * {@link legacyMigrateAndSeed}'s own `experimental` gate (`internal/migration/apply/ - * apply.go:19`); this module has no other use for it. + * The internal Docker-network address the PG15+ one-shot jobs connect through — Go's + * `host` parameter to `SetupDatabase(ctx, conn, host, w, fsys)` (`start.go:383`). The real + * local `db` container's own caller (`legacyRunFreshDbSetup`) passes + * `legacyServiceContainerName("db", projectId)` (Go's `utils.DbId`, threaded straight + * through unchanged from before CLI-1956); the shadow-database variant + * (`shadow-database.ts`) passes the shadow container's own 12-char short id instead (Go's + * `container[:12]`, `apps/cli-go/internal/db/diff/diff.go:172` / `internal/migration/ + * squash/squash.go:96`) — empirically verified to resolve via Docker's embedded DNS even + * though the shadow container has no name/alias at all (see `shadow-database.ts`'s + * header). This field was hardcoded inside this module prior to CLI-1956; it is now the + * caller's responsibility, the one genuine parameterization this port needed for shadow + * provisioning to reuse `SetupDatabase` at all. */ - readonly experimental: boolean; + readonly dbHost: string; + /** + * Go's `Config.ProjectId` — labels the PG15+ one-shot job containers + * (`com.supabase.cli.project`/`com.docker.compose.project`, see {@link + * legacyRunStartMigrateJob}), matching Go's `DockerStart`, which sets both + * unconditionally for every container it starts (`docker.go:371-376`). Independent of + * {@link dbHost}: this labels the one-shot job containers THEMSELVES, not the (possibly + * different) container `dbHost` addresses. + */ + readonly projectId: string; /** The `start` run's Docker network id (Go's `utils.NetId` or the `--network-id` override) — every PG15+ one-shot job joins it, matching `DockerStart`'s own default (`docker.go:379-383`). */ readonly networkId: string; /** `LegacyLocalConfigValues.dbUrl` — reused (not recomputed) to derive the internal DB password via `legacyStartInternalDbPassword`, matching every other `start/services/*.service.ts` builder. */ @@ -286,6 +352,25 @@ export interface LegacyStartSetupLocalDatabaseInput { * `utils.GetDebugLogger()` as the job's stderr writer (`start.go:349-353`). */ readonly debug: boolean; + /** `toml.baseline.apiAutoExposeNewTables` — Go's `api.auto_expose_new_tables` tri-state, threaded straight into {@link legacyApplyApiPrivileges}. */ + readonly apiAutoExposeNewTables: Option.Option; + /** `toml.vault` — Go's `utils.Config.Db.Vault`, threaded straight into {@link legacyUpsertVaultSecrets}. */ + readonly vault: ReadonlyArray; +} + +/** Input to {@link legacyStartSetupLocalDatabase}. */ +export interface LegacyStartSetupLocalDatabaseInput extends Omit< + LegacySetupDatabaseInput, + "apiAutoExposeNewTables" | "vault" +> { + /** + * `--experimental`/`SUPABASE_EXPERIMENTAL`, resolved by the caller (Go's + * `viper.GetBool("EXPERIMENTAL")`) — threaded straight into + * {@link legacyMigrateAndSeed}'s own `experimental` gate (`internal/migration/apply/ + * apply.go:19`); `legacySetupDatabase`/Go's own `SetupDatabase` have no use for it — + * only this function's own trailing `MigrateAndSeed` call does. + */ + readonly experimental: boolean; /** * The migration version to reapply (Go's `apply.MigrateAndSeed(ctx, version, ...)`). * `db start`'s own caller always passes `""` (Go's `SetupLocalDatabase(ctx, "", ...)`, @@ -574,9 +659,9 @@ function legacyStartAuthMigrateEnv(input: { */ const legacyStartInitSchema15 = Effect.fnUntraced(function* ( spawner: Spawner, - input: LegacyStartSetupLocalDatabaseInput, + input: LegacySetupDatabaseInput, ) { - const dbHost = legacyServiceContainerName("db", input.projectId); + const dbHost = input.dbHost; const dbPassword = legacyStartInternalDbPassword(input.dbUrl); if (input.config.realtime.enabled) { @@ -669,7 +754,7 @@ const legacyStartInitSchema15 = Effect.fnUntraced(function* ( */ const legacyStartInitSchema = Effect.fnUntraced(function* ( spawner: Spawner, - input: LegacyStartSetupLocalDatabaseInput, + input: LegacySetupDatabaseInput, tmpDir: string, ) { const output = yield* Output; @@ -769,37 +854,23 @@ export const legacyStartInitCurrentBranch = Effect.fnUntraced(function* ( }); /** - * Runs the full `SetupLocalDatabase`-equivalent sequence — see this module's - * header for the exact Go call chain and line-range citations. Call once, right - * after the `db` container's healthcheck passes on a fresh volume (Go's - * `NoBackupVolume` gate); the caller decides that gating, this function performs - * no health/readiness checks of its own. + * Runs Go's EXPORTED `SetupDatabase(ctx, conn, host, w, fsys)` (`start.go:383-399`) — + * see {@link LegacySetupDatabaseInput}'s own doc comment for exactly what's in and out of + * scope. Extracted out of {@link legacyStartSetupLocalDatabase} (CLI-1956) so shadow-database + * provisioning can reuse this exact sequence without also reaching `apply.MigrateAndSeed`. */ -export const legacyStartSetupLocalDatabase = ( +export const legacySetupDatabase = ( spawner: Spawner, - input: LegacyStartSetupLocalDatabaseInput, + input: LegacySetupDatabaseInput, ): Effect.Effect< void, - LegacyStartSetupLocalDatabaseError, + LegacyDbSetupError | LegacyMigrationVaultError | LegacyImagePrepullError, Output | LegacyDockerRun | RuntimeInfo > => Effect.gen(function* () { const { session, fs, path, workdir } = input; - // `warnOnUnresolvedEnv: false` — both `start.handler.ts` and `db/start/ - // start.handler.ts` already ran an earlier, same-invocation `legacyCheckDbToml` - // purely for its Go-parity validation side effect (their own callers discard the - // result) before ever reaching this fresh-volume setup, so that earlier call - // already printed Go's single `assertEnvLoaded` OrioleDB S3 WARN, if any. Without - // this, this module's own accepted duplicate config-load pass (see this module's - // header) would print the SAME warning a second time — a real, observable stderr - // divergence from Go's exactly-once `flags.LoadConfig`, unlike the harmless - // resolved-value duplication the header describes. - const toml = yield* legacyCheckDbToml(fs, path, workdir, undefined, { - warnOnUnresolvedEnv: false, - }); - - // SetupDatabase: initSchema -> ApplyApiPrivileges (start.go:383-389). + // initSchema -> ApplyApiPrivileges (start.go:383-389). yield* Effect.scoped( Effect.gen(function* () { const tmpDir = yield* fs @@ -813,18 +884,12 @@ export const legacyStartSetupLocalDatabase = ( ), ); yield* legacyStartInitSchema(spawner, input, tmpDir); - yield* legacyApplyApiPrivileges( - session, - fs, - path, - tmpDir, - toml.baseline.apiAutoExposeNewTables, - ); + yield* legacyApplyApiPrivileges(session, fs, path, tmpDir, input.apiAutoExposeNewTables); }), ); // "Create vault secrets first so roles.sql can reference them" (start.go:390). - yield* legacyUpsertVaultSecrets(session, toml.vault); + yield* legacyUpsertVaultSecrets(session, input.vault); // Custom-roles seed (start.go:394-398, pkg/migration/seed.go:84-97): Go's // `SeedGlobals` prints "Seeding globals from roles.sql..." BEFORE attempting @@ -857,6 +922,44 @@ export const legacyStartSetupLocalDatabase = ( (message) => new LegacyDbSetupError({ message }), ); } + }); + +/** + * Runs the full `SetupLocalDatabase`-equivalent sequence — see this module's + * header for the exact Go call chain and line-range citations. Call once, right + * after the `db` container's healthcheck passes on a fresh volume (Go's + * `NoBackupVolume` gate); the caller decides that gating, this function performs + * no health/readiness checks of its own. + */ +export const legacyStartSetupLocalDatabase = ( + spawner: Spawner, + input: LegacyStartSetupLocalDatabaseInput, +): Effect.Effect< + void, + LegacyStartSetupLocalDatabaseError, + Output | LegacyDockerRun | RuntimeInfo +> => + Effect.gen(function* () { + const { session, fs, path, workdir } = input; + + // `warnOnUnresolvedEnv: false` — both `start.handler.ts` and `db/start/ + // start.handler.ts` already ran an earlier, same-invocation `legacyCheckDbToml` + // purely for its Go-parity validation side effect (their own callers discard the + // result) before ever reaching this fresh-volume setup, so that earlier call + // already printed Go's single `assertEnvLoaded` OrioleDB S3 WARN, if any. Without + // this, this module's own accepted duplicate config-load pass (see this module's + // header) would print the SAME warning a second time — a real, observable stderr + // divergence from Go's exactly-once `flags.LoadConfig`, unlike the harmless + // resolved-value duplication the header describes. + const toml = yield* legacyCheckDbToml(fs, path, workdir, undefined, { + warnOnUnresolvedEnv: false, + }); + + yield* legacySetupDatabase(spawner, { + ...input, + apiAutoExposeNewTables: toml.baseline.apiAutoExposeNewTables, + vault: toml.vault, + }); // apply.MigrateAndSeed(ctx, version, conn, fsys) — `db start`'s own caller always // passes `version: ""` (every pending migration, matching `SetupLocalDatabase`'s @@ -910,7 +1013,7 @@ export interface LegacyFreshDbSetupInput { readonly experimental: boolean; readonly dbUrl: string; readonly jwtSecret: string; - /** Lazy — evaluated only when reached AND `realtimeEnabledForSetup`. See `start-database.ts`'s header for why this is caller-supplied rather than resolved here unconditionally. */ + /** Lazy — evaluated only when reached AND `majorVersion >= 15` AND `realtimeEnabledForSetup` (see {@link legacyResolveDbSetupPrelude}'s own doc comment for the Go citation). See `start-database.ts`'s header for why this is caller-supplied rather than resolved here unconditionally. */ readonly jwks: Effect.Effect; readonly apiUrl: string; readonly authExternalUrl: string | undefined; @@ -936,11 +1039,9 @@ export interface LegacyFreshDbSetupInput { * healthcheck passes on a fresh database (`db start`'s fresh-volume branch and * `db reset`'s PG15 recreate, see {@link LegacyFreshDbSetupInput}'s own doc * comment): dial the host-facing session (Go's `ConnectLocalPostgres`), resolve - * JWKS lazily (only when `majorVersion >= 15` AND `realtimeEnabledForSetup` — Go's - * `initSchema`, `start.go:243-254`, only ever reaches `initSchema15`'s - * `ResolveJWKS` call on PG15+; the PG13/14 branch, `InitSchema14`, never touches - * JWKS at all), compute the three PG15+ one-shot job images' PINNED names via - * {@link legacyResolveDbSetupImages}, then run {@link legacyStartSetupLocalDatabase} + * JWKS + the three PG15+ one-shot job images' PINNED names via {@link + * legacyResolveDbSetupPrelude} (the same hoisted prelude the shadow-database variant + * uses), then run {@link legacyStartSetupLocalDatabase} * itself. `version`/`seedFlags` are the one genuine difference between the two * callers (`db start` always passes `""`/`{noSeed:false, sqlPaths:[]}`; `db * reset` passes its own resolved reset version/flags) — threaded straight @@ -982,14 +1083,7 @@ export const legacyRunFreshDbSetup = ( { isLocal: true, dnsResolver: "native" }, ); - // Go's `initSchema` (`start.go:243-254`) branches to `initSchema15` — the ONLY place - // `ResolveJWKS` is ever called — solely on `majorVersion >= 15`; the PG13/14 branch - // (`InitSchema14`) never touches JWKS, so a PG13/14 database with realtime enabled - // must not pay for (or fail on) an external JWKS fetch it will never use. - const jwks = - setup.majorVersion >= 15 && setup.realtimeEnabledForSetup ? yield* setup.jwks : ""; - - const dbSetupImages = legacyResolveDbSetupImages(setup.serviceVersionOverrides); + const { jwks, images: dbSetupImages } = yield* legacyResolveDbSetupPrelude(setup); yield* legacyStartSetupLocalDatabase(spawner, { session, @@ -999,6 +1093,10 @@ export const legacyRunFreshDbSetup = ( config: setup.config, experimental: setup.experimental, majorVersion: setup.majorVersion, + // Go's `utils.DbId` — the internal Docker-network address the PG15+ one-shot + // jobs connect through. Unchanged from before CLI-1956, just now an explicit + // parameter on `LegacySetupDatabaseInput` instead of computed inside it. + dbHost: localDbContainerId(input.projectId), projectId: input.projectId, networkId: input.networkId, dbUrl: setup.dbUrl, diff --git a/apps/cli/src/legacy/shared/db-bootstrap/db-setup.unit.test.ts b/apps/cli/src/legacy/shared/db-bootstrap/db-setup.unit.test.ts index 386fa512ad..53d7e746f8 100644 --- a/apps/cli/src/legacy/shared/db-bootstrap/db-setup.unit.test.ts +++ b/apps/cli/src/legacy/shared/db-bootstrap/db-setup.unit.test.ts @@ -148,6 +148,7 @@ function baseInput( config: defaultConfig, experimental: false, majorVersion: 17, + dbHost: "supabase_db_proj", projectId: "proj", networkId: "supabase_network_proj", dbUrl: "postgresql://postgres:postgrespassword@127.0.0.1:54322/postgres", @@ -345,7 +346,7 @@ describe("legacyStartSetupLocalDatabase", () => { baseInput(workdir, session, { majorVersion: 15, config, - projectId: "myproj", + dbHost: "supabase_db_myproj", jwks: '{"keys":["stub"]}', }), out, diff --git a/apps/cli/src/legacy/shared/db-bootstrap/local-container-inputs.ts b/apps/cli/src/legacy/shared/db-bootstrap/local-container-inputs.ts index f8d44fc43a..7d44f39115 100644 --- a/apps/cli/src/legacy/shared/db-bootstrap/local-container-inputs.ts +++ b/apps/cli/src/legacy/shared/db-bootstrap/local-container-inputs.ts @@ -103,6 +103,14 @@ export const legacyBuildLocalDbContainerInputs = ( networkIdFlag: Option.Option, platform: string, debug: boolean, + // The resolved `--linked` ref, when the caller already has one (`db diff`/`db pull` — + // CLI-1956) — threaded straight through to `legacyLoadLocalProjectContext` so the shadow's + // OWN container-spec fields (image, `db.major_version`, JWT secret, root key, + // `db.settings`, service enabled-for-setup flags) reflect the matching `[remotes.]` + // override, the same way `legacyReadDbToml(..., ref)` already does for those commands' + // other config read. `db start`/`db reset` never pass this — see that function's own doc + // comment. + projectRef?: string, ): Effect.Effect< LegacyLocalDbContainerInputs, LegacyDbConfigLoadError, @@ -113,7 +121,7 @@ export const legacyBuildLocalDbContainerInputs = ( const path = yield* Path.Path; const mapError = (message: string) => new LegacyDbConfigLoadError({ message }); - const context = yield* legacyLoadLocalProjectContext(workdir, mapError); + const context = yield* legacyLoadLocalProjectContext(workdir, mapError, projectRef); const { config, projectEnvValues, loaded, hostname, projectId } = context; // Go's `viper.GetBool("EXPERIMENTAL")` (`internal/migration/apply/apply.go:19`), read deep // inside `legacyRunFreshDbSetup`'s fresh-volume setup pipeline — see this field's own doc diff --git a/apps/cli/src/legacy/shared/db-bootstrap/postgres.service.ts b/apps/cli/src/legacy/shared/db-bootstrap/postgres.service.ts index e32fc8d34e..369c606f6c 100644 --- a/apps/cli/src/legacy/shared/db-bootstrap/postgres.service.ts +++ b/apps/cli/src/legacy/shared/db-bootstrap/postgres.service.ts @@ -250,20 +250,25 @@ function legacyPostgresExtraEnv( * {@link legacyBuildPostgresStartContainerSpec}), so it never appears in this * process's own `docker create` argv (CWE-214/522). * - * Otherwise byte-for-byte derived from Go's raw-string concatenation - * (including the trailing space after `/etc/postgresql` — Go's - * `NewContainerConfig(args ...string)` joins its variadic `args` there, - * always empty for `supabase start`, so the space survives as-is); built via - * explicit `"...\n" +` concatenation rather than a multi-line template - * literal so that trailing space stays a visible, lint/format-proof string - * character instead of invisible end-of-line whitespace. + * Otherwise byte-for-byte derived from Go's raw-string concatenation — + * `NewContainerConfig(args ...string)` splices `strings.Join(args, " ")` + * straight after the literal trailing space following `/etc/postgresql` + * (`start.go:95`): `supabase start`'s own Postgres container always calls it + * with zero args (`args` here defaults to `""`, so the trailing space + * survives on its own, unchanged from before), while the shadow-database + * variant (`CreateShadowDatabase`, `apps/cli-go/internal/db/diff/diff.go:140`) + * passes {@link LEGACY_SHADOW_ENTRYPOINT_ARGS} — see + * {@link legacyBuildShadowPostgresContainerSpec}. Built via explicit + * `"...\n" +` concatenation rather than a multi-line template literal so that + * the trailing space (when `args` is empty) stays a visible, lint/format-proof + * string character instead of invisible end-of-line whitespace. */ -function legacyPostgresEntrypointScriptPg15(postgresConfig: string): string { +function legacyPostgresEntrypointScriptPg15(postgresConfig: string, args = ""): string { return ( "\n" + "cat <<'EOF' > /etc/postgresql.schema.sql && \\\n" + "cat <<'EOF' >> /etc/postgresql/postgresql.conf && \\\n" + - "docker-entrypoint.sh postgres -D /etc/postgresql \n" + + `docker-entrypoint.sh postgres -D /etc/postgresql ${args}\n` + `${LEGACY_START_DB_SCHEMA_SQL}\n` + `${LEGACY_START_DB_WEBHOOK_SQL}\n` + `${LEGACY_START_DB_SUPABASE_SQL}\n` + @@ -280,14 +285,15 @@ function legacyPostgresEntrypointScriptPg15(postgresConfig: string): string { * only), appends `postgresConfig` to `postgresql.conf`, then execs * `docker-entrypoint.sh`. See {@link legacyPostgresEntrypointScriptPg15}'s doc * comment for why this is explicit concatenation rather than a template - * literal. + * literal, and for the `args` parameter (same trailing-space splice, same + * default). */ -function legacyPostgresEntrypointScriptPg14(postgresConfig: string): string { +function legacyPostgresEntrypointScriptPg14(postgresConfig: string, args = ""): string { return ( "\n" + "cat <<'EOF' > /docker-entrypoint-initdb.d/supabase_schema.sql && \\\n" + "cat <<'EOF' >> /etc/postgresql/postgresql.conf && \\\n" + - "docker-entrypoint.sh postgres -D /etc/postgresql \n" + + `docker-entrypoint.sh postgres -D /etc/postgresql ${args}\n` + `${LEGACY_START_DB_SUPABASE_SQL}\n` + "EOF\n" + `${postgresConfig}\n` + @@ -397,3 +403,119 @@ export function legacyBuildPostgresStartContainerSpec( labels: {}, }; } + +/** + * Go's `NewContainerConfig("-c", "max_worker_processes=0")` (`CreateShadowDatabase`, + * `apps/cli-go/internal/db/diff/diff.go:140`) — disables background workers in the + * shadow database. Not a docker flag: it is spliced into the entrypoint script's own + * `docker-entrypoint.sh postgres -D /etc/postgresql ` line, exactly like every + * other `args` value {@link legacyPostgresEntrypointScriptPg15}/`Pg14` accept. + */ +export const LEGACY_SHADOW_ENTRYPOINT_ARGS = "-c max_worker_processes=0"; + +/** + * Input to {@link legacyBuildShadowPostgresContainerSpec} — the subset of + * {@link LegacyPostgresStartServiceInput} the shadow variant actually needs (no + * `projectId`/`fromBackup`: the shadow container has no name and never restores from a + * backup) plus the shadow's own host port. + */ +export interface LegacyShadowPostgresContainerSpecInput { + readonly db: Pick; + readonly experimental: ProjectConfig["experimental"]; + readonly jwtSecret: string; + readonly jwtExpiry: number; + readonly networkId: string; + readonly image: string; + readonly configImage: string; + readonly rootKey?: string; + /** `utils.Config.Db.ShadowPort` — the shadow's own host port, published to `5432/tcp` in-container. */ + readonly shadowPort: number; + /** + * `[db] password` (already resolved from `config.toml`, `DEFAULT_DB_PASSWORD`/"postgres" when + * unset) — matches Go's `NewContainerConfig`, which sources `POSTGRES_PASSWORD` from the SAME + * `utils.Config.Db.Password` for both the real local container and the shadow + * (`CreateShadowDatabase` reuses `NewContainerConfig` verbatim, `diff.go:140`). Must be threaded + * through so the shadow's actual Postgres password matches what + * `legacyShadowRunInputFromLocalContainerInputs`'s caller connects with — otherwise a + * non-default `[db] password` authenticates against the wrong secret. + */ + readonly password: string; +} + +/** + * Builds the {@link LegacyStartContainerSpec} for the shadow database container. Port of + * Go's `CreateShadowDatabase` (`apps/cli-go/internal/db/diff/diff.go:138-151`) — reuses + * the EXACT SAME `NewContainerConfig` (image/env/healthcheck/entrypoint-script shape) the + * real local `db` container uses, just with {@link LEGACY_SHADOW_ENTRYPOINT_ARGS} spliced + * into the entrypoint and a materially different `container.HostConfig`/networking: + * + * - **Empty `containerName`** (Go passes `""` to `DockerStart`, letting Docker + * auto-generate one) — see {@link LegacyStartContainerSpec.containerName}'s own doc + * comment for how the arg-builder and secret-file staging handle this. + * - **`autoRemove: true`** — Go's `hostConfig.AutoRemove` (`--rm`). + * - **No volume bind** — the shadow is throwaway; Go's `hostConfig` sets no `Binds` at all. + * - **No `restartPolicy`** — Go's `hostConfig` sets no `RestartPolicy` either. + * - **No `networkAliases`** — Go's `networkingConfig` is a bare, empty + * `network.NetworkingConfig{}` (no `db`/`db.supabase.internal` aliases). The shadow + * still joins the network via `DockerStart`'s own default `NetworkMode` (confirmed + * empirically: Docker's embedded DNS resolves a container on a user-defined network by + * BOTH its auto-generated name and its 12-char short container id, with no alias + * needed — see `shadow-database.ts`'s header for why this matters). + * - **Tmpfs on PG <= 14 IS still applied** — same `isPg14OrEarlier` condition as the real + * `db` container. + * - **The pgsodium root key `secretFiles` entry is still applied on PG >= 15** — the + * shadow's entrypoint script is the SAME `legacyPostgresEntrypointScriptPg15`, which + * still heredocs it in Go (splice point unaffected by `args`), so this port still needs + * the bind-mounted host temp file. `legacyCreateContainer`'s caller must supply + * {@link LegacyContainerOpts.secretDirId} for the shadow case (empty `containerName`). + * - **Labels ARE still applied** (merged in by `legacyCreateContainer`, same as every + * other container) so `supabase stop`'s label-filtered sweep catches an orphaned shadow + * too — Go's `DockerStart` sets `CliProjectLabel`/`composeProjectLabel` unconditionally, + * regardless of the `container.Config` literal passed in. + */ +export function legacyBuildShadowPostgresContainerSpec( + input: LegacyShadowPostgresContainerSpecInput, +): LegacyStartContainerSpec { + const rootKeyValue = input.rootKey ?? LEGACY_POSTGRES_DEFAULT_ROOT_KEY; + const postgresConfig = legacyPostgresSettingsToPostgresConfig(input.db.settings); + const isPg14OrEarlier = input.db.major_version <= 14; + + const env: Record = { + POSTGRES_PASSWORD: input.password, + POSTGRES_HOST: "/var/run/postgresql", + JWT_SECRET: input.jwtSecret, + JWT_EXP: String(input.jwtExpiry), + ...legacyPostgresExtraEnv(input.experimental, input.configImage), + }; + + const script = isPg14OrEarlier + ? legacyPostgresEntrypointScriptPg14(postgresConfig, LEGACY_SHADOW_ENTRYPOINT_ARGS) + : legacyPostgresEntrypointScriptPg15(postgresConfig, LEGACY_SHADOW_ENTRYPOINT_ARGS); + + return { + image: input.image, + containerName: "", + env, + entrypoint: "sh", + cmd: ["-c", script], + binds: [], + autoRemove: true, + ...(isPg14OrEarlier ? { tmpfs: { "/docker-entrypoint-initdb.d": "" } } : {}), + ...(isPg14OrEarlier + ? {} + : { + secretFiles: [ + { containerPath: LEGACY_POSTGRES_PGSODIUM_ROOT_KEY_PATH, content: rootKeyValue }, + ], + }), + ports: [{ hostPort: String(input.shadowPort), containerPort: "5432" }], + healthcheck: { + test: ["CMD", "pg_isready", "-U", "postgres", "-h", "127.0.0.1", "-p", "5432"], + intervalSeconds: LEGACY_POSTGRES_HEALTHCHECK_INTERVAL_SECONDS, + timeoutSeconds: LEGACY_POSTGRES_HEALTHCHECK_TIMEOUT_SECONDS, + retries: LEGACY_POSTGRES_HEALTHCHECK_RETRIES, + }, + networkId: input.networkId, + labels: {}, + }; +} diff --git a/apps/cli/src/legacy/shared/db-bootstrap/postgres.service.unit.test.ts b/apps/cli/src/legacy/shared/db-bootstrap/postgres.service.unit.test.ts index 3b88393d98..2864609075 100644 --- a/apps/cli/src/legacy/shared/db-bootstrap/postgres.service.unit.test.ts +++ b/apps/cli/src/legacy/shared/db-bootstrap/postgres.service.unit.test.ts @@ -7,11 +7,14 @@ import { LEGACY_START_DB_SUPABASE_SQL } from "./templates/db-supabase.sql.ts"; import { LEGACY_START_DB_WEBHOOK_SQL } from "./templates/db-webhook.sql.ts"; import { LEGACY_POSTGRES_DEFAULT_ROOT_KEY } from "../legacy-local-config-values.ts"; import { + LEGACY_SHADOW_ENTRYPOINT_ARGS, legacyBuildPostgresStartContainerSpec, + legacyBuildShadowPostgresContainerSpec, legacyPostgresImageVersionTag, legacyPostgresSettingsToPostgresConfig, legacyPostgresVersionCompare, type LegacyPostgresStartServiceInput, + type LegacyShadowPostgresContainerSpecInput, } from "./postgres.service.ts"; const POSTGRES_CONFIG_HEADER = "\n# supabase [db.settings] configuration\n"; @@ -377,3 +380,75 @@ describe("legacyPostgresImageVersionTag", () => { expect(legacyPostgresImageVersionTag("supabase/postgres")).toBe("supabase/postgres"); }); }); + +function baseShadowInput( + overrides: Partial = {}, +): LegacyShadowPostgresContainerSpecInput { + return { + db: { major_version: 17, settings: {} }, + experimental: baseExperimental(), + jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", + jwtExpiry: 3600, + networkId: "supabase_network_myproj", + image: "public.ecr.aws/supabase/postgres:17.4.1.030", + configImage: "supabase/postgres:17.4.1.030", + shadowPort: 54320, + password: "postgres", + ...overrides, + }; +} + +describe("legacyBuildShadowPostgresContainerSpec", () => { + test("PG >= 15: splices the shadow entrypoint args into the SAME trailing-space join point the real db container uses, and still carries the pgsodium root key as a secretFile", () => { + const spec = legacyBuildShadowPostgresContainerSpec( + baseShadowInput({ db: { major_version: 17, settings: {} } }), + ); + const script = spec.cmd?.[1]; + expect(script).toContain( + `docker-entrypoint.sh postgres -D /etc/postgresql ${LEGACY_SHADOW_ENTRYPOINT_ARGS}\n`, + ); + expect(spec.secretFiles).toEqual([ + { + containerPath: "/etc/postgresql-custom/pgsodium_root.key", + content: LEGACY_POSTGRES_DEFAULT_ROOT_KEY, + }, + ]); + expect(spec.tmpfs).toBeUndefined(); + }); + + test("PG <= 14: splices the same args, no pgsodium secretFile, and sets the initdb tmpfs mount", () => { + const spec = legacyBuildShadowPostgresContainerSpec( + baseShadowInput({ db: { major_version: 14, settings: {} } }), + ); + const script = spec.cmd?.[1]; + expect(script).toContain( + `docker-entrypoint.sh postgres -D /etc/postgresql ${LEGACY_SHADOW_ENTRYPOINT_ARGS}\n`, + ); + expect(spec.secretFiles).toBeUndefined(); + expect(spec.tmpfs).toEqual({ "/docker-entrypoint-initdb.d": "" }); + }); + + test("has no name (Docker auto-generates one), no network aliases, no volume bind, and no restart policy — unlike the real db container", () => { + const spec = legacyBuildShadowPostgresContainerSpec(baseShadowInput()); + expect(spec.containerName).toBe(""); + expect(spec.networkAliases).toBeUndefined(); + expect(spec.binds).toEqual([]); + expect(spec.restartPolicy).toBeUndefined(); + }); + + test("sets autoRemove and publishes the shadow port to 5432/tcp", () => { + const spec = legacyBuildShadowPostgresContainerSpec(baseShadowInput({ shadowPort: 54399 })); + expect(spec.autoRemove).toBe(true); + expect(spec.ports).toEqual([{ hostPort: "54399", containerPort: "5432" }]); + }); + + test("labels are still applied (empty map here — the caller merges project/compose labels in, same as every other container)", () => { + const spec = legacyBuildShadowPostgresContainerSpec(baseShadowInput()); + expect(spec.labels).toEqual({}); + }); + + test("initializes POSTGRES_PASSWORD from the resolved [db] password, not a hardcoded literal — matching Go's NewContainerConfig, which sources it from utils.Config.Db.Password for both the real container and the shadow", () => { + const spec = legacyBuildShadowPostgresContainerSpec(baseShadowInput({ password: "hunter2" })); + expect(spec.env?.["POSTGRES_PASSWORD"]).toBe("hunter2"); + }); +}); diff --git a/apps/cli/src/legacy/shared/db-bootstrap/shadow-database.ts b/apps/cli/src/legacy/shared/db-bootstrap/shadow-database.ts new file mode 100644 index 0000000000..cc62fffe24 --- /dev/null +++ b/apps/cli/src/legacy/shared/db-bootstrap/shadow-database.ts @@ -0,0 +1,566 @@ +/** + * Native TypeScript port of Go's shadow-database provisioning primitives + * (`apps/cli-go/internal/db/diff/diff.go:138-209`) — CLI-1956. These are the low-level + * building blocks; `legacyPrepareRawShadow` below (create -> health-wait, no platform + * baseline) is one of the two composed shapes `db diff`/`db pull` actually call (Go's + * `PrepareRawShadow`, `apps/cli-go/internal/db/diff/shadow.go:93-116`) — it has zero + * pg-delta/declarative dependency, so it lives here rather than in + * `commands/db/shared/legacy-shadow-source.ts`, which owns the OTHER composed shape + * (`legacyPrepareShadowSource`, Go's `PrepareShadowSource`) precisely because that one also + * needs the `--target-local` declarative-schema branch and pg-delta, which this module — + * deliberately kept dependency-light, like every other `shared/db-bootstrap/` module — does + * not. + * + * Exposed separately (not fused into one monolithic function) because the composed shapes + * Go itself has are NOT all the same: `migration squash` (a future port, CLI-1969) only ever + * needs create -> health-wait -> connect -> `SetupDatabase` (no `CREATE_TEMPLATE`, no + * migrations at that point — `apps/cli-go/internal/migration/squash/squash.go:83-96`), while + * `db diff --use-pgadmin` (CLI-1968) needs create -> health-wait -> `MigrateShadowDatabase` + * (`apps/cli-go/internal/db/diff/pgadmin.go:70-78`). Exposing every primitive individually + * lets each future caller compose exactly the subset it needs, matching Go's own module shape + * 1:1 rather than forcing every caller through one shape only `db diff`/`db pull` happen to + * need. + * + * A note on the shadow container's own addressing, since it's the one genuinely surprising + * empirical fact this whole module depends on: the shadow container is created with NO name + * (Docker auto-generates one) and NO network alias (`legacyBuildShadowPostgresContainerSpec`), + * unlike every other container this codebase creates. The PG15+ one-shot setup jobs + * (`legacySetupDatabase` -> `initSchema15`) still need SOME hostname to reach it over the + * shared Docker network, though — Go passes `container[:12]` (the container id's own 12-char + * short form) as that hostname (`diff.go:172`, `squash.go:96`). This was verified empirically + * against a real Docker daemon (matching Go's exact container-creation shape: no `--name`, no + * `--network-alias`, joined to a user-defined network via `NetworkMode` alone): `docker + * inspect`'s `NetworkSettings.Networks..DNSNames` lists BOTH the auto-generated name AND + * the 12-char short id, and a sibling container on the same network successfully resolved and + * authenticated against Postgres using ONLY the short id as hostname. So `dbHost: + * container.slice(0, 12)` below is not a guess — it is the exact mechanism Go itself relies on. + */ + +import { randomUUID } from "node:crypto"; +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import { Data, Effect, Schedule, type FileSystem, type Path, type Scope } from "effect"; +import type * as HttpClient from "effect/unstable/http/HttpClient"; +import type { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner"; + +import { Output } from "../../../shared/output/output.service.ts"; +import type { RuntimeInfo } from "../../../shared/runtime/runtime-info.service.ts"; +import { + collectText, + legacyDescribeContainerCliFailure, + spawnContainerCli, +} from "../legacy-container-cli.ts"; +import { LegacyDbConnection, type LegacyDbSession } from "../legacy-db-connection.service.ts"; +import type { LegacyPgConnInput } from "../legacy-db-connection.service.ts"; +import { LEGACY_CLI_PROJECT_LABEL } from "../legacy-docker-ids.ts"; +import type { LegacyDockerRun } from "../legacy-docker-run.service.ts"; +import { legacyApplyMigrations } from "../legacy-migration-apply.ts"; +import { + legacyEnsureNetwork, + legacyCreateContainer, + LEGACY_COMPOSE_PROJECT_LABEL, + type LegacyContainerOpts, +} from "../containers/container-lifecycle.ts"; +import type { LegacyImagePrepullError } from "../containers/image-prepull.ts"; +import type { LegacyHealthCheckTimeoutError } from "../containers/health-check.ts"; +import { legacyWaitForHealthyServices } from "../containers/health-check.ts"; +import { legacyListLocalMigrationPaths } from "../legacy-migration-history.ts"; +import { legacyToPostgresURL } from "../legacy-postgres-url.ts"; +import { + type LegacyFreshDbSetupInput, + type LegacySetupDatabaseInput, + type LegacyStartDbSetupImages, + type LegacyStartSetupLocalDatabaseError, + legacyResolveDbSetupPrelude, + legacySetupDatabase, +} from "./db-setup.ts"; +import { + LEGACY_SHADOW_ENTRYPOINT_ARGS, + legacyBuildShadowPostgresContainerSpec, + type LegacyShadowPostgresContainerSpecInput, +} from "./postgres.service.ts"; + +// Re-exported for convenience — the entrypoint-args constant lives on `postgres.service.ts` +// alongside the container-spec builder it feeds, but it documents THIS module's own Go +// citation (`CreateShadowDatabase`, `diff.go:140`) just as much. +export { LEGACY_SHADOW_ENTRYPOINT_ARGS }; + +type Spawner = ChildProcessSpawner["Service"]; + +const errMessage = (e: unknown): string => + typeof e === "object" && e !== null && "message" in e && typeof e.message === "string" + ? e.message + : String(e); + +/** + * Creating, connecting to, setting up, or migrating the shadow database failed. Kept in + * `shared/db-bootstrap/` (not `commands/db/shared/legacy-pgdelta.errors.ts`'s + * `LegacyDeclarativeShadowDbError`) so these primitives stay usable by future callers outside + * the `db diff`/`db pull` family (`migration squash`, `db diff --use-pgadmin`) without pulling + * in a pg-delta-family-specific error type — see this module's own header. + */ +export class LegacyShadowDbError extends Data.TaggedError("LegacyShadowDbError")<{ + readonly message: string; +}> {} + +/** + * Required to bypass the pg_cron check + * (https://github.com/citusdata/pg_cron/blob/main/pg_cron.sql#L3). Go's `CREATE_TEMPLATE` + * (`apps/cli-go/internal/db/diff/diff.go:164`). + */ +export const LEGACY_SHADOW_CREATE_TEMPLATE_SQL = + "CREATE DATABASE contrib_regression TEMPLATE postgres"; + +/** + * Go's `ConnectShadowDatabase`'s fixed timeout — 10 seconds, EVERY real Go caller + * (`apps/cli-go/internal/db/diff/diff.go:187,200`, `internal/migration/squash/squash.go:91`) + * passes the same `10*time.Second` literal. + */ +export const LEGACY_SHADOW_CONNECT_TIMEOUT_SECONDS = 10; + +/** + * Go's `NewBackoffPolicy(ctx, timeout)` (`apps/cli-go/internal/db/start/start.go:192-198`): a + * 1-second constant delay, capped at `timeout` (in whole seconds) retries after the initial + * attempt. + */ +const LEGACY_SHADOW_CONNECT_SCHEDULE = Schedule.max([ + Schedule.spaced("1 seconds"), + Schedule.recurs(LEGACY_SHADOW_CONNECT_TIMEOUT_SECONDS), +]); + +/** + * Port of Go's `ConnectShadowDatabase` (`apps/cli-go/internal/db/diff/diff.go:153-161`): a + * SECOND, independent connect-retry loop layered ON TOP OF the container health wait the + * caller already ran (`start.WaitForHealthyService`) — a healthy Postgres healthcheck doesn't + * guarantee the very next connection attempt succeeds instantly, so Go retries the connect + * itself too, constant 1s backoff, up to {@link LEGACY_SHADOW_CONNECT_TIMEOUT_SECONDS} retries. + * Scoped: the returned session's connection closes when the caller's scope closes, matching + * Go's `defer conn.Close(context.Background())` at each real call site. + */ +export const legacyConnectShadowDatabase = ( + cfg: LegacyPgConnInput, +): Effect.Effect => + Effect.gen(function* () { + const dbConnection = yield* LegacyDbConnection; + return yield* dbConnection.connect(cfg, { isLocal: true, dnsResolver: "native" }).pipe( + Effect.mapError((cause) => new LegacyShadowDbError({ message: cause.message })), + Effect.retry({ schedule: LEGACY_SHADOW_CONNECT_SCHEDULE }), + ); + }); + +/** + * Input to {@link legacyCreateShadowDatabase} — the subset of the real `db` container's own + * bootstrap inputs the shadow variant needs, plus its own host port. See + * {@link LegacyShadowPostgresContainerSpecInput} (the container-spec shape this wraps) for + * the field-by-field Go citations. + */ +export interface LegacyCreateShadowDatabaseInput extends LegacyShadowPostgresContainerSpecInput { + /** Go's `Config.ProjectId` — merged onto the shadow's own labels (`DockerStart`'s unconditional label assignment) and the network-create call, matching every other container this codebase creates. */ + readonly projectId: string; + readonly isBitbucketPipeline: boolean; + readonly workdir: string; + readonly extraHosts: ReadonlyArray; +} + +/** Resolved by {@link legacyCreateShadowDatabase} — everything a caller needs to both use and later tear down the shadow. */ +export interface LegacyShadowDatabaseHandle { + /** Docker always returns the id from `docker create`, regardless of whether `--name` was passed. */ + readonly containerId: string; + /** See {@link legacyCreateShadowDatabase}'s own doc comment for why this is generated per-call rather than fixed. Threaded through by the caller to {@link legacyRemoveShadowDatabase} so the staged secret directory (see {@link LegacyContainerOpts.secretDirId}) is reclaimed at teardown. */ + readonly secretDirId: string; +} + +/** + * Port of Go's `CreateShadowDatabase` (`apps/cli-go/internal/db/diff/diff.go:138-151`): + * ensures the local Docker network exists (Go's `DockerStart` calls + * `DockerNetworkCreateIfNotExists` on EVERY invocation, unlike the `start`/`reset` + * compositions, which hoist this to run once per orchestrated run — `db diff`/`db pull` have + * no such orchestrator, so this mirrors Go's own per-call behavior instead), then creates + + * starts the shadow container. + */ +export const legacyCreateShadowDatabase = ( + spawner: Spawner, + input: LegacyCreateShadowDatabaseInput, +): Effect.Effect => + Effect.gen(function* () { + const labels = { + [LEGACY_CLI_PROJECT_LABEL]: input.projectId, + [LEGACY_COMPOSE_PROJECT_LABEL]: input.projectId, + }; + yield* legacyEnsureNetwork(spawner, input.networkId, labels).pipe( + Effect.mapError((cause) => new LegacyShadowDbError({ message: cause.message })), + ); + const spec = legacyBuildShadowPostgresContainerSpec(input); + // The shadow container has no name (Docker auto-generates one) — key its staged secret + // files (the pgsodium root key, PG15+ only) off a fallback identifier instead of + // `spec.containerName` (see `LegacyContainerOpts.secretDirId`'s own doc comment). + // Randomized (not a fixed `"shadow"` string): `legacyStageStartSecretFiles` `rm -rf`s + // its target directory FIRST on every call, so two concurrent `db diff`/`db pull` runs + // in the same workdir (two terminals, a CI matrix) sharing a fixed identifier could have + // one wipe the other's staged root key mid-flight, BEFORE either container's own + // `docker start` even runs (so a shared `shadowPort` — which would itself collide, since + // both runs read the same config.toml — can't be used as the differentiator either). + // Reclaimed once the shadow container itself is torn down, not eagerly right after + // `docker start` returns: Postgres's entrypoint actually reads the file at + // postmaster-start, seconds later — safe only if the bind mount's source inode is pinned + // by then, which is guaranteed on native Linux dockerd but NOT on Docker Desktop + // (macOS/Windows). {@link legacyRemoveShadowDatabase} `rm -rf`s this exact directory + // (keyed off `secretDirId`, returned below) once the container is gone, so a randomized + // per-call id doesn't leak a directory per invocation — see that function's own doc + // comment. + const secretDirId = `shadow-${randomUUID()}`; + const containerOpts: LegacyContainerOpts = { + projectId: input.projectId, + isBitbucketPipeline: input.isBitbucketPipeline, + workdir: input.workdir, + extraHosts: input.extraHosts, + secretDirId, + }; + const containerId = yield* legacyCreateContainer(spawner, spec, containerOpts).pipe( + Effect.mapError((cause) => new LegacyShadowDbError({ message: cause.message })), + ); + return { containerId, secretDirId }; + }); + +/** Input to {@link legacyRemoveShadowDatabase} — everything needed to tear down both halves of a shadow ({@link legacyCreateShadowDatabase}'s container AND its staged secret directory). */ +export interface LegacyRemoveShadowDatabaseInput { + readonly containerId: string; + /** {@link LegacyShadowDatabaseHandle.secretDirId} — the ONLY reclaim path for the shadow's staged secret directory (see {@link legacyCreateShadowDatabase}'s own doc comment): the shadow container has no name, so neither `legacyCleanupStartSecrets` (keyed off a container's own name/label) nor `legacyStageStartSecretFiles`'s self-healing `rm -rf` (keyed off the SAME directory being reused across calls) can ever find it. */ + readonly secretDirId: string; + readonly workdir: string; +} + +/** + * Best-effort `rm -rf` of the shadow's OWN staged secret directory + * (`/supabase/.temp/start-secrets//`, PG15+ only — see + * {@link legacyCreateShadowDatabase}'s own doc comment) — a no-op when nothing was ever + * staged (PG<=14, or `secretDirId` empty). Never fails: a missing directory is already the + * desired end state, and a real deletion error is not worth failing the caller's diff/pull + * over. + */ +const legacyCleanupShadowSecretDir = ( + secretDirId: string, + workdir: string, +): Effect.Effect => { + if (secretDirId.length === 0) return Effect.void; + return Effect.tryPromise(() => + rm(join(workdir, "supabase", ".temp", "start-secrets", secretDirId), { + recursive: true, + force: true, + }), + ).pipe( + Effect.asVoid, + Effect.orElseSucceed(() => undefined), + ); +}; + +/** + * Port of Go's `utils.DockerRemove(shadow)` as called by every shadow caller + * (`apps/cli-go/internal/db/diff/diff.go:217`, `shadow.go:45,103`, + * `internal/migration/squash/squash.go:87`): `RemoveOptions{RemoveVolumes: true, Force: + * true}` via `docker rm -f -v `. Best-effort for the OVERALL operation — Go's own + * `DockerRemove` swallows the removal's ERROR RETURN (it has no return value at all), so a + * failure here must never mask whatever the caller was doing with the shadow — but it does + * NOT swallow the message: Go prints `"Failed to remove container:", containerId, err` to + * stderr on failure (`apps/cli-go/internal/utils/docker.go:442-449`), so this does the same + * before continuing. That includes a failure to even launch/collect the removal itself (the + * container CLI missing, a disconnected runtime, a stream-read error) — Go's single + * `Docker.ContainerRemove` SDK call folds every one of those causes into the same `err` it + * prints, so this catches {@link spawnContainerCli}/exit-code-collection failures the same way + * {@link legacyRestartSatelliteService} does (`restart-services.ts`), via + * {@link legacyDescribeContainerCliFailure}, rather than discarding them unreported. Also + * reclaims the shadow's staged secret directory (see {@link legacyCleanupShadowSecretDir}), + * since nothing else in this codebase can find it once the container is gone (`secretDirId` is + * randomized per shadow, not the container's name). + */ +export const legacyRemoveShadowDatabase = ( + spawner: Spawner, + input: LegacyRemoveShadowDatabaseInput, +): Effect.Effect => + Effect.gen(function* () { + const { containerId, secretDirId, workdir } = input; + if (containerId.length > 0) { + const failureMessage = yield* Effect.scoped( + Effect.gen(function* () { + const child = yield* spawnContainerCli(spawner, ["rm", "-f", "-v", containerId], { + stdin: "ignore", + stdout: "ignore", + stderr: "pipe", + extendEnv: true, + }); + const [exitCode, stderr] = yield* Effect.all( + [child.exitCode.pipe(Effect.map(Number)), collectText(child.stderr)], + { concurrency: "unbounded" }, + ); + return exitCode === 0 ? undefined : stderr.trim(); + }), + ).pipe(Effect.catch((cause) => Effect.succeed(legacyDescribeContainerCliFailure(cause)))); + if (failureMessage !== undefined) { + const output = yield* Output; + yield* output.raw( + `Failed to remove container: ${containerId} ${failureMessage}\n`, + "stderr", + ); + } + } + yield* legacyCleanupShadowSecretDir(secretDirId, workdir); + }); + +/** A live shadow database left running for the caller to diff against and remove. Mirrors Go's `ShadowSource`. */ +export interface LegacyShadowSourceResult { + /** Container id; the caller MUST remove it (`legacyRemoveShadowDatabase`) when done. */ + readonly container: string; + /** {@link LegacyShadowDatabaseHandle.secretDirId} — the caller MUST also thread this (and the shadow's own `workdir`) into `legacyRemoveShadowDatabase` so the staged secret directory is reclaimed alongside the container. No Go equivalent (Go never stages this on host disk at all). */ + readonly secretDirId: string; + /** The diff source Postgres URL (the provisioned shadow). */ + readonly sourceUrl: string; + /** + * When set, replaces the diff target with a second database on the SAME shadow container + * (`contrib_regression`, cloned from `postgres` by `CREATE_TEMPLATE` during shadow setup — + * see {@link legacySetupShadowConn}) with declarative schemas applied. Mirrors Go's + * local-target declarative branch, where the user's local DB is not diffed. Only ever set + * by `legacy-shadow-source.ts`'s `legacyPrepareShadowSource` — {@link legacyPrepareRawShadow} + * below always leaves this `undefined`. + */ + readonly targetUrlOverride: string | undefined; +} + +/** Fields shared by `legacy-shadow-source.ts`'s `LegacyPrepareShadowSourceInput`/{@link LegacyPrepareRawShadowInput}. */ +export interface LegacyShadowConnectionInput extends LegacyCreateShadowDatabaseInput { + readonly fs: FileSystem.FileSystem; + readonly path: Path.Path; + readonly hostname: string; + /** `[db] password` (already resolved from `config.toml`) — the shadow's own connect password. */ + readonly password: string; + readonly healthTimeoutSeconds: number; +} + +export type LegacyPrepareRawShadowInput = LegacyShadowConnectionInput; + +/** + * Port of Go's `PrepareRawShadow` (`apps/cli-go/internal/db/diff/shadow.go:93-116`): a bare + * shadow (created + healthy, no platform baseline or migrations applied) — used inline + * (`db pull --declarative`'s empty declarative-export source), not the `ok`-sentinel + * error-path pattern `legacy-shadow-source.ts`'s `legacyPrepareShadowSource` uses, since there + * is only ONE step after creation that can fail (the health wait) rather than several. Lives + * here (not `legacy-shadow-source.ts`) because it has zero pg-delta/declarative dependency — + * see this module's own header. + */ +export const legacyPrepareRawShadow = ( + spawner: Spawner, + input: LegacyPrepareRawShadowInput, +): Effect.Effect< + LegacyShadowSourceResult, + LegacyShadowDbError | LegacyHealthCheckTimeoutError, + Output | LegacyDockerRun | RuntimeInfo | HttpClient.HttpClient +> => + Effect.gen(function* () { + const { containerId, secretDirId } = yield* legacyCreateShadowDatabase(spawner, input); + yield* legacyWaitForHealthyServices(spawner, [containerId], { + timeoutSeconds: input.healthTimeoutSeconds, + }).pipe( + Effect.onError(() => + legacyRemoveShadowDatabase(spawner, { + containerId, + secretDirId, + workdir: input.workdir, + }), + ), + ); + const connConfig: LegacyPgConnInput = { + host: input.hostname, + port: input.shadowPort, + user: "postgres", + password: input.password, + database: "postgres", + }; + return { + container: containerId, + secretDirId, + sourceUrl: legacyToPostgresURL(connConfig), + targetUrlOverride: undefined, + }; + }); + +/** + * Port of Go's `setupShadowConn` (`apps/cli-go/internal/db/diff/diff.go:171-179`): + * {@link legacySetupDatabase} (Go's `SetupDatabase`) against an already-connected shadow, + * dialed at `input.dbHost` = `container.slice(0, 12)` (see this module's own header), then + * optionally {@link LEGACY_SHADOW_CREATE_TEMPLATE_SQL}. `withTemplate` is `true` for every + * real Go caller of `setupShadowConn` itself (`SetupShadowDatabase`/`MigrateShadowDatabase` + * below); exposed as a parameter (not hardcoded) so a future caller that only needs the bare + * `SetupDatabase` step (`migration squash`, which calls `start.SetupDatabase` DIRECTLY, + * bypassing `setupShadowConn` entirely — `squash.go:96`) can call {@link legacySetupDatabase} + * on its own instead, while this function stays the exact `setupShadowConn` shape. + */ +export const legacySetupShadowConn = ( + spawner: Spawner, + input: LegacySetupDatabaseInput, + withTemplate: boolean, +): Effect.Effect< + void, + LegacyStartSetupLocalDatabaseError | LegacyShadowDbError, + Output | LegacyDockerRun | RuntimeInfo +> => + Effect.gen(function* () { + yield* legacySetupDatabase(spawner, input); + if (!withTemplate) return; + yield* input.session.exec(LEGACY_SHADOW_CREATE_TEMPLATE_SQL).pipe( + Effect.mapError( + (cause) => + new LegacyShadowDbError({ + message: `failed to create template database: ${errMessage(cause)}`, + }), + ), + ); + }); + +/** + * Shared fields both {@link legacySetupShadowDatabase} and {@link legacyMigrateShadowDatabase} + * need to resolve JWKS/images and run {@link legacySetupDatabase} — derived from `db-setup.ts`'s + * `LegacyFreshDbSetupInput` (the exact same shape `legacyRunFreshDbSetup` resolves for the real + * local `db` container) rather than hand-copied, so the two never silently drift: swap + * `experimental` (which only `legacyStartSetupLocalDatabase`'s trailing `MigrateAndSeed` call + * needs — irrelevant to the shadow's `SetupDatabase`-only pipeline, see {@link + * LegacySetupDatabaseInput}'s own doc comment) for the two fields the shadow's own caller + * (`legacy-shadow-source.ts`) resolves from an already-loaded `config.toml` instead + * (`apiAutoExposeNewTables`/`vault`), threaded straight through here rather than re-read. + */ +export type LegacyShadowDbSetupInput = Omit, "experimental"> & { + readonly apiAutoExposeNewTables: LegacySetupDatabaseInput["apiAutoExposeNewTables"]; + readonly vault: LegacySetupDatabaseInput["vault"]; +}; + +/** Common caller-supplied plumbing for {@link legacySetupShadowDatabase}/{@link legacyMigrateShadowDatabase}. */ +interface LegacyShadowSetupRunInput { + readonly fs: FileSystem.FileSystem; + readonly path: Path.Path; + readonly workdir: string; + /** Go's `Config.ProjectId` — labels the shadow's own PG15+ one-shot migrate job containers, same as the real local `db` container's — see {@link LegacySetupDatabaseInput.projectId}'s own doc comment. */ + readonly projectId: string; + readonly container: string; + readonly networkId: string; + /** The shadow's own connect target — host/port/user/password/database (`postgres`/`postgres`). */ + readonly connConfig: LegacyPgConnInput; + readonly setup: LegacyShadowDbSetupInput; +} + +/** + * Builds a {@link LegacySetupDatabaseInput} for {@link legacySetupDatabase} out of an + * already-connected shadow session plus the resolved images/JWKS prelude — exported so a + * future caller that only needs `SetupDatabase` directly (`migration squash`, which calls + * Go's `start.SetupDatabase` without going through `setupShadowConn` at all — see {@link + * legacySetupShadowConn}'s own doc comment) can build this same shape without duplicating the + * `container[:12]` dbHost derivation. + */ +export const legacyBuildShadowSetupDatabaseInput = ( + input: LegacyShadowSetupRunInput, + session: LegacyDbSession, + resolved: { readonly jwks: string; readonly images: LegacyStartDbSetupImages }, +): LegacySetupDatabaseInput => ({ + session, + fs: input.fs, + path: input.path, + workdir: input.workdir, + config: input.setup.config, + majorVersion: input.setup.majorVersion, + // Go's `container[:12]` — see this module's own header for why this resolves as a + // hostname at all despite the shadow container having no name/alias. + dbHost: input.container.slice(0, 12), + projectId: input.projectId, + networkId: input.networkId, + dbUrl: input.setup.dbUrl, + jwtSecret: input.setup.jwtSecret, + jwks: resolved.jwks, + apiUrl: input.setup.apiUrl, + authExternalUrl: input.setup.authExternalUrl, + siteUrl: input.setup.siteUrl, + anonKey: input.setup.anonKey, + serviceRoleKey: input.setup.serviceRoleKey, + storageTargetMigration: input.setup.storageTargetMigration, + images: resolved.images, + projectEnvValues: input.setup.projectEnvValues, + debug: input.setup.debug, + apiAutoExposeNewTables: input.setup.apiAutoExposeNewTables, + vault: input.setup.vault, +}); + +/** + * Port of Go's `SetupShadowDatabase` (`apps/cli-go/internal/db/diff/diff.go:181-193`): + * connects to the shadow (Go's `ConnectShadowDatabase`, {@link legacyConnectShadowDatabase}) + * FIRST, THEN resolves the setup prelude (JWKS/pinned image names, {@link + * legacyResolveDbSetupPrelude}) and runs {@link legacySetupShadowConn} WITH the template + * database — the platform baseline only, no user migrations. Connect-then-setup, matching Go's + * own `SetupShadowDatabase` (which dials `ConnectShadowDatabase` before ever calling + * `start.SetupDatabase`, `diff.go:186-192`) and this same module's `legacyRunFreshDbSetup` + * (`db-setup.ts`) for the real local `db` container: an unconnectable shadow must surface a + * connect error immediately, not pay for JWKS work first. The connection is closed once this + * resolves (Go's `defer conn.Close(...)`), matching `Effect.scoped`'s finalizer running at the + * end of this function rather than leaking a `Scope.Scope` requirement to the caller. + */ +export const legacySetupShadowDatabase = ( + spawner: Spawner, + input: LegacyShadowSetupRunInput, +): Effect.Effect< + void, + LegacyStartSetupLocalDatabaseError | LegacyShadowDbError | LegacyImagePrepullError | E, + Output | LegacyDockerRun | RuntimeInfo | LegacyDbConnection +> => + Effect.scoped( + Effect.gen(function* () { + const session = yield* legacyConnectShadowDatabase(input.connConfig); + const resolved = yield* legacyResolveDbSetupPrelude(input.setup); + yield* legacySetupShadowConn( + spawner, + legacyBuildShadowSetupDatabaseInput(input, session, resolved), + true, + ); + }), + ); + +/** + * Port of Go's `MigrateShadowDatabase` (`apps/cli-go/internal/db/diff/diff.go:195-209`): + * lists local migrations FIRST (Go's `migration.ListLocalMigrations`, fails fast on a bad + * migrations directory before any DB connection is even attempted), THEN connects (Go's + * `ConnectShadowDatabase`), THEN resolves the setup prelude (JWKS/pinned image names, {@link + * legacyResolveDbSetupPrelude}) and sets up the platform baseline + template database ({@link + * legacySetupShadowConn}, `withTemplate: true`), then applies every listed migration (Go's + * `migration.ApplyMigrations`). Connect-then-setup (not the reverse) matches Go's own + * `MigrateShadowDatabase` (`diff.go:195-209`) and this same module's `legacyRunFreshDbSetup` + * (`db-setup.ts`) for the real local `db` container — see {@link legacySetupShadowDatabase}'s + * own doc comment for why the ordering matters. Connection closed once this resolves, matching + * Go's `defer conn.Close(...)`. + */ +export const legacyMigrateShadowDatabase = ( + spawner: Spawner, + input: LegacyShadowSetupRunInput, +): Effect.Effect< + void, + LegacyStartSetupLocalDatabaseError | LegacyShadowDbError | LegacyImagePrepullError | E, + Output | LegacyDockerRun | RuntimeInfo | LegacyDbConnection +> => + Effect.scoped( + Effect.gen(function* () { + const migrationsDir = input.path.join(input.workdir, "supabase", "migrations"); + const pending = yield* legacyListLocalMigrationPaths( + input.fs, + input.path, + migrationsDir, + ).pipe(Effect.mapError((cause) => new LegacyShadowDbError({ message: cause.message }))); + + const session = yield* legacyConnectShadowDatabase(input.connConfig); + const resolved = yield* legacyResolveDbSetupPrelude(input.setup); + yield* legacySetupShadowConn( + spawner, + legacyBuildShadowSetupDatabaseInput(input, session, resolved), + true, + ); + yield* legacyApplyMigrations( + session, + input.fs, + input.path, + pending, + (message) => new LegacyShadowDbError({ message }), + ); + }), + ); diff --git a/apps/cli/src/legacy/shared/db-bootstrap/shadow-database.unit.test.ts b/apps/cli/src/legacy/shared/db-bootstrap/shadow-database.unit.test.ts new file mode 100644 index 0000000000..ded60ce863 --- /dev/null +++ b/apps/cli/src/legacy/shared/db-bootstrap/shadow-database.unit.test.ts @@ -0,0 +1,637 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { ProjectConfig } from "@supabase/config"; +import { ProjectConfigSchema } from "@supabase/config"; +import { BunServices } from "@effect/platform-bun"; +import { describe, expect, it } from "@effect/vitest"; +import { Effect, FileSystem, Layer, Option, Path, Schema, Sink, Stream } from "effect"; +import { ChildProcessSpawner } from "effect/unstable/process"; + +import { mockOutput, mockRuntimeInfo } from "../../../../tests/helpers/mocks.ts"; +import type { LegacyDbSession } from "../legacy-db-connection.service.ts"; +import { LegacyDbConnection } from "../legacy-db-connection.service.ts"; +import { LegacyDockerRun, type LegacyDockerRunOpts } from "../legacy-docker-run.service.ts"; +import type { LegacySetupDatabaseInput } from "./db-setup.ts"; +import { + LEGACY_SHADOW_CONNECT_TIMEOUT_SECONDS, + LEGACY_SHADOW_CREATE_TEMPLATE_SQL, + LEGACY_SHADOW_ENTRYPOINT_ARGS, + legacyBuildShadowSetupDatabaseInput, + legacyConnectShadowDatabase, + legacyCreateShadowDatabase, + legacyMigrateShadowDatabase, + legacyRemoveShadowDatabase, + legacySetupShadowConn, + legacySetupShadowDatabase, + type LegacyCreateShadowDatabaseInput, + type LegacyShadowDbSetupInput, +} from "./shadow-database.ts"; + +const decodeConfig = Schema.decodeUnknownSync(ProjectConfigSchema); +const defaultConfig: ProjectConfig = decodeConfig({}); + +function fakeSession() { + const calls: Array<{ kind: "exec" | "query"; sql: string }> = []; + const session: LegacyDbSession = { + exec: (sql) => + Effect.sync(() => { + calls.push({ kind: "exec", sql }); + }), + query: (sql) => + Effect.sync(() => { + calls.push({ kind: "query", sql }); + return []; + }), + extensionExists: () => Effect.succeed(false), + copyToCsv: () => Effect.succeed(new Uint8Array()), + queryRaw: () => Effect.succeed({ fields: [], rows: [], commandTag: "" }), + }; + return { session, calls }; +} + +function mockDbConnection(session: LegacyDbSession) { + return Layer.succeed(LegacyDbConnection, { connect: () => Effect.succeed(session) }); +} + +function mockDockerRun() { + const runs: Array = []; + return Layer.succeed(LegacyDockerRun, { + run: () => Effect.succeed(0), + runCapture: (runOpts) => { + runs.push(runOpts); + return Effect.succeed({ exitCode: 0, stdout: new Uint8Array(), stderr: "" }); + }, + // The shadow's own PG15+ one-shot platform-baseline jobs (`legacyRunStartMigrateJob`) + // go through `runStream`, not `runCapture` — see `db-setup.ts`'s own doc comment. + runStream: (runOpts) => { + runs.push(runOpts); + return Effect.succeed({ exitCode: 0, stderr: "" }); + }, + }); +} + +/** Fakes `docker image inspect` (always cached), `network create`, `create` (returns a fixed id), `start`, and `rm`. */ +function mockSpawner() { + const spawned: Array> = []; + const encoder = new TextEncoder(); + const spawner = ChildProcessSpawner.make((command) => + Effect.sync(() => { + const args = command._tag === "StandardCommand" ? command.args : []; + spawned.push(args); + const stdout = args[0] === "create" ? "shadow-container-id-0123456789abcdef" : ""; + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(1), + stdout: Stream.fromIterable(stdout.length > 0 ? [encoder.encode(`${stdout}\n`)] : []), + stderr: Stream.empty, + all: Stream.empty, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + stdin: Sink.drain, + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ); + return { spawner, spawned }; +} + +function baseCreateInput( + overrides: Partial = {}, +): LegacyCreateShadowDatabaseInput { + return { + db: { major_version: 17, settings: {} }, + experimental: defaultConfig.experimental, + jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", + jwtExpiry: 3600, + networkId: "supabase_network_proj", + image: "public.ecr.aws/supabase/postgres:17.4.1.030", + configImage: "supabase/postgres:17.4.1.030", + shadowPort: 54320, + password: "postgres", + projectId: "proj", + isBitbucketPipeline: false, + workdir: mkdtempSync(join(tmpdir(), "legacy-shadow-database-")), + extraHosts: [], + ...overrides, + }; +} + +describe("legacyCreateShadowDatabase / legacyRemoveShadowDatabase", () => { + it.effect( + "creates the network then the container with no --name, and returns the created id + a fresh secretDirId", + () => { + const mock = mockSpawner(); + return legacyCreateShadowDatabase(mock.spawner, baseCreateInput()).pipe( + Effect.map(({ containerId, secretDirId }) => { + expect(containerId).toBe("shadow-container-id-0123456789abcdef"); + expect(secretDirId).toMatch(/^shadow-/); + const networkCreateIdx = mock.spawned.findIndex((a) => a[0] === "network"); + const createIdx = mock.spawned.findIndex((a) => a[0] === "create"); + expect(networkCreateIdx).toBeGreaterThanOrEqual(0); + expect(networkCreateIdx).toBeLessThan(createIdx); + expect(mock.spawned[createIdx]).not.toContain("--name"); + expect(mock.spawned[createIdx]).toContain("--rm"); + }), + ); + }, + ); + + it.effect("legacyRemoveShadowDatabase issues docker rm -f -v against the given id", () => { + const mock = mockSpawner(); + return legacyRemoveShadowDatabase(mock.spawner, { + containerId: "shadow-container-id-0123456789abcdef", + secretDirId: "", + workdir: "/proj", + }).pipe( + Effect.map(() => { + expect(mock.spawned).toContainEqual([ + "rm", + "-f", + "-v", + "shadow-container-id-0123456789abcdef", + ]); + }), + Effect.provide(mockOutput().layer), + ); + }); + + it.effect( + "legacyRemoveShadowDatabase is a pure no-op (no spawn at all) for an empty container id", + () => { + const mock = mockSpawner(); + return legacyRemoveShadowDatabase(mock.spawner, { + containerId: "", + secretDirId: "", + workdir: "/proj", + }).pipe( + Effect.map(() => { + expect(mock.spawned).toEqual([]); + }), + Effect.provide(mockOutput().layer), + ); + }, + ); + + it.effect( + "legacyRemoveShadowDatabase rm -rf's the staged secret directory keyed off secretDirId", + () => { + const workdir = mkdtempSync(join(tmpdir(), "legacy-shadow-database-")); + const mock = mockSpawner(); + const secretDir = join(workdir, "supabase", ".temp", "start-secrets", "shadow-abc123"); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fs.makeDirectory(secretDir, { recursive: true }); + yield* fs.writeFileString(path.join(secretDir, "secret-0"), "root-key"); + yield* legacyRemoveShadowDatabase(mock.spawner, { + containerId: "shadow-container-id-0123456789abcdef", + secretDirId: "shadow-abc123", + workdir, + }); + const stillExists = yield* fs.exists(secretDir); + expect(stillExists).toBe(false); + rmSync(workdir, { recursive: true, force: true }); + }).pipe(Effect.provide(Layer.mergeAll(BunServices.layer, mockOutput().layer))); + }, + ); +}); + +describe("legacyConnectShadowDatabase", () => { + it.effect( + "dials the shadow's own connect config and returns the session on the first successful attempt", + () => { + const { session } = fakeSession(); + return legacyConnectShadowDatabase({ + host: "127.0.0.1", + port: 54320, + user: "postgres", + password: "postgres", + database: "postgres", + }).pipe( + Effect.scoped, + Effect.map((resolvedSession) => { + expect(resolvedSession).toBe(session); + }), + Effect.provide(mockDbConnection(session)), + ); + }, + ); + + it("the retry timeout constant matches Go's fixed 10-second ConnectShadowDatabase literal", () => { + expect(LEGACY_SHADOW_CONNECT_TIMEOUT_SECONDS).toBe(10); + }); +}); + +function baseSetupDatabaseInput( + session: LegacyDbSession, + fs: FileSystem.FileSystem, + path: Path.Path, + workdir: string, +): LegacySetupDatabaseInput { + return { + session, + fs, + path, + workdir, + config: defaultConfig, + majorVersion: 17, + dbHost: "abcdef012345", + projectId: "proj", + networkId: "supabase_network_proj", + dbUrl: "postgresql://postgres:postgrespassword@127.0.0.1:54322/postgres", + jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", + jwks: '{"keys":[]}', + apiUrl: "http://127.0.0.1:54321", + siteUrl: defaultConfig.auth.site_url, + anonKey: "anon-key", + serviceRoleKey: "service-role-key", + storageTargetMigration: "", + images: { + realtime: "public.ecr.aws/supabase/realtime:v2.34.7", + storage: "public.ecr.aws/supabase/storage-api:v1.0.0", + auth: "public.ecr.aws/supabase/gotrue:v2.170.0", + }, + projectEnvValues: undefined, + debug: false, + apiAutoExposeNewTables: Option.some(true), + vault: [], + }; +} + +describe("legacySetupShadowConn", () => { + it.effect("runs SetupDatabase, then execs CREATE_TEMPLATE when withTemplate is true", () => { + const { session, calls } = fakeSession(); + const workdir = mkdtempSync(join(tmpdir(), "legacy-shadow-database-")); + const mock = mockSpawner(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* legacySetupShadowConn( + mock.spawner, + baseSetupDatabaseInput(session, fs, path, workdir), + true, + ); + expect(calls.some((c) => c.sql === LEGACY_SHADOW_CREATE_TEMPLATE_SQL)).toBe(true); + rmSync(workdir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll(BunServices.layer, mockOutput().layer, mockDockerRun(), mockRuntimeInfo()), + ), + ); + }); + + it.effect("skips CREATE_TEMPLATE when withTemplate is false", () => { + const { session, calls } = fakeSession(); + const workdir = mkdtempSync(join(tmpdir(), "legacy-shadow-database-")); + const mock = mockSpawner(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* legacySetupShadowConn( + mock.spawner, + baseSetupDatabaseInput(session, fs, path, workdir), + false, + ); + expect(calls.some((c) => c.sql === LEGACY_SHADOW_CREATE_TEMPLATE_SQL)).toBe(false); + rmSync(workdir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll(BunServices.layer, mockOutput().layer, mockDockerRun(), mockRuntimeInfo()), + ), + ); + }); +}); + +function baseShadowSetup( + overrides: Partial> = {}, +): LegacyShadowDbSetupInput { + return { + majorVersion: 17, + config: defaultConfig, + dbUrl: "postgresql://postgres:postgrespassword@127.0.0.1:54322/postgres", + jwtSecret: "super-secret-jwt-token-with-at-least-32-characters-long", + jwks: Effect.succeed('{"keys":[]}') as Effect.Effect, + apiUrl: "http://127.0.0.1:54321", + authExternalUrl: undefined, + siteUrl: defaultConfig.auth.site_url, + anonKey: "anon-key", + serviceRoleKey: "service-role-key", + storageTargetMigration: "", + realtimeEnabledForSetup: false, + storageEnabledForSetup: false, + authEnabledForSetup: false, + serviceVersionOverrides: {}, + projectEnvValues: undefined, + debug: false, + apiAutoExposeNewTables: Option.some(true), + vault: [], + ...overrides, + }; +} + +describe("legacyBuildShadowSetupDatabaseInput", () => { + it.effect( + "derives dbHost from the container's own 12-char short id and threads every field through", + () => { + const { session } = fakeSession(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const built = legacyBuildShadowSetupDatabaseInput( + { + fs, + path, + workdir: "/proj", + projectId: "proj", + container: "shadow-container-id-0123456789abcdef", + networkId: "supabase_network_proj", + connConfig: { + host: "127.0.0.1", + port: 54320, + user: "postgres", + password: "postgres", + database: "postgres", + }, + setup: baseShadowSetup(), + }, + session, + { + jwks: '{"keys":[]}', + images: { + realtime: "public.ecr.aws/supabase/realtime:v2.34.7", + storage: "public.ecr.aws/supabase/storage-api:v1.0.0", + auth: "public.ecr.aws/supabase/gotrue:v2.170.0", + }, + }, + ); + // Go's `container[:12]` — the future callers this was exported for (`migration + // squash`) need this exact same derivation, not a re-implementation. + expect(built.dbHost).toBe("shadow-conta"); + expect(built.session).toBe(session); + expect(built.workdir).toBe("/proj"); + expect(built.networkId).toBe("supabase_network_proj"); + expect(built.majorVersion).toBe(17); + expect(built.jwks).toBe('{"keys":[]}'); + expect(built.images.realtime).toBe("public.ecr.aws/supabase/realtime:v2.34.7"); + expect(built.apiAutoExposeNewTables).toEqual(Option.some(true)); + expect(built.vault).toEqual([]); + }).pipe(Effect.provide(BunServices.layer)); + }, + ); +}); + +describe("legacySetupShadowDatabase / legacyMigrateShadowDatabase", () => { + it.effect( + "legacySetupShadowDatabase connects, sets up the platform baseline, and creates the template database", + () => { + const { session, calls } = fakeSession(); + const workdir = mkdtempSync(join(tmpdir(), "legacy-shadow-database-")); + const mock = mockSpawner(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* legacySetupShadowDatabase(mock.spawner, { + fs, + path, + workdir, + projectId: "proj", + container: "shadow-container-id-0123456789abcdef", + networkId: "supabase_network_proj", + connConfig: { + host: "127.0.0.1", + port: 54320, + user: "postgres", + password: "postgres", + database: "postgres", + }, + setup: baseShadowSetup(), + }); + expect(calls.some((c) => c.sql === LEGACY_SHADOW_CREATE_TEMPLATE_SQL)).toBe(true); + rmSync(workdir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + mockOutput().layer, + mockDockerRun(), + mockRuntimeInfo(), + mockDbConnection(session), + ), + ), + ); + }, + ); + + it.effect( + "legacyMigrateShadowDatabase applies pending local migrations after the platform baseline", + () => { + const { session, calls } = fakeSession(); + const workdir = mkdtempSync(join(tmpdir(), "legacy-shadow-database-")); + const mock = mockSpawner(); + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* fs.makeDirectory(path.join(workdir, "supabase", "migrations"), { recursive: true }); + yield* fs.writeFileString( + path.join(workdir, "supabase", "migrations", "20240101000000_init.sql"), + "create table t ();", + ); + yield* legacyMigrateShadowDatabase(mock.spawner, { + fs, + path, + workdir, + projectId: "proj", + container: "shadow-container-id-0123456789abcdef", + networkId: "supabase_network_proj", + connConfig: { + host: "127.0.0.1", + port: 54320, + user: "postgres", + password: "postgres", + database: "postgres", + }, + setup: baseShadowSetup(), + }); + expect(calls.some((c) => c.sql === LEGACY_SHADOW_CREATE_TEMPLATE_SQL)).toBe(true); + expect(calls.some((c) => c.sql.includes("create table t ()"))).toBe(true); + rmSync(workdir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + mockOutput().layer, + mockDockerRun(), + mockRuntimeInfo(), + mockDbConnection(session), + ), + ), + ); + }, + ); + + it.effect( + "does not resolve JWKS on PG14 even when realtime is enabled (Go's initSchema never reaches ResolveJWKS for MajorVersion <= 14)", + () => { + const { session } = fakeSession(); + const workdir = mkdtempSync(join(tmpdir(), "legacy-shadow-database-")); + const mock = mockSpawner(); + let jwksEvaluated = false; + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* legacySetupShadowDatabase(mock.spawner, { + fs, + path, + workdir, + projectId: "proj", + container: "shadow-container-id-0123456789abcdef", + networkId: "supabase_network_proj", + connConfig: { + host: "127.0.0.1", + port: 54320, + user: "postgres", + password: "postgres", + database: "postgres", + }, + setup: baseShadowSetup({ + majorVersion: 14, + realtimeEnabledForSetup: true, + jwks: Effect.sync(() => { + jwksEvaluated = true; + return '{"keys":[]}'; + }), + }), + }); + expect(jwksEvaluated).toBe(false); + rmSync(workdir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + mockOutput().layer, + mockDockerRun(), + mockRuntimeInfo(), + mockDbConnection(session), + ), + ), + ); + }, + ); + + it.effect("resolves JWKS on PG15+ when realtime is enabled", () => { + const { session } = fakeSession(); + const workdir = mkdtempSync(join(tmpdir(), "legacy-shadow-database-")); + const mock = mockSpawner(); + let jwksEvaluated = false; + return Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + yield* legacySetupShadowDatabase(mock.spawner, { + fs, + path, + workdir, + projectId: "proj", + container: "shadow-container-id-0123456789abcdef", + networkId: "supabase_network_proj", + connConfig: { + host: "127.0.0.1", + port: 54320, + user: "postgres", + password: "postgres", + database: "postgres", + }, + setup: baseShadowSetup({ + majorVersion: 17, + realtimeEnabledForSetup: true, + jwks: Effect.sync(() => { + jwksEvaluated = true; + return '{"keys":[]}'; + }), + }), + }); + expect(jwksEvaluated).toBe(true); + rmSync(workdir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + mockOutput().layer, + mockDockerRun(), + mockRuntimeInfo(), + mockDbConnection(session), + ), + ), + ); + }); + + it.effect( + "legacyMigrateShadowDatabase lists local migrations BEFORE connecting, tolerating a missing migrations directory as an empty list rather than a failure", + () => { + const workdir = mkdtempSync(join(tmpdir(), "legacy-shadow-database-")); + const mock = mockSpawner(); + // One shared, ordered log — recording both events into separate booleans (the prior + // version of this test) would still pass if the two steps were swapped, since both + // would still end up `true`; only an ordered log actually proves the sequence. + const events: Array = []; + const dbConnection = Layer.succeed(LegacyDbConnection, { + connect: () => + Effect.sync(() => { + events.push("connect"); + return fakeSession().session; + }), + }); + return Effect.gen(function* () { + const realFs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const migrationsDir = path.join(workdir, "supabase", "migrations"); + const fs = FileSystem.FileSystem.of({ + ...realFs, + readDirectory: (dir, opts) => { + if (dir === migrationsDir) events.push("list"); + return realFs.readDirectory(dir, opts); + }, + }); + // No `supabase/migrations` directory exists — Go's `ListLocalMigrations` on a + // missing dir resolves to an empty list (not an error), so this exercises the + // ordering guarantee (list BEFORE connect) rather than a failure path. + yield* legacyMigrateShadowDatabase(mock.spawner, { + fs, + path, + workdir, + projectId: "proj", + container: "shadow-container-id-0123456789abcdef", + networkId: "supabase_network_proj", + connConfig: { + host: "127.0.0.1", + port: 54320, + user: "postgres", + password: "postgres", + database: "postgres", + }, + setup: baseShadowSetup(), + }); + expect(events).toEqual(["list", "connect"]); + rmSync(workdir, { recursive: true, force: true }); + }).pipe( + Effect.provide( + Layer.mergeAll( + BunServices.layer, + mockOutput().layer, + mockDockerRun(), + mockRuntimeInfo(), + dbConnection, + ), + ), + ); + }, + ); +}); + +describe("LEGACY_SHADOW_ENTRYPOINT_ARGS", () => { + it("matches Go's -c max_worker_processes=0 args exactly", () => { + expect(LEGACY_SHADOW_ENTRYPOINT_ARGS).toBe("-c max_worker_processes=0"); + }); +}); diff --git a/apps/cli/src/legacy/shared/legacy-local-project-context.ts b/apps/cli/src/legacy/shared/legacy-local-project-context.ts index 8e2cbc70b3..ae81fab4a3 100644 --- a/apps/cli/src/legacy/shared/legacy-local-project-context.ts +++ b/apps/cli/src/legacy/shared/legacy-local-project-context.ts @@ -55,6 +55,14 @@ export interface LegacyLocalProjectContext { export const legacyLoadLocalProjectContext = ( workdir: string, mapConfigLoadError: (message: string) => E, + // The resolved `--linked` ref, when the caller already has one in scope (`db diff`/`db + // pull`'s shadow-provisioning prelude — CLI-1956) — threaded straight into + // `loadProjectConfig`'s own `projectRef` option so the matching `[remotes.]` block + // merges over the base config, exactly like `legacyReadDbToml(..., ref)` already does for + // those same commands' OTHER config read. `db start`/`db reset` never pass this (neither + // operates against a linked target), so it defaults to `undefined` — no remote merge, + // unchanged from before. + projectRef?: string, ): Effect.Effect => Effect.gen(function* () { // `search: false`: `workdir` already IS Go's fully-resolved chdir target (`legacy-cli-config. @@ -152,6 +160,7 @@ export const legacyLoadLocalProjectContext = ( // `config.toml`. tomlOnly: true, goViperCompat: true, + projectRef, }).pipe( Effect.mapError((cause) => mapConfigLoadError(`failed to read config: ${String(cause)}`)), ); diff --git a/apps/cli/tests/helpers/legacy-mocks.ts b/apps/cli/tests/helpers/legacy-mocks.ts index 1242bd1d44..c3c74dc946 100644 --- a/apps/cli/tests/helpers/legacy-mocks.ts +++ b/apps/cli/tests/helpers/legacy-mocks.ts @@ -4,8 +4,9 @@ import { join } from "node:path"; import { BunServices } from "@effect/platform-bun"; import { type ApiClient, makeApiClient, type SupabaseApiConfigError } from "@supabase/api/effect"; -import { Effect, FileSystem, Layer, Option, Redacted } from "effect"; +import { Effect, FileSystem, Layer, Option, Redacted, Sink, Stream } from "effect"; import { PlatformError, SystemError } from "effect/PlatformError"; +import { ChildProcessSpawner } from "effect/unstable/process"; import * as HttpClient from "effect/unstable/http/HttpClient"; import * as HttpClientError from "effect/unstable/http/HttpClientError"; import * as HttpClientResponse from "effect/unstable/http/HttpClientResponse"; @@ -702,6 +703,81 @@ export function legacyFailWriteStringOnNthCallFsLayer( ).pipe(Layer.provide(BunServices.layer)); } +// --------------------------------------------------------------------------- +// Shadow-database container-CLI spawner — shared by `db diff`/`db pull`'s native +// shadow-provisioning integration tests (CLI-1956). Hoisted here (it was a verbatim +// ~55-line duplicate in both `diff.integration.test.ts` and `pull.integration.test.ts`) +// per `apps/cli/CLAUDE.md`'s "Hoist Before You Duplicate" rule. +// --------------------------------------------------------------------------- + +/** The shadow container's fake id — used both as `docker create`'s stdout and the `dbHost` `.slice(0, 12)` derives from. */ +export const LEGACY_FAKE_SHADOW_CONTAINER_ID = "abc123456789shadow0".padEnd(64, "0").slice(0, 64); + +/** Go's `container.HealthConfig`-shaped inspect JSON for a healthy container. */ +const LEGACY_SHADOW_HEALTHY_STATE = + '{"Running":true,"Status":"running","Health":{"Status":"healthy"}}'; + +/** + * Fakes every `docker`/`podman` subprocess call the native shadow-provisioning path issues + * (`legacyBuildLocalDbContainerInputs`'s image-cache check, `legacyCreateShadowDatabase`'s + * network-create + container create/start, `legacyWaitForHealthyServices`'s container + * inspect, and `legacyRemoveShadowDatabase`'s cleanup) — scoped-down port of + * `start.integration.test.ts`'s own `mockContainerCliSpawner`, since both callers only ever + * create one (shadow) container, never named. + */ +export function mockLegacyShadowContainerCliSpawner(): { + readonly layer: Layer.Layer; + readonly spawned: ReadonlyArray<{ readonly args: ReadonlyArray }>; +} { + const spawned: Array<{ readonly args: ReadonlyArray }> = []; + const encoder = new TextEncoder(); + + const layer = Layer.succeed( + ChildProcessSpawner.ChildProcessSpawner, + ChildProcessSpawner.make((command) => + Effect.gen(function* () { + const args = command._tag === "StandardCommand" ? command.args : []; + spawned.push({ args }); + if (command._tag !== "StandardCommand") { + return yield* Effect.fail( + new PlatformError( + new SystemError({ + _tag: "NotFound", + module: "ChildProcess", + method: "spawn", + description: "spawn failed", + }), + ), + ); + } + let stdoutLines: ReadonlyArray = []; + if (args[0] === "create") { + stdoutLines = [LEGACY_FAKE_SHADOW_CONTAINER_ID]; + } else if (args[0] === "container" && args[1] === "inspect") { + stdoutLines = [LEGACY_SHADOW_HEALTHY_STATE]; + } + // "image inspect", "network create", "start", "rm -f -v" all succeed with no output. + const stdoutBytes = stdoutLines.map((line) => encoder.encode(`${line}\n`)); + return ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(7000 + spawned.length), + stdout: Stream.fromIterable(stdoutBytes), + stderr: Stream.empty, + all: Stream.empty, + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(0)), + isRunning: Effect.succeed(false), + stdin: Sink.drain, + kill: () => Effect.void, + unref: Effect.succeed(Effect.void), + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + }); + }), + ), + ); + + return { layer, spawned }; +} + // --------------------------------------------------------------------------- // Runtime composition — bundles the entire Layer.mergeAll(...) graph that // every native-port integration test re-builds, including the easy-to-mis-wire