From 9488b4d2bfc352e033e0a87f9c2b7262ae0ca5fe Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Fri, 12 Jul 2024 21:07:49 +0200 Subject: [PATCH 1/3] WIP dump of data dir to a tar.gz --- packages/pglite/examples/dumpDataDir.html | 1 + packages/pglite/examples/dumpDataDir.js | 26 ++++++ packages/pglite/src/fs/idbfs.ts | 5 ++ packages/pglite/src/fs/memoryfs.ts | 7 +- packages/pglite/src/fs/nodefs.ts | 7 +- packages/pglite/src/fs/tarUtils.ts | 104 ++++++++++++++++++++++ packages/pglite/src/fs/types.ts | 14 +-- packages/pglite/src/interface.ts | 7 ++ packages/pglite/src/pglite.ts | 13 +++ packages/pglite/src/worker/index.ts | 4 + packages/pglite/src/worker/process.ts | 8 ++ 11 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 packages/pglite/examples/dumpDataDir.html create mode 100644 packages/pglite/examples/dumpDataDir.js create mode 100644 packages/pglite/src/fs/tarUtils.ts diff --git a/packages/pglite/examples/dumpDataDir.html b/packages/pglite/examples/dumpDataDir.html new file mode 100644 index 000000000..f02cfc466 --- /dev/null +++ b/packages/pglite/examples/dumpDataDir.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/pglite/examples/dumpDataDir.js b/packages/pglite/examples/dumpDataDir.js new file mode 100644 index 000000000..bc67eee56 --- /dev/null +++ b/packages/pglite/examples/dumpDataDir.js @@ -0,0 +1,26 @@ +import { PGlite } from "../dist/index.js"; + +const pg = new PGlite(); +await pg.exec(` + CREATE TABLE IF NOT EXISTS test ( + id SERIAL PRIMARY KEY, + name TEXT + ); +`); +await pg.exec("INSERT INTO test (name) VALUES ('test');"); + +const { tarball, filename } = await pg.dumpDataDir(); + +if (typeof window !== 'undefined') { + // Download the dump + const blob = new Blob([tarball], { type: "application/octet-stream" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = filename; + a.click(); +} else { + // Save the dump to a file using node fs + const fs = await import("fs"); + fs.writeFileSync(filename, tarball); +} diff --git a/packages/pglite/src/fs/idbfs.ts b/packages/pglite/src/fs/idbfs.ts index 7a34fae1f..bce5149e8 100644 --- a/packages/pglite/src/fs/idbfs.ts +++ b/packages/pglite/src/fs/idbfs.ts @@ -1,6 +1,7 @@ import { FilesystemBase } from "./types.js"; import type { FS, PostgresMod } from "../postgres.js"; import { PGDATA } from "./index.js"; +import { dumpTar } from "./tarUtils.js"; export class IdbFs extends FilesystemBase { async emscriptenOpts(opts: Partial) { @@ -48,4 +49,8 @@ export class IdbFs extends FilesystemBase { }); }); } + + async dumpTar(mod: FS) { + return dumpTar(mod); + } } diff --git a/packages/pglite/src/fs/memoryfs.ts b/packages/pglite/src/fs/memoryfs.ts index ee7aa7a29..0242da740 100644 --- a/packages/pglite/src/fs/memoryfs.ts +++ b/packages/pglite/src/fs/memoryfs.ts @@ -1,9 +1,14 @@ import { FilesystemBase } from "./types.js"; -import type { PostgresMod } from "../postgres.js"; +import type { PostgresMod, FS } from "../postgres.js"; +import { dumpTar } from "./tarUtils.js"; export class MemoryFS extends FilesystemBase { async emscriptenOpts(opts: Partial) { // Nothing to do for memoryfs return opts; } + + async dumpTar(mod: FS) { + return dumpTar(mod); + } } diff --git a/packages/pglite/src/fs/nodefs.ts b/packages/pglite/src/fs/nodefs.ts index 2670f6139..887c4e74d 100644 --- a/packages/pglite/src/fs/nodefs.ts +++ b/packages/pglite/src/fs/nodefs.ts @@ -2,7 +2,8 @@ import * as fs from "fs"; import * as path from "path"; import { FilesystemBase } from "./types.js"; import { PGDATA } from "./index.js"; -import type { PostgresMod } from "../postgres.js"; +import type { PostgresMod, FS } from "../postgres.js"; +import { dumpTar } from "./tarUtils.js"; export class NodeFS extends FilesystemBase { protected rootDir: string; @@ -29,4 +30,8 @@ export class NodeFS extends FilesystemBase { }; return options; } + + async dumpTar(mod: FS) { + return dumpTar(mod); + } } diff --git a/packages/pglite/src/fs/tarUtils.ts b/packages/pglite/src/fs/tarUtils.ts new file mode 100644 index 000000000..51f3f0fed --- /dev/null +++ b/packages/pglite/src/fs/tarUtils.ts @@ -0,0 +1,104 @@ +import { tar, type TarFile } from "tinytar"; +import { FS } from "../postgres.js"; +import { PGDATA } from "./index.js"; + +export interface DumpTarResult { + tarball: Uint8Array; + extension: ".tar" | ".tgz"; +} + +export async function dumpTar(FS: FS): Promise { + const tarball = createTarball(FS, PGDATA); + const [compressed, zipped] = await maybeZip(tarball); + return { + tarball: compressed, + extension: zipped ? ".tgz" : ".tar", + }; +} + +function readDirectory(FS: FS, path: string) { + let files: TarFile[] = []; + + const traverseDirectory = (currentPath: string) => { + const entries = FS.readdir(currentPath); + entries.forEach((entry) => { + if (entry === "." || entry === "..") { + return; + } + const fullPath = currentPath + "/" + entry; + const stats = FS.stat(fullPath); + if (FS.isDir(stats.mode)) { + traverseDirectory(fullPath); + } else if (FS.isFile(stats.mode)) { + const data = FS.readFile(fullPath, { encoding: "binary" }); + files.push({ + name: fullPath.substring(path.length), // remove the root path + mode: stats.mode, + size: stats.size, + modifyTime: stats.mtime, + data: new Uint8Array(data), + }); + } + }); + }; + + traverseDirectory(path); + return files; +} + +export function createTarball(FS: FS, directoryPath: string) { + const files = readDirectory(FS, directoryPath); + const tarball = tar(files); + return tarball; +} + +export async function maybeZip( + file: Uint8Array, +): Promise<[Uint8Array, boolean]> { + if (typeof window !== "undefined" && "CompressionStream" in window) { + return [await zipBrowser(file), true]; + } else if ( + typeof process !== "undefined" && + process.versions && + process.versions.node + ) { + return [await zipNode(file), true]; + } else { + return [file, false]; + } +} + +export async function zipBrowser(file: Uint8Array): Promise { + const cs = new CompressionStream("gzip"); + const writer = cs.writable.getWriter(); + const reader = cs.readable.getReader(); + + writer.write(file); + writer.close(); + + const chunks: Uint8Array[] = []; + + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value) chunks.push(value); + } + + const compressed = new Uint8Array( + chunks.reduce((acc, chunk) => acc + chunk.length, 0), + ); + let offset = 0; + chunks.forEach((chunk) => { + compressed.set(chunk, offset); + offset += chunk.length; + }); + + return compressed; +} + +export async function zipNode(file: Uint8Array): Promise { + const { promisify } = await import("util"); + const { gzip } = await import("zlib"); + const gzipPromise = promisify(gzip); + return await gzipPromise(file); +} diff --git a/packages/pglite/src/fs/types.ts b/packages/pglite/src/fs/types.ts index a9dcae5f7..d3dc76a39 100644 --- a/packages/pglite/src/fs/types.ts +++ b/packages/pglite/src/fs/types.ts @@ -1,4 +1,5 @@ import type { PostgresMod, FS } from "../postgres.js"; +import type { DumpTarResult } from "./tarUtils.js"; export type FsType = "nodefs" | "idbfs" | "memoryfs"; @@ -15,15 +16,17 @@ export interface Filesystem { /** * Sync the filesystem to the emscripten filesystem. */ - syncToFs(mod: FS): Promise; + syncToFs(FS: FS): Promise; /** * Sync the emscripten filesystem to the filesystem. */ - initialSyncFs(mod: FS): Promise; + initialSyncFs(FS: FS): Promise; - // on_mount(): Function; - // load_extension(ext: string): Promise; + /** + * Dump the PGDATA dir from the filesystem to a gziped tarball. + */ + dumpTar(FS: FS): Promise; } export abstract class FilesystemBase implements Filesystem { @@ -34,6 +37,7 @@ export abstract class FilesystemBase implements Filesystem { abstract emscriptenOpts( opts: Partial, ): Promise>; - async syncToFs(mod: FS) {} + async syncToFs(FS: FS) {} async initialSyncFs(mod: FS) {} + abstract dumpTar(mod: FS): Promise; } diff --git a/packages/pglite/src/interface.ts b/packages/pglite/src/interface.ts index 1eb895f15..2b067b905 100644 --- a/packages/pglite/src/interface.ts +++ b/packages/pglite/src/interface.ts @@ -43,6 +43,12 @@ export type Extensions = { [namespace: string]: Extension | URL; }; +export interface DumpDataDirResult { + tarball: Uint8Array; + extension: ".tar" | ".tgz"; + filename: string; +} + export interface PGliteOptions { dataDir?: string; fs?: Filesystem; @@ -83,6 +89,7 @@ export type PGliteInterface = { callback: (channel: string, payload: string) => void, ): () => void; offNotification(callback: (channel: string, payload: string) => void): void; + dumpDataDir(): Promise; }; export type PGliteInterfaceExtensions = E extends Extensions diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index 4d094d606..31396b80c 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -34,6 +34,8 @@ export class PGlite implements PGliteInterface { fs?: Filesystem; protected mod?: PostgresMod; + readonly dataDir?: string; + #ready = false; #closing = false; #closed = false; @@ -92,6 +94,7 @@ export class PGlite implements PGliteInterface { } else { options = dataDirOrPGliteOptions; } + this.dataDir = options.dataDir; // Enable debug logging if requested if (options?.debug !== undefined) { @@ -677,4 +680,14 @@ export class PGlite implements PGliteInterface { ): PGlite & PGliteInterfaceExtensions { return new PGlite(options) as any; } + + /** + * Dump the PGDATA dir from the filesystem to a gziped tarball. + */ + async dumpDataDir() { + const { tarball, extension } = await this.fs!.dumpTar(this.mod!.FS); + let filename = this.dataDir ? this.dataDir.split("/").pop() : "pgdata"; + filename = `${filename}${extension}`; + return { tarball, extension, filename }; + } } diff --git a/packages/pglite/src/worker/index.ts b/packages/pglite/src/worker/index.ts index a2b5a9c8d..b69be732a 100644 --- a/packages/pglite/src/worker/index.ts +++ b/packages/pglite/src/worker/index.ts @@ -144,4 +144,8 @@ export class PGliteWorker implements PGliteInterface { queueMicrotask(() => listener(channel, payload)); } } + + async dumpDataDir() { + return this.#worker.dumpDataDir(); + } } diff --git a/packages/pglite/src/worker/process.ts b/packages/pglite/src/worker/process.ts index d9695b94f..118327783 100644 --- a/packages/pglite/src/worker/process.ts +++ b/packages/pglite/src/worker/process.ts @@ -34,6 +34,14 @@ const worker = { async execProtocol(message: Uint8Array) { return await db.execProtocol(message); }, + async dumpDataDir() { + const ret = await db.dumpDataDir(); + return { + tarball: Comlink.transfer(ret.tarball, [ret.tarball.buffer]), + extension: ret.extension, + filename: ret.filename, + }; + }, }; Comlink.expose(worker); From 33314058ef3196e96bc1c0c199dd53d0110545ac Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Sat, 13 Jul 2024 12:48:13 +0200 Subject: [PATCH 2/3] Dump and loading of a datadir to a tarball + tests --- packages/pglite/examples/dumpDataDir.js | 12 +- packages/pglite/src/definitions/tinytar.d.ts | 5 + packages/pglite/src/fs/tarUtils.ts | 117 +++++++++++++++++-- packages/pglite/src/fs/types.ts | 6 +- packages/pglite/src/interface.ts | 7 ++ packages/pglite/src/pglite.ts | 16 +++ packages/pglite/tests/dump.test.js | 30 +++++ 7 files changed, 176 insertions(+), 17 deletions(-) create mode 100644 packages/pglite/tests/dump.test.js diff --git a/packages/pglite/examples/dumpDataDir.js b/packages/pglite/examples/dumpDataDir.js index bc67eee56..2dc42d1ef 100644 --- a/packages/pglite/examples/dumpDataDir.js +++ b/packages/pglite/examples/dumpDataDir.js @@ -9,9 +9,9 @@ await pg.exec(` `); await pg.exec("INSERT INTO test (name) VALUES ('test');"); -const { tarball, filename } = await pg.dumpDataDir(); +const { tarball, filename, extension } = await pg.dumpDataDir(); -if (typeof window !== 'undefined') { +if (typeof window !== "undefined") { // Download the dump const blob = new Blob([tarball], { type: "application/octet-stream" }); const url = URL.createObjectURL(blob); @@ -24,3 +24,11 @@ if (typeof window !== 'undefined') { const fs = await import("fs"); fs.writeFileSync(filename, tarball); } + +const pg2 = new PGlite({ + // debug: 1, + loadDataDir: { tarball, extension }, +}); + +const rows = await pg2.query("SELECT * FROM test;"); +console.log(rows); diff --git a/packages/pglite/src/definitions/tinytar.d.ts b/packages/pglite/src/definitions/tinytar.d.ts index 677f02556..90e4f7609 100644 --- a/packages/pglite/src/definitions/tinytar.d.ts +++ b/packages/pglite/src/definitions/tinytar.d.ts @@ -34,6 +34,8 @@ declare module "tinytar" { const NULL_CHAR: string; const TMAGIC: string; const OLDGNU_MAGIC: string; + + // Values used in typeflag field const REGTYPE: number; const LNKTYPE: number; const SYMTYPE: number; @@ -42,6 +44,8 @@ declare module "tinytar" { const DIRTYPE: number; const FIFOTYPE: number; const CONTTYPE: number; + + // Bits used in the mode field, values in octal const TSUID: number; const TSGID: number; const TSVTX: number; @@ -54,6 +58,7 @@ declare module "tinytar" { const TOREAD: number; const TOWRITE: number; const TOEXEC: number; + const TPERMALL: number; const TPERMMASK: number; } diff --git a/packages/pglite/src/fs/tarUtils.ts b/packages/pglite/src/fs/tarUtils.ts index 51f3f0fed..699f44646 100644 --- a/packages/pglite/src/fs/tarUtils.ts +++ b/packages/pglite/src/fs/tarUtils.ts @@ -1,13 +1,13 @@ -import { tar, type TarFile } from "tinytar"; +import { tar, untar, type TarFile, REGTYPE, DIRTYPE } from "tinytar"; import { FS } from "../postgres.js"; import { PGDATA } from "./index.js"; -export interface DumpTarResult { +export interface DumpedTar { tarball: Uint8Array; extension: ".tar" | ".tgz"; } -export async function dumpTar(FS: FS): Promise { +export async function dumpTar(FS: FS): Promise { const tarball = createTarball(FS, PGDATA); const [compressed, zipped] = await maybeZip(tarball); return { @@ -16,6 +16,40 @@ export async function dumpTar(FS: FS): Promise { }; } +export async function loadTar(FS: FS, dump: DumpedTar): Promise { + let tarball = dump.tarball; + console.log("loading tarball"); + if (dump.extension === ".tgz") { + tarball = await unzip(tarball); + } + + const files = untar(tarball); + for (const file of files) { + const filePath = PGDATA + file.name; + + // Ensure the directory structure exists + const dirPath = filePath.split("/").slice(0, -1); + for (let i = 1; i <= dirPath.length; i++) { + const dir = dirPath.slice(0, i).join("/"); + if (!FS.analyzePath(dir).exists) { + FS.mkdir(dir); + } + } + + // Write the file or directory + if (file.type == REGTYPE) { + FS.writeFile(filePath, file.data); + FS.utime( + filePath, + dateToUnixTimestamp(file.modifyTime), + dateToUnixTimestamp(file.modifyTime), + ); + } else if (file.type == DIRTYPE) { + FS.mkdir(filePath); + } + } +} + function readDirectory(FS: FS, path: string) { let files: TarFile[] = []; @@ -27,17 +61,19 @@ function readDirectory(FS: FS, path: string) { } const fullPath = currentPath + "/" + entry; const stats = FS.stat(fullPath); + const data = FS.isFile(stats.mode) + ? FS.readFile(fullPath, { encoding: "binary" }) + : new Uint8Array(0); + files.push({ + name: fullPath.substring(path.length), // remove the root path + mode: stats.mode, + size: stats.size, + type: FS.isFile(stats.mode) ? REGTYPE : DIRTYPE, + modifyTime: stats.mtime, + data, + }); if (FS.isDir(stats.mode)) { traverseDirectory(fullPath); - } else if (FS.isFile(stats.mode)) { - const data = FS.readFile(fullPath, { encoding: "binary" }); - files.push({ - name: fullPath.substring(path.length), // remove the root path - mode: stats.mode, - size: stats.size, - modifyTime: stats.mtime, - data: new Uint8Array(data), - }); } }); }; @@ -102,3 +138,60 @@ export async function zipNode(file: Uint8Array): Promise { const gzipPromise = promisify(gzip); return await gzipPromise(file); } + +export async function unzip(file: Uint8Array): Promise { + if (typeof window !== "undefined" && "DecompressionStream" in window) { + return await unzipBrowser(file); + } else if ( + typeof process !== "undefined" && + process.versions && + process.versions.node + ) { + return await unzipNode(file); + } else { + throw new Error("Unsupported environment for decompression"); + } +} + +export async function unzipBrowser(file: Uint8Array): Promise { + const ds = new DecompressionStream("gzip"); + const writer = ds.writable.getWriter(); + const reader = ds.readable.getReader(); + + writer.write(file); + writer.close(); + + const chunks: Uint8Array[] = []; + + while (true) { + const { value, done } = await reader.read(); + if (done) break; + if (value) chunks.push(value); + } + + const decompressed = new Uint8Array( + chunks.reduce((acc, chunk) => acc + chunk.length, 0), + ); + let offset = 0; + chunks.forEach((chunk) => { + decompressed.set(chunk, offset); + offset += chunk.length; + }); + + return decompressed; +} + +export async function unzipNode(file: Uint8Array): Promise { + const { promisify } = await import("util"); + const { gunzip } = await import("zlib"); + const gunzipPromise = promisify(gunzip); + return await gunzipPromise(file); +} + +function dateToUnixTimestamp(date: Date | number | undefined): number { + if (!date) { + return Math.floor(Date.now() / 1000); + } else { + return typeof date === "number" ? date : Math.floor(date.getTime() / 1000); + } +} diff --git a/packages/pglite/src/fs/types.ts b/packages/pglite/src/fs/types.ts index d3dc76a39..790c02962 100644 --- a/packages/pglite/src/fs/types.ts +++ b/packages/pglite/src/fs/types.ts @@ -1,5 +1,5 @@ import type { PostgresMod, FS } from "../postgres.js"; -import type { DumpTarResult } from "./tarUtils.js"; +import type { DumpedTar } from "./tarUtils.js"; export type FsType = "nodefs" | "idbfs" | "memoryfs"; @@ -26,7 +26,7 @@ export interface Filesystem { /** * Dump the PGDATA dir from the filesystem to a gziped tarball. */ - dumpTar(FS: FS): Promise; + dumpTar(FS: FS): Promise; } export abstract class FilesystemBase implements Filesystem { @@ -39,5 +39,5 @@ export abstract class FilesystemBase implements Filesystem { ): Promise>; async syncToFs(FS: FS) {} async initialSyncFs(mod: FS) {} - abstract dumpTar(mod: FS): Promise; + abstract dumpTar(mod: FS): Promise; } diff --git a/packages/pglite/src/interface.ts b/packages/pglite/src/interface.ts index 2b067b905..4ec661f87 100644 --- a/packages/pglite/src/interface.ts +++ b/packages/pglite/src/interface.ts @@ -49,12 +49,19 @@ export interface DumpDataDirResult { filename: string; } +export interface LoadDataDir { + tarball: Uint8Array; + extension: ".tar" | ".tgz" | ".tar.gz"; + filename?: string; +} + export interface PGliteOptions { dataDir?: string; fs?: Filesystem; debug?: DebugLevel; relaxedDurability?: boolean; extensions?: Extensions; + loadDataDir?: LoadDataDir; } export type PGliteInterface = { diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index 31396b80c..d1971d1e7 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -16,6 +16,7 @@ import type { Extensions, } from "./interface.js"; import { loadExtensionBundle, loadExtensions } from "./extensionUtils.js"; +import { loadTar } from "./fs/tarUtils.js"; import { PGDATA, WASM_PREFIX } from "./fs/index.js"; @@ -194,6 +195,21 @@ export class PGlite implements PGliteInterface { // Sync the filesystem from any previous store await this.fs!.initialSyncFs(this.mod.FS); + // If the user has provided a tarball to load the database from, do that now. + // We do this after the initial sync so that we can throw if the database + // already exists. + if (options.loadDataDir) { + if (this.mod.FS.analyzePath(PGDATA + "/PG_VERSION").exists) { + throw new Error("Database already exists, cannot load from tarball"); + } + this.#log("pglite: loading data from tarball"); + const { tarball, extension } = options.loadDataDir; + await loadTar(this.mod.FS, { + tarball, + extension: extension === ".tar.gz" ? ".tgz" : extension, + }); + } + // Check and log if the database exists if (this.mod.FS.analyzePath(PGDATA + "/PG_VERSION").exists) { this.#log("pglite: found DB, resuming"); diff --git a/packages/pglite/tests/dump.test.js b/packages/pglite/tests/dump.test.js new file mode 100644 index 000000000..9e7536f91 --- /dev/null +++ b/packages/pglite/tests/dump.test.js @@ -0,0 +1,30 @@ +import test from "ava"; +import { PGlite } from "../dist/index.js"; + +test("dump data dir and load it", async (t) => { + const pg1 = new PGlite(); + await pg1.exec(` + CREATE TABLE IF NOT EXISTS test ( + id SERIAL PRIMARY KEY, + name TEXT + ); + `); + pg1.exec("INSERT INTO test (name) VALUES ('test');"); + + const ret1 = await pg1.query("SELECT * FROM test;"); + + const { tarball, filename, extension } = await pg1.dumpDataDir(); + + t.is(typeof tarball, "object"); + t.is(typeof filename, "string"); + t.is(typeof extension, "string"); + + const pg2 = new PGlite({ + // debug: 1, + loadDataDir: { tarball, extension }, + }); + + const ret2 = await pg2.query("SELECT * FROM test;"); + + t.deepEqual(ret1, ret2); +}); From 2a886db1773318fa0f1653151233b6e4fb1bebf7 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Sat, 13 Jul 2024 13:40:56 +0200 Subject: [PATCH 3/3] Swap to using File object for dump/load --- packages/pglite/examples/dumpDataDir.js | 12 ++++----- packages/pglite/src/fs/idbfs.ts | 4 +-- packages/pglite/src/fs/memoryfs.ts | 4 +-- packages/pglite/src/fs/nodefs.ts | 4 +-- packages/pglite/src/fs/tarUtils.ts | 34 +++++++++++++++---------- packages/pglite/src/fs/types.ts | 5 ++-- packages/pglite/src/interface.ts | 10 ++------ packages/pglite/src/pglite.ts | 12 +++------ packages/pglite/src/worker/process.ts | 8 ++---- packages/pglite/tests/dump.test.js | 9 +++---- 10 files changed, 43 insertions(+), 59 deletions(-) diff --git a/packages/pglite/examples/dumpDataDir.js b/packages/pglite/examples/dumpDataDir.js index 2dc42d1ef..35bdd52ee 100644 --- a/packages/pglite/examples/dumpDataDir.js +++ b/packages/pglite/examples/dumpDataDir.js @@ -9,25 +9,23 @@ await pg.exec(` `); await pg.exec("INSERT INTO test (name) VALUES ('test');"); -const { tarball, filename, extension } = await pg.dumpDataDir(); +const file = await pg.dumpDataDir(); if (typeof window !== "undefined") { // Download the dump - const blob = new Blob([tarball], { type: "application/octet-stream" }); - const url = URL.createObjectURL(blob); + const url = URL.createObjectURL(file); const a = document.createElement("a"); a.href = url; - a.download = filename; + a.download = file.name; a.click(); } else { // Save the dump to a file using node fs const fs = await import("fs"); - fs.writeFileSync(filename, tarball); + fs.writeFileSync(file.name, await file.arrayBuffer()); } const pg2 = new PGlite({ - // debug: 1, - loadDataDir: { tarball, extension }, + loadDataDir: file, }); const rows = await pg2.query("SELECT * FROM test;"); diff --git a/packages/pglite/src/fs/idbfs.ts b/packages/pglite/src/fs/idbfs.ts index bce5149e8..29fe13e47 100644 --- a/packages/pglite/src/fs/idbfs.ts +++ b/packages/pglite/src/fs/idbfs.ts @@ -50,7 +50,7 @@ export class IdbFs extends FilesystemBase { }); } - async dumpTar(mod: FS) { - return dumpTar(mod); + async dumpTar(mod: FS, dbname: string) { + return dumpTar(mod, dbname); } } diff --git a/packages/pglite/src/fs/memoryfs.ts b/packages/pglite/src/fs/memoryfs.ts index 0242da740..82b7ffdcf 100644 --- a/packages/pglite/src/fs/memoryfs.ts +++ b/packages/pglite/src/fs/memoryfs.ts @@ -8,7 +8,7 @@ export class MemoryFS extends FilesystemBase { return opts; } - async dumpTar(mod: FS) { - return dumpTar(mod); + async dumpTar(mod: FS, dbname: string) { + return dumpTar(mod, dbname); } } diff --git a/packages/pglite/src/fs/nodefs.ts b/packages/pglite/src/fs/nodefs.ts index 887c4e74d..8e846e5c7 100644 --- a/packages/pglite/src/fs/nodefs.ts +++ b/packages/pglite/src/fs/nodefs.ts @@ -31,7 +31,7 @@ export class NodeFS extends FilesystemBase { return options; } - async dumpTar(mod: FS) { - return dumpTar(mod); + async dumpTar(mod: FS, dbname: string) { + return dumpTar(mod, dbname); } } diff --git a/packages/pglite/src/fs/tarUtils.ts b/packages/pglite/src/fs/tarUtils.ts index 699f44646..0c80d43e3 100644 --- a/packages/pglite/src/fs/tarUtils.ts +++ b/packages/pglite/src/fs/tarUtils.ts @@ -2,24 +2,30 @@ import { tar, untar, type TarFile, REGTYPE, DIRTYPE } from "tinytar"; import { FS } from "../postgres.js"; import { PGDATA } from "./index.js"; -export interface DumpedTar { - tarball: Uint8Array; - extension: ".tar" | ".tgz"; -} - -export async function dumpTar(FS: FS): Promise { +export async function dumpTar(FS: FS, dbname?: string): Promise { const tarball = createTarball(FS, PGDATA); const [compressed, zipped] = await maybeZip(tarball); - return { - tarball: compressed, - extension: zipped ? ".tgz" : ".tar", - }; + const filename = (dbname || "pgdata") + (zipped ? ".tar.gz" : ".tar"); + return new File([compressed], filename, { + type: zipped ? "application/x-gtar" : "application/x-tar", + }); } -export async function loadTar(FS: FS, dump: DumpedTar): Promise { - let tarball = dump.tarball; - console.log("loading tarball"); - if (dump.extension === ".tgz") { +const compressedMimeTypes = [ + "application/x-gtar", + "application/x-tar+gzip", + "application/x-gzip", + "application/gzip", +]; + +export async function loadTar(FS: FS, file: File | Blob): Promise { + let tarball = new Uint8Array(await file.arrayBuffer()); + const filename = file instanceof File ? file.name : undefined; + const compressed = + compressedMimeTypes.includes(file.type) || + filename?.endsWith(".tgz") || + filename?.endsWith(".tar.gz"); + if (compressed) { tarball = await unzip(tarball); } diff --git a/packages/pglite/src/fs/types.ts b/packages/pglite/src/fs/types.ts index 790c02962..ed09167d9 100644 --- a/packages/pglite/src/fs/types.ts +++ b/packages/pglite/src/fs/types.ts @@ -1,5 +1,4 @@ import type { PostgresMod, FS } from "../postgres.js"; -import type { DumpedTar } from "./tarUtils.js"; export type FsType = "nodefs" | "idbfs" | "memoryfs"; @@ -26,7 +25,7 @@ export interface Filesystem { /** * Dump the PGDATA dir from the filesystem to a gziped tarball. */ - dumpTar(FS: FS): Promise; + dumpTar(FS: FS, dbname: string): Promise; } export abstract class FilesystemBase implements Filesystem { @@ -39,5 +38,5 @@ export abstract class FilesystemBase implements Filesystem { ): Promise>; async syncToFs(FS: FS) {} async initialSyncFs(mod: FS) {} - abstract dumpTar(mod: FS): Promise; + abstract dumpTar(mod: FS, dbname: string): Promise; } diff --git a/packages/pglite/src/interface.ts b/packages/pglite/src/interface.ts index 4ec661f87..15f3c544a 100644 --- a/packages/pglite/src/interface.ts +++ b/packages/pglite/src/interface.ts @@ -49,19 +49,13 @@ export interface DumpDataDirResult { filename: string; } -export interface LoadDataDir { - tarball: Uint8Array; - extension: ".tar" | ".tgz" | ".tar.gz"; - filename?: string; -} - export interface PGliteOptions { dataDir?: string; fs?: Filesystem; debug?: DebugLevel; relaxedDurability?: boolean; extensions?: Extensions; - loadDataDir?: LoadDataDir; + loadDataDir?: Blob | File; } export type PGliteInterface = { @@ -96,7 +90,7 @@ export type PGliteInterface = { callback: (channel: string, payload: string) => void, ): () => void; offNotification(callback: (channel: string, payload: string) => void): void; - dumpDataDir(): Promise; + dumpDataDir(): Promise; }; export type PGliteInterfaceExtensions = E extends Extensions diff --git a/packages/pglite/src/pglite.ts b/packages/pglite/src/pglite.ts index d1971d1e7..c184884fc 100644 --- a/packages/pglite/src/pglite.ts +++ b/packages/pglite/src/pglite.ts @@ -203,11 +203,7 @@ export class PGlite implements PGliteInterface { throw new Error("Database already exists, cannot load from tarball"); } this.#log("pglite: loading data from tarball"); - const { tarball, extension } = options.loadDataDir; - await loadTar(this.mod.FS, { - tarball, - extension: extension === ".tar.gz" ? ".tgz" : extension, - }); + await loadTar(this.mod.FS, options.loadDataDir); } // Check and log if the database exists @@ -701,9 +697,7 @@ export class PGlite implements PGliteInterface { * Dump the PGDATA dir from the filesystem to a gziped tarball. */ async dumpDataDir() { - const { tarball, extension } = await this.fs!.dumpTar(this.mod!.FS); - let filename = this.dataDir ? this.dataDir.split("/").pop() : "pgdata"; - filename = `${filename}${extension}`; - return { tarball, extension, filename }; + let dbname = this.dataDir?.split("/").pop() ?? "pgdata"; + return this.fs!.dumpTar(this.mod!.FS, dbname); } } diff --git a/packages/pglite/src/worker/process.ts b/packages/pglite/src/worker/process.ts index 118327783..cdaf91e6a 100644 --- a/packages/pglite/src/worker/process.ts +++ b/packages/pglite/src/worker/process.ts @@ -35,12 +35,8 @@ const worker = { return await db.execProtocol(message); }, async dumpDataDir() { - const ret = await db.dumpDataDir(); - return { - tarball: Comlink.transfer(ret.tarball, [ret.tarball.buffer]), - extension: ret.extension, - filename: ret.filename, - }; + const file = await db.dumpDataDir(); + return Comlink.transfer(file, [await file.arrayBuffer()]); }, }; diff --git a/packages/pglite/tests/dump.test.js b/packages/pglite/tests/dump.test.js index 9e7536f91..24eff1f44 100644 --- a/packages/pglite/tests/dump.test.js +++ b/packages/pglite/tests/dump.test.js @@ -13,15 +13,12 @@ test("dump data dir and load it", async (t) => { const ret1 = await pg1.query("SELECT * FROM test;"); - const { tarball, filename, extension } = await pg1.dumpDataDir(); + const file = await pg1.dumpDataDir(); - t.is(typeof tarball, "object"); - t.is(typeof filename, "string"); - t.is(typeof extension, "string"); + t.is(typeof file, "object"); const pg2 = new PGlite({ - // debug: 1, - loadDataDir: { tarball, extension }, + loadDataDir: file, }); const ret2 = await pg2.query("SELECT * FROM test;");