diff --git a/.changeset/minidb-durability-hardening.md b/.changeset/minidb-durability-hardening.md new file mode 100644 index 0000000000..1a3a441494 --- /dev/null +++ b/.changeset/minidb-durability-hardening.md @@ -0,0 +1,6 @@ +--- +"@moonshot-ai/minidb": patch +"@moonshot-ai/kimi-code": patch +--- + +Harden the embedded key-value engine's durability: WAL compaction now always terminates under sustained write storms instead of chasing the tail forever, a committed write can no longer slip through a compaction rotation undetected, torn WAL tails no longer misplace later disk-mode value pointers, read-only opens never create or modify database files or compact under a live writer, corrupt index-definition files no longer force a full rebuild, stale compaction temp files are cleaned on open, and the process lock can no longer be taken over by several processes at once. diff --git a/.changeset/minidb-perf-hardening.md b/.changeset/minidb-perf-hardening.md new file mode 100644 index 0000000000..fbff2a2e20 --- /dev/null +++ b/.changeset/minidb-perf-hardening.md @@ -0,0 +1,6 @@ +--- +"@moonshot-ai/minidb": patch +"@moonshot-ai/kimi-code": patch +--- + +Speed up the embedded key-value engine under stress: queries with skip/limit now stream candidates instead of decoding every match first, LRU eviction picks victims in O(1) instead of scanning every key, bursts of simultaneously expired TTL keys are drained within seconds, existence checks and size counting no longer read values when they only need metadata, and one oversized token can no longer poison the full-text index. diff --git a/.changeset/minidb-reader-catchup.md b/.changeset/minidb-reader-catchup.md new file mode 100644 index 0000000000..72f91bc451 --- /dev/null +++ b/.changeset/minidb-reader-catchup.md @@ -0,0 +1,6 @@ +--- +"@moonshot-ai/minidb": minor +"@moonshot-ai/kimi-code": patch +--- + +Cluster readers of the embedded key-value engine now catch up incrementally by replaying only newly appended WAL frames after another process writes, instead of fully reopening the shard on every read; cross-process read latency drops by orders of magnitude at larger shard sizes, and readers still fall back to a full reopen after WAL rotation or truncation. diff --git a/.changeset/minidb-review-hardening.md b/.changeset/minidb-review-hardening.md new file mode 100644 index 0000000000..a85dbef338 --- /dev/null +++ b/.changeset/minidb-review-hardening.md @@ -0,0 +1,6 @@ +--- +"@moonshot-ai/minidb": patch +"@moonshot-ai/kimi-code": patch +--- + +Keep the embedded key-value engine writable when a WAL compaction rotation fails mid-way instead of wedging it until reopen, stop a rolled-back write from erasing a concurrently committed value for the same key, let the RESP server survive aborted connections, recover after oversized requests, and answer each pipelined command independently, and keep the previous full-text index intact when a postings rebuild fails. diff --git a/packages/agent-core-v2/test/persistence/backends/minidb/miniDbQueryStore.test.ts b/packages/agent-core-v2/test/persistence/backends/minidb/miniDbQueryStore.test.ts index bc367ad8dd..fe02ef2ee3 100644 --- a/packages/agent-core-v2/test/persistence/backends/minidb/miniDbQueryStore.test.ts +++ b/packages/agent-core-v2/test/persistence/backends/minidb/miniDbQueryStore.test.ts @@ -161,18 +161,24 @@ describe('MiniDbQueryStore', () => { } }); - it('rebuilds a clean store after on-disk corruption', async () => { + it('preserves data and drops a corrupt index sidecar on reopen', async () => { const first = build(); - await first.put(COLLECTION, 'a', { v: 1 }); + await first.put(COLLECTION, 'a', { id: 'a', v: 1 }); await first.ensureIndex(COLLECTION, { kind: 'value', name: 'byV', field: 'v' }); await first.close(); disposeHost?.(); disposeHost = undefined; + // A corrupt index-definition sidecar holds only derived metadata. The + // opener must not wipe the database over it: the sidecar is dropped, the + // data is preserved, and the caller can re-register the definition. const indexFile = join(homeDir, 'cache', 'query-store', 'db.indexes.json'); await fsp.writeFile(indexFile, '{ definitely not valid json'); const second = build(); - expect(await second.get(COLLECTION, 'a')).toBeUndefined(); + expect(await second.get(COLLECTION, 'a')).toEqual({ id: 'a', v: 1 }); + await second.ensureIndex(COLLECTION, { kind: 'value', name: 'byV', field: 'v' }); + const page = await second.query<{ id: string; v: number }>(COLLECTION).where({ v: 1 }).execute(); + expect(page.items).toEqual([{ id: 'a', v: 1 }]); }); }); diff --git a/packages/minidb/DESIGN_NOTES.md b/packages/minidb/DESIGN_NOTES.md index 984b2b479f..3590215141 100644 --- a/packages/minidb/DESIGN_NOTES.md +++ b/packages/minidb/DESIGN_NOTES.md @@ -111,7 +111,8 @@ See `src/wal.ts`. Borrowed from Redis RDB/`BGREWRITEAOF` and Bitcask's merge. Compaction is **non-blocking for writes**: the live WAL doubles as a Redis-style rewrite buffer (`aof_rewrite_buf`), so the (slow) snapshot is written while writers -keep appending, and they pause only for a brief final rotation. +keep appending, and they pause only for the final rotation — bounded, but the +pause scales with the remaining tail when a write storm outruns the pre-copy. - When the WAL exceeds a size threshold, compact (rewrite state): 1. **Fence.** Flush the WAL and record `baseOffset = wal.size`. Every write @@ -124,28 +125,40 @@ keep appending, and they pause only for a brief final rotation. store while we iterate. The snapshot need not be point-in-time — the WAL tail below repairs any fuzziness on replay. 3. **Pre-copy the tail.** Stream `WAL[baseOffset .. head]` into `db.wal.tmp`, - looping until the remaining delta is small. Also non-blocking: post-fence + while the copy is CONVERGING: each pass must shrink the remaining delta + meaningfully, bounded by a small pass cap. Also non-blocking: post-fence writes accumulate in the live WAL instead of a separate in-memory buffer. - 4. **Rotate** (the only blocking phase, and brief): set `_rotateLock` so new - writers park, flush, copy the tiny remaining tail delta, close the old - WAL, `rename` the snapshot into place, `rename` the new WAL into place, - and reopen it. The two renames are ordered **snapshot-first** so a crash - between them pairs the new snapshot with the old full WAL — replaying the - whole old WAL on top of the new snapshot is idempotent for pre-fence - frames and correct for post-fence frames, so the state stays consistent. - Reversing the order would pair an old snapshot with a truncated new WAL - and lose pre-fence data. + Under sustained writes whose append rate approaches the copy rate the gap + stops shrinking, and looping until it was small enough would never + terminate — observed in stress testing as compactions stalling for as long + as a write storm lasted (WAL unbounded, disk amplification ~20x). Giving + up to the rotation phase is what keeps compaction live in that regime. + 4. **Rotate** (the only blocking phase): set `_rotateLock` so new writers + park, then **seal** the old WAL — post-seal appends fail fast with a + retryable error and the op re-runs against the new WAL — flush, and copy + the remaining tail in a drain loop. With the WAL sealed the head no + longer moves, so the loop provably terminates; the pause scales with the + tail the pre-copy could not drain (the same bounded end-of-rewrite pause + Redis accepts for its AOF diff flush). Finish by `rename`ing the snapshot + into place, `rename`ing the new WAL into place, and reopening it. The two + renames are ordered **snapshot-first** so a crash between them pairs the + new snapshot with the old full WAL — replaying the whole old WAL on top + of the new snapshot is idempotent for pre-fence frames and correct for + post-fence frames, so the state stays consistent. Reversing the order + would pair an old snapshot with a truncated new WAL and lose pre-fence + data. - Reads are unaffected throughout. Writes pause only for the rotation critical - section (one flush, a small copy, two renames + two `fsyncDir`s); the bulk - snapshot + tail copy happen concurrently with writes. Recovery is always - `load snapshot + replay WAL`, last-writer-wins, which is exactly what makes - the non-point-in-time snapshot converge to the correct latest state. -- A writer's gate check (`if (_rotateLock) await _rotateLock`) and its - `wal.append()` are in the same synchronous segment, and `_rotateLock` is set - synchronously before the rotation flush — so a writer either enqueued its - frame before the lock (and is drained by the flush) or parks on the lock - without appending. The event loop cannot interleave between the two, which - is why the critical section is quiescent after the flush and cannot deadlock. + section (one flush, the remaining tail copy, two renames + two `fsyncDir`s); + the bulk snapshot + tail copy happen concurrently with writes. Recovery is + always `load snapshot + replay WAL`, last-writer-wins, which is exactly what + makes the non-point-in-time snapshot converge to the correct latest state. +- The rotation cannot slip a committed write: a writer's gate check + (`if (_rotateLock) await _rotateLock`) and its `wal.append()` are NOT in the + same synchronous segment (memory-budget checks yield microtasks in between), + so an in-flight op could previously append between the final flush and the + close and lose a resolved write into the abandoned old file. Sealing the old + WAL before the final flush closes that hole: such an op fails fast against + the sealed WAL and transparently retries against the new one. - The snapshot encoder runs on the main thread but **yields to the event loop every N entries** (chunked), so large snapshots stay responsive without a Worker. Offloading the encode to a `worker_thread` is the planned diff --git a/packages/minidb/README.md b/packages/minidb/README.md index adb55ea97c..977003848a 100644 --- a/packages/minidb/README.md +++ b/packages/minidb/README.md @@ -26,6 +26,8 @@ Written in **TypeScript**, with strict types and zero runtime dependencies. backed by a Redis-style **skip list** with O(log N) rank/range; unique and sparse supported; array fields indexed per element. - **RESP server** (optional): talk to it with `redis-cli` / `ioredis`. +- **ClusterDb** (optional sharding layer): multi-process concurrent read/write + over N sharded MiniDb directories, scaling write throughput with shard count. - **Value codecs**: `buffer`, `string`, or `json`. - **Zero dependencies**, pure `node:*` built-ins. @@ -261,6 +263,14 @@ bound the metadata/index footprint; it does not delete value bytes from disk because values are already stored outside RAM. `db.stats.evictions` and `db.stats.maxMemoryRejections` track the behavior. +The budget is tracked in **approximate logical bytes** (key + value + dt +metadata), not real JS heap usage. Actual per-key overhead is higher — on the +order of several hundred bytes per key for the Map entry, the ordered index +node, and bookkeeping — so a database of many small keys needs far more RAM +than `store.bytes` suggests. (Measured: ~0.6 KB of heap per key for tiny +records, so 1M small keys ≈ 600 MB of heap.) For millions of small keys, +prefer `valueMode: 'disk'` and/or budget accordingly. + ### Online backup / restore ```js @@ -350,6 +360,14 @@ sorting in memory. On 10k sessions in one workspace, paginating with `compoundRange` is ~20–40× faster than "fetch-all + sort", and stays sub-millisecond at any offset. +Multi-process `ClusterDb` scaling across process counts × shard counts (the +headline: shard-affinity writes scale ~linearly until the single-shard speed +is reached; uniform cross-shard traffic pays lock handoffs): + +```bash +pnpm bench:cluster # bench/cluster.ts — spawns real writer/reader processes +``` + ## Testing Three layers of tests, run with the built-in `node:test` runner (no deps): @@ -378,6 +396,13 @@ long-run behavior: | `boundary.test.js` | key-length limits, large values, many keys, empty db | | `soak.test.js` | sustained ops + heap stability (opt-in: `SOAK=30 npm run test:e2e`) | +The **cluster suite** (`test/cluster/*.test.ts`) covers the `ClusterDb` +sharding layer: topology/routing, merged scans, lock contention and lease +renewal in-process, cross-shard indexes/compaction, true multi-process +scenarios (concurrent writers on disjoint and shared shards, live cross-process +read visibility, read/write storms) and crash takeovers (`kill -9` → contiguous +recovery + stale-lock handoff). + ## Design in one paragraph Log-structured engine: all writes append to a CRC-framed **WAL** (group @@ -414,6 +439,49 @@ const r = await MiniDb.open({ dir: './data', readOnly: true }); For many clients, run the **RESP server** (single minidb process, many TCP clients) — that is the intended concurrent-access model, like Redis. +### ClusterDb: multi-process sharding + +When several **processes** must read and write the same logical database +without a server, `ClusterDb` shards the key space over N ordinary minidb +directories (each with its own WAL, snapshot, and `db.lock`): + +```js +import { ClusterDb } from '@moonshot-ai/minidb/cluster'; + +const db = await ClusterDb.open({ dir: './data', shardCount: 16, valueCodec: 'json' }); +await db.set('user:1', { name: 'alice' }); // routed by hash to one shard +await db.mset([['a:1', v1], ['b:2', v2]]); // grouped per shard, atomic per shard +const all = await db.scan({ prefix: 'user:' }); // merged over all shards +await db.close(); +``` + +- **Writes** never meet a single global writer: each process acquires shard + write locks on demand (with retry up to `lockAcquireTimeoutMs`), caches them + briefly (`lockPoolMaxShards`, LRU), and yields a held shard after + `lockHoldMs` (default 250ms) so other processes are never starved. Throughput + scales with the number of distinct shards being written — up to the + single-shard speed of MiniDb per shard. +- **Reads** never take locks: they use the cached writer when the local + process holds the shard, else a read-only instance that is revalidated + against the shard files on every use, so a read that starts after another + process's commit always observes it. +- **Consistency**: single-key and same-shard batch ops are strongly + consistent (single writer per shard, atomic WAL frames). Cross-shard + `mset`/`mdel`/`batch` are best-effort (atomic per shard, not globally; + `crossShard: 'none'` rejects them instead). Scans merge per-shard snapshots, + so entries from different shards may reflect different points in time. +- **Indexes** (`createIndex`/`createTextIndex`) are recorded in a cluster-wide + registry and applied by every shard writer on open; `findEq`/`findRange`/ + `search` merge per-shard results (text scores are per-shard). Index + management acquires every shard writer — run it from one process, off the + hot path. +- Crash recovery is per shard: a lock left by a dead PID is taken over by the + next opener, exactly like single MiniDb. + +`crossShard: '2pc'` is reserved for a future two-phase commit and is rejected +today. Performance numbers across process/shard counts: run +`pnpm bench:cluster` (see `bench/cluster.ts`). + For a **rebuildable cache**, use `openOrRebuild`: a corrupt cache is discarded and reopened empty, while a live-locked db is never destroyed: @@ -435,11 +503,20 @@ const db = await MiniDb.openOrRebuild( cold, very large postings list can briefly block the event loop. - Compaction is **non-blocking for writes**: the WAL itself acts as a `BGREWRITEAOF`-style rewrite buffer, so the (slow) snapshot is written while - writers keep appending. Writes pause only for a brief final rotation (a - flush, a small tail copy, and two atomic renames). + writers keep appending. Writes pause only for the final rotation (a flush, a + tail copy, and two atomic renames). A pre-copy drains most of the tail + beforehand when writes are slow enough; under sustained writes that outrun + the pre-copy, the rotation absorbs a larger tail — the same bounded + end-of-rewrite pause Redis accepts for its AOF diff flush — so compaction + always terminates. Mid-compaction crashes leave `db.*.tmp` files behind, + which the next writer open removes automatically. - Snapshot encoding runs on the main thread (chunked + yielding); offloading to a `worker_thread` is a planned optimization. -- Single process / single writer. +- Single minidb directory = single process / single writer. For multi-process + access use `ClusterDb` (above): sharding scales writes, but hash routing + means whole-range scans fan out to all shards, and uniform cross-shard write + patterns pay shard-lock handoff costs (`lockHoldMs` per yield) — workloads + with per-process shard affinity scale best. ## Credits diff --git a/packages/minidb/bench/cluster-worker.ts b/packages/minidb/bench/cluster-worker.ts new file mode 100644 index 0000000000..75c5863d8a --- /dev/null +++ b/packages/minidb/bench/cluster-worker.ts @@ -0,0 +1,91 @@ +// bench/cluster-worker.ts +// +// Worker process for the cluster concurrency benchmark. Spawned by +// bench/cluster.ts; prints one JSON report line and exits. +// +// write +// Writes n keys `${prefix}:${i}` (json values). `allow` is either 'all' +// or a comma-separated shard-id list; keys routing outside the allowed +// shards are skipped (shard-affinity workloads). +// read + +import { ClusterDb } from '../src/cluster/index.js'; +import { LockError } from '../src/lockfile.js'; + +const [, , mode, ...rest] = process.argv; + +function out(report: Record): void { + process.stdout.write(JSON.stringify(report) + '\n'); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function main(): Promise { + const [dir, shards, prefix, n, valueBytes = '64', lockHoldMs = '250', allow = 'all'] = rest; + const shardCount = Number(shards); + const allowed = allow === 'all' ? null : new Set(allow.split(',').map(Number)); + const want = Number(n); + + let retries = 0; + const withRetry = async (fn: () => Promise): Promise => { + for (;;) { + try { + return await fn(); + } catch (e) { + if ((e as { code?: string }).code !== 'ELOCKED' && !(e instanceof LockError)) throw e; + retries++; + await sleep(15 + Math.floor(Math.random() * 45)); + } + } + }; + + if (mode === 'write') { + const db = await ClusterDb.open({ + dir: dir!, + shardCount, + valueCodec: 'json', + fsyncPolicy: 'everysec', + lockHoldMs: Number(lockHoldMs), + }); + const pad = 'x'.repeat(Number(valueBytes)); + let written = 0; + const t0 = performance.now(); + for (let i = 0; written < want; i++) { + const key = `${prefix}:${i}`; + if (allowed && !allowed.has(db.shardOf(key))) continue; + await withRetry(() => db.set(key, { p: prefix, i, pad })); + written++; + } + const ms = performance.now() - t0; + out({ ok: 1, mode, n: written, ms, retries, lockWaits: db.stats().lockWaits }); + await db.close(); + return; + } + + if (mode === 'read') { + const db = await ClusterDb.open({ dir: dir!, shardCount, valueCodec: 'json', readOnly: true }); + let found = 0; + const t0 = performance.now(); + for (let i = 0; found < want; i++) { + if (i > want * shardCount * 8 + 1000) throw new Error(`read found only ${found}/${want} keys`); + const key = `${prefix}:${i}`; + if (allowed && !allowed.has(db.shardOf(key))) continue; + const v = (await db.get(key)) as { i?: number } | undefined; + if (v !== undefined && v.i === i) found++; + } + const ms = performance.now() - t0; + out({ ok: 1, mode, n: want, found, ms, retries }); + await db.close(); + return; + } + + out({ ok: 0, error: `unknown mode: ${mode}` }); + process.exit(1); +} + +main().catch((e) => { + out({ ok: 0, mode, error: String(e && (e as Error).stack ? (e as Error).stack : e) }); + process.exit(1); +}); diff --git a/packages/minidb/bench/cluster.ts b/packages/minidb/bench/cluster.ts new file mode 100644 index 0000000000..9b42fa370d --- /dev/null +++ b/packages/minidb/bench/cluster.ts @@ -0,0 +1,236 @@ +// bench/cluster.ts +// +// Multi-process concurrency benchmark for ClusterDb. +// +// For every (processes P, shards S) cell of the matrix: +// WRITE-AFFINITY P processes, each confined to its own block of shards +// (best case: measure sharding scale-out with zero lock +// contention while P <= S). +// WRITE-SCATTER P processes writing uniformly random keys across all +// shards (worst case: every lock handoff costs up to the +// hold window). +// READ P read-only processes, each reading back another +// process's keyspace (cross-process, mostly cross-shard). +// A raw single-process MiniDb baseline is measured first for reference. +// +// Run: pnpm --filter @moonshot-ai/minidb bench:cluster +// Env: CLUSTER_BENCH_PROCESSES=1,2,4,8 CLUSTER_BENCH_SHARDS=1,4,16 +// CLUSTER_BENCH_KEYS=3000 CLUSTER_BENCH_VALUE_BYTES=64 +// CLUSTER_BENCH_HOLD_MS=250 + +import { spawn } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { MiniDb } from '../src/index.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const WORKER = path.join(__dirname, 'cluster-worker.ts'); + +const PROCESSES = (process.env.CLUSTER_BENCH_PROCESSES ?? '1,2,4,8').split(',').map(Number); +const SHARDS = (process.env.CLUSTER_BENCH_SHARDS ?? '1,4,16').split(',').map(Number); +const KEYS = Number(process.env.CLUSTER_BENCH_KEYS ?? 3000); // per process +const VALUE_BYTES = Number(process.env.CLUSTER_BENCH_VALUE_BYTES ?? 64); +const HOLD_MS = Number(process.env.CLUSTER_BENCH_HOLD_MS ?? 250); + +const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); + +interface Report { + ok: number; + mode: string; + n: number; + ms: number; + found?: number; + retries: number; + lockWaits?: number; + error?: string; +} + +function runWorker(args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ['--import', 'tsx', WORKER, ...args], { stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (d) => (stdout += d)); + child.stderr.on('data', (d) => (stderr += d)); + child.on('error', reject); + child.on('exit', (code) => { + const lines = stdout.trim().split('\n').filter((l) => l.startsWith('{')); + if (code !== 0 || lines.length === 0) { + reject(new Error(`worker failed (code=${code}) args=${args.join(' ')}\n${stderr}\n${stdout}`)); + return; + } + const report = JSON.parse(lines[lines.length - 1]!) as Report; + if (!report.ok) reject(new Error(`worker error args=${args.join(' ')}: ${report.error}`)); + else resolve(report); + }); + }); +} + +async function rmrf(dir: string): Promise { + await fs.rm(dir, { recursive: true, force: true }); +} + +/** Partition S shards into P contiguous blocks (overlapping when P > S). */ +function shardBlock(p: number, procs: number, shards: number): string { + const s = Math.floor((p * shards) / procs); + const e = Math.max(s + 1, Math.floor(((p + 1) * shards) / procs)); + const ids: number[] = []; + for (let i = s; i < Math.min(e, shards); i++) ids.push(i); + return ids.join(',') || 'all'; +} + +interface CellResult { + procs: number; + shards: number; + affinityOps: number; + affinityPerProc: number; + scatterOps: number; + scatterPerProc: number; + scatterRetries: number; + readOps: number; + readPerProc: number; +} + +async function benchCell(procs: number, shards: number): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-cluster-bench-')); + try { + // ---- phase 1: affinity writes ------------------------------------------ + const affinityArgs = Array.from({ length: procs }, (_, p) => [ + 'write', + dir, + String(shards), + `w${p}`, + String(KEYS), + String(VALUE_BYTES), + String(HOLD_MS), + shardBlock(p, procs, shards), + ]); + const affinity = await Promise.all(affinityArgs.map((args) => runWorker(args))); + const affinityTotal = affinity.reduce((s, r) => s + r.n, 0); + const affinityWall = Math.max(...affinity.map((r) => r.ms)); + + // ---- phase 2: scatter writes ------------------------------------------- + const scatterArgs = Array.from({ length: procs }, (_, p) => [ + 'write', + dir, + String(shards), + `s${p}`, + String(KEYS), + String(VALUE_BYTES), + String(HOLD_MS), + 'all', + ]); + const scatter = await Promise.all(scatterArgs.map((args) => runWorker(args))); + const scatterTotal = scatter.reduce((s, r) => s + r.n, 0); + const scatterWall = Math.max(...scatter.map((r) => r.ms)); + const scatterRetries = scatter.reduce((s, r) => s + (r.lockWaits ?? r.retries), 0); + + // ---- phase 3: cross-process reads of the scatter keyspace --------------- + const readArgs = Array.from({ length: procs }, (_, p) => [ + 'read', + dir, + String(shards), + `s${(p + 1) % procs}`, + String(KEYS), + String(VALUE_BYTES), + String(HOLD_MS), + 'all', + ]); + const reads = await Promise.all(readArgs.map((args) => runWorker(args))); + for (const r of reads) { + if (r.found !== r.n) throw new Error(`read verification failed: found ${r.found}/${r.n}`); + } + const readTotal = reads.reduce((s, r) => s + r.found!, 0); + const readWall = Math.max(...reads.map((r) => r.ms)); + + const agg = (total: number, wall: number) => (total / wall) * 1000; + const perProc = (reports: Report[]) => (KEYS / (reports.reduce((s, r) => s + r.ms, 0) / reports.length)) * 1000; + return { + procs, + shards, + affinityOps: agg(affinityTotal, affinityWall), + affinityPerProc: perProc(affinity), + scatterOps: agg(scatterTotal, scatterWall), + scatterPerProc: perProc(scatter), + scatterRetries, + readOps: agg(readTotal, readWall), + readPerProc: perProc(reads), + }; + } finally { + await rmrf(dir); + } +} + +async function baseline(): Promise<{ writeOps: number; readOps: number }> { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-bench-baseline-')); + try { + const db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'everysec' }); + const pad = 'x'.repeat(VALUE_BYTES); + let t0 = performance.now(); + for (let i = 0; i < KEYS; i++) await db.set(`w0:${i}`, { p: 'w0', i, pad }); + const writeMs = performance.now() - t0; + t0 = performance.now(); + let found = 0; + for (let i = 0; i < KEYS; i++) { + const v = db.get(`w0:${i}`) as { i: number } | undefined; + if (v?.i === i) found++; + } + const readMs = performance.now() - t0; + if (found !== KEYS) throw new Error(`baseline verification failed ${found}/${KEYS}`); + await db.close(); + return { writeOps: (KEYS / writeMs) * 1000, readOps: (KEYS / readMs) * 1000 }; + } finally { + await rmrf(dir); + } +} + +function printMatrix(title: string, cells: CellResult[], pick: (c: CellResult) => number): void { + console.log(`\n${title}`); + const header = ['P \\ S', ...SHARDS.map((s) => String(s).padStart(12))].join(' | '); + console.log(` ${header}`); + for (const p of PROCESSES) { + const row = [ + String(p).padEnd(5), + ...SHARDS.map((s) => fmt(pick(cells.find((c) => c.procs === p && c.shards === s)!)).padStart(12)), + ].join(' | '); + console.log(` ${row}`); + } +} + +async function main(): Promise { + console.log( + `\nClusterDb concurrency benchmark (keys/proc=${fmt(KEYS)}, value=${VALUE_BYTES}B pad, codec=json, fsync=everysec, lockHoldMs=${HOLD_MS}, node ${process.version})`, + ); + + const base = await baseline(); + console.log(`baseline raw MiniDb (1 process, in-process, awaited writes):`); + console.log(` write ${fmt(base.writeOps)} ops/s | read ${fmt(base.readOps)} ops/s`); + + const cells: CellResult[] = []; + for (const shards of SHARDS) { + for (const procs of PROCESSES) { + process.stdout.write(`running P=${procs} S=${shards} ... `); + const cell = await benchCell(procs, shards); + cells.push(cell); + console.log( + `affinity ${fmt(cell.affinityOps)} (${fmt(cell.affinityPerProc)}/proc) | ` + + `scatter ${fmt(cell.scatterOps)} (${fmt(cell.scatterPerProc)}/proc, lockWaits=${fmt(cell.scatterRetries)}) | ` + + `read ${fmt(cell.readOps)} (${fmt(cell.readPerProc)}/proc)`, + ); + } + } + + console.log(`\nall numbers are aggregate ops/s across processes (spawn/teardown excluded)`); + printMatrix('WRITE-AFFINITY', cells, (c) => c.affinityOps); + printMatrix('WRITE-SCATTER ', cells, (c) => c.scatterOps); + printMatrix('READ ', cells, (c) => c.readOps); + printMatrix('SCATTER lock waits', cells, (c) => c.scatterRetries); + console.log(''); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/reader-catchup.ts b/packages/minidb/bench/reader-catchup.ts new file mode 100644 index 0000000000..e6aa6c3738 --- /dev/null +++ b/packages/minidb/bench/reader-catchup.ts @@ -0,0 +1,237 @@ +// bench/reader-catchup.ts +// +// Reader-side catch-up benchmark for ClusterDb: one writer child process +// hammers a single hot shard while this process reads keys on that shard +// through a second, read-only ClusterDb. Measures per-read latency and how +// the reader pool keeps its cached shard instance current. +// +// BEFORE (full reopen per WAL change): +// every read whose fingerprint check notices a WAL append closes the +// cached reader and replays snapshot+WAL from scratch — O(shard size). +// AFTER (incremental WAL catch-up): +// only the appended WAL frames are scanned and applied — O(delta). +// +// Run: node --import tsx bench/reader-catchup.ts +// Env: READER_BENCH_KEYS=10000,50000 READER_BENCH_WINDOW_MS=8000 +// READER_BENCH_VALUE_BYTES=150 READER_BENCH_SHARDS=4 +// READER_BENCH_PACE_MS=5 (writer sleep between ops ~190 ops/s; the WAL +// then grows steadily enough that a full-reopen-per-read reader can +// never catch a quiet fingerprint, which pins the disputed cost) +// +// --------------------------------------------------------------------------- +// RESULTS (this machine, node v24.15.0, valueBytes=150, poll=20ms, window +// 8000ms, 4 shards, hot shard = 1, writer pace=5ms): +// +// BEFORE (2026-07-17, pre-change; run: node --import tsx bench/reader-catchup.ts): +// keys | build | reads | qps | p50 ms | p95 ms | p99 ms | reopens | catchups | frames +// 10,000 | 1,054ms | 52 | 6/s | 130.5 | 157.2 | 238.7 | 52 | 0 | 0 +// 50,000 | 5,054ms | 14 | 2/s | 532.2 | 860.8 | 860.8 | 14 | 0 | 0 +// Every 20ms poll observes fresh WAL appends, so every read pays a full +// replay of the whole shard (snapshot + rebuild of all derived indexes): +// read QPS collapses to ~1/reopen-cost and p50 = the reopen cost itself. +// +// AFTER (same command, with incremental catch-up): +// keys | build | reads | qps | p50 ms | p95 ms | p99 ms | reopens | catchups | frames +// 10,000 | 868ms | 374 | 47/s | 0.4 | 1.3 | 5.7 | 0 | 374 | 1,388 +// 50,000 | 5,671ms | 368 | 46/s | 0.4 | 2.7 | 6.2 | 0 | 368 | 1,345 +// Every poll applies only the appended WAL frames (catchupFrames == the +// writer's ops) and never reopens: p50 drops ~330x (@10k) / ~1,330x (@50k); +// read QPS rises 6→47/s and 2→46/s (bounded only by the 20ms poll grid). +// --------------------------------------------------------------------------- + +import { spawn } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { ClusterDb, shardDirName } from '../src/cluster/index.js'; +import { shardFor } from '../src/cluster/utils.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const WORKER = path.join(__dirname, 'reader-worker.ts'); + +const SIZES = (process.env.READER_BENCH_KEYS ?? '10000,50000').split(',').map(Number); +const WINDOW_MS = Number(process.env.READER_BENCH_WINDOW_MS ?? 8000); +const VALUE_BYTES = Number(process.env.READER_BENCH_VALUE_BYTES ?? 150); +const SHARDS = Number(process.env.READER_BENCH_SHARDS ?? 4); +const PACE_MS = Number(process.env.READER_BENCH_PACE_MS ?? 5); +const POLL_MS = Number(process.env.READER_BENCH_POLL_MS ?? 20); +const HOT_SHARD = 1; + +const fmt = (n: number) => n.toLocaleString('en-US', { maximumFractionDigits: 0 }); + +interface Report { + ok: number; + mode: string; + n: number; + ms: number; + compactMs?: number; + error?: string; +} + +function spawnWorker(args: string[]): { child: ReturnType; done: Promise } { + const child = spawn(process.execPath, ['--import', 'tsx', WORKER, ...args], { stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (d) => (stdout += d)); + child.stderr.on('data', (d) => (stderr += d)); + const done = new Promise((resolve, reject) => { + child.on('error', reject); + child.on('close', (code) => { + const lines = stdout.trim().split('\n').filter((l) => l.startsWith('{')); + if (code !== 0 || lines.length === 0) { + reject(new Error(`worker failed (code=${code}) args=${args.join(' ')}\n${stderr}`)); + return; + } + const report = JSON.parse(lines[lines.length - 1]!) as Report; + if (!report.ok) reject(new Error(`worker error args=${args.join(' ')}: ${report.error}`)); + else resolve(report); + }); + }); + return { child, done }; +} + +async function runWorker(args: string[]): Promise { + const { done } = spawnWorker(args); + return done; +} + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +function percentile(sorted: number[], p: number): number { + if (!sorted.length) return NaN; + const idx = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1); + return sorted[Math.max(0, idx)]!; +} + +/** The exact preload key list (same generator as reader-worker 'preload'). */ +function preloadKeys(n: number): string[] { + const keys: string[] = []; + for (let seq = 0; keys.length < n; seq++) { + const key = `pre:${seq}`; + if (shardFor(key, SHARDS) === HOT_SHARD) keys.push(key); + } + return keys; +} + +interface Row { + keys: number; + buildMs: number; + reads: number; + qps: number; + p50: number; + p95: number; + p99: number; + writerOps: number; + readerReopens: number; + incrementalCatchups: number; + catchupFramesApplied: number; +} + +async function benchSize(n: number): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-reader-bench-')); + const keys = preloadKeys(n); + let db: ClusterDb | null = null; + try { + // Build the shard (writes + compaction), then start the hot writer. + const t0 = performance.now(); + await runWorker(['preload', dir, String(SHARDS), String(HOT_SHARD), String(n), String(VALUE_BYTES)]); + const buildMs = performance.now() - t0; + + db = await ClusterDb.open({ dir, readOnly: true }); + // Warm the reader cache (this first open pays the full replay once). + for (let i = 0; i < 3; i++) await db.get(keys[i]!); + const stats0 = db.stats(); + + const hammer = spawnWorker(['hammer', dir, String(SHARDS), String(HOT_SHARD), String(VALUE_BYTES), String(PACE_MS)]); + // The hammer's own shard open replays the (large) shard first, so it only + // starts appending hundreds of ms after spawn. Wait for the WAL to grow + // once before measuring, or the early window samples a quiet file. + const walPath = path.join(dir, shardDirName(HOT_SHARD, SHARDS), 'db.wal'); + const walBase = await fs.stat(walPath).then((s) => s.size, () => -1); + for (let waited = 0; ; waited += 50) { + const cur = await fs.stat(walPath).then((s) => s.size, () => -1); + if (cur !== walBase || waited > 15_000) break; + await sleep(50); + } + + // Poll at a fixed interval rather than busy-looping: with a hot writer the + // WAL changes between any two polls, so every poll pays exactly one + // refresh — a full reopen before, an incremental catch-up after. A busy + // loop instead degenerates into a scheduling lottery between the two + // processes and measures mostly stale-fingerprint fast paths. + const lat: number[] = []; + const w0 = performance.now(); + const deadline = w0 + WINDOW_MS; + let i = 0; + while (performance.now() < deadline) { + await sleep(POLL_MS); + const t = performance.now(); + await db.get(keys[i % keys.length]!); + lat.push(performance.now() - t); + i++; + } + const windowMs = performance.now() - w0; + + hammer.child.kill('SIGTERM'); + const report = await Promise.race([ + hammer.done, + sleep(10_000).then(() => { + hammer.child.kill('SIGKILL'); + return hammer.done; + }), + ]); + + lat.sort((a, b) => a - b); + const stats1 = db.stats(); + return { + keys: n, + buildMs, + reads: lat.length, + qps: (lat.length / windowMs) * 1000, + p50: percentile(lat, 50), + p95: percentile(lat, 95), + p99: percentile(lat, 99), + writerOps: report.n, + readerReopens: stats1.readerReopens - stats0.readerReopens, + incrementalCatchups: (stats1.incrementalCatchups ?? 0) - (stats0.incrementalCatchups ?? 0), + catchupFramesApplied: (stats1.catchupFramesApplied ?? 0) - (stats0.catchupFramesApplied ?? 0), + }; + } finally { + if (db) await db.close().catch(() => {}); + await fs.rm(dir, { recursive: true, force: true }); + } +} + +async function main(): Promise { + console.log( + `\nClusterDb reader catch-up benchmark (value=${VALUE_BYTES}B pad, window=${WINDOW_MS}ms, poll=${POLL_MS}ms, S=${SHARDS}, hot shard=${HOT_SHARD}, writer pace=${PACE_MS}ms, fsync=no, node ${process.version})`, + ); + const rows: Row[] = []; + for (const n of SIZES) { + process.stdout.write(`building shard with ${fmt(n)} keys ... `); + const row = await benchSize(n); + rows.push(row); + console.log( + `built in ${fmt(row.buildMs)}ms; reads=${fmt(row.reads)} (${fmt(row.qps)}/s), ` + + `p50=${row.p50.toFixed(1)}ms p95=${row.p95.toFixed(1)}ms p99=${row.p99.toFixed(1)}ms, ` + + `writerOps=${fmt(row.writerOps)}, fullReopens=${fmt(row.readerReopens)}, ` + + `incrementalCatchups=${fmt(row.incrementalCatchups)}, catchupFrames=${fmt(row.catchupFramesApplied)}`, + ); + } + + console.log(`\n ${'keys'.padStart(8)} | ${'build'.padStart(8)} | ${'reads'.padStart(7)} | ${'qps'.padStart(8)} | ${'p50 ms'.padStart(8)} | ${'p95 ms'.padStart(8)} | ${'p99 ms'.padStart(8)} | ${'reopens'.padStart(8)} | ${'catchups'.padStart(9)} | ${'frames'.padStart(9)}`); + for (const r of rows) { + console.log( + ` ${fmt(r.keys).padStart(8)} | ${fmt(r.buildMs).padStart(8)} | ${fmt(r.reads).padStart(7)} | ${fmt(r.qps).padStart(8)} | ${r.p50.toFixed(1).padStart(8)} | ${r.p95.toFixed(1).padStart(8)} | ${r.p99.toFixed(1).padStart(8)} | ${fmt(r.readerReopens).padStart(8)} | ${fmt(r.incrementalCatchups).padStart(9)} | ${fmt(r.catchupFramesApplied).padStart(9)}`, + ); + } + console.log(''); +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/packages/minidb/bench/reader-worker.ts b/packages/minidb/bench/reader-worker.ts new file mode 100644 index 0000000000..e26819f29b --- /dev/null +++ b/packages/minidb/bench/reader-worker.ts @@ -0,0 +1,104 @@ +// bench/reader-worker.ts +// +// Child-process entrypoint for bench/reader-catchup.ts. Not run directly. +// +// preload +// Write n keys (`pre:`, all routed to ) then compact, so the +// shard holds a large snapshot + a nearly-empty WAL when the reader +// benchmark starts. +// hammer +// Overwrite `hot:` keys on as fast as possible until SIGTERM, +// keeping the shard's WAL hot so the parent's reads never see a quiet +// fingerprint. + +import { ClusterDb } from '../src/cluster/index.js'; +import { shardFor } from '../src/cluster/utils.js'; + +const [, , mode, ...rest] = process.argv; + +function out(report: Record): void { + process.stdout.write(JSON.stringify(report) + '\n'); +} + +/** Keys `${seed}:${n}` (n = 0,1,2,...) that route to `shard`, in order. */ +function* keysOnShard(seed: string, shard: number, shards: number): Generator { + for (let n = 0; ; n++) { + const key = `${seed}:${n}`; + if (shardFor(key, shards) === shard) yield key; + } +} + +function value(i: number, valueBytes: number): { p: string; i: number; pad: string } { + return { p: 'b', i, pad: 'x'.repeat(valueBytes) }; +} + +async function main(): Promise { + if (mode === 'preload') { + const [dir, shards, shard, n, valueBytes] = rest; + const db = await ClusterDb.open({ + dir: dir!, + shardCount: Number(shards), + valueCodec: 'json', + fsyncPolicy: 'no', + autoCompact: false, + }); + const gen = keysOnShard('pre', Number(shard), Number(shards)); + const t0 = performance.now(); + const count = Number(n); + for (let i = 0; i < count; i++) { + await db.set(gen.next().value!, value(i, Number(valueBytes))); + } + const writeMs = performance.now() - t0; + const t1 = performance.now(); + await db.compact(); + const compactMs = performance.now() - t1; + out({ ok: 1, mode, n: count, ms: writeMs, compactMs }); + await db.close(); + return; + } + + if (mode === 'hammer') { + const [dir, shards, shard, valueBytes, paceMs = '5'] = rest; + const db = await ClusterDb.open({ + dir: dir!, + shardCount: Number(shards), + valueCodec: 'json', + fsyncPolicy: 'no', + // No compaction during the measured window: rotation costs are covered + // by tests, mixing one into a latency sample only muddies the numbers. + autoCompact: false, + // Keep the shard lock held: the default 250ms hold window would force a + // writer reopen (full WAL replay) several times a second, which would + // throttle the WAL drip this benchmark is meant to produce. + lockHoldMs: 0, + }); + let stop = false; + process.on('SIGTERM', () => { + stop = true; + }); + // A small yield between writes paces the WAL into a steady drip: un-paced, + // the writer's own group commit coalesces hundreds of frames into one + // writev, so a reader only observes the file change a few times a second + // instead of on (almost) every read. + const pace = Number(paceMs); + const gen = keysOnShard('hot', Number(shard), Number(shards)); + let i = 0; + const t0 = performance.now(); + while (!stop) { + await db.set(gen.next().value!, value(i, Number(valueBytes))); + i++; + if (pace > 0) await new Promise((r) => setTimeout(r, pace)); + } + out({ ok: 1, mode, n: i, ms: performance.now() - t0 }); + await db.close(); + return; + } + + out({ ok: 0, error: `unknown mode: ${mode}` }); + process.exit(1); +} + +main().catch((e) => { + out({ ok: 0, mode, error: String(e && (e as Error).stack ? (e as Error).stack : e) }); + process.exit(1); +}); diff --git a/packages/minidb/package.json b/packages/minidb/package.json index 0113961e54..1974c8466f 100644 --- a/packages/minidb/package.json +++ b/packages/minidb/package.json @@ -33,6 +33,10 @@ ".": { "types": "./src/index.ts", "default": "./src/index.ts" + }, + "./cluster": { + "types": "./src/cluster/index.ts", + "default": "./src/cluster/index.ts" } }, "scripts": { @@ -40,6 +44,7 @@ "typecheck": "tsc -p tsconfig.json --noEmit", "test": "vitest run", "bench": "node --import tsx bench/bench.ts", + "bench:cluster": "node --import tsx bench/cluster.ts", "clean": "rm -rf dist" } } diff --git a/packages/minidb/src/cluster/coordinator.ts b/packages/minidb/src/cluster/coordinator.ts new file mode 100644 index 0000000000..54a70abe63 --- /dev/null +++ b/packages/minidb/src/cluster/coordinator.ts @@ -0,0 +1,101 @@ +// src/cluster/coordinator.ts +// +// Cross-shard write coordination. Ops are grouped by shard; groups are +// executed in ascending shard-id order (the fixed lock-acquisition order that +// avoids deadlocks between processes). Within one shard the group commits as +// a single MiniDb batch (one WAL frame, all-or-nothing); across shards the +// semantics depend on the configured CrossShardMode. + +import type { BatchInputOp, MiniDb } from '../index.js'; +import type { CrossShardMode } from './types.js'; +import type { Router } from './router.js'; + +export class Coordinator { + constructor( + private readonly router: Router, + private readonly runOnShard: (shardId: number, fn: (db: MiniDb) => T | Promise) => Promise, + private readonly mode: CrossShardMode, + ) {} + + private checkMode(shardIds: number[]): void { + if (shardIds.length <= 1) return; + if (this.mode === 'none') { + throw new Error(`operation spans ${shardIds.length} shards but crossShard mode is 'none'`); + } + if (this.mode === '2pc') { + throw new Error("crossShard mode '2pc' is reserved but not implemented yet"); + } + } + + /** Group items by their key's shard, shards in ascending id order. */ + private group(items: readonly T[], keyOf: (item: T) => string): [number, T[]][] { + const byShard = new Map(); + for (const item of items) { + const id = this.router.shardFor(keyOf(item)); + const arr = byShard.get(id); + if (arr) arr.push(item); + else byShard.set(id, [item]); + } + return [...byShard.entries()].sort((a, b) => a[0] - b[0]); + } + + private async runGroups( + groups: [number, T[]][], + run: (shardId: number, items: T[]) => Promise, + opName: string, + ): Promise { + this.checkMode(groups.map(([id]) => id)); + const errors: unknown[] = []; + for (const [id, items] of groups) { + try { + await run(id, items); + } catch (e) { + // best-effort: earlier groups may be committed already; report, don't hide. + errors.push(e); + } + } + if (errors.length > 0) { + throw new AggregateError(errors, `${opName} failed on ${errors.length}/${groups.length} shard(s); partial writes possible`); + } + } + + /** Multi-shard mset. Atomic within each shard (single WAL batch frame). */ + async mset(entries: readonly (readonly [string, V])[]): Promise { + const groups = this.group(entries, ([key]) => key); + await this.runGroups( + groups, + (id, items) => + this.runOnShard(id, (db) => db.batch(items.map(([key, value]) => ({ op: 'set' as const, key, value })))), + 'mset', + ); + } + + /** Multi-shard delete. Returns the number of keys that actually existed. */ + async mdel(keys: readonly string[]): Promise { + const groups = this.group(keys, (k) => k); + this.checkMode(groups.map(([id]) => id)); + let removed = 0; + const errors: unknown[] = []; + for (const [id, ks] of groups) { + try { + removed += await this.runOnShard(id, async (db) => { + const existing = ks.filter((k) => db.has(k)); + if (existing.length > 0) await db.batch(existing.map((key) => ({ op: 'del' as const, key }))); + return existing.length; + }); + } catch (e) { + errors.push(e); + } + } + if (errors.length > 0) { + throw new AggregateError(errors, `mdel failed on ${errors.length}/${groups.length} shard(s); partial writes possible`); + } + return removed; + } + + /** Multi-shard atomic-per-shard batch (set/del ops with optional ttl/dt). */ + async batch(ops: readonly BatchInputOp[]): Promise { + const groups = this.group(ops, (op) => op.key); + await this.runGroups(groups, (id, items) => this.runOnShard(id, (db) => db.batch(items)), 'batch'); + } +} diff --git a/packages/minidb/src/cluster/index.ts b/packages/minidb/src/cluster/index.ts new file mode 100644 index 0000000000..c16b867dd4 --- /dev/null +++ b/packages/minidb/src/cluster/index.ts @@ -0,0 +1,581 @@ +// src/cluster/index.ts +// +// ClusterDb: a logically single MiniDb database sharded into N independent +// MiniDb directories so multiple processes can read and write concurrently. +// +// - Placement is a pure hash (see router.ts): every process agrees on the +// shard for a key without coordination. +// - Each shard keeps minidb's single-writer model (its own db.lock), so +// concurrency scales with the number of distinct shards being written. +// - Writes route through a per-process writer pool (lock-pool.ts): cached +// shard writers hold their lock and renew its timestamp; acquisition +// retries live holders up to lockAcquireTimeoutMs. +// - Reads never take write locks: they use the cached writer when this +// process holds the shard, else a read-only MiniDb revalidated against +// the shard files' fingerprint on every use. +// - Index definitions live in a cluster-wide registry (cluster.indexes.json) +// as the source of truth; every shard writer applies missing definitions +// right after opening, and create/drop operations fan out to all shards. +// +// Consistency: single-key and same-shard batch ops are strongly consistent +// (single writer per shard, atomic WAL frames). Cross-shard mset/mdel/batch +// are best-effort (atomic per shard, not globally). scan/prefix results are a +// per-shard snapshot merged globally, so entries from different shards may +// reflect different points in time. + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { BatchInputOp, IndexDef, IndexInfo, MiniDb, ScanEntry, SetOptions } from '../index.js'; +import { LockError } from '../lockfile.js'; +import { Coordinator } from './coordinator.js'; +import { ShardLockPool } from './lock-pool.js'; +import { Router } from './router.js'; +import { Topology } from './topology.js'; +import type { + ClusterIndexRegistry, + ClusterOpenOptions, + ClusterStats, + CompactResult, + ScanOptions, +} from './types.js'; +import { CLUSTER_INDEX_FILE, sleep } from './utils.js'; + +export type { + ClusterIndexRegistry, + ClusterMeta, + ClusterOpenOptions, + ClusterStats, + CompactResult, + CrossShardMode, + ScanOptions, +} from './types.js'; +export { Router } from './router.js'; +export { Topology } from './topology.js'; +export { LockError } from '../lockfile.js'; +export type { ShardOpenOptions } from './shard.js'; +export { shardDirName, shardFor, stableHash32 } from './utils.js'; + +/** Compare ScanEntry keys by UTF-8 byte order, matching the per-shard store + * ordering so a globally sorted merge is consistent with local scans. */ +function compareEntries(a: ScanEntry, b: ScanEntry): number { + return Buffer.compare(Buffer.from(a.key, 'utf8'), Buffer.from(b.key, 'utf8')); +} + +export class ClusterDb { + private closed = false; + + private constructor( + private readonly topology: Topology, + private readonly router: Router, + private readonly pool: ShardLockPool, + private readonly coordinator: Coordinator, + private readonly indexPath: string, + readonly readOnly: boolean, + ) {} + + static async open(opts: ClusterOpenOptions): Promise> { + if (!opts || !opts.dir) throw new TypeError('ClusterDb.open: opts.dir is required'); + if ((opts.crossShard ?? 'best-effort') === '2pc') { + throw new Error("crossShard: '2pc' is reserved for a future release and is not implemented yet"); + } + const topology = await Topology.open(opts.dir, opts); + await topology.ensureShardDirs(); + const router = new Router(opts.dir, topology.meta); + const readOnly = !!opts.readOnly; + const indexPath = path.join(opts.dir, CLUSTER_INDEX_FILE); + + const pool = new ShardLockPool({ + writerOpts: { + valueCodec: topology.meta.valueCodec, + fsyncPolicy: topology.meta.fsyncPolicy, + valueMode: opts.valueMode, + compactThresholdBytes: opts.compactThresholdBytes, + autoCompact: opts.autoCompact, + activeExpireIntervalMs: opts.activeExpireIntervalMs, + recovery: opts.recovery, + maxMemoryBytes: opts.maxMemoryBytes, + maxMemoryPolicy: opts.maxMemoryPolicy, + }, + readerOpts: { + valueCodec: topology.meta.valueCodec, + valueMode: opts.valueMode, + recovery: opts.recovery, + }, + lockRenewMs: opts.lockRenewMs ?? 10_000, + lockAcquireTimeoutMs: opts.lockAcquireTimeoutMs ?? 30_000, + lockHoldMs: opts.lockHoldMs ?? 250, + maxWriters: opts.lockPoolMaxShards ?? 16, + maxReaders: opts.readersMaxShards ?? topology.shardCount, + readOnly, + applyDefs: async (db) => { + const reg = await ClusterDb.loadRegistry(indexPath); + for (const { name, def } of reg.indexes) { + if (!db.listIndexes().some((i) => i.name === name)) await db.createIndex(name, def); + } + for (const { name, fields } of reg.textIndexes) { + try { + await db.createTextIndex(name, { fields: fields ?? undefined }); + } catch (e) { + // Idempotent apply: the def may already exist on this shard. + if (!(e instanceof Error) || !e.message.includes('already exists')) throw e; + } + } + }, + }); + const coordinator = new Coordinator( + router, + (shardId, fn) => pool.withWriter(shardId, router.shardDir(shardId), (db) => fn(db as MiniDb)), + opts.crossShard ?? 'best-effort', + ); + return new ClusterDb(topology, router, pool, coordinator, indexPath, readOnly); + } + + private ensureOpen(): void { + if (this.closed) throw new Error('ClusterDb is closed'); + } + + get dir(): string { + return this.topology.dir; + } + + get shardCount(): number { + return this.router.shardCount; + } + + /** Which shard a key lives on. */ + shardOf(key: string): number { + return this.router.shardFor(key); + } + + private writer(shardId: number, fn: (db: MiniDb) => T | Promise): Promise { + return this.pool.withWriter(shardId, this.router.shardDir(shardId), (db) => fn(db as MiniDb)); + } + + private reader(shardId: number, fn: (db: MiniDb) => T | Promise): Promise { + return this.pool.withReader(shardId, this.router.shardDir(shardId), (db) => fn(db as MiniDb)); + } + + // ---- single-key ops ------------------------------------------------------- + + async get(key: string): Promise { + this.ensureOpen(); + return this.reader(this.router.shardFor(key), (db) => db.get(key)); + } + + async set(key: string, value: V, opts?: SetOptions): Promise { + this.ensureOpen(); + await this.writer(this.router.shardFor(key), (db) => db.set(key, value, opts)); + } + + async del(key: string): Promise { + this.ensureOpen(); + return this.writer(this.router.shardFor(key), (db) => db.del(key)); + } + + async has(key: string): Promise { + this.ensureOpen(); + return this.reader(this.router.shardFor(key), (db) => db.has(key)); + } + + /** Remaining TTL in ms; -2 when the key does not exist, -1 when it has no TTL. */ + async ttl(key: string): Promise { + this.ensureOpen(); + return this.reader(this.router.shardFor(key), (db) => db.ttl(key)); + } + + async expire(key: string, ttlMs: number): Promise { + this.ensureOpen(); + return this.writer(this.router.shardFor(key), (db) => db.expire(key, ttlMs)); + } + + // ---- multi-key ops -------------------------------------------------------- + + async mget(keys: readonly string[]): Promise<(V | undefined)[]> { + this.ensureOpen(); + const out = Array.from({ length: keys.length }); + const groups = new Map(); + keys.forEach((key, idx) => { + const id = this.router.shardFor(key); + const group = groups.get(id); + if (group) group.push({ key, idx }); + else groups.set(id, [{ key, idx }]); + }); + for (const [id, items] of groups) { + await this.reader(id, (db) => { + for (const { key, idx } of items) out[idx] = db.get(key); + }); + } + return out; + } + + /** Atomic per shard; best-effort across shards (see CrossShardMode). */ + async mset(entries: readonly (readonly [string, V])[]): Promise { + this.ensureOpen(); + await this.coordinator.mset(entries); + } + + /** Returns the number of keys that existed and were deleted. */ + async mdel(keys: readonly string[]): Promise { + this.ensureOpen(); + return this.coordinator.mdel(keys); + } + + /** Atomic per shard (single WAL batch frame); best-effort across shards. */ + async batch(ops: readonly BatchInputOp[]): Promise { + this.ensureOpen(); + await this.coordinator.batch(ops); + } + + // ---- scans ---------------------------------------------------------------- + + /** Merged scan over all shards, sorted by key bytes. Hash sharding means + * every range scan fans out to all shards; entries are materialized, + * merged, then limited. */ + async scan(opts: ScanOptions = {}): Promise[]> { + this.ensureOpen(); + // A reverse scan must see the tail per shard, so per-shard limits only + // apply to forward scans; slicing happens after the global merge either way. + const perShardLimit = opts.reverse ? Infinity : (opts.limit ?? Infinity); + const range = { gte: opts.gte, gt: opts.gt, lte: opts.lte, lt: opts.lt, count: perShardLimit }; + const usePrefix = opts.prefix !== undefined; + const all: ScanEntry[] = []; + for (const id of this.router.shardIds()) { + const entries = await this.reader(id, (db) => + usePrefix ? db.prefix(opts.prefix!, perShardLimit) : db.scan(range), + ); + for (const e of entries) all.push(e); + } + all.sort(compareEntries); + if (opts.reverse) all.reverse(); + const limit = opts.limit ?? Infinity; + return limit === Infinity ? all : all.slice(0, limit); + } + + async prefix(p: string, limit = Infinity): Promise[]> { + return this.scan({ prefix: p, limit }); + } + + // ---- secondary indexes ------------------------------------------------------ + + private static async loadRegistry(file: string): Promise { + try { + const raw = JSON.parse(await fs.readFile(file, 'utf8')) as Partial; + return { indexes: raw.indexes ?? [], textIndexes: raw.textIndexes ?? [] }; + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return { indexes: [], textIndexes: [] }; + throw e; + } + } + + private static async saveRegistry(file: string, reg: ClusterIndexRegistry): Promise { + const tmp = `${file}.tmp-${process.pid}`; + await fs.writeFile(tmp, JSON.stringify(reg, null, 2)); + await fs.rename(tmp, file); + } + + /** Compare two index definitions by their effective values (defaults + * applied), so a raced create of the same definition is a no-op while a + * genuinely different one keeps the "already exists" error. */ + private static sameIndexDef(a: IndexDef, b: IndexDef): boolean { + return ( + a.field === b.field && + (a.type ?? 'equality') === (b.type ?? 'equality') && + !!a.unique === !!b.unique && + (a.sparse ?? true) === (b.sparse ?? true) + ); + } + + /** Per-process serialization of registry read-modify-publish cycles, keyed + * by registry file. Cross-process safety comes from the CAS loop in + * mutateRegistry; this only keeps ClusterDb instances in THIS process from + * interleaving their load/save/verify steps. */ + private static readonly registryLocks = new Map>(); + + private static async withRegistryLock(file: string, fn: () => Promise): Promise { + const prev = ClusterDb.registryLocks.get(file) ?? Promise.resolve(); + let release!: () => void; + const current = new Promise((resolve) => { + release = resolve; + }); + const tail = prev.then(() => current); + ClusterDb.registryLocks.set(file, tail); + await prev; + try { + return await fn(); + } finally { + release(); + if (ClusterDb.registryLocks.get(file) === tail) ClusterDb.registryLocks.delete(file); + } + } + + /** Publish one registry mutation as a compare-and-swap loop: every attempt + * re-loads the registry, re-applies the mutation idempotently + * (append-if-absent for creates, remove-if-present for drops; mutate + * returns false when the effect is already there, ending the loop), + * writes via tmp+rename, and accepts the write only when a post-save + * re-read still equals what was published. A concurrent rename from + * another process fails that check, so the attempt is retried with + * jittered backoff. */ + private async mutateRegistry(mutate: (reg: ClusterIndexRegistry) => boolean): Promise { + for (let attempt = 0; ; attempt++) { + const done = await ClusterDb.withRegistryLock(this.indexPath, async () => { + const reg = await ClusterDb.loadRegistry(this.indexPath); + if (!mutate(reg)) return true; // The effect is already published. + const published = JSON.stringify(reg); + await ClusterDb.saveRegistry(this.indexPath, reg); + const reread = await ClusterDb.loadRegistry(this.indexPath); + return JSON.stringify(reread) === published; + }); + if (done) return; + if (attempt >= 19) throw new Error('cluster index registry update keeps losing write races; retry the operation'); + await sleep(10 + Math.floor(Math.random() * 41)); + } + } + + private requireJsonCodec(what: string): void { + if (this.topology.meta.valueCodec !== 'json') { + throw new Error(`${what} require valueCodec: "json"`); + } + } + + /** Run fn against a writer of every shard in ascending id order. Index + * management needs this so definitions stay consistent cluster-wide; it + * waits (up to lockAcquireTimeoutMs per shard) for shards held by other + * processes and throws LockError when they cannot be acquired in time. */ + private async forEachShardWriter(fn: (db: MiniDb, shardId: number) => void | Promise): Promise { + for (const id of this.router.shardIds()) { + await this.writer(id, (db) => fn(db, id)); + } + } + + /** Best-effort cleanup after a failed index fan-out: run fn on the shards + * the fan-out had completed on, swallowing errors (a shard that cannot be + * re-acquired is left as-is). */ + private async rollbackShards(shardIds: number[], fn: (db: MiniDb) => void | Promise): Promise { + for (const id of shardIds) { + try { + await this.writer(id, fn); + } catch { + // Best effort only; the original fan-out error is what propagates. + } + } + } + + /** Create a secondary index on every shard and record it in the cluster + * registry. Applies to existing data (per-shard backfill) and is picked up + * by future shard opens via the registry. */ + async createIndex(name: string, def: IndexDef): Promise { + this.ensureOpen(); + this.requireJsonCodec('secondary indexes'); + const reg = await ClusterDb.loadRegistry(this.indexPath); + if (reg.indexes.some((i) => i.name === name)) throw new Error(`index "${name}" already exists`); + const createdOn: number[] = []; + try { + await this.forEachShardWriter(async (db, shardId) => { + if (!db.listIndexes().some((i) => i.name === name)) { + await db.createIndex(name, def); + createdOn.push(shardId); + } + }); + } catch (e) { + // Roll back the partial fan-out: drop the index from exactly the shards + // this call created it on, so no shard keeps enforcing an index the + // registry never recorded. + await this.rollbackShards(createdOn, async (db) => { + await db.dropIndex(name); + }); + throw e; + } + await this.mutateRegistry((current) => { + const existing = current.indexes.find((i) => i.name === name); + if (existing) { + // A raced create of the same definition already published. + if (ClusterDb.sameIndexDef(existing.def, def)) return false; + throw new Error(`index "${name}" already exists`); + } + current.indexes.push({ name, def }); + return true; + }); + } + + async dropIndex(name: string): Promise { + this.ensureOpen(); + const reg = await ClusterDb.loadRegistry(this.indexPath); + const existed = reg.indexes.some((i) => i.name === name); + await this.forEachShardWriter(async (db) => { + if (db.listIndexes().some((i) => i.name === name)) await db.dropIndex(name); + }); + if (!existed) return false; + await this.mutateRegistry((current) => { + if (!current.indexes.some((i) => i.name === name)) return false; + current.indexes = current.indexes.filter((i) => i.name !== name); + return true; + }); + return true; + } + + /** Cluster-wide index definitions from the registry (source of truth). */ + async listIndexes(): Promise { + this.ensureOpen(); + const reg = await ClusterDb.loadRegistry(this.indexPath); + return reg.indexes.map(({ name, def }) => ({ + name, + field: def.field, + type: def.type ?? 'equality', + unique: !!def.unique, + sparse: !!def.sparse, + })); + } + + async findEq(name: string, value: unknown): Promise<{ key: string; value: V | undefined }[]> { + this.ensureOpen(); + await this.requireIndex(name); + const out: { key: string; value: V | undefined }[] = []; + for (const id of this.router.shardIds()) { + const rows = await this.reader(id, (db) => + db.listIndexes().some((i) => i.name === name) ? db.findEq(name, value) : [], + ); + out.push(...rows); + } + out.sort((a, b) => compareEntries({ key: a.key, value: a.value }, { key: b.key, value: b.value })); + return out; + } + + async findRange( + name: string, + opts: Parameters['findRange']>[1], + ): Promise<{ key: string; value: V | undefined; field: number }[]> { + this.ensureOpen(); + await this.requireIndex(name); + // Only the numeric bounds go to the shards; offset/count/reverse must + // apply to the globally merged result, not per shard. + const bounds = { min: opts?.min, max: opts?.max, minExclusive: opts?.minExclusive, maxExclusive: opts?.maxExclusive }; + const out: { key: string; value: V | undefined; field: number }[] = []; + for (const id of this.router.shardIds()) { + const rows = await this.reader(id, (db) => + db.listIndexes().some((i) => i.name === name) ? db.findRange(name, bounds) : [], + ); + out.push(...rows); + } + out.sort((a, b) => a.field - b.field || compareEntries({ key: a.key, value: a.value }, { key: b.key, value: b.value })); + if (opts?.reverse) out.reverse(); + const offset = opts?.offset ?? 0; + const sliced = offset > 0 ? out.slice(offset) : out; + return opts?.count === undefined ? sliced : sliced.slice(0, opts.count); + } + + private async requireIndex(name: string): Promise { + const reg = await ClusterDb.loadRegistry(this.indexPath); + if (!reg.indexes.some((i) => i.name === name)) throw new Error(`no such index: ${name}`); + } + + // ---- full-text search ------------------------------------------------------- + + async createTextIndex(name: string, opts: { fields?: readonly string[] } = {}): Promise { + this.ensureOpen(); + this.requireJsonCodec('text indexes'); + const reg = await ClusterDb.loadRegistry(this.indexPath); + if (reg.textIndexes.some((t) => t.name === name)) throw new Error(`text index "${name}" already exists`); + const createdOn: number[] = []; + try { + await this.forEachShardWriter(async (db, shardId) => { + try { + await db.createTextIndex(name, opts); + createdOn.push(shardId); + } catch (e) { + if (!(e instanceof Error) || !e.message.includes('already exists')) throw e; + } + }); + } catch (e) { + // Roll back the partial fan-out: drop the text index only from the + // shards this call created it on (see createIndex). + await this.rollbackShards(createdOn, async (db) => { + await db.dropTextIndex(name); + }); + throw e; + } + const fields = opts.fields ?? null; + await this.mutateRegistry((current) => { + if (current.textIndexes.some((t) => t.name === name)) return false; // A raced create already published. + current.textIndexes.push({ name, fields }); + return true; + }); + } + + async dropTextIndex(name: string): Promise { + this.ensureOpen(); + const reg = await ClusterDb.loadRegistry(this.indexPath); + const existed = reg.textIndexes.some((t) => t.name === name); + await this.forEachShardWriter(async (db) => { + try { + await db.dropTextIndex(name); + } catch (e) { + if (!(e instanceof Error) || !e.message.includes('no such text index')) throw e; + } + }); + if (!existed) return false; + await this.mutateRegistry((current) => { + if (!current.textIndexes.some((t) => t.name === name)) return false; + current.textIndexes = current.textIndexes.filter((t) => t.name !== name); + return true; + }); + return existed; + } + + /** Search every shard and merge by score (desc), key (asc). Scores are + * computed per shard (per-shard idf), so global ranking is approximate. */ + async search(name: string, q: string, opts: { op?: 'AND' | 'OR'; limit?: number } = {}): Promise<{ key: string; value: V | undefined; score: number }[]> { + this.ensureOpen(); + const reg = await ClusterDb.loadRegistry(this.indexPath); + if (!reg.textIndexes.some((t) => t.name === name)) throw new Error(`no such text index: ${name}`); + const out: { key: string; value: V | undefined; score: number }[] = []; + for (const id of this.router.shardIds()) { + const rows = await this.reader(id, (db) => { + try { + return db.search(name, q, opts); + } catch (e) { + if (e instanceof Error && e.message.includes('no such text index')) return []; + throw e; + } + }); + out.push(...rows); + } + out.sort((a, b) => b.score - a.score || compareEntries({ key: a.key, value: a.value }, { key: b.key, value: b.value })); + return opts.limit === undefined ? out : out.slice(0, opts.limit); + } + + // ---- maintenance ------------------------------------------------------------ + + /** Compact every shard this process can acquire. Shards whose write lock is + * held elsewhere (beyond lockAcquireTimeoutMs) are skipped, not errored. */ + async compact(): Promise { + this.ensureOpen(); + const compacted: number[] = []; + const skipped: number[] = []; + for (const id of this.router.shardIds()) { + try { + await this.writer(id, (db) => db.compact()); + compacted.push(id); + } catch (e) { + if (e instanceof LockError) skipped.push(id); + else throw e; + } + } + return { compacted, skipped }; + } + + stats(): ClusterStats { + return { + shardCount: this.router.shardCount, + writersCached: this.pool.writersCached, + readersCached: this.pool.readersCached, + ...this.pool.stats, + }; + } + + async close(): Promise { + if (this.closed) return; + await this.pool.closeAll(); + this.closed = true; + } +} diff --git a/packages/minidb/src/cluster/lock-pool.ts b/packages/minidb/src/cluster/lock-pool.ts new file mode 100644 index 0000000000..236046362e --- /dev/null +++ b/packages/minidb/src/cluster/lock-pool.ts @@ -0,0 +1,379 @@ +// src/cluster/lock-pool.ts +// +// Per-process cache of opened shards. +// +// Writers: MiniDb instances holding the shard's db.lock. While a process +// holds a shard's write lock it is by definition the only writer of that +// shard, so the cached writer's in-memory view is authoritative and current. +// Cached writers are LRU-evicted (only when idle) down to a soft cap. +// +// Readers: read-only MiniDb instances used for keys whose shard this process +// does not currently hold. A MiniDb reader replays snapshot+WAL only at open +// time and would go stale afterwards, so every reader use is guarded by a +// cheap file fingerprint (mtime+size of the shard's WAL, snapshot and index +// definition files). A change refreshes the reader first: +// - when only the WAL changed as pure appends on the same inode (tracked by +// a {dev, ino, size} watermark), the appended frames are scanned and +// applied incrementally (MiniDb.catchUpFromWal) — O(delta); +// - anything else (rotation, truncation, snapshot/index-def changes, an +// offset that turns out not to be a frame boundary) falls back to a close +// + full reopen — O(shard size). +// Because a writer's WAL append is complete before its set() resolves, a read +// that starts after another process's write resolved always observes it. + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { MiniDb } from '../index.js'; +import { LockError } from '../lockfile.js'; +import { ShardHandle } from './shard.js'; +import type { ShardOpenOptions } from './shard.js'; +import { sleep } from './utils.js'; + +export interface LockPoolOptions { + writerOpts: ShardOpenOptions; + readerOpts: ShardOpenOptions; + lockRenewMs: number; + lockAcquireTimeoutMs: number; + lockHoldMs: number; + maxWriters: number; + maxReaders: number; + /** Cluster-wide read-only: withWriter rejects, withReader never uses cached writers. */ + readOnly: boolean; + /** Apply cluster-wide index definitions to a freshly opened writer. */ + applyDefs: (db: MiniDb) => Promise; +} + +interface WriterEntry { + handle: ShardHandle; + lastUsedAt: number; + busy: number; + /** Set once the hold window expired while busy; closed at the next idle point. */ + retire: boolean; + /** When the process should yield this shard's lock (Infinity: never by time). */ + expiresAt: number; +} + +interface ReaderEntry { + handle: ShardHandle; + fingerprint: string; + /** Per-file fingerprint parts, in FINGERPRINT_FILES order. */ + fpParts: string[]; + /** WAL watermark the instance's data represents: the {dev, ino} anchor of + * the WAL inode recovery (or the last catch-up) read, and the byte offset + * up to which frames were applied. null when the shard has no WAL yet. */ + walMark: { dev: number; ino: number; size: number } | null; + lastUsedAt: number; + busy: number; +} + +async function statFingerprint(file: string): Promise { + try { + const s = await fs.stat(file); + return `${s.mtimeMs}:${s.size}`; + } catch { + return '-'; + } +} + +/** Cheap change detector for a shard directory. WAL appends change size (and + * usually mtime); compaction swaps both snapshot and WAL; index definition + * changes rewrite their JSON files. */ +const FINGERPRINT_FILES = ['db.wal', 'db.snapshot', 'db.indexes.json', 'db.textindexes.json'] as const; + +async function shardFingerprint(dir: string): Promise { + return Promise.all(FINGERPRINT_FILES.map((f) => statFingerprint(path.join(dir, f)))); +} + +/** WAL watermark for a freshly opened reader: the exact inode recovery + * scanned and the offset up to which frames were replayed — never an offset + * beyond the replayed state, so frames appended during/after the open are + * picked up by a later incremental catch-up instead of being skipped. */ +function readerWalMark(handle: ShardHandle): ReaderEntry['walMark'] { + const ri = handle.db.recoveryInfo; + if (!ri || !ri.walIno) return null; + return { dev: ri.walDev, ino: ri.walIno, size: ri.walScanEnd }; +} + +export class ShardLockPool { + private readonly writers = new Map(); + private readonly readers = new Map(); + private readonly openingWriters = new Map>(); + private readonly openingReaders = new Map>(); + private closed = false; + + readonly stats = { + writerOpens: 0, + readerOpens: 0, + readerReopens: 0, + incrementalCatchups: 0, + catchupFramesApplied: 0, + lockWaits: 0, + evictions: 0, + }; + + constructor(private readonly opts: LockPoolOptions) {} + + get writersCached(): number { + return this.writers.size; + } + get readersCached(): number { + return this.readers.size; + } + + /** Run fn against the shard's writer, opening it (with lock retry) if this + * process does not hold it yet. The writer cannot be evicted while busy. */ + async withWriter(shardId: number, dir: string, fn: (db: MiniDb) => T | Promise): Promise { + if (this.opts.readOnly) throw new Error('ClusterDb is open in read-only mode'); + if (this.closed) throw new Error('ClusterDb is closed'); + const entry = await this.acquireWriter(shardId, dir); + entry.busy++; + try { + return await fn(entry.handle.db); + } finally { + entry.busy--; + entry.lastUsedAt = Date.now(); + if (entry.retire && entry.busy === 0) { + // The hold window expired while ops were in flight: yield the lock so + // other processes can take the shard over. + if (this.writers.get(shardId) === entry) this.writers.delete(shardId); + await entry.handle.close().catch(() => {}); + } + await this.evictWriters(); + } + } + + /** Run fn against the best available read view of the shard: the cached + * writer when this process holds the shard (current and lock-free), else a + * fingerprint-revalidated read-only instance. */ + async withReader(shardId: number, dir: string, fn: (db: MiniDb) => T | Promise): Promise { + if (this.closed) throw new Error('ClusterDb is closed'); + if (!this.opts.readOnly) { + const w = this.writers.get(shardId); + if (w) { + w.busy++; + try { + return await fn(w.handle.db); + } finally { + w.busy--; + w.lastUsedAt = Date.now(); + } + } + } + const entry = await this.acquireReader(shardId, dir); + entry.busy++; + try { + return await fn(entry.handle.db); + } finally { + entry.busy--; + entry.lastUsedAt = Date.now(); + await this.evictReaders(); + } + } + + async closeAll(): Promise { + if (this.closed) return; + this.closed = true; + const writers = [...this.writers.values()]; + const readers = [...this.readers.values()]; + this.writers.clear(); + this.readers.clear(); + const results = await Promise.allSettled([...writers, ...readers].map((e) => e.handle.close())); + const failed = results.find((r): r is PromiseRejectedResult => r.status === 'rejected'); + if (failed) throw failed.reason; + } + + // ---- writers ------------------------------------------------------------ + + private async acquireWriter(shardId: number, dir: string): Promise { + const cached = this.writers.get(shardId); + if (cached) { + if (cached.expiresAt > Date.now()) { + cached.lastUsedAt = Date.now(); + return cached; + } + // Hold window expired: yield. If ops are in flight, serve this one too + // (it is short) and close at the next idle point. + if (cached.busy > 0) { + cached.retire = true; + return cached; + } + if (this.writers.get(shardId) === cached) this.writers.delete(shardId); + await cached.handle.close().catch(() => {}); + } + const inflight = this.openingWriters.get(shardId); + if (inflight) return inflight; + const opening = this.openWriter(shardId, dir).finally(() => this.openingWriters.delete(shardId)); + this.openingWriters.set(shardId, opening); + return opening; + } + + private async openWriter(shardId: number, dir: string): Promise { + const deadline = Date.now() + this.opts.lockAcquireTimeoutMs; + let delay = 10; + for (;;) { + try { + const handle = await ShardHandle.openWriter(shardId, dir, this.opts.writerOpts, this.opts.lockRenewMs); + this.stats.writerOpens++; + try { + await this.opts.applyDefs(handle.db); + } catch (e) { + await handle.close().catch(() => {}); + throw e; + } + const entry: WriterEntry = { + handle, + lastUsedAt: Date.now(), + busy: 0, + retire: false, + expiresAt: this.opts.lockHoldMs > 0 ? Date.now() + this.opts.lockHoldMs : Infinity, + }; + if (this.opts.lockHoldMs > 0) { + // Proactively yield the lock at the end of the hold window even if + // this process goes idle, so a waiting process is never starved by + // a holder that stopped writing but kept the process alive. + const timer = setTimeout(() => { + if (entry.busy > 0) { + entry.retire = true; + return; + } + if (this.writers.get(shardId) === entry) this.writers.delete(shardId); + void entry.handle.close().catch(() => {}); + }, this.opts.lockHoldMs); + timer.unref(); + } + this.writers.set(shardId, entry); + return entry; + } catch (e) { + // Apply-time failures (e.g. a unique index that does not backfill) are + // permanent; only lock contention is retried, until the deadline. + if (!(e instanceof LockError) || Date.now() + delay > deadline) throw e; + this.stats.lockWaits++; + await sleep(delay + Math.floor(Math.random() * delay)); + delay = Math.min(delay * 2, 250); + } + } + } + + private async evictWriters(): Promise { + while (this.writers.size > this.opts.maxWriters) { + let victim: WriterEntry | null = null; + for (const entry of this.writers.values()) { + if (entry.busy > 0) continue; + if (!victim || entry.lastUsedAt < victim.lastUsedAt) victim = entry; + } + if (!victim) return; // everything busy: soft cap exceeded temporarily + this.writers.delete(victim.handle.shardId); + this.stats.evictions++; + await victim.handle.close().catch(() => {}); + } + } + + // ---- readers ------------------------------------------------------------ + + private async acquireReader(shardId: number, dir: string): Promise { + const inflight = this.openingReaders.get(shardId); + if (inflight) return inflight; + const opening = this.refreshReader(shardId, dir).finally(() => this.openingReaders.delete(shardId)); + this.openingReaders.set(shardId, opening); + return opening; + } + + private async refreshReader(shardId: number, dir: string): Promise { + const cached = this.readers.get(shardId); + const parts = await shardFingerprint(dir); + const fp = parts.join('|'); + if (cached && cached.fingerprint === fp) { + cached.lastUsedAt = Date.now(); + return cached; + } + if (cached) { + // Something in the shard changed. When the change is confined to WAL + // appends on the same inode, apply just those frames instead of paying + // for a full replay (fallback: a clean full reopen below). + if (parts[1] === cached.fpParts[1] && parts[2] === cached.fpParts[2] && parts[3] === cached.fpParts[3]) { + if (await this.tryCatchUpReader(cached, dir, parts)) return cached; + } + // A change the watermark cannot advance over (rotation, truncation, + // snapshot/index-def rewrite, broken boundary): reopen to see it. Wait + // for any in-flight user of the stale instance to drain first. + while (cached.busy > 0) await sleep(5); + this.readers.delete(shardId); + this.stats.readerReopens++; + await cached.handle.close().catch(() => {}); + } + // Retry a few times: a compaction by another process briefly swaps files, + // which can make an otherwise-fine open race with the rotation. + let lastErr: unknown; + for (let attempt = 0; attempt < 3; attempt++) { + try { + const handle = await ShardHandle.openReader(shardId, dir, this.opts.readerOpts); + this.stats.readerOpens++; + const openedParts = await shardFingerprint(dir); + const entry: ReaderEntry = { + handle, + fingerprint: openedParts.join('|'), + fpParts: openedParts, + walMark: readerWalMark(handle), + lastUsedAt: Date.now(), + busy: 0, + }; + this.readers.set(shardId, entry); + return entry; + } catch (e) { + lastErr = e; + await sleep(25); + } + } + throw lastErr; + } + + /** Incremental reader refresh: apply the WAL frames appended after the + * cached reader's watermark. Only safe when the fingerprint change is + * WAL-only (caller checked), the WAL is still the inode the watermark is + * anchored to, and it never shrank below the applied offset. Returns false + * on every divergence, leaving the cached reader untouched for the caller + * to fully reopen. */ + private async tryCatchUpReader(cached: ReaderEntry, dir: string, parts: string[]): Promise { + const mark = cached.walMark; + if (!mark) return false; + const st = await fs.stat(path.join(dir, 'db.wal')).catch(() => null); + if (!st || st.dev !== mark.dev || st.ino !== mark.ino || st.size < mark.size) return false; + // Serialize against in-flight users of the cached instance (the reopen + // path does the same drain) and hold it busy so eviction skips it. + while (cached.busy > 0) await sleep(5); + cached.busy++; + let res: { offset: number; appliedFrames: number } | null = null; + try { + res = await cached.handle.db.catchUpFromWal(mark.size); + } catch { + res = null; // raced a rotation/truncation: caller falls back to a full reopen + } finally { + cached.busy--; + } + if (res === null) return false; + // Advance the watermark only to what was actually applied — a frame + // appended after this stat (or after catch-up scanned) sits beyond + // res.offset and is picked up by the next fingerprint miss. + cached.walMark = { dev: st.dev, ino: st.ino, size: res.offset }; + cached.fpParts = parts; + cached.fingerprint = parts.join('|'); + cached.lastUsedAt = Date.now(); + this.stats.incrementalCatchups++; + this.stats.catchupFramesApplied += res.appliedFrames; + return true; + } + + private async evictReaders(): Promise { + while (this.readers.size > this.opts.maxReaders) { + let victim: ReaderEntry | null = null; + for (const entry of this.readers.values()) { + if (entry.busy > 0) continue; + if (!victim || entry.lastUsedAt < victim.lastUsedAt) victim = entry; + } + if (!victim) return; + this.readers.delete(victim.handle.shardId); + this.stats.evictions++; + await victim.handle.close().catch(() => {}); + } + } +} diff --git a/packages/minidb/src/cluster/router.ts b/packages/minidb/src/cluster/router.ts new file mode 100644 index 0000000000..7d8a4b879f --- /dev/null +++ b/packages/minidb/src/cluster/router.ts @@ -0,0 +1,31 @@ +// src/cluster/router.ts +// +// Key -> shard routing. Pure functions of (key, shardCount) so every process +// agrees on placement without any coordination. + +import path from 'node:path'; +import type { ClusterMeta } from './types.js'; +import { shardDirName, shardFor } from './utils.js'; + +export class Router { + constructor( + private readonly baseDir: string, + private readonly meta: ClusterMeta, + ) {} + + get shardCount(): number { + return this.meta.shardCount; + } + + shardFor(key: string): number { + return shardFor(key, this.meta.shardCount); + } + + shardDir(shardId: number): string { + return path.join(this.baseDir, shardDirName(shardId, this.meta.shardCount)); + } + + shardIds(): number[] { + return Array.from({ length: this.meta.shardCount }, (_, i) => i); + } +} diff --git a/packages/minidb/src/cluster/shard.ts b/packages/minidb/src/cluster/shard.ts new file mode 100644 index 0000000000..bbd5ea7f14 --- /dev/null +++ b/packages/minidb/src/cluster/shard.ts @@ -0,0 +1,65 @@ +// src/cluster/shard.ts +// +// A single shard: one MiniDb instance in either writer mode (holding the +// shard's db.lock, with a lease timer refreshing the lock timestamp) or +// reader mode (read-only, no lock, coexisting with another process's writer). + +import { MiniDb } from '../index.js'; +import type { OpenOptions } from '../index.js'; + +/** MiniDb options for a shard open; everything except the identity fields. */ +export type ShardOpenOptions = Omit; + +export class ShardHandle { + private leaseTimer: NodeJS.Timeout | null = null; + + private constructor( + readonly shardId: number, + readonly dir: string, + readonly db: MiniDb, + readonly writer: boolean, + ) {} + + /** Open the shard for writing: acquires db.lock via MiniDb.open and starts + * refreshing the lock timestamp every renewMs (0 disables renewal). Throws + * LockError if the lock is held by a live process. */ + static async openWriter( + shardId: number, + dir: string, + opts: ShardOpenOptions, + renewMs: number, + ): Promise { + const db = await MiniDb.open({ ...opts, dir }); + const handle = new ShardHandle(shardId, dir, db as MiniDb, true); + if (renewMs > 0) { + handle.leaseTimer = setInterval(() => { + void db.renewLock().catch(() => {}); + }, renewMs); + // Never keep a worker process alive just for lease renewal. + handle.leaseTimer.unref(); + } + return handle; + } + + /** Open the shard read-only. Does not touch db.lock, never fsyncs, and + * never auto-compacts (a read-only open must not rewrite a live writer's + * directory). */ + static async openReader(shardId: number, dir: string, opts: ShardOpenOptions): Promise { + const db = await MiniDb.open({ + ...opts, + dir, + readOnly: true, + autoCompact: false, + fsyncPolicy: 'no', + }); + return new ShardHandle(shardId, dir, db as MiniDb, false); + } + + async close(): Promise { + if (this.leaseTimer) { + clearInterval(this.leaseTimer); + this.leaseTimer = null; + } + await this.db.close(); + } +} diff --git a/packages/minidb/src/cluster/topology.ts b/packages/minidb/src/cluster/topology.ts new file mode 100644 index 0000000000..675e75b1d5 --- /dev/null +++ b/packages/minidb/src/cluster/topology.ts @@ -0,0 +1,94 @@ +// src/cluster/topology.ts +// +// Cluster topology: the cluster.meta.json file that pins shardCount (and the +// codec/fsync defaults) for every process that opens the database. Creation +// races are resolved with O_EXCL: exactly one process writes the meta file, +// everyone else loads and validates against it. + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { CLUSTER_META_FILE, shardDirName } from './utils.js'; +import type { ClusterMeta, ClusterOpenOptions } from './types.js'; + +const META_VERSION = 1; +const DEFAULT_SHARD_COUNT = 16; + +type TopologyOpts = Pick; + +export class Topology { + private constructor( + readonly dir: string, + readonly meta: ClusterMeta, + ) {} + + /** Create the cluster if dir has no meta file yet, otherwise load and + * validate the existing topology. Caller-specified values must match what + * is on disk; unspecified values inherit the disk topology. */ + static async open(dir: string, opts: TopologyOpts): Promise { + if (!dir) throw new TypeError('Topology.open: dir is required'); + if (opts.shardCount !== undefined && (!Number.isInteger(opts.shardCount) || opts.shardCount < 1)) { + throw new RangeError(`shardCount must be a positive integer, got ${opts.shardCount}`); + } + await fs.mkdir(dir, { recursive: true }); + const metaPath = path.join(dir, CLUSTER_META_FILE); + + const requested: ClusterMeta = { + version: META_VERSION, + shardCount: opts.shardCount ?? DEFAULT_SHARD_COUNT, + createdAt: new Date().toISOString(), + valueCodec: opts.valueCodec ?? 'buffer', + fsyncPolicy: opts.fsyncPolicy ?? 'everysec', + }; + // Atomic creation race: write to a temp file, then hard-link it into + // place. link(2) fails with EEXIST if another process won the race, and + // unlike the O_EXCL-create-then-write pattern, readers can never observe + // a meta file whose content has not been fully written yet. + const tmpPath = `${metaPath}.tmp-${process.pid}`; + try { + await fs.writeFile(tmpPath, JSON.stringify(requested, null, 2)); + await fs.link(tmpPath, metaPath); + return new Topology(dir, requested); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e; + } finally { + await fs.unlink(tmpPath).catch(() => {}); + } + + const loaded = JSON.parse(await fs.readFile(metaPath, 'utf8')) as ClusterMeta; + if (loaded.version !== META_VERSION) { + throw new Error(`unsupported cluster meta version ${loaded.version} in ${metaPath}`); + } + if (!Number.isInteger(loaded.shardCount) || loaded.shardCount < 1) { + throw new Error(`invalid shardCount in ${metaPath}`); + } + if (opts.shardCount !== undefined && opts.shardCount !== loaded.shardCount) { + throw new RangeError(`cluster was created with shardCount=${loaded.shardCount}, got ${opts.shardCount}`); + } + if (opts.valueCodec !== undefined && opts.valueCodec !== loaded.valueCodec) { + throw new RangeError(`cluster was created with valueCodec=${loaded.valueCodec}, got ${opts.valueCodec}`); + } + if (opts.fsyncPolicy !== undefined && opts.fsyncPolicy !== loaded.fsyncPolicy) { + throw new RangeError(`cluster was created with fsyncPolicy=${loaded.fsyncPolicy}, got ${opts.fsyncPolicy}`); + } + return new Topology(dir, loaded); + } + + get shardCount(): number { + return this.meta.shardCount; + } + + shardDir(shardId: number): string { + return path.join(this.dir, shardDirName(shardId, this.meta.shardCount)); + } + + allShardDirs(): string[] { + return Array.from({ length: this.meta.shardCount }, (_, i) => this.shardDir(i)); + } + + /** Create any missing shard directories. Runs on every open so a cluster + * interrupted between meta creation and directory creation heals on the + * next open. */ + async ensureShardDirs(): Promise { + await Promise.all(this.allShardDirs().map((d) => fs.mkdir(d, { recursive: true }))); + } +} diff --git a/packages/minidb/src/cluster/types.ts b/packages/minidb/src/cluster/types.ts new file mode 100644 index 0000000000..9897be641c --- /dev/null +++ b/packages/minidb/src/cluster/types.ts @@ -0,0 +1,122 @@ +// src/cluster/types.ts +// +// Public option and result types for ClusterDb, the sharded multi-process +// layer on top of MiniDb. See plan/minidb-cluster-plan.md for the design. + +import type { ValueCodecName } from '../index.js'; +import type { IndexDef } from '../index-manager.js'; +import type { FsyncPolicy } from '../wal.js'; + +/** Cross-shard write semantics. + * - 'best-effort': shard groups are written one by one in shard-id order; + * a failure may leave earlier shards written and later ones untouched. + * - 'none': an operation spanning more than one shard throws. + * - '2pc': reserved for a future two-phase-commit implementation; rejected + * at open time for now. */ +export type CrossShardMode = 'best-effort' | '2pc' | 'none'; + +export interface ClusterOpenOptions { + dir: string; + /** Shard count; only used when creating a new cluster (default 16). An + * existing cluster's count is read from cluster.meta.json and an explicit + * mismatching value is rejected. */ + shardCount?: number; + valueCodec?: ValueCodecName; + fsyncPolicy?: FsyncPolicy; + valueMode?: 'memory' | 'disk' | 'auto'; + compactThresholdBytes?: number; + autoCompact?: boolean; + activeExpireIntervalMs?: number; + recovery?: 'resync' | 'strict'; + maxMemoryBytes?: number; + maxMemoryPolicy?: 'reject' | 'evict-lru'; + + /** Open the whole cluster without ever acquiring a write lock. Writes + * throw; reads always go through revalidated read-only shard instances. */ + readOnly?: boolean; + + /** How long a lock timestamp may go unrefreshed before the lock would be + * considered abandoned (default 30000). Reserved: the underlying LockFile + * currently takes a lock over only when the recorded owner PID is dead, + * never merely because it is old; lockRenewMs keeps the timestamp fresh + * for observability and for a future lease-based check. */ + lockLeaseMs?: number; + /** How often a cached shard writer refreshes its lock timestamp + * (default 10000). Set to 0 to disable renewal. */ + lockRenewMs?: number; + /** Maximum number of shard write locks cached in this process; least + * recently used, non-busy shards are evicted beyond the cap (default 16). */ + lockPoolMaxShards?: number; + /** How long a shard writer may stay cached before this process yields the + * lock so other processes get a chance to write that shard (default 250ms; + * 0 holds the lock until pool eviction or close). Yielding costs a shard + * reopen (WAL replay) but prevents a continuously-writing process from + * starving everyone else on a hot shard. */ + lockHoldMs?: number; + /** Maximum number of read-only shard instances cached in this process + * (default: shardCount). */ + readersMaxShards?: number; + /** Maximum time to wait for a contended shard write lock before throwing + * LockError (default 30000). */ + lockAcquireTimeoutMs?: number; + /** Cross-shard write semantics (default 'best-effort'). */ + crossShard?: CrossShardMode; +} + +export interface ClusterMeta { + version: number; + shardCount: number; + createdAt: string; + valueCodec: ValueCodecName; + fsyncPolicy: FsyncPolicy; +} + +export interface ScanOptions { + gte?: string; + gt?: string; + lte?: string; + lt?: string; + /** When set, range bounds are ignored and a pure prefix scan runs. */ + prefix?: string; + limit?: number; + /** Return entries in descending key order (after global merge). */ + reverse?: boolean; +} + +/** Registry of cluster-wide index definitions (cluster.indexes.json). The + * registry is the source of truth; every shard writer applies missing + * definitions right after it is opened, so a shard that was never opened + * when an index was created catches up later. */ +export interface ClusterIndexRegistry { + indexes: { name: string; def: IndexDef }[]; + textIndexes: { name: string; fields: readonly string[] | null }[]; +} + +export interface CompactResult { + /** Shard ids that were compacted. */ + compacted: number[]; + /** Shard ids whose write lock could not be acquired (held elsewhere). */ + skipped: number[]; +} + +export interface ClusterStats { + shardCount: number; + writersCached: number; + readersCached: number; + /** Total shard writer opens performed by this process. */ + writerOpens: number; + /** Total shard reader opens performed by this process. */ + readerOpens: number; + /** Reader cache reopens triggered by on-disk changes from other processes + * (the full-replay path; contrast incrementalCatchups). */ + readerReopens: number; + /** Reader refreshes satisfied by applying only the appended WAL frames + * instead of a full reopen. */ + incrementalCatchups: number; + /** Total WAL frames applied by incremental reader catch-ups. */ + catchupFramesApplied: number; + /** Times a write-lock acquisition had to wait/retry on a live holder. */ + lockWaits: number; + /** LRU evictions of cached writers/readers. */ + evictions: number; +} diff --git a/packages/minidb/src/cluster/utils.ts b/packages/minidb/src/cluster/utils.ts new file mode 100644 index 0000000000..e220b835ac --- /dev/null +++ b/packages/minidb/src/cluster/utils.ts @@ -0,0 +1,70 @@ +// src/cluster/utils.ts +// +// Hashing, shard directory naming, and shared small helpers. + +export const CLUSTER_META_FILE = 'cluster.meta.json'; +export const CLUSTER_INDEX_FILE = 'cluster.indexes.json'; +const SHARD_DIR_PREFIX = 'shard-'; + +/** Zero-padded shard directory name, e.g. shard-03 for shardCount <= 100. The + * width grows with the shard count so directory listings stay sorted. */ +export function shardDirName(shardId: number, shardCount: number): string { + const width = Math.max(2, String(Math.max(shardCount - 1, 0)).length); + return `${SHARD_DIR_PREFIX}${String(shardId).padStart(width, '0')}`; +} + +/** MurmurHash3 x86 32-bit: stable, fast, and with a proper avalanche so the + * low bits (which is all `hash % shardCount` uses) stay uniform even for + * sequential or highly-similar keys. Zero dependencies. */ +export function stableHash32(key: string, seed = 0): number { + const data = Buffer.from(key, 'utf8'); + const len = data.length; + let h1 = seed >>> 0; + const c1 = 0xcc9e2d51; + const c2 = 0x1b873593; + + const roundedEnd = len & ~3; + for (let i = 0; i < roundedEnd; i += 4) { + let k1 = data.readUInt32LE(i); + k1 = Math.imul(k1, c1); + k1 = (k1 << 15) | (k1 >>> 17); + k1 = Math.imul(k1, c2); + h1 ^= k1; + h1 = (h1 << 13) | (h1 >>> 19); + h1 = (Math.imul(h1, 5) + 0xe6546b64) >>> 0; + } + + let k1 = 0; + const tail = len & 3; + if (tail === 3) k1 ^= data[roundedEnd + 2]! << 16; + if (tail >= 2) k1 ^= data[roundedEnd + 1]! << 8; + if (tail >= 1) { + k1 ^= data[roundedEnd]!; + k1 = Math.imul(k1, c1); + k1 = (k1 << 15) | (k1 >>> 17); + k1 = Math.imul(k1, c2); + h1 ^= k1; + } + + h1 ^= len; + // fmix32 finalizer. + h1 ^= h1 >>> 16; + h1 = Math.imul(h1, 0x85ebca6b); + h1 ^= h1 >>> 13; + h1 = Math.imul(h1, 0xc2b2ae35); + h1 ^= h1 >>> 16; + return h1 >>> 0; +} + +/** Route a key to its shard. Pure function of (key, shardCount), so every + * process agrees on placement without coordination. */ +export function shardFor(key: string, shardCount: number): number { + if (!(shardCount >= 1) || !Number.isInteger(shardCount)) { + throw new RangeError(`shardCount must be a positive integer, got ${shardCount}`); + } + return stableHash32(key) % shardCount; +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/packages/minidb/src/codec.ts b/packages/minidb/src/codec.ts index 77488af4aa..623205869e 100644 --- a/packages/minidb/src/codec.ts +++ b/packages/minidb/src/codec.ts @@ -393,15 +393,17 @@ function findMagicSync(fd: number, start: number, size: number): number { return -1; } -/** Scan an open snapshot/WAL fd into frame refs without copying values. */ +/** Scan an open snapshot/WAL fd into frame refs without copying values. + * `startOffset` restricts the scan to [startOffset, EOF) — used by replica + * catch-up, which resumes at a known frame boundary. */ export function scanFrameRefsFd( fd: number, - { onCorrupt = 'resync' }: { onCorrupt?: 'resync' | 'strict' } = {}, + { onCorrupt = 'resync', startOffset = 0 }: { onCorrupt?: 'resync' | 'strict'; startOffset?: number } = {}, ): ScanFrameRefsResult { const size = fs.fstatSync(fd).size; const frames: FrameRef[] = []; const corruptRanges: [number, number][] = []; - let pos = 0; + let pos = startOffset; while (pos < size) { const r = readFrameRefAt(fd, pos, size); diff --git a/packages/minidb/src/compaction.ts b/packages/minidb/src/compaction.ts index 88db531b86..3da409fb16 100644 --- a/packages/minidb/src/compaction.ts +++ b/packages/minidb/src/compaction.ts @@ -5,8 +5,8 @@ // This is a NON-BLOCKING variant, modelled on Redis's BGREWRITEAOF and // Bitcask's merge: while the (potentially large) snapshot is being written, // writers keep appending to the live WAL — the WAL itself acts as the -// "rewrite buffer". Writes are blocked only for a short *rotation* critical -// section at the very end (a flush + a tiny tail copy + two renames). +// "rewrite buffer". Writes are blocked only for the *rotation* critical +// section at the very end (a flush + a bounded tail copy + two renames). // // Phases: // 1. fence — flush the WAL, record baseOffset = wal.size. Every write @@ -19,13 +19,20 @@ // tail copied below is replayed last-writer-wins on top of // it, repairing any fuzziness. // 2.5 pre-copy — stream WAL[baseOffset .. head] into db.wal.tmp, draining -// the bulk of the post-fence tail. NON-BLOCKING. Loops -// until the remaining delta is small, so the critical -// section stays bounded. -// 3. rotation — SHORT BLOCKING critical section: set _rotateLock so new -// writers park, flush, copy the tiny remaining tail delta, -// close the old WAL, rename snapshot then WAL (crash-safe -// order), reopen the new WAL. +// the bulk of the post-fence tail. NON-BLOCKING. Loops while +// the copy is CONVERGING (the remaining delta shrinks fast +// enough) and gives up after a few passes otherwise — +// chasing a tail under writes that append as fast as the +// copy drains would otherwise never terminate, stalling +// compaction for as long as the write storm lasts. +// 3. rotation — BLOCKING critical section: set _rotateLock so new writers +// park, seal the old WAL (post-seal appends fail fast and +// are retried by the op against the new WAL), flush, then +// copy the remaining tail. With the WAL sealed and writers +// parked the head no longer moves, so this copy provably +// finishes; the pause scales with the tail the pre-copy did +// not drain — the same bounded end-of-rewrite pause Redis +// accepts for its AOF diff flush. // 4. bookkeeping — stats + onCompacted() (rebuild derived text postings). // // Crash safety: recovery is `load db.snapshot` + `replay db.wal`, last-writer @@ -40,6 +47,7 @@ import fs from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import path from 'node:path'; import { WAL } from './wal.js'; +import { renameReplace } from './rename-replace.js'; import { writeSnapshot } from './snapshot.js'; import type { Store, ValueLoc } from './store.js'; import type { FsyncPolicy } from './wal.js'; @@ -58,10 +66,11 @@ export interface CompactionTarget { * Null outside rotation, so the snapshot phase is fully non-blocking. */ _rotateLock: Promise | null; lastCompactError: unknown; - stats: { compactions: number; walBytesWritten: number; walFsyncs: number; snapshotBytesWritten: number }; + stats: { compactions: number; walBytesWritten: number; walFsyncs: number; snapshotBytesWritten: number; compactErrors?: number }; /** Reader for disk-backed values; reopened after snapshot/WAL rotation so - * remapped value pointers read from the new files. */ - valueReader?: { reopenBoth(): void }; + * remapped value pointers read from the new files. On Windows it is also + * closed before the rotation renames (see rotateReplace). */ + valueReader?: { reopenBoth(): void; close?(): void }; /** Optional hook invoked after the snapshot + WAL rotation succeeds, so the * owner can rewrite derived on-disk state (e.g. text postings) against the * new live set. */ @@ -77,6 +86,17 @@ const COPY_CHUNK = 1 << 20; // 1 MiB read/write coalescing // the rotation critical section, so the pre-copy loop stops draining. const SMALL_DELTA = 64 * 1024; // 64 KiB +// Windows cannot rename over an open destination; rotation uses the shared +// retrying replace helper (see rename-replace.ts). +const rotateReplace = (src: string, dst: string): Promise => renameReplace(src, dst); +// Pre-copy convergence bounds: each pass costs roughly `gap / copyRate` and +// appends `gap * (appendRate / copyRate)` new bytes during the copy. Give up +// when a pass fails to shrink the gap meaningfully (appendRate ≳ copyRate), +// or after this many passes regardless — the rotation critical section (with +// the WAL sealed and writers parked) then absorbs the remaining tail. +const MAX_PRECOPY_PASSES = 5; +const CONVERGE_RATIO = 0.7; + export async function fsyncDir(dir: string): Promise { let fh: FileHandle | null = null; try { @@ -139,10 +159,13 @@ export async function compact(db: CompactionTarget): Promise { db._compactDone = (async () => { try { await runCompaction(db); + // The onCompacted hook is part of the compaction: a run whose hook + // throws is counted as a compactError, not a successful compaction. + db.onCompacted?.(); db.stats.compactions++; db.lastCompactError = null; - db.onCompacted?.(); } catch (err) { + db.stats.compactErrors = (db.stats.compactErrors ?? 0) + 1; db.lastCompactError = err; throw err; } finally { @@ -171,63 +194,127 @@ async function runCompaction(db: CompactionTarget): Promise { db.stats.snapshotBytesWritten += snapRes.bytes; // Phase 2.5: pre-copy the post-fence WAL tail into db.wal.tmp. NON-BLOCKING. - // Drain until the remaining delta is small enough to copy inside the critical - // section. Each iteration flushes to get a stable `head`, then copies the - // bytes that landed since the previous iteration. The loop converges because - // sequential copy is far faster than fsync-bound writes; if writes ever - // outrun the copy, the critical section simply absorbs a larger delta (still - // correct, just a longer pause — no worse than the old stop-the-world). + // Each pass flushes to get a stable `head`, then copies the bytes that landed + // since the previous pass. The loop only continues while it is CONVERGING: + // under sustained writes whose append rate approaches the copy rate the gap + // stops shrinking, and looping until it was small enough would never + // terminate (stalling compaction for the whole write storm — observed in the + // field as compactions=0 forever while the WAL grew unboundedly). Give up to + // the rotation critical section instead, which finishes because the sealed + // WAL + parked writers freeze the head. let copiedUpTo = baseOffset; let appended = false; - for (;;) { + let prevGap = Number.POSITIVE_INFINITY; + for (let pass = 0; pass < MAX_PRECOPY_PASSES; pass++) { await db.wal.flush(); const head = db.wal.size; - if (head - copiedUpTo <= SMALL_DELTA) break; + const gap = head - copiedUpTo; + if (gap <= SMALL_DELTA) break; + if (pass > 0 && gap > prevGap * CONVERGE_RATIO) break; // not converging: rotate with a parked writer set await copyFileRange(db.walPath, walTmp, copiedUpTo, head, { append: appended }); appended = true; copiedUpTo = head; + prevGap = gap; } - // Phase 3: rotation. SHORT BLOCKING critical section. + // Phase 3: rotation. BLOCKING critical section. // - // Setting _rotateLock is synchronous and happens-before the flush below. A - // writer's gate check (`if (_rotateLock) await _rotateLock`) and its - // wal.append() are in the same synchronous segment, so either the writer saw - // the old (null) lock and already enqueued its frame (→ drained by flush), or - // it sees the new lock and parks without appending. The event loop cannot - // interleave between the two, so after flush() the queue is quiescent and - // endOffset is final. + // Setting _rotateLock is synchronous and happens-before the seal below. New + // writers park on the lock; an in-flight writer that passed the gate check + // just before the lock landed cannot have its append slip between the final + // flush and close(), because seal() makes any post-seal append fail fast + // (the op retries against the new WAL once the rotation is done). With the + // old WAL sealed, its head no longer moves after this drain loop, so the + // loop provably terminates — at the cost of a write pause proportional to + // the tail the pre-copy could not drain. + // + // Recovery: the seal is one-way and the old WAL object is single-use, so a + // failure anywhere between the seal and the new WAL's open would leave every + // later write hitting WAL_SEALED/'WAL is closed' forever. The catch below + // rolls the db forward to a writable state by swapping in a FRESH WAL on + // db.walPath (it appends at the real EOF of whatever file the path now + // holds). `rotated` tracks the commit point: before the WAL rename the old + // full WAL is still at db.walPath and the old store pointers stay valid (a + // renamed-in new snapshot paired with the old WAL is consistent — see the + // crash-safety note above); past it, the new layout is on disk and recovery + // must additionally apply the store-pointer remap + reader reopen. If the + // rollback itself also fails (e.g. persistent EMFILE), the db stays + // unwritable but the on-disk snapshot/WAL pair is consistent either way, so + // the next process open still recovers. let releaseRotation!: () => void; db._rotateLock = new Promise((resolve) => { releaseRotation = resolve; }); + let rotated = false; + let remapped = false; + // Remap disk-backed value pointers to the new snapshot/WAL files. Guarded + // against double application: the wal-offset shift is NOT idempotent. + const remap = (): void => { + if (remapped) return; + const snapLocs = snapRes.locs; + db.store.remapLocs((k: string, loc: ValueLoc) => { + if (loc.file === 'wal' && loc.off >= baseOffset) { + return { file: 'wal', off: loc.off - baseOffset, len: loc.len }; + } + return snapLocs.get(k); + }); + remapped = true; + }; try { - await db.wal.flush(); - const endOffset = db.wal.size; - await copyFileRange(db.walPath, walTmp, copiedUpTo, endOffset, { append: appended }); + db.wal.seal(); + for (;;) { + await db.wal.flush(); + const endOffset = db.wal.size; + // `!appended` guarantees the (possibly empty) new WAL file is created + // even when there is no post-fence tail to copy. + if (endOffset === copiedUpTo && appended) break; + await copyFileRange(db.walPath, walTmp, copiedUpTo, endOffset, { append: appended }); + appended = true; + copiedUpTo = endOffset; + } await db.wal.close(); + // Windows cannot rename over an open destination, so our own ValueReader + // must let go of the old snapshot/WAL before the renames below. POSIX + // keeps the handles across the rotation (old fd reads the unlinked old + // inode) — no close needed there; after the remap segment below, + // reopenBoth() re-attaches both handles on every platform. + if (process.platform === 'win32') db.valueReader?.close?.(); + // Snapshot first, then WAL — see the crash-safety note in the file header. - await fs.rename(tmp, snap); + await rotateReplace(tmp, snap); await fsyncDir(db.dir); - await fs.rename(walTmp, db.walPath); + await rotateReplace(walTmp, db.walPath); + rotated = true; await fsyncDir(db.dir); - db.wal = new WAL(db.walPath, { fsyncPolicy: db.fsyncPolicy, stats: db.stats }); - await db.wal.open(); + const fresh = new WAL(db.walPath, { fsyncPolicy: db.fsyncPolicy, stats: db.stats }); + db.wal = fresh; + await fresh.open(); - // Remap disk-backed value pointers to the new snapshot/WAL files. Do this in - // the same synchronous segment as the fd reopen, so synchronous readers can - // never observe a new pointer against an old fd or vice versa. - const snapLocs = snapRes.locs; - db.store.remapLocs((k: string, loc: ValueLoc) => { - if (loc.file === 'wal' && loc.off >= baseOffset) { - return { file: 'wal', off: loc.off - baseOffset, len: loc.len }; - } - return snapLocs.get(k); - }); + // Commit the in-memory view to the new files. Do this in the same + // synchronous segment as the fd reopen, so synchronous readers can never + // observe a new pointer against an old fd or vice versa. + remap(); db.valueReader?.reopenBoth(); + } catch (err) { + try { + // Swap the sealed/closed WAL for a fresh handle on db.walPath. The swap + // comes first: it both restores appendability and stops late in-flight + // writers from publishing old-file value pointers against the fresh WAL. + await db.wal.close().catch(() => {}); + const fresh = new WAL(db.walPath, { fsyncPolicy: db.fsyncPolicy, stats: db.stats }); + await fresh.open(); + db.wal = fresh; + if (rotated) { + remap(); + db.valueReader?.reopenBoth(); + } + } catch { + // Best-effort recovery only — on-disk state is consistent regardless. + } + throw err; } finally { releaseRotation(); db._rotateLock = null; diff --git a/packages/minidb/src/index.ts b/packages/minidb/src/index.ts index 2f52afef30..a6ea101928 100644 --- a/packages/minidb/src/index.ts +++ b/packages/minidb/src/index.ts @@ -13,7 +13,7 @@ import { Store } from './store.js'; import type { StoreRecord, ValueLoc } from './store.js'; import { WAL } from './wal.js'; import { ValueReader } from './value-reader.js'; -import { recover } from './recovery.js'; +import { recover, catchUpWal, frameToOps } from './recovery.js'; import { compact, shouldCompact } from './compaction.js'; import { IndexManager, UniqueViolationError } from './index-manager.js'; import { DtIndex } from './dt-index.js'; @@ -22,9 +22,9 @@ import { CompoundIndexManager } from './compound-index.js'; import { getPath, match, project } from './query.js'; import { LockFile, LockError } from './lockfile.js'; import { encodeFrame, encodeBatchOps, scanBatchOpRefs, HEADER_SIZE, TYPE_SET, TYPE_DEL, TYPE_BATCH } from './codec.js'; -import type { BatchOp as EncodedBatchOp } from './codec.js'; +import type { BatchOp as EncodedBatchOp, FrameRef } from './codec.js'; import type { FsyncPolicy } from './wal.js'; -import type { RecoveryMode, RecoveryInfo, ValueMode } from './recovery.js'; +import type { RecoveryMode, RecoveryInfo, ValueMode, RecoveredOp } from './recovery.js'; import type { IndexDef, IndexInfo } from './index-manager.js'; import type { CompoundIndexDef, CompoundIndexInfo } from './compound-index.js'; import type { DtRangeEntry } from './dt-index.js'; @@ -35,6 +35,8 @@ export { LockError } from './lockfile.js'; export type { RecoveryInfo } from './recovery.js'; export type { IndexDef, IndexInfo, IndexType } from './index-manager.js'; export type { CompoundIndexDef, CompoundIndexInfo } from './compound-index.js'; +// ClusterDb (the multi-process sharding layer) lives at the './cluster' +// subpath export to keep this module free of import cycles. export type ValueCodecName = 'buffer' | 'string' | 'json'; @@ -91,6 +93,13 @@ function canonRange(opts: RangeOptions): RangeOptions { if (out.lt !== undefined) out.lt = toKStr(out.lt); return out; } + +/** Lazy one-shot candidate filter — keeps query pipelines streaming so a + * bounded query stops after `skip + limit` matches instead of materializing + * every candidate. */ +function* filterKeys(keys: Iterable, pred: (k: string) => boolean): Generator { + for (const k of keys) if (pred(k)) yield k; +} function normDt(dt?: Record | null): Record | null { if (!dt) return null; const out: Record = {}; @@ -112,6 +121,14 @@ async function fileSize(file: string): Promise { } } +/** Write a small metadata file atomically (tmp + rename), so a crash cannot + * leave a torn definition file that would force openers into error/rebuild. */ +async function writeFileAtomic(file: string, data: string): Promise { + const tmp = `${file}.tmp`; + await fs.writeFile(tmp, data, 'utf8'); + await fs.rename(tmp, file); +} + async function resolveValueMode(mode: ValueModeSetting, dir: string, maxMemoryBytes: number | null): Promise { if (mode !== 'auto') return mode; if (maxMemoryBytes === null) return 'memory'; @@ -203,6 +220,10 @@ export class MiniDb { fsyncPolicy: FsyncPolicy = 'everysec'; private closed = false; recoveryInfo: RecoveryInfo | null = null; + /** Continuation watermark for catchUpFromWal: the WAL inode + applied + * offset as advanced by the last successful catch-up (recoveryInfo's scan + * endpoint anchors the first call). */ + private walTail: { dev: number; ino: number; size: number } | null = null; readOnly = false; private lock: LockFile | null = null; @@ -217,11 +238,11 @@ export class MiniDb { lastCompactError: unknown = null; maxMemoryBytes: number | null = null; maxMemoryPolicy: 'reject' | 'evict-lru' = 'reject'; - private access = new Map(); // pk -> last access seq, for LRU eviction - private accessSeq = 0; + private access = new Set(); // pk, insertion-ordered by last touch (Map/Set iteration order): front = LRU private uniqueWriteLock: Promise = Promise.resolve(); readonly stats = { compactions: 0, + compactErrors: 0, walBytesWritten: 0, walFsyncs: 0, snapshotBytesWritten: 0, @@ -277,6 +298,29 @@ export class MiniDb { } } + // Remove stale temp files left behind by an interrupted previous run (a + // compaction's snapshot/WAL temps, sidecar-definition temps). Only the + // sole writer may delete them — a read-only opener must never touch a live + // writer's in-flight temps. + if (!db.readOnly) { + for (const tmp of [ + 'db.snapshot.tmp', + 'db.wal.tmp', + 'db.indexes.json.tmp', + 'db.textindexes.json.tmp', + 'db.compound-indexes.json.tmp', + ]) { + await fs.rm(path.join(db.dir, tmp), { force: true }); + } + // A failed postings rebuild orphans `db.text-*.postings.tmp` (its atomic + // rename never ran). Postings are pure derived state — rebuilt from the + // Store on open and after compaction — so such temps are always safe to + // delete, for any index name. + for (const f of await fs.readdir(db.dir)) { + if (/^db\.text-.*\.postings\.tmp$/.test(f)) await fs.rm(path.join(db.dir, f), { force: true }); + } + } + db.store = new Store({ activeExpireIntervalMs: opts.activeExpireIntervalMs ?? 100, onExpire: (k, rec) => db.onStoreExpire(k, rec), @@ -287,7 +331,11 @@ export class MiniDb { }); try { db.wal = new WAL(db.walPath, { fsyncPolicy: db.fsyncPolicy, stats: db.stats }); - await db.wal.open(); + // A read-only instance must not create or modify any file: the WAL is + // constructed but never opened (opening with 'a' would create db.wal on + // disk). Writes are already rejected by ensureWritable, and the unopened + // WAL's size stays 0, so shouldCompact never fires for it. + if (!db.readOnly) await db.wal.open(); db.recoveryInfo = await recover({ dir: db.dir, @@ -296,8 +344,19 @@ export class MiniDb { truncate: !db.readOnly, valueMode: db.valueMode, }); - db.valueReader = new ValueReader(db.dir); - db.valueReader.open(); + // Recovery may have truncated a torn WAL tail behind the WAL's back; + // re-sync its size bookkeeping so later appends (and their disk-mode + // value pointers) are computed against the real, truncated file size. + if (db.recoveryInfo.truncatedWal) await db.wal.refreshSize(); + // Disk-backed values need the positioned reader; in valueMode 'memory' + // no record ever carries a disk loc, so opening the files would only + // hold handles for no benefit. (On Windows those idle handles would + // additionally block compaction's rename-over-path rotation — rename + // over an open destination is EPERM there.) + if (db.valueMode === 'disk') { + db.valueReader = new ValueReader(db.dir); + db.valueReader.open(); + } db.seedAccessFromStore(); await db.loadIndexDefinitions(); @@ -305,7 +364,9 @@ export class MiniDb { await db.loadTextIndexDefinitions(); db.rebuildAllIndexes(); - if (db.autoCompact && shouldCompact(db)) await compact(db); + // A read-only instance never compacts: rotation would rename the live + // writer's snapshot/WAL out from under it and lose its acknowledged data. + if (!db.readOnly && db.autoCompact && shouldCompact(db)) await compact(db); } catch (err) { // Release every resource acquired so far: an open that fails after the // WAL/store are set up must not leak a file handle or keep the everysec / @@ -342,6 +403,23 @@ export class MiniDb { const rebuildable = err instanceof SyntaxError || (err as { name?: string }).name === 'CorruptFrameError'; if (!rebuildable) throw err; if (hooks.onRebuild) hooks.onRebuild(err); + if (err instanceof SyntaxError) { + // A corrupted index-definition sidecar holds only derived metadata and + // must not cost the whole database: drop the sidecars (indexes can be + // recreated by the caller) and retry once before falling back to a + // full rebuild. If the SyntaxError came from the data files themselves + // (e.g. a corrupt frame meta), the retry fails the same way and the + // full rebuild below runs anyway. + try { + for (const f of ['db.indexes.json', 'db.textindexes.json', 'db.compound-indexes.json']) { + await fs.rm(path.join(opts.dir, f), { force: true }); + await fs.rm(path.join(opts.dir, `${f}.tmp`), { force: true }); + } + return await MiniDb.open(opts); + } catch { + /* fall through to a full rebuild */ + } + } await fs.rm(opts.dir, { recursive: true, force: true }); return MiniDb.open(opts); } @@ -409,7 +487,7 @@ export class MiniDb { } } private async persistIndexDefinitions(): Promise { - await fs.writeFile(this.indexPath, JSON.stringify(this.indexes.list()), 'utf8'); + await writeFileAtomic(this.indexPath, JSON.stringify(this.indexes.list())); } private async loadTextIndexDefinitions(): Promise { try { @@ -431,7 +509,7 @@ export class MiniDb { } } private async persistTextIndexDefinitions(): Promise { - await fs.writeFile(this.textIndexPath, JSON.stringify(this.textDefs), 'utf8'); + await writeFileAtomic(this.textIndexPath, JSON.stringify(this.textDefs)); } private async loadCompoundIndexDefinitions(): Promise { try { @@ -444,7 +522,7 @@ export class MiniDb { } } private async persistCompoundIndexDefinitions(): Promise { - await fs.writeFile(this.compoundIndexPath, JSON.stringify(this.compound.list()), 'utf8'); + await writeFileAtomic(this.compoundIndexPath, JSON.stringify(this.compound.list())); } /** Drop every derived index entry for a key that just expired in the Store. */ @@ -479,13 +557,41 @@ export class MiniDb { } } + /** + * Run a write-op commit body, transparently retrying once when the commit + * raced a compaction rotation: an op that passed the _rotateLock gate check + * just before it was set can hit the freshly-sealed old WAL (code + * 'WAL_SEALED') between the gate and its append, or — one step later in the + * rotation — the already-closed but not-yet-replaced old WAL (the untyped + * 'WAL is closed'; only retried while a rotation is actually in flight, so a + * write after db.close() still fails). The op rolls its in-memory side + * effects back on a failed append, so re-running the (idempotent) commit + * body against the post-rotation WAL is safe. + */ + private async retryOnWalSeal(commit: () => Promise): Promise { + try { + await commit(); + } catch (e) { + const sealed = (e as { code?: string }).code === 'WAL_SEALED'; + const closedMidRotation = + this._rotateLock !== null && e instanceof Error && e.message === 'WAL is closed'; + if (!sealed && !closedMidRotation) throw e; + if (this._rotateLock) await this._rotateLock; + await commit(); + } + } + private touchAccess(pk: string): void { - this.access.set(pk, ++this.accessSeq); + // Re-insert so the iteration order of `access` is LRU..MRU: delete()+add() + // moves the key to the most-recently-used end (a plain set() on an existing + // key would keep its old position, which forced O(N) victim scans). + this.access.delete(pk); + this.access.add(pk); } private seedAccessFromStore(): void { this.access.clear(); - for (const [k] of this.store.map) this.touchAccess(k); + for (const [k] of this.store.map) this.access.add(k); } private projectedBytesForOps(ops: readonly PreparedOp[]): number { @@ -504,17 +610,13 @@ export class MiniDb { return projected; } + /** O(1) LRU victim: `access` is insertion-ordered by last touch, so the + * first entry that is a live, non-skipped key is the least-recently-used one. */ private pickEvictionVictim(skip: Set): string | undefined { - let best: string | undefined; - let bestSeq = Infinity; - for (const [k, seq] of this.access) { + for (const k of this.access) { if (skip.has(k) || !this.store.map.has(k)) continue; - if (seq < bestSeq) { - best = k; - bestSeq = seq; - } + return k; } - if (best) return best; for (const [k] of this.store.map) if (!skip.has(k)) return k; return undefined; } @@ -523,15 +625,25 @@ export class MiniDb { const bytes = this.store.recordBytes(pk); if (!bytes) return; const op = this.prepareDel(Buffer.from(pk, 'binary')); - const appended = this.wal.append(encodeFrame({ type: TYPE_DEL, key: op.key })); - const prev = this.applyOp(op); - try { - await appended; - this.stats.evictions++; - } catch (e) { - this.restoreKey(op.pk, prev); - throw e; - } + // Committed through retryOnWalSeal like any other write: an evict that + // passed the writer gate just before a compaction rotation can land its + // DEL on the freshly-sealed (or just-closed, soon-to-be-replaced) old WAL, + // and the user write that triggered the eviction must never see that race. + // A failed attempt restores the victim via restoreKey, so re-running the + // idempotent DEL body against the post-rotation WAL is safe. + const commit = async (): Promise => { + const appended = this.wal.append(encodeFrame({ type: TYPE_DEL, key: op.key })); + const prev = this.applyOp(op); + const seq = this.store.map.get(op.pk)?.seq; + try { + await appended; + this.stats.evictions++; + } catch (e) { + this.restoreKey(op.pk, prev, seq); + throw e; + } + }; + await this.retryOnWalSeal(commit); } private async ensureMemoryFor(ops: readonly PreparedOp[]): Promise { @@ -634,7 +746,7 @@ export class MiniDb { try { await appended.done; } catch (e) { - this.restoreKey(op.pk, prev); + this.restoreKey(op.pk, prev, seq); throw e; } if (this.valueMode === 'disk') { @@ -650,27 +762,31 @@ export class MiniDb { this.maybeAutoCompact(); }; - if (this.hasUniqueIndexes()) await this.withUniqueWriteLock(commit); - else await commit(); + if (this.hasUniqueIndexes()) await this.withUniqueWriteLock(() => this.retryOnWalSeal(commit)); + else await this.retryOnWalSeal(commit); } async del(key: string | Buffer): Promise { this.ensureOpen(); this.ensureWritable(); if (this._rotateLock) await this._rotateLock; - const existed = this.store.get(toKStr(key)) !== undefined; + const existed = this.store.has(toKStr(key)); if (!existed) return false; const op = this.prepareDel(key); await this.ensureMemoryFor([op]); - const appended = this.wal.append(encodeFrame({ type: TYPE_DEL, key: op.key })); - const prev = this.applyOp(op); - try { - await appended; - } catch (e) { - this.restoreKey(op.pk, prev); - throw e; - } - this.maybeAutoCompact(); + const commit = async (): Promise => { + const appended = this.wal.append(encodeFrame({ type: TYPE_DEL, key: op.key })); + const prev = this.applyOp(op); + const seq = this.store.map.get(op.pk)?.seq; + try { + await appended; + } catch (e) { + this.restoreKey(op.pk, prev, seq); + throw e; + } + this.maybeAutoCompact(); + }; + await this.retryOnWalSeal(commit); return true; } @@ -706,6 +822,11 @@ export class MiniDb { const prev = this.applyOp(op); if (!prevs.has(op.pk)) prevs.set(op.pk, prev); } + // Seq identity of each record as this batch left it (undefined where the + // batch's last op deleted the key): guards both the rollback and the WAL + // pointer publish against interleaved same-key commits. + const seqs = new Map(); + for (const pk of prevs.keys()) seqs.set(pk, this.store.map.get(pk)?.seq); // In valueMode 'disk' the applied records hold in-memory refs for now // (see set()); their WAL pointers are published after `done` resolves. // Only the LAST set per key may publish — an earlier op's frame range @@ -718,15 +839,14 @@ export class MiniDb { const op = prepared[i]!; const ref = opRefs[i]; if (op.type === TYPE_SET && ref) { - lastSet.set(op.pk, { op, loc: { file: 'wal', off: bodyOff + ref.valueOff, len: ref.valLen }, seq: undefined }); + lastSet.set(op.pk, { op, loc: { file: 'wal', off: bodyOff + ref.valueOff, len: ref.valLen }, seq: seqs.get(op.pk) }); } } - for (const [pk, e] of lastSet) e.seq = this.store.map.get(pk)?.seq; } try { await appended.done; } catch (e) { - for (const [pk, prev] of prevs) this.restoreKey(pk, prev); + for (const [pk, prev] of prevs) this.restoreKey(pk, prev, seqs.get(pk)); throw e; } for (const [pk, { op, loc, seq }] of lastSet) { @@ -735,8 +855,8 @@ export class MiniDb { this.maybeAutoCompact(); }; - if (this.hasUniqueIndexes()) await this.withUniqueWriteLock(commit); - else await commit(); + if (this.hasUniqueIndexes()) await this.withUniqueWriteLock(() => this.retryOnWalSeal(commit)); + else await this.retryOnWalSeal(commit); } private prepareOp(o: BatchInputOp): PreparedOp { @@ -802,8 +922,16 @@ export class MiniDb { } /** Roll a key back to its pre-op record across the store and every derived - * index. Used when a WAL write fails after applyOp already mutated state. */ - private restoreKey(pk: string, prev: StoreRecord | undefined): void { + * index. Used when a WAL write fails after applyOp already mutated state. + * `appliedSeq` is the store record's seq captured right after THIS attempt's + * own apply (undefined when the op left the key absent, i.e. a DEL). The + * restore is skipped when the key's current state no longer matches it — + * the same seq-identity guard publishWalRef uses — because a later same-key + * op committed (or an expiry reaped the key) meanwhile, and rolling back + * over it would wipe state that is already durable. */ + private restoreKey(pk: string, prev: StoreRecord | undefined, appliedSeq: number | undefined): void { + const cur = this.store.map.get(pk); + if (appliedSeq === undefined ? cur !== undefined : cur?.seq !== appliedSeq) return; if (this.indexes.indexes.size) this.indexes.remove(pk, undefined); for (const ti of this.text.values()) ti.remove(pk); this.dt.del(pk); @@ -824,6 +952,56 @@ export class MiniDb { } } + /** Apply one recovered WAL frame during catchUpFromWal: the same ops + * open-time recovery derives from it (frameToOps), plus the incremental + * derived-index maintenance applyOp performs on the write path — minus + * unique checks: the writer already validated, and intermediate frame + * states must apply literally (LWW). */ + private applyRecoveredFrame(f: FrameRef, fd: number): void { + for (const op of frameToOps(f, 'wal', fd, this.valueMode)) this.applyRecoveredOp(op); + } + + private applyRecoveredOp(op: RecoveredOp): void { + const pk = this.pk(op.key); + // Old doc for derived-index removal; decoded before the overwrite, like + // applyOp. This get also lazy-reaps an expired old record, whose onExpire + // hook then removes its derived entries for us. + const oldDoc = this.indexes.indexes.size ? this.decode(this.store.get(pk)) : undefined; + if (op.type === TYPE_DEL) { + if (!this.store.del(pk)) return; + this.access.delete(pk); + this.dt.del(pk); + this.compound.remove(pk); + if (this.indexes.indexes.size && this.indexable(oldDoc)) this.indexes.remove(pk, oldDoc); + for (const ti of this.text.values()) ti.remove(pk); + return; + } + this.store.setRef(op.key, op.ref!, op.expireAt, op.dt); + // Re-read through the store: a TTL too short to survive the few + // microseconds since the replay-time expiry check was already reaped + // here, with onExpire dropping derived state — exactly what a fresh + // reopen leaves for the key. Otherwise dt/compound/secondary/text indexes + // would be resurrected for a key the store no longer holds. + const buf = this.store.get(pk); + if (buf === undefined) return; + this.dt.set(pk, op.dt); + // Values are only decoded when a value-derived index exists (all of them + // require the json codec): with none, recovery never copies them either. + if (this.indexes.indexes.size || this.text.size || this.compound.list().length) { + const doc = this.decode(buf)!; + this.compound.add(pk, doc, op.dt); + if (this.indexes.indexes.size) { + if (this.indexable(oldDoc)) this.indexes.remove(pk, oldDoc); + if (this.indexable(doc)) this.indexes.add(pk, doc); + } + for (const ti of this.text.values()) { + if (this.indexable(doc)) ti.add(pk, doc); + else ti.remove(pk); + } + } + this.touchAccess(pk); + } + has(key: string | Buffer): boolean { this.ensureOpen(); return this.store.has(toKStr(key)); @@ -855,31 +1033,35 @@ export class MiniDb { const meta = cur.dt ? Buffer.from(JSON.stringify({ dt: cur.dt })) : null; const keyBuf = toBuf(key); const frame = encodeFrame({ type: TYPE_SET, key: keyBuf, value: curValue, meta, expireAt }); - const wal = this.wal; - const appended = wal.appendLoc(frame); - // In-memory ref first (see set()); the disk pointer is published once the - // frame's bytes are durably in db.wal. - this.store.set(k, curValue, expireAt, cur.dt); - const seq = this.store.map.get(k)?.seq; - try { - await appended.done; - } catch (e) { - // Value/dt are unchanged (only the TTL moved), so restoring the store - // record is enough; derived indexes were never touched. - this.store.setRef(k, cur.ref, cur.expireAt, cur.dt); - throw e; - } - if (this.valueMode === 'disk') { - this.publishWalRef( - k, - wal, - seq, - { file: 'wal', off: appended.offset + HEADER_SIZE + keyBuf.length, len: curValue.length }, - expireAt, - cur.dt, - ); - } - this.maybeAutoCompact(); + const commit = async (): Promise => { + const wal = this.wal; + const appended = wal.appendLoc(frame); + // In-memory ref first (see set()); the disk pointer is published once the + // frame's bytes are durably in db.wal. prev/seq are captured per attempt + // (as in set()): a rotation retry can find a different record in place, + // and restoreKey's seq guard then leaves that newer durable state alone. + const prev = this.store.map.get(k); + this.store.set(k, curValue, expireAt, cur.dt); + const seq = this.store.map.get(k)?.seq; + try { + await appended.done; + } catch (e) { + this.restoreKey(k, prev, seq); + throw e; + } + if (this.valueMode === 'disk') { + this.publishWalRef( + k, + wal, + seq, + { file: 'wal', off: appended.offset + HEADER_SIZE + keyBuf.length, len: curValue.length }, + expireAt, + cur.dt, + ); + } + this.maybeAutoCompact(); + }; + await this.retryOnWalSeal(commit); return true; } @@ -1026,11 +1208,25 @@ export class MiniDb { if (this.codecName !== 'json') throw new Error('text indexes require valueCodec: "json"'); if (this.text.has(name)) throw new Error(`text index "${name}" already exists`); const ti = new TextIndex({ fields, postingsPath: this.textPostingsPath(name) }); - this.text.set(name, ti); const def = { name, fields: fields ?? null }; - this.textDefs.push(def); + // Build BEFORE registering: a failed build must leave no phantom index + // behind — a registered-but-unbuilt index would both poison every write + // path that walks this.text and make a retry fail with "already exists". ti.build(this.textRecords()); - await this.persistTextIndexDefinitions(); + this.text.set(name, ti); + this.textDefs.push(def); + try { + await this.persistTextIndexDefinitions(); + } catch (e) { + // Unwind so the in-memory state and the definition sidecar (which does + // not name this index) do not diverge; drop the derived postings file + // with it, exactly like dropTextIndex would. + this.text.delete(name); + this.textDefs = this.textDefs.filter((d) => d.name !== name); + ti.close(); + await fs.rm(this.textPostingsPath(name), { force: true }).catch(() => {}); + throw e; + } } async dropTextIndex(name: string): Promise { this.ensureOpen(); @@ -1168,6 +1364,10 @@ export class MiniDb { if (dtCols.length !== 1) return null; const col = dtCols[0]!; const cond = q.dt[col]!; + // A dt condition carrying its own offset/count has slice semantics this + // fast path cannot reproduce exactly (it honors range bounds only) — the + // general path handles it. + if (cond.offset !== undefined || cond.count !== undefined) return null; // Result order must be the dt column's order. let reverse = false; @@ -1181,8 +1381,6 @@ export class MiniDb { const limit = q.limit; const skip = q.skip ?? 0; - // Only the range bounds are honored here; ignore any stray count/offset a - // caller put on the dt cond so they cannot truncate the walk prematurely. const iterOpts: RangeOptions = { reverse }; if (cond.gte !== undefined) iterOpts.gte = cond.gte; if (cond.gt !== undefined) iterOpts.gt = cond.gt; @@ -1231,23 +1429,30 @@ export class MiniDb { const fast = this.tryDtOrderedLimit(q); if (fast !== null) return fast; - let keys: string[] | null = null; + // Candidate collection never decodes values and stays lazy (a one-shot + // iterable) wherever possible: key scans walk the ordered index directly, + // and intersections filter as they go. A bounded query below then decodes + // only the rows it returns instead of materializing the whole candidate + // set first. + let keys: Iterable | null = null; if (typeof q.key === 'string') { keys = [toKStr(q.key)]; } else if (q.key && typeof q.key === 'object') { - if ((q.key as { prefix?: string }).prefix) keys = this.prefix((q.key as { prefix: string }).prefix).map((r) => toKStr(r.key)); - else { + if ((q.key as { prefix?: string }).prefix) { + const p = toKStr((q.key as { prefix: string }).prefix); + keys = this.store.rawKeys({ gte: p, lt: p + '\uffff' }); + } else { const opts: RangeOptions = {}; for (const b of ['gte', 'gt', 'lte', 'lt'] as const) if ((q.key as Record)[b] !== undefined) opts[b] = (q.key as Record)[b] as string; - keys = this.scan(opts).map((r) => toKStr(r.key)); + keys = this.store.rawKeys(canonRange(opts)); } } if (q.dt) { for (const [col, cond] of Object.entries(q.dt)) { const set = new Set(this.dt.range(col, cond).map((r) => r.key)); - keys = keys === null ? [...set] : keys.filter((k) => set.has(k)); + keys = keys === null ? set : filterKeys(keys, (k) => set.has(k)); } } @@ -1258,25 +1463,40 @@ export class MiniDb { const hits = ti.search(q.text.q, { op: q.text.op, limit: q.text.limit ?? 1_000_000 }); textOrder = hits; const set = new Set(hits.map((h) => h.key)); - keys = keys === null ? hits.map((h) => h.key) : keys.filter((k) => set.has(k)); + keys = keys === null ? hits.map((h) => h.key) : filterKeys(keys, (k) => set.has(k)); } const indexed = this.indexedCandidateKeys(q.filter); if (indexed) { const set = new Set(indexed); - keys = keys === null ? indexed : keys.filter((k) => set.has(k)); + keys = keys === null ? indexed : filterKeys(keys, (k) => set.has(k)); } - if (keys === null) keys = this.scan().map((r) => toKStr(r.key)); + if (keys === null) keys = this.store.rawKeys({}); + const skip = q.skip ?? 0; + const limit = q.limit === undefined ? Infinity : q.limit; + // Without an explicit sort or text ranking, result order is the candidate + // iteration order, so skip/limit can be applied while iterating: a bounded + // query decodes only the rows it returns instead of the whole candidate + // set (an indexed equality query with limit previously decoded every + // candidate and sliced at the end). + const early = !q.sort && !textOrder; const docs: ScanEntry[] = []; + let seen = 0; for (const k of keys) { const buf = this.store.get(k); if (buf === undefined) continue; const r = this.store.map.get(k); const value = this.decode(buf)!; if (q.filter && !match(value, q.filter)) continue; - docs.push({ key: k, value, dt: r?.dt ?? undefined }); + if (early) { + if (seen++ < skip) continue; + docs.push({ key: k, value, dt: r?.dt ?? undefined }); + if (docs.length >= limit) break; + } else { + docs.push({ key: k, value, dt: r?.dt ?? undefined }); + } } if (textOrder && !q.sort) { @@ -1297,9 +1517,7 @@ export class MiniDb { }); } - const skip = q.skip ?? 0; - const limit = q.limit === undefined ? Infinity : q.limit; - const sliced = skip || limit !== Infinity ? docs.slice(skip, skip + limit) : docs; + const sliced = early ? docs : skip || limit !== Infinity ? docs.slice(skip, skip + limit) : docs; if (q.project) { return sliced.map((d) => ({ key: fromKStr(d.key), value: project(d.value, q.project) as V, dt: d.dt })); @@ -1387,6 +1605,40 @@ export class MiniDb { // ---- maintenance -------------------------------------------------------- + /** Refresh the write lock's timestamp (see {@link LockFile.renew}). No-op + * for a read-only instance. Exposed for lease-style holders such as the + * cluster shard pool, which renew on a timer to prove liveness. */ + async renewLock(): Promise { + await this.lock?.renew(); + } + + /** Advanced/internal (read-replica owners such as the cluster shard pool): + * incrementally apply WAL frames appended to db.wal after `offset` — the + * same frames open-time recovery would replay, interpreted identically + * (frameToOps: valueMode memory/disk refs, expired-SET drop with LWW, + * TYPE_BATCH unrolling, dt meta) — plus incremental maintenance of every + * derived index (dt, compound, secondary, text). Unique constraints are + * NOT checked: the writer already validated, and intermediate frame states + * must apply literally, last-writer-wins. + * + * The instance tracks its own continuation: the first call must pass + * recoveryInfo.walScanEnd, every later call the previous call's returned + * offset, and the fs identity of the WAL opened for reading must match the + * inode recovery (or the last catch-up) scanned — an offset too old/new, a + * rotated file and a shrunken one all return null, meaning: reopen from + * scratch. A partial/torn tail left by a writer mid-writev is NOT an + * error: the scan stops at the last fully-valid frame; call again later + * and its CRC validates once the writev landed. */ + async catchUpFromWal(offset: number): Promise<{ offset: number; appliedFrames: number } | null> { + this.ensureOpen(); + const ri = this.recoveryInfo; + const anchor = this.walTail ?? (ri && ri.walIno ? { dev: ri.walDev, ino: ri.walIno, size: ri.walScanEnd } : null); + if (!anchor || offset !== anchor.size) return null; + const res = catchUpWal(this.walPath, offset, anchor, (f, fd) => this.applyRecoveredFrame(f, fd)); + if (res) this.walTail = { dev: anchor.dev, ino: anchor.ino, size: res.offset }; + return res; + } + async compact(): Promise { this.ensureOpen(); this.ensureWritable(); diff --git a/packages/minidb/src/lockfile.ts b/packages/minidb/src/lockfile.ts index 9fd624e47d..43727f1a41 100644 --- a/packages/minidb/src/lockfile.ts +++ b/packages/minidb/src/lockfile.ts @@ -6,7 +6,9 @@ // recorded owner PID is no longer alive — never merely because it is old. import fs from 'node:fs/promises'; -import { unlinkSync } from 'node:fs'; +import fsSync from 'node:fs'; +import path from 'node:path'; +import { renameReplace } from './rename-replace.js'; export class LockError extends Error { readonly code = 'ELOCKED'; @@ -28,7 +30,27 @@ function pidAlive(pid: unknown): boolean { // Track held locks so we can release them on process exit as a safety net. const HELD = new Set(); +// Distinct sidecar names per acquire attempt: two lock users in the same +// process (e.g. independent shard pools) must never share a tmp/bid/watch +// path, or one user's cleanup would delete the other's in-flight file. +let sidecarSeq = 0; +const nextSidecarSeq = (): number => ++sidecarSeq; let exitHooked = false; +// Co-bidders all replace the stale corpse within the same wave (they woke on +// the same event); the settle pause before the winner claims the lock must +// outlast that wave so the last bidder to land is unambiguous. A fixed value +// (even a generous one) loses on shared CI runners that deschedule a bidder +// for hundreds of milliseconds inside its own atomic-op sequence, so the +// settle is ADAPTIVE: it scales with how long our own takeover attempt took +// (4x the wall clock of inspect+writeBid+rename — a stalled machine stalls +// every bidder), floored at 60ms and capped at 2s so a healthy takeover stays +// fast. Residual (bounded-delay, inherent to file-based takeover): a bidder +// whose writeBid is delayed past the winner's final verify can still +// double-win — the bid-file sweep at verification shrinks this window to +// "competitor had not even started writing their bid yet", which requires a +// full-attempt+settle-sized skew and is effectively a process-level pause. +const TAKEOVER_SETTLE_BASE_MS = 60; +const TAKEOVER_SETTLE_MAX_MS = 2_000; function hookExit(): void { if (exitHooked) return; exitHooked = true; @@ -45,32 +67,197 @@ export class LockFile { this.path = path; } - /** Try to acquire the lock. Returns true if acquired, false if held by a live process. */ + /** Try to acquire the lock exactly once. Returns true when this call created + * the lock file, either directly or by winning a stale-lock takeover. Returns + * false whenever the lock was already held at attempt time — by a live owner + * or by a competing takeover. After observing a held lock this call never + * re-races: callers that want to wait retry acquire() at a higher level + * (see the cluster lock pool). */ async acquire(): Promise { + // Register a "watch" BEFORE touching the lock: every contender is visible + // to every other for its whole attempt, regardless of where the scheduler + // stalls it. (Settle-window heuristics alone could not survive a bidder + // descheduled before its bid write on a shard-parallel CI runner — see the + // takeover loop below; a stalled contender is only in the way, not + // invisible.) + const watch = `${this.path}.watch-${process.pid}-${nextSidecarSeq()}`; + await fs.writeFile(watch, JSON.stringify({ pid: process.pid, ts: Date.now() })); try { - const fh = await fs.open(this.path, 'wx'); // O_CREAT | O_EXCL | O_WRONLY - await fh.writeFile(JSON.stringify({ pid: process.pid, ts: Date.now() })); - await fh.close(); + await this.reapDeadWatches(); + + if (await this.tryCreate()) return true; + + // The lock exists. Only a DEAD owner's lock may be taken over; everything + // else (a live owner, or a takeover bid made by another racer in the + // meantime) is respected. + const seen = await this.inspect(); + if (seen === null || seen.alive) return false; + + // Takeover via atomic bid-replace, NOT unlink-then-create. Unlinking a + // stale lock and then racing to re-create it left a window in which a + // loser could delete the winner's just-linked file, after which several + // processes all believed they held the lock. Rename atomically replaces + // the corpse with our bid. + // + // Windows cannot rename over a destination while ANY process holds it + // open (co-racers reading/stat'ing the corpse make the rename EPERM), so + // the rename is retried with jitter. Crucially, each retry re-inspects + // the corpse first: a blind retry loop could land our bid seconds late, + // OVERWRITING an already-verified winner's lock line and double-holding + // (exactly the failure this loop is careful not to reintroduce). + const bid = `${this.path}.bid-${process.pid}-${nextSidecarSeq()}`; + const attemptStart = Date.now(); + try { + await fs.writeFile(bid, JSON.stringify({ pid: process.pid, ts: Date.now() })); + for (let attempt = 0; ; attempt++) { + // The corpse must still be there and dead. A competitor who landed + // wins by being alive in the file now — back off instead of + // overwriting their lock. (Unconditional, not just win32: the same + // overwrite hazard exists on POSIX when a co-bidder is descheduled + // between its first inspect and its rename.) + const gate = await this.inspect(); + if (gate === null || gate.alive || gate.mine) { + await fs.unlink(bid).catch(() => {}); + return false; + } + try { + await fs.rename(bid, this.path); + break; + } catch (e) { + const code = (e as NodeJS.ErrnoException).code; + const epermRetryable = + code === 'EPERM' && process.platform === 'win32' && attempt < 50; + if (!epermRetryable) { + await fs.unlink(bid).catch(() => {}); + // EEXIST races another creator; a persistent EPERM (Windows + // retries exhausted) means some holder kept the path pinned — + // either way the corpse could not be displaced this round, so + // decline like a live lock and let callers retry higher up. + if (code === 'EEXIST' || code === 'EPERM') return false; + throw e; + } + await new Promise((r) => setTimeout(r, 20 + Math.floor(Math.random() * 30))); + } + } + } catch (e) { + await fs.unlink(bid).catch(() => {}); + throw e; + } + + // Adaptive settle: scale with how long our own attempt took (a stalled + // machine stalls every bidder), floored and capped (see the constants). + const elapsedMs = Date.now() - attemptStart; + let settleMs = Math.min(TAKEOVER_SETTLE_MAX_MS, Math.max(TAKEOVER_SETTLE_BASE_MS, elapsedMs * 4)); + for (;;) { + await new Promise((resolve) => setTimeout(resolve, settleMs)); + const cur = await this.inspect(); + if (cur === null || !cur.mine) return false; + // Any live foreign watch means a contender is still in flight (its + // registration precedes its whole attempt): wait for its loop to + // finish instead of claiming on stale evidence. This is the check + // that makes exactly-one a construction, not a timing bet. + if (!(await this.hasLiveForeignWatch())) break; + settleMs = Math.min(TAKEOVER_SETTLE_MAX_MS, settleMs * 2); + } + this.markHeld(); + return true; + } finally { + await fs.unlink(watch).catch(() => {}); + } + } + + /** Delete watch registrations whose owner pid is no longer alive. */ + private async reapDeadWatches(): Promise { + const dir = path.dirname(this.path); + const prefix = `${path.basename(this.path)}.watch-`; + for (const f of await fs.readdir(dir).catch(() => [] as string[])) { + if (!f.startsWith(prefix)) continue; + const pid = Number(f.slice(prefix.length).split('-')[0]); + if (Number.isInteger(pid) && pid !== process.pid && !pidAlive(pid)) { + await fs.unlink(path.join(dir, f)).catch(() => {}); + } + } + } + + /** True when any OTHER process's liveness watch exists (reaping dead ones on sight). */ + private async hasLiveForeignWatch(): Promise { + const dir = path.dirname(this.path); + const prefix = `${path.basename(this.path)}.watch-`; + for (const f of await fs.readdir(dir).catch(() => [] as string[])) { + if (!f.startsWith(prefix)) continue; + const pid = Number(f.slice(prefix.length).split('-')[0]); + if (!Number.isInteger(pid) || pid === process.pid) continue; + if (pidAlive(pid)) return true; + await fs.unlink(path.join(dir, f)).catch(() => {}); + } + return false; + } + + /** Atomic create-if-absent publish: tmp write + hard link (EEXIST-safe). */ + private async tryCreate(): Promise { + const tmp = `${this.path}.tmp-${process.pid}-${nextSidecarSeq()}`; + try { + await fs.writeFile(tmp, JSON.stringify({ pid: process.pid, ts: Date.now() })); + await fs.link(tmp, this.path); this.markHeld(); return true; } catch (e) { if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e; + return false; + } finally { + await fs.unlink(tmp).catch(() => {}); } - if (await this.isStale()) { - await fs.unlink(this.path).catch(() => {}); - return this.acquire(); + } + + /** Read the lock file and decide its state. null = the file vanished. */ + private async inspect(): Promise<{ ino: number | bigint; alive: boolean; mine: boolean } | null> { + let raw: string; + let st: { ino: number | bigint }; + try { + [raw, st] = await Promise.all([fs.readFile(this.path, 'utf8'), fs.stat(this.path)]); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw e; } - return false; + let pid: number | undefined; + try { + pid = (JSON.parse(raw) as { pid?: number }).pid; + } catch { + pid = undefined; // unparsable content looks abandoned, same as a dead PID + } + return { ino: st.ino, alive: pidAlive(pid), mine: pid === process.pid }; } - private async isStale(): Promise { + private inspectSync(): { ino: number | bigint; alive: boolean; mine: boolean } | null { + let raw: string; + let st: { ino: number | bigint }; + try { + raw = fsSync.readFileSync(this.path, 'utf8'); + st = fsSync.statSync(this.path); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw e; + } + let pid: number | undefined; try { - const raw = await fs.readFile(this.path, 'utf8'); - const { pid } = JSON.parse(raw) as { pid?: number }; - return !pidAlive(pid); + pid = (JSON.parse(raw) as { pid?: number }).pid; } catch { - return true; + pid = undefined; } + return { ino: st.ino, alive: pidAlive(pid), mine: pid === process.pid }; + } + + /** Refresh the lock timestamp (proves liveness to processes inspecting the + * lock file). No-op when the lock is not held. Uses write-tmp-then-rename + * so a crash mid-renew cannot leave a truncated, "stale-looking" lock file + * behind for a lock that is actually still owned. */ + async renew(): Promise { + if (!this.held) return; + const tmp = `${this.path}.tmp-${process.pid}-${nextSidecarSeq()}`; + await fs.writeFile(tmp, JSON.stringify({ pid: process.pid, ts: Date.now() })); + // Windows: replacing our own lock can still clash with a co-process's + // readFile/stat of it (EPERM) — the helper rides out such transients. + await renameReplace(tmp, this.path, { retries: 20 }); } private markHeld(): void { @@ -81,7 +268,12 @@ export class LockFile { async release(): Promise { if (!this.held) return; - await fs.unlink(this.path).catch(() => {}); + // Unlink ONLY the file this instance actually owns. The content at this + // path may have been replaced since we acquired it (a supervisor re-plant a + // dead-man's marker, a concurrent takeover…), and deleting such a file + // would drop a lock that no longer belongs to us. + const cur = await this.inspect(); + if (cur?.mine) await fs.unlink(this.path).catch(() => {}); this.held = false; HELD.delete(this); } @@ -90,7 +282,8 @@ export class LockFile { releaseSync(): void { if (!this.held) return; try { - unlinkSync(this.path); + const cur = this.inspectSync(); + if (cur?.mine) fsSync.unlinkSync(this.path); } catch { /* ignore */ } diff --git a/packages/minidb/src/recovery.ts b/packages/minidb/src/recovery.ts index c25064ed8c..b44309f558 100644 --- a/packages/minidb/src/recovery.ts +++ b/packages/minidb/src/recovery.ts @@ -3,11 +3,15 @@ // Startup recovery: load the latest snapshot (if any) then replay the WAL on // top, last-writer-wins. In valueMode:'disk' recovery scans frames without // copying values and stores { file, off, len } pointers instead. +// +// The per-frame interpretation (expiry drop, batch unrolling, value refs, dt +// meta) lives in frameToOps so that open-time recovery and read-replica WAL +// catch-up (catchUpWal) can never drift apart. import fs from 'node:fs/promises'; import fsSync from 'node:fs'; import path from 'node:path'; -import { scanFrameRefsFd, scanBatchOpRefs, TYPE_SET, TYPE_DEL, TYPE_BATCH } from './codec.js'; +import { scanFrameRefsFd, scanBatchOpRefs, TYPE_SET, TYPE_DEL, TYPE_BATCH, MAGIC } from './codec.js'; import type { FrameRef } from './codec.js'; import type { Store, ValueLoc, ValueRef } from './store.js'; @@ -21,6 +25,13 @@ export interface RecoveryInfo { corruptRanges: [number, number][]; snapshotCorruptRanges: [number, number][]; lostBytes: number; + /** Byte offset in db.wal up to which recovery replayed frames (the scan + * endpoint; a torn/corrupt tail beyond it was NOT applied). 0 without WAL. + * Anchored by walDev/walIno to the inode that was scanned. */ + walScanEnd: number; + /** dev/ino of the WAL inode recovery scanned (both 0 when there was none). */ + walDev: number; + walIno: number; } function readAtSync(fd: number, off: number, len: number): Buffer { @@ -41,13 +52,22 @@ function parseMeta(meta: Buffer | null): Record | null { return parsed.dt ?? null; } -function applySetRef( +/** A recovered frame unrolled into one primitive store op; the shared + * interpretation layer between open-time recovery and replica catch-up. */ +export interface RecoveredOp { + type: number; // TYPE_SET | TYPE_DEL + key: Buffer; + ref: ValueRef | null; + expireAt: number; + dt: Record | null; +} + +function* setRefToOps( f: { key: Buffer; valueOff: number; valLen: number; meta: Buffer | null; expireAt: number }, file: ValueLoc['file'], fd: number, - store: Store, valueMode: ValueMode, -): void { +): Generator { // A record whose TTL already elapsed while the db was closed must not be // replayed as a live key: that would make `size` count a key scan/get hide, // and would rebuild indexes without it (inconsistency). @@ -60,45 +80,50 @@ function applySetRef( // semantics: a later op (another SET, or a DEL) will re-establish the key if // needed; otherwise the key stays gone, as its expired TTL dictates. if (f.expireAt && f.expireAt <= Date.now()) { - store.del(f.key); + yield { type: TYPE_DEL, key: f.key, ref: null, expireAt: 0, dt: null }; return; } const dt = parseMeta(f.meta); - if (valueMode === 'disk') { - const ref: ValueRef = { kind: 'disk', loc: { file, off: f.valueOff, len: f.valLen } }; - store.setRef(f.key, ref, f.expireAt, dt); - } else { - store.set(f.key, readAtSync(fd, f.valueOff, f.valLen), f.expireAt, dt); - } + const ref: ValueRef = + valueMode === 'disk' + ? { kind: 'disk', loc: { file, off: f.valueOff, len: f.valLen } } + : { kind: 'memory', value: readAtSync(fd, f.valueOff, f.valLen) }; + yield { type: TYPE_SET, key: f.key, ref, expireAt: f.expireAt, dt }; } -function applyBatchRef( - f: FrameRef, - file: ValueLoc['file'], - fd: number, - store: Store, - valueMode: ValueMode, -): void { - let ops; - try { - ops = scanBatchOpRefs(readAtSync(fd, f.valueOff, f.valLen), f.valueOff); - } catch { - // A malformed body with a valid outer CRC can only come from an encoder - // bug. Skip the whole batch rather than half-apply it, preserving the - // all-or-nothing guarantee. - return; - } - for (const op of ops) { - if (op.type === TYPE_SET) applySetRef(op, file, fd, store, valueMode); - else if (op.type === TYPE_DEL) store.del(op.key); +/** Unroll one recovered frame into primitive ops. SET frames carry their value + * ref (inline bytes in memory mode, a {file, off, len} pointer in disk mode); + * expired-at-replay SETs become DELs (see setRefToOps). A BATCH frame yields + * its sub-ops in order; a malformed body with a valid outer CRC skips the + * whole batch rather than half-applying it. Unknown frame types yield nothing. */ +export function* frameToOps(f: FrameRef, file: ValueLoc['file'], fd: number, valueMode: ValueMode): Generator { + if (f.type === TYPE_SET) { + yield* setRefToOps(f, file, fd, valueMode); + } else if (f.type === TYPE_DEL) { + yield { type: TYPE_DEL, key: f.key, ref: null, expireAt: 0, dt: null }; + } else if (f.type === TYPE_BATCH) { + let ops; + try { + ops = scanBatchOpRefs(readAtSync(fd, f.valueOff, f.valLen), f.valueOff); + } catch { + // A malformed body with a valid outer CRC can only come from an encoder + // bug. Skip the whole batch rather than half-apply it, preserving the + // all-or-nothing guarantee. + return; + } + for (const op of ops) { + if (op.type === TYPE_SET) yield* setRefToOps(op, file, fd, valueMode); + else if (op.type === TYPE_DEL) yield { type: TYPE_DEL, key: op.key, ref: null, expireAt: 0, dt: null }; + } } } function applyFrames(frames: FrameRef[], file: ValueLoc['file'], fd: number, store: Store, valueMode: ValueMode): void { for (const f of frames) { - if (f.type === TYPE_SET) applySetRef(f, file, fd, store, valueMode); - else if (f.type === TYPE_DEL) store.del(f.key); - else if (f.type === TYPE_BATCH) applyBatchRef(f, file, fd, store, valueMode); + for (const op of frameToOps(f, file, fd, valueMode)) { + if (op.type === TYPE_SET) store.setRef(op.key, op.ref!, op.expireAt, op.dt); + else if (op.type === TYPE_DEL) store.del(op.key); + } } } @@ -135,15 +160,22 @@ export async function recover({ let walFrames = 0; let walCorrupt: [number, number][] = []; let truncatedWal = false; + let walScanEnd = 0; + let walDev = 0; + let walIno = 0; if (fsSync.existsSync(walPath)) { const fd = fsSync.openSync(walPath, 'r'); let walSize = 0; try { - walSize = fsSync.fstatSync(fd).size; + const st = fsSync.fstatSync(fd); + walSize = st.size; + walDev = st.dev; + walIno = st.ino; const r = scanFrameRefsFd(fd, { onCorrupt: mode }); applyFrames(r.frames, 'wal', fd, store, valueMode); walFrames = r.frames.length; walCorrupt = r.corruptRanges; + walScanEnd = r.eofOffset; const last = r.corruptRanges[r.corruptRanges.length - 1]; if (last && last[1] === walSize) { // A torn/corrupt tail is normally truncated so the next writer appends @@ -167,5 +199,56 @@ export async function recover({ corruptRanges: walCorrupt, snapshotCorruptRanges: snapshotCorrupt, lostBytes: [...walCorrupt, ...snapshotCorrupt].reduce((a, [s, e]) => a + (e - s), 0), + walScanEnd, + walDev, + walIno, }; } + +/** Continue a replica from a WAL watermark: strictly scan frames in + * [offset, EOF) of `walPath` and hand each to `apply(f, fd)` in order. + * + * Returns the new continuation offset (end of the last fully-valid frame) + * and how many frames were applied. An invalid/partial frame anywhere stops + * the scan WITHOUT error (a writer mid-writev leaves such a tail; everything + * applied before it stands and the next call re-validates from the stopped + * offset — its CRC passes once the writev lands). Returns null when `offset` + * cannot be a clean frame-boundary continuation (negative, beyond EOF, + * pointing at bytes that start no frame, or the file is not the `anchor` + * inode — rotation swapped it in the microseconds between the caller's stat + * and this open): the caller must fully reopen. */ +export function catchUpWal( + walPath: string, + offset: number, + anchor: { dev: number; ino: number }, + apply: (f: FrameRef, fd: number) => void, +): { offset: number; appliedFrames: number } | null { + let fd: number; + try { + fd = fsSync.openSync(walPath, 'r'); + } catch (e) { + if ((e as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw e; + } + try { + const st = fsSync.fstatSync(fd); + if (st.dev !== anchor.dev || st.ino !== anchor.ino) return null; + const size = st.size; + if (offset < 0 || offset > size) return null; + const r = scanFrameRefsFd(fd, { onCorrupt: 'strict', startOffset: offset }); + if (r.frames.length === 0 && r.eofOffset < size) { + // Bytes at `offset` start no valid frame. An append-only file grows + // whole frames sequentially, so a torn tail is always a frame PREFIX + // (starts with the magic): retry next time. Anything else means offset + // was not a boundary of this file — no valid continuation exists here. + const n = Math.min(MAGIC.length, size - offset); + const head = readAtSync(fd, offset, n); + if (!MAGIC.subarray(0, n).equals(head)) return null; + return { offset, appliedFrames: 0 }; + } + for (const f of r.frames) apply(f, fd); + return { offset: r.eofOffset, appliedFrames: r.frames.length }; + } finally { + fsSync.closeSync(fd); + } +} diff --git a/packages/minidb/src/rename-replace.ts b/packages/minidb/src/rename-replace.ts new file mode 100644 index 0000000000..f3072fd560 --- /dev/null +++ b/packages/minidb/src/rename-replace.ts @@ -0,0 +1,36 @@ +// src/rename-replace.ts +// +// `fs.rename(src, dst)` with Windows replace-open-destination semantics. +// +// Windows, unlike POSIX, refuses to rename over a destination that ANY +// process still holds open (no delete sharing by default): libuv returns +// EPERM. Transient openers — co-process readers doing a split-second +// readFile/stat, antivirus, file indexers — come and go within single-digit +// milliseconds, so on Windows the caller-side facility retries EPERM with +// jitter before giving up. POSIX renames over open files directly and never +// takes the retry path. + +import fs from 'node:fs/promises'; + +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); + +export interface RenameReplaceOptions { + /** Max EPERM retries before the error propagates (Windows only). */ + retries?: number; + /** Base delay between retries in ms (a like-sized random jitter is added). */ + baseDelayMs?: number; +} + +export async function renameReplace(src: string, dst: string, opts: RenameReplaceOptions = {}): Promise { + if (process.platform !== 'win32') return fs.rename(src, dst); + const retries = opts.retries ?? 100; + const base = opts.baseDelayMs ?? 20; + for (let attempt = 0; ; attempt++) { + try { + return await fs.rename(src, dst); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'EPERM' || attempt >= retries) throw e; + await sleep(base + Math.floor(Math.random() * (base + 10))); + } + } +} diff --git a/packages/minidb/src/server.ts b/packages/minidb/src/server.ts index 5f00a8ed06..6124f31352 100644 --- a/packages/minidb/src/server.ts +++ b/packages/minidb/src/server.ts @@ -42,6 +42,10 @@ class RespParser { *feed(chunk: Buffer): Generator { this.buf = this.buf.length ? Buffer.concat([this.buf, chunk]) : chunk; if (this.buf.length > this.maxBuf) { + // Drop the buffered oversized request before reporting: without the + // reset every later chunk would fail with the same error and the giant + // buffer would be retained for the life of the connection. + this.buf = Buffer.alloc(0); throw new Error(`RESP request too large (>${this.maxBuf} bytes)`); } while (this.buf.length) { @@ -164,21 +168,48 @@ export async function startServer({ dir, port = 6379, host = '127.0.0.1', fsyncP const db = (await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy })) as MiniDb; const server = net.createServer((socket: Socket) => { const parser = new RespParser(); + // Serialize per-connection processing: a new chunk's commands are queued + // behind the previous chunk's in-flight work, so replies always leave in + // request order. Without this, a slow command in one packet (e.g. SET with + // fsync 'always') let replies from the next packet overtake it, breaking + // pipelined clients. + let queue: Promise = Promise.resolve(); + // A client that resets the connection while a large reply is being written + // makes the next write fail with EPIPE/ECONNRESET. Without an 'error' + // listener that event becomes an uncaught exception and takes the whole + // process down, so swallow it: the connection is dead either way, and the + // queued work below skips further writes to it. + socket.on('error', () => {}); + // Never write to a destroyed socket: write-after-destroy would just + // surface as another 'error' event on the dead connection. + const send = (res: string | Buffer): void => { + if (!socket.destroyed) socket.write(res); + }; socket.on('data', (chunk: Buffer) => { - void (async () => { + queue = queue.then(async () => { try { for (const args of parser.feed(chunk)) { - const res = await handle(db, args); + if (socket.destroyed) return; + let res: string | Buffer | null; + try { + res = await handle(db, args); + } catch (e) { + // One failing command must not starve the replies of the + // commands already parsed from the same chunk. + res = reply.err((e as Error).message); + } if (res === null) { socket.end(); return; } - socket.write(res); + send(res); } } catch (e) { - socket.write(reply.err((e as Error).message)); + // Parser-level failure (e.g. oversized request): feed() has already + // reset its buffer, so the connection can keep serving new commands. + send(reply.err((e as Error).message)); } - })(); + }); }); }); diff --git a/packages/minidb/src/store.ts b/packages/minidb/src/store.ts index ed04eafdaa..1cd8b6682f 100644 --- a/packages/minidb/src/store.ts +++ b/packages/minidb/src/store.ts @@ -89,6 +89,12 @@ class MinHeap { export interface StoreOptions { activeExpireIntervalMs?: number; activeExpireMaxPerTick?: number; + /** Time budget (ms) per active-expiration tick, used once + * `activeExpireMaxPerTick` entries have been reaped: a simultaneously-expired + * burst keeps being reaped within this budget, so TTL storms drain far + * faster than the fixed per-tick count alone, while the event-loop pause per + * tick stays bounded. */ + activeExpireTimeBudgetMs?: number; /** Read a value back from disk for disk-backed records. Required whenever a * StoreRecord may hold a disk ref. */ readValue?: ValueReader; @@ -106,13 +112,22 @@ export class Store { /** Approximate bytes held by live + expired-not-yet-reaped records. In * valueMode:'disk' this counts keys/metadata/refs, not the value bulk. */ bytes = 0; + /** Number of records with an expiry set. Enables an O(1) size fast path when + * TTL is not in use. */ + private expiring = 0; private readonly maxPerTick: number; + private readonly expireTimeBudgetMs: number; + /** Sticky flag: set when the last tick reaped ≥ maxPerTick (i.e. there is an + * expiry backlog), making the next ticks aggressive until the storm drains + * — like Redis's aggressive expire cycle. */ + private expireAggressive = false; private timer: ReturnType | null = null; private readonly onExpire?: (key: string, record: StoreRecord) => void; private readonly readValue?: ValueReader; constructor(opts: StoreOptions = {}) { this.maxPerTick = opts.activeExpireMaxPerTick ?? 100; + this.expireTimeBudgetMs = opts.activeExpireTimeBudgetMs ?? 2; this.onExpire = opts.onExpire; this.readValue = opts.readValue; const interval = opts.activeExpireIntervalMs ?? 100; @@ -121,6 +136,8 @@ export class Store { } get size(): number { + // O(1) fast path: with no TTL ever set, every map entry is live. + if (this.expiring === 0) return this.map.size; // Count only logically-live keys: expired-but-not-yet-reaped entries stay // in the map until lazy/active expiration removes them, so map.size would // otherwise over-report. @@ -172,7 +189,10 @@ export class Store { const r = this.map.get(k); const ok = this.map.delete(k); if (ok) { - if (r) this.bytes -= Buffer.byteLength(k, 'binary') + this.refBytes(r.ref) + this.metaBytes(r.dt); + if (r) { + this.bytes -= Buffer.byteLength(k, 'binary') + this.refBytes(r.ref) + this.metaBytes(r.dt); + if (r.expireAt) this.expiring--; + } this.order.delete(k, k); } return ok; @@ -190,14 +210,18 @@ export class Store { setRef(key: string | Buffer, ref: ValueRef, expireAt = 0, dt: Record | null = null): void { const k = toKStr(key); - const existed = this.map.has(k); - if (existed) this.bytes -= this.recordBytes(k); + const prev = this.map.get(k); + if (prev) { + if (prev.expireAt) this.expiring--; + this.bytes -= this.recordBytes(k); + } const seq = ++this.seq; const stored = this.cloneRef(ref); this.map.set(k, { ref: stored, expireAt: expireAt || 0, seq, dt }); this.bytes += Buffer.byteLength(k, 'binary') + this.refBytes(stored) + this.metaBytes(dt); - if (!existed) this.order.insert(k, k); + if (!prev) this.order.insert(k, k); if (expireAt) { + this.expiring++; this.heap.push({ t: expireAt, k, seq }); // Overwriting a TTL key leaves a stale heap entry that is only reaped // when its (possibly far-future) timestamp passes, so the heap can grow @@ -245,8 +269,18 @@ export class Store { return this.remove(toKStr(key)); } + /** Metadata-only existence check: no value materialization (no buffer copy in + * memory mode, no positioned disk read for disk-backed records). Applies the + * same lazy-expiration semantics as get(). */ has(key: string | Buffer): boolean { - return this.get(key) !== undefined; + const k = toKStr(key); + const r = this.map.get(k); + if (!r) return false; + if (r.expireAt && r.expireAt <= Date.now()) { + this.expireKey(k, r); // lazy expiration, same as get() + return false; + } + return true; } *entries(): Generator { @@ -272,6 +306,16 @@ export class Store { yield* this.scan({ gte: pk, lt: pk + '\uffff', count: limit }); } + /** Ordered scan yielding canonical keys only, without materializing values + * (no buffer copies in memory mode, no positioned reads in disk mode). + * Expired records are lazily reaped, exactly as in scan(). */ + *rawKeys(opts: RangeOptions = {}): Generator { + for (const n of this.order.range(opts) as Iterable>) { + if (!this.getRecord(n.key)) continue; + yield n.key; + } + } + /** Rewrite disk-backed value locations after compaction rotates the * snapshot/WAL files. Memory refs are left untouched. */ remapLocs(remap: (k: string, loc: ValueLoc, rec: StoreRecord) => ValueLoc | undefined): void { @@ -284,14 +328,27 @@ export class Store { private activeExpire(): void { const now = Date.now(); + // Normal ticks stay within the small budget; a tick that still finds a + // full quota of expired keys flips to aggressive mode (larger budget, like + // Redis's fast expire cycle) until the storm drains. + const deadline = now + (this.expireAggressive ? Math.max(this.expireTimeBudgetMs, 10) : this.expireTimeBudgetMs); let n = 0; - while (n++ < this.maxPerTick && this.heap.size && this.heap.peek()!.t <= now) { + let reaped = 0; + while (this.heap.size && this.heap.peek()!.t <= now) { + // Once the guaranteed per-tick quota is reaped, keep draining within the + // time budget: a fixed ~1000/s rate let a large simultaneous-expiry storm + // (e.g. 100k keys) linger in memory for minutes. The budget bounds the + // pause each tick instead of bounding the work. + if (n >= this.maxPerTick && (n & 15) === 0 && Date.now() >= deadline) break; const e = this.heap.pop()!; const r = this.map.get(e.k); if (r && r.seq === e.seq && r.expireAt && r.expireAt <= now) { this.expireKey(e.k, r); + reaped++; } + n++; } + this.expireAggressive = reaped >= this.maxPerTick; } /** Synchronously reap every expired record. Returns the number removed. */ diff --git a/packages/minidb/src/text-index.ts b/packages/minidb/src/text-index.ts index d49867f413..bbe59d598f 100644 --- a/packages/minidb/src/text-index.ts +++ b/packages/minidb/src/text-index.ts @@ -24,6 +24,12 @@ import type { PostingEntry } from './text-postings.js'; const LATIN = /[a-z0-9]+/g; const CJK = /[\u3400-\u9fff\u3040-\u30ff\uff00-\uffef]+/g; +// Postings records store the term length in a uint16. A single document with a +// longer token previously made every postings rebuild throw after the index had +// already been cleared, permanently poisoning the index (and compaction). Such +// tokens can never be real query terms — drop them at tokenization so one +// pathological document cannot destroy the index. +const MAX_TERM_CHARS = 0xffff; /** Tokenize text into terms (lowercased latin words + CJK uni/bigrams). */ export function tokenize(str: unknown): string[] { @@ -32,7 +38,8 @@ export function tokenize(str: unknown): string[] { const latin = s.match(LATIN); // Loop-push instead of `terms.push(...latin)`: spreading a large match array // (hundreds of thousands of tokens from a big doc) overflows the call stack. - if (latin) for (const t of latin) terms.push(t); + // Latin matches are ASCII, so chars == utf8 bytes for the length guard. + if (latin) for (const t of latin) if (t.length <= MAX_TERM_CHARS) terms.push(t); const runs = s.match(CJK) ?? []; for (const r of runs) { for (let i = 0; i < r.length; i++) { @@ -143,23 +150,24 @@ export class TextIndex { * Assigns fresh dense docIDs, writes a new postings file (disk mode) or * replaces the in-memory base (memory mode), and clears the delta + * tombstones. Called on open and on compaction. + * + * Atomic on failure: everything is staged off to the side first and swapped + * in only after the new postings file is durably renamed (disk mode), so a + * failed rebuild (e.g. a transient ENOSPC/EMFILE inside rebuildSync) leaves + * the PREVIOUS index fully functional instead of silently emptying it until + * the next successful build. */ build(entries: Iterable<{ key: string; value: unknown }>): void { - this.postings.clear(); - this.docLen.clear(); - this.keys.length = 0; - this.keyToId.clear(); - this.delta.clear(); - this.deltaCount = 0; - this.removed.clear(); - this.cache.clear(); - this.N = 0; - + // Staged state. const agg = new Map>(); // term -> (docID -> freq) + const newKeys: (string | undefined)[] = []; // docID -> key + const newKeyToId = new Map(); // key -> docID + const newDocLen = new Map(); // docID -> token count + let n = 0; for (const { key, value } of entries) { - const docID = this.keys.length; - this.keys.push(key); - this.keyToId.set(key, docID); + const docID = newKeys.length; + newKeys.push(key); + newKeyToId.set(key, docID); const tokens = tokenize(this.extract(value)); const counts = new Map(); for (const t of tokens) counts.set(t, (counts.get(t) ?? 0) + 1); @@ -168,23 +176,63 @@ export class TextIndex { if (!m) agg.set(t, (m = new Map())); m.set(docID, c); // docIDs increase monotonically -> insertion order is sorted } - this.docLen.set(docID, tokens.length); - this.N++; + newDocLen.set(docID, tokens.length); + n++; } if (this.path) { - // Disk mode: rewrite the postings file, then reopen for reads. - if (this.pf) { - this.pf.close(); + // Disk mode: write the new postings file (tmp + fsync + atomic rename in + // rebuildSync). The old read handle is closed before the rename (an open + // fd would block the rename on Windows), so the in-memory state below is + // NOT touched until the new file is in place — if rebuildSync throws, + // the old file is still intact and is simply re-attached. + const oldPf = this.pf; + if (oldPf) { + oldPf.close(); this.pf = null; } - const dict = PostingsFile.rebuildSync(this.path, aggToSorted(agg)); + let dict: Map; + try { + dict = PostingsFile.rebuildSync(this.path, aggToSorted(agg)); + } catch (e) { + // rebuildSync failed before the atomic rename, so the old file is + // intact: re-attach it and keep serving the previous index until the + // next successful build. + if (oldPf) { + try { + this.pf = PostingsFile.open(this.path); + } catch { + /* old handle unrecoverable; the next successful build fixes it */ + } + } + throw e; + } + // The rename happened — the old postings are replaced on disk, so from + // here the swap commits to the new index. A failed reopen (EMFILE & co.) + // is not special-cased: readBase treats a null pf as an empty base, so + // reads degrade to delta-only until the next build instead of reading + // through a stale dictionary. + const newPf = PostingsFile.open(this.path); + this.postings.clear(); for (const [t, e] of dict) this.postings.set(t, e); - this.pf = PostingsFile.open(this.path); + this.pf = newPf; } else { - // Memory mode. + // Memory mode: pure in-memory staging, no fallible I/O involved. this.memBase = agg; } + + // Swap in the staged per-doc state and drop the write buffer. + this.docLen.clear(); + for (const [id, len] of newDocLen) this.docLen.set(id, len); + this.keys.length = 0; + for (const k of newKeys) this.keys.push(k); + this.keyToId.clear(); + for (const [k, id] of newKeyToId) this.keyToId.set(k, id); + this.delta.clear(); + this.deltaCount = 0; + this.removed.clear(); + this.cache.clear(); + this.N = n; } /** Add or replace a document. Overwrites tombstone the old docID. */ diff --git a/packages/minidb/src/wal.ts b/packages/minidb/src/wal.ts index ff23e5872e..9ad4a04d87 100644 --- a/packages/minidb/src/wal.ts +++ b/packages/minidb/src/wal.ts @@ -45,6 +45,11 @@ export class WAL { private flushing = false; private inflight: Promise | null = null; private scheduled = false; + /** When sealed, appendLoc rejects new frames (code 'WAL_SEALED') while the + * already-queued frames can still be flushed. Compaction rotation seals the + * old WAL so no append can slip between the final flush and close(): any + * frame that will ever land in the old file is durable after one flush. */ + private sealed = false; private timer: ReturnType | null = null; private closed = false; private readonly stats: { walBytesWritten: number; walFsyncs: number } | null; @@ -72,6 +77,12 @@ export class WAL { } } + /** Reject new appends from now on; already-queued frames stay flushable. + * Idempotent. */ + seal(): void { + this.sealed = true; + } + /** Append one frame and return its predicted absolute file offset. The offset * is known synchronously because frames are flushed strictly in append order. * NOTE: the frame's bytes are NOT in the file yet — they sit in the in-memory @@ -80,6 +91,11 @@ export class WAL { * that window would hit a short read past the current end of the file. */ appendLoc(frame: Buffer): { offset: number; done: Promise } { if (this.closed) return { offset: -1, done: Promise.reject(new Error('WAL is closed')) }; + if (this.sealed) { + const err = new Error('WAL is sealed by a compaction rotation; retry against the new WAL'); + (err as { code?: string }).code = 'WAL_SEALED'; + return { offset: -1, done: Promise.reject(err) }; + } if (!Buffer.isBuffer(frame)) return { offset: -1, done: Promise.reject(new TypeError('frame must be a Buffer')) }; const offset = this.nextOffset; this.nextOffset += frame.length; @@ -153,6 +169,18 @@ export class WAL { return this.inflight; } + /** Re-sync size/nextOffset with the file on disk. Required after recovery + * truncates a torn WAL tail: the truncate happens on the path behind this + * WAL's back, and stale bookkeeping would otherwise make later appends + * publish value pointers offset by the torn byte count (reads then hit the + * wrong frames). */ + async refreshSize(): Promise { + if (!this.fh) return; + const st = await this.fh.stat(); + this.size = st.size; + this.nextOffset = st.size; + } + /** Force an fsync of the underlying file. */ async sync(): Promise { if (this.fh) { @@ -179,11 +207,19 @@ export class WAL { clearInterval(this.timer); this.timer = null; } - await this.flush(); - if (this.fh) { - await this.sync(); - await this.fh.close(); + // Release the file handle even when the final flush/fsync fails: the error + // still propagates to the caller, but a half-closed WAL must not leak its + // fd (a compaction rotation recovering from a failed close swaps in a + // fresh WAL on the same path and abandons this handle). An fh.close() + // error itself is swallowed: with the fsync above already durable there + // is nothing actionable left to report. + try { + await this.flush(); + if (this.fh) await this.sync(); + } finally { + const fh = this.fh; this.fh = null; + if (fh) await fh.close().catch(() => {}); } } } diff --git a/packages/minidb/test/cluster/basic.test.ts b/packages/minidb/test/cluster/basic.test.ts new file mode 100644 index 0000000000..2130b77294 --- /dev/null +++ b/packages/minidb/test/cluster/basic.test.ts @@ -0,0 +1,222 @@ +// test/cluster/basic.test.js +// +// Single-process ClusterDb behavior: topology creation/validation, basic KV +// across shards, multi-key ops, merged scans, hash distribution, stats. + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { ClusterDb } from '../../src/cluster/index.js'; +import { shardFor } from '../../src/cluster/utils.js'; +import { tmpDir, rmrf } from '../e2e/helpers/tmp.js'; +import { keyOnShard, keysByShard } from './helpers.js'; + +interface User { + name: string; + age: number; +} + +test('cluster open creates meta + shard dirs; reload validates topology', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + assert.equal(db.shardCount, 4); + const meta = JSON.parse(await fs.readFile(path.join(dir, 'cluster.meta.json'), 'utf8')); + assert.equal(meta.version, 1); + assert.equal(meta.shardCount, 4); + assert.equal(meta.valueCodec, 'json'); + const entries = await fs.readdir(dir); + for (let i = 0; i < 4; i++) assert.ok(entries.includes(`shard-0${i}`), `shard-0${i} exists`); + await db.close(); + + // Reopen without options: inherits the on-disk topology. + const db2 = await ClusterDb.open({ dir }); + assert.equal(db2.shardCount, 4); + await db2.close(); + + // Explicit mismatches are rejected. + await assert.rejects(() => ClusterDb.open({ dir, shardCount: 8 }), /shardCount=4/); + await assert.rejects(() => ClusterDb.open({ dir, valueCodec: 'string' }), /valueCodec=json/); + + // Invalid creation parameter is rejected upfront. + const dir2 = await tmpDir('minidb-cluster-'); + try { + await assert.rejects(() => ClusterDb.open({ dir: dir2, shardCount: 0 }), RangeError); + } finally { + await rmrf(dir2); + } + } finally { + await rmrf(dir); + } +}); + +test('single-key ops round-trip across shards', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + // Keys crafted to hit two distinct shards, proving reads/writes route. + const k0 = keyOnShard('round', 0, 4); + const k1 = keyOnShard('round', 1, 4); + await db.set(k0, { name: 'alice', age: 30 }); + await db.set(k1, { name: 'bob', age: 25 }); + assert.deepEqual(await db.get(k0), { name: 'alice', age: 30 }); + assert.deepEqual(await db.get(k1), { name: 'bob', age: 25 }); + assert.equal(await db.has(k0), true); + assert.equal(await db.has('round:nope'), false); + assert.equal(await db.del(k0), true); + assert.equal(await db.del(k0), false); + assert.equal(await db.get(k0), undefined); + await db.close(); + } finally { + await rmrf(dir); + } +}); + +test('ttl and expire work per key', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db = await ClusterDb.open({ dir, shardCount: 2, valueCodec: 'string' }); + await db.set('temp', 'v', { ttl: 50 }); + const left = await db.ttl('temp'); + assert.ok(left > 0 && left <= 50, `ttl in range, got ${left}`); + assert.equal(await db.ttl('missing'), -2); + await db.set('forever', 'v'); + assert.equal(await db.ttl('forever'), -1); + assert.equal(await db.expire('forever', 30), true); + assert.equal(await db.expire('missing', 30), false); + await new Promise((r) => setTimeout(r, 80)); + assert.equal(await db.get('temp'), undefined); + assert.equal(await db.get('forever'), undefined); + await db.close(); + } finally { + await rmrf(dir); + } +}); + +test('mset/mget/mdel span shards (atomic per shard)', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db = await ClusterDb.open({ dir, shardCount: 8, valueCodec: 'json' }); + const grouped = keysByShard('multi', 200, 8); + assert.ok(grouped.size >= 3, `keys hit multiple shards, got ${grouped.size}`); + + const entries = [...grouped.values()].flat().map((k, i) => [k, i] as [string, number]); + await db.mset(entries); + + const keys = entries.map(([k]) => k); + const got = await db.mget(keys); + assert.deepEqual(got, entries.map(([, v]) => v)); + + // mdel counts real deletions and spans shards. + const some = keys.filter((_, i) => i % 2 === 0); + const removed = await db.mdel([...some, 'multi:never-existed']); + assert.equal(removed, some.length); + const after = await db.mget(some); + assert.ok(after.every((v) => v === undefined)); + const rest = await db.mget(keys.filter((_, i) => i % 2 === 1)); + assert.ok(rest.every((v) => typeof v === 'number')); + await db.close(); + } finally { + await rmrf(dir); + } +}); + +test('batch applies per-shard atomically with ttl/dt', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + const grouped = keysByShard('batch', 40, 4); + const ops = [...grouped.values()].flat().flatMap((k, i) => [ + { op: 'set' as const, key: k, value: { name: `u${i}`, age: i }, dt: { created: 1000 + i } }, + { op: 'del' as const, key: `ghost:${i}` }, + ]); + await db.batch(ops); + const all = await db.scan({ prefix: 'batch:' }); + assert.equal(all.length, 40); + assert.ok(all.every((e) => e.dt && typeof e.dt.created === 'number')); + await db.close(); + } finally { + await rmrf(dir); + } +}); + +test('scan/prefix merge across shards: sorted, bounded, limited, reversible', { timeout: 60_000 }, async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db = await ClusterDb.open({ dir, shardCount: 8, valueCodec: 'json' }); + const keys: string[] = []; + for (let i = 0; i < 300; i++) { + const k = `s:${String(i).padStart(4, '0')}`; + keys.push(k); + await db.set(k, i); + } + const byteSorted = [...keys].sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b))); + + const all = await db.scan(); + assert.equal(all.length, 300); + assert.deepEqual( + all.map((e) => e.key), + byteSorted, + 'globally sorted by key bytes', + ); + + // Interleaved writes land in every shard; the merge must be complete. + const range = await db.scan({ gte: 's:0010', lt: 's:0020' }); + assert.deepEqual( + range.map((e) => e.key), + byteSorted.filter((k) => k >= 's:0010' && k < 's:0020'), + ); + + const pref = await db.prefix('s:01', 10); + assert.equal(pref.length, 10); + assert.deepEqual( + pref.map((e) => e.key), + byteSorted.filter((k) => k.startsWith('s:01')).slice(0, 10), + ); + + const limited = await db.scan({ limit: 7 }); + assert.equal(limited.length, 7); + assert.deepEqual(limited.map((e) => e.key), byteSorted.slice(0, 7)); + + const rev = await db.scan({ reverse: true, limit: 5 }); + assert.deepEqual(rev.map((e) => e.key), [...byteSorted].reverse().slice(0, 5)); + + // Prefix within ScanOptions beats range bounds. + const prefOpt = await db.scan({ prefix: 's:029', gte: 'zzz' }); + assert.deepEqual(prefOpt.map((e) => e.key), ['s:0290', 's:0291', 's:0292', 's:0293', 's:0294', 's:0295', 's:0296', 's:0297', 's:0298', 's:0299'].filter((k) => k.startsWith('s:029'))); + await db.close(); + } finally { + await rmrf(dir); + } +}); + +test('hash routing distributes keys over all shards and is deterministic', async () => { + const counts = Array.from({ length: 16 }, () => 0); + for (let i = 0; i < 4000; i++) counts[shardFor(`dist:${i}`, 16)]++; + const avg = 4000 / 16; + for (let s = 0; s < 16; s++) { + assert.ok(counts[s]! > 0, `shard ${s} got keys`); + assert.ok(counts[s]! < avg * 3, `shard ${s} not hot (${counts[s]} vs avg ${avg})`); + } + // Stable across calls (crypto-free pure function). + assert.equal(shardFor('dist:42', 16), shardFor('dist:42', 16)); + assert.equal(shardFor('', 16), shardFor('', 16)); // empty key hashes fine +}); + +test('stats reflect writer usage', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'string', lockPoolMaxShards: 2 }); + for (let i = 0; i < 20; i++) await db.set(`st:${i}`, 'v'); + const s = db.stats(); + assert.equal(s.shardCount, 4); + assert.ok(s.writerOpens >= 2, `writers opened, got ${s.writerOpens}`); + assert.ok(s.writersCached <= 2, `pool bounded, got ${s.writersCached}`); + assert.ok(s.evictions >= 1, 'LRU eviction happened with tiny pool'); + await db.close(); + await assert.rejects(() => db.get('st:0'), /closed/); + } finally { + await rmrf(dir); + } +}); diff --git a/packages/minidb/test/cluster/concurrent.test.ts b/packages/minidb/test/cluster/concurrent.test.ts new file mode 100644 index 0000000000..61e51f1781 --- /dev/null +++ b/packages/minidb/test/cluster/concurrent.test.ts @@ -0,0 +1,279 @@ +// test/cluster/concurrent.test.js +// +// True multi-process concurrent read/write tests. Each scenario spawns real +// child processes (node --import tsx mp-worker.ts) that open the same cluster +// directory concurrently, and verifies data integrity afterwards. + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { MiniDb } from '../../src/index.js'; +import { ClusterDb, shardDirName } from '../../src/cluster/index.js'; +import { shardFor } from '../../src/cluster/utils.js'; +import { tmpDir } from '../e2e/helpers/tmp.js'; +import { keyOnShard, runWorker, runWorkerOk, rmrf, sleep } from './helpers.js'; + +test( + 'P=4 processes write disjoint keyspaces concurrently (S=8); all data survives', + { timeout: 180_000 }, + async () => { + const dir = await tmpDir('minidb-cluster-mp-'); + try { + const P = 4; + const N = 150; + const runs = Array.from({ length: P }, (_, p) => + runWorkerOk(['write', dir, '8', `p${p}`, String(N)], { timeoutMs: 120_000 }), + ); + const reports = await Promise.all(runs); + for (const r of reports) assert.equal(r.n, N); + + // Verify from a brand-new process: every key, every value. + await Promise.all( + Array.from({ length: P }, (_, p) => runWorkerOk(['verify', dir, '8', `p${p}`, String(N)], { timeoutMs: 120_000 })), + ); + + // Double-check through a read-only cluster in this process. + const db = await ClusterDb.open<{ p: string; i: number }>({ dir, readOnly: true }); + let count = 0; + for (const e of await db.scan()) { + assert.equal(e.key, `${e.value.p}:${e.value.i}`); + count++; + } + assert.equal(count, P * N); + await db.close(); + } finally { + await rmrf(dir); + } + }, +); + +test( + 'concurrent writers on the SAME shard serialize safely with no lost writes', + { timeout: 180_000 }, + async () => { + const dir = await tmpDir('minidb-cluster-mp-'); + try { + const shards = 4; + const targetShard = 2; + const procs = 3; + const perProc = 40; + // Precompute disjoint keys that all route to the same shard. + const keysPerProc = Array.from({ length: procs }, (_, p) => + Array.from({ length: perProc }, (_, i) => keyOnShard(`hot${p}:${i}`, targetShard, shards)), + ); + const runs = keysPerProc.map((keys) => + runWorkerOk(['writekeys', dir, String(shards), keys.join(',')], { timeoutMs: 150_000 }), + ); + const reports = await Promise.all(runs); + // Retries are expected under contention but not required; log for insight. + const retries = reports.map((r) => r.retries); + + const db = await ClusterDb.open<{ key: string }>({ dir, shardCount: shards, readOnly: true }); + const allKeys = keysPerProc.flat(); + const got = await db.mget(allKeys); + assert.deepEqual( + got.map((v) => v?.key), + allKeys, + `no lost writes on a hot shard (retries: ${retries.join('/')})`, + ); + await db.close(); + } finally { + await rmrf(dir); + } + }, +); + +test( + 'a reader process observes commits from a concurrently running writer process', + { timeout: 180_000 }, + async () => { + const dir = await tmpDir('minidb-cluster-mp-'); + try { + // Start a long-polling reader BEFORE the data exists. The inner wait + // budget must absorb writer process startup on heavily loaded CI + // runners (writer spawn + first open can take tens of seconds there). + const waiter = runWorkerOk(['wait-read', dir, '4', 'live:k', '42', '120000'], { timeoutMs: 150_000 }); + // Give the reader a head start so it really polls with a cold cache. + await new Promise((r) => setTimeout(r, 500)); + + const db = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + await db.set('live:k', { i: 42 }); + const report = await waiter; + assert.ok(typeof report.waitedMs === 'number'); + await db.close(); + } finally { + await rmrf(dir); + } + }, +); + +test( + 'mixed read/write storm across processes keeps all committed data readable', + { timeout: 240_000 }, + async () => { + const dir = await tmpDir('minidb-cluster-mp-'); + try { + const writers = Array.from({ length: 3 }, (_, p) => + runWorkerOk(['write', dir, '8', `storm${p}`, '100'], { timeoutMs: 150_000 }), + ); + // Concurrent read-only verifiers race the writers; they may legitimately + // see partial progress, so only assert they never error out. + const racingReaders = Array.from({ length: 2 }, () => + runWorker(['wait-read', dir, '8', 'storm0:0', '0', '90000'], { timeoutMs: 120_000 }), + ); + const writerReports = await Promise.all(writers); + const readerResults = await Promise.all(racingReaders); + for (const w of writerReports) assert.equal(w.n, 100); + for (const r of readerResults) assert.equal(r.code, 0, `racing reader exited cleanly: ${r.stderr}`); + + // After quiesce, the full dataset is visible to a new process. + for (let p = 0; p < 3; p++) { + await runWorkerOk(['verify', dir, '8', `storm${p}`, '100'], { timeoutMs: 120_000 }); + } + } finally { + await rmrf(dir); + } + }, +); + + +/** Dual-check: every view of the cached reader on the hot shard must equal a + * from-scratch full open of the same shard files. */ +type Doc = Record; + +async function assertSameAsFreshOpen(db: ClusterDb, dir: string, hot: number): Promise { + const sortByKey = (a: { key: string }, b: { key: string }): number => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0); + const ref = await MiniDb.open({ dir: path.join(dir, shardDirName(hot, db.shardCount)), readOnly: true, valueCodec: 'json' }); + try { + assert.deepEqual(await db.scan(), ref.scan(), 'scan equality with a fresh full open'); + assert.deepEqual((await db.findEq('c', 'c3')).sort(sortByKey), ref.findEq('c', 'c3').sort(sortByKey), 'findEq equality'); + const [crange, rrange] = [await db.findRange('n', { min: 100, max: 900 }), ref.findRange('n', { min: 100, max: 900 })]; + assert.deepEqual(crange.sort(sortByKey), rrange.sort(sortByKey), 'findRange equality'); + assert.deepEqual(await db.findEq('u', 'pre-u42'), ref.findEq('u', 'pre-u42'), 'unique-index equality'); + const [cs, rs] = [await db.search('t', 'alpha'), ref.search('t', 'alpha')]; + assert.deepEqual(cs.sort(sortByKey), rs.sort(sortByKey), 'text search equality'); + } finally { + await ref.close(); + } +} + +test( + 'reader WAL catch-up: incremental tail apply equals a full reopen (multi-process)', + { timeout: 180_000 }, + async () => { + const dir = await tmpDir('minidb-cluster-mp-'); + try { + const shards = 4; + const hot = 1; + // Preload 30k keys on the hot shard (secondary eq+range indexes, one + // unique index, one text index, dt column on every key), then release + // all locks so the storm child can write. + const setup = await ClusterDb.open({ dir, shardCount: shards, valueCodec: 'json', fsyncPolicy: 'no', lockHoldMs: 0 }); + const pre: string[] = []; + for (let seq = 0; pre.length < 30_000; seq++) { + const key = `pre:${seq}`; + if (shardFor(key, shards) === hot) pre.push(key); + } + for (let j = 0; j < pre.length; j++) { + await setup.set(pre[j]!, { n: j, c: `c${j % 11}`, u: `pre-u${j}`, t: `alpha beta pre w${j % 17}` }, { dt: { created: 1700000100000 + j } }); + } + await setup.createIndex('c', { field: 'c' }); + await setup.createIndex('n', { field: 'n', type: 'range' }); + await setup.createIndex('u', { field: 'u', unique: true }); + await setup.createTextIndex('t', { fields: ['t'] }); + await setup.close(); + + // The parent's read-only cluster: first read opens and warms the cached + // reader of the hot shard (records the WAL watermark). + const db = await ClusterDb.open({ dir, readOnly: true }); + assert.equal((await db.get(pre[0]!))?.n, 0); + const stats0 = db.stats(); + assert.equal(stats0.readerReopens, 0); + assert.equal(stats0.incrementalCatchups, 0); + + // Mixed storm from a child process: sets, updates, dels, TYPE_BATCH + // batches, short/long TTL sets, dt changes — all on the hot shard. + const report = await runWorkerOk(['storm', dir, String(shards), String(hot), 'storm', '4000'], { timeoutMs: 120_000 }); + // Outlive the 50ms short TTLs so replay-side expiry is deterministic. + await sleep(150); + + // The first read after the storm catches the cached reader up by + // applying ONLY the appended frames (exactly the storm's frame count); + // no full reader reopen is allowed on this pure-append path. + await db.get(pre[1]!); + const stats1 = db.stats(); + assert.ok(stats1.incrementalCatchups > 0, 'incremental catch-up happened'); + assert.equal(stats1.readerReopens - stats0.readerReopens, 0, 'no full reopen on the pure-append path'); + assert.equal(stats1.catchupFramesApplied - stats0.catchupFramesApplied, report.frames, 'every storm frame applied exactly once'); + + // Every view equals a from-scratch full open of the shard. + await assertSameAsFreshOpen(db, dir, hot); + + // Control: a quiet shard (no writes) — reads cost only the fingerprint. + const quiet = keyOnShard('quiet', 3, shards); + await db.get(quiet); // first read caches the shard's reader + const stats2 = db.stats(); + for (let k = 0; k < 200; k++) assert.equal(await db.get(quiet), undefined); + const stats3 = db.stats(); + assert.equal(stats3.incrementalCatchups, stats2.incrementalCatchups, 'no catch-ups without writes'); + assert.equal(stats3.readerReopens, stats2.readerReopens, 'no reopens without writes'); + await db.close(); + } finally { + await rmrf(dir); + } + }, +); + +test( + 'reader falls back to a full reopen on compaction rotation, then resumes incrementally', + { timeout: 180_000 }, + async () => { + const dir = await tmpDir('minidb-cluster-mp-'); + try { + const shards = 4; + const hot = 1; + const setup = await ClusterDb.open({ dir, shardCount: shards, valueCodec: 'json', fsyncPolicy: 'no', lockHoldMs: 0 }); + const pre: string[] = []; + for (let seq = 0; pre.length < 1_000; seq++) { + const key = `pre:${seq}`; + if (shardFor(key, shards) === hot) pre.push(key); + } + for (let j = 0; j < pre.length; j++) { + await setup.set(pre[j]!, { n: j, c: `c${j % 11}`, u: `pre-u${j}`, t: `alpha beta pre w${j % 17}` }, { dt: { created: 1700000100000 + j } }); + } + await setup.createIndex('c', { field: 'c' }); + await setup.createIndex('n', { field: 'n', type: 'range' }); + await setup.createIndex('u', { field: 'u', unique: true }); + await setup.createTextIndex('t', { fields: ['t'] }); + await setup.close(); + + const db = await ClusterDb.open({ dir, readOnly: true }); + await db.get(pre[0]!); // warm the cached reader + const stats0 = db.stats(); + + // Storm with a tiny compaction threshold: the WAL/snapshot are rotated + // (possibly several times) while the parent's reader is cached. + await runWorkerOk(['storm', dir, String(shards), String(hot), 'rot', '1500', '60000', '1'], { timeoutMs: 120_000 }); + await sleep(150); + + await db.get(pre[1]!); + const stats1 = db.stats(); + assert.ok(stats1.readerReopens - stats0.readerReopens >= 1, 'rotation forced a full reopen'); + assert.equal(stats1.incrementalCatchups - stats0.incrementalCatchups, 0, 'no incremental progress across a rotation'); + await assertSameAsFreshOpen(db, dir, hot); + + // A follow-up append-only storm is caught incrementally again. + const report2 = await runWorkerOk(['storm', dir, String(shards), String(hot), 'rot2', '800', '0', '0'], { timeoutMs: 120_000 }); + await sleep(150); + await db.get(pre[2]!); + const stats2 = db.stats(); + assert.ok(stats2.incrementalCatchups - stats1.incrementalCatchups > 0, 'incremental catch-up resumed after the reopen'); + assert.equal(stats2.readerReopens - stats1.readerReopens, 0, 'no further reopen'); + assert.equal(stats2.catchupFramesApplied - stats1.catchupFramesApplied, report2.frames, 'second storm applied frame-exactly'); + await assertSameAsFreshOpen(db, dir, hot); + await db.close(); + } finally { + await rmrf(dir); + } + }, +); diff --git a/packages/minidb/test/cluster/cross-shard.test.ts b/packages/minidb/test/cluster/cross-shard.test.ts new file mode 100644 index 0000000000..062c18b886 --- /dev/null +++ b/packages/minidb/test/cluster/cross-shard.test.ts @@ -0,0 +1,269 @@ +// test/cluster/cross-shard.test.js +// +// Cross-shard semantics: best-effort mset, 'none' mode rejection, per-shard +// secondary/text indexes with global merge, and cluster-wide compaction. + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { MiniDb } from '../../src/index.js'; +import { ClusterDb, shardDirName } from '../../src/cluster/index.js'; +import { tmpDir, rmrf } from '../e2e/helpers/tmp.js'; +import { keyOnShard, keysByShard } from './helpers.js'; + +test("crossShard 'none' rejects multi-shard writes but allows single-shard ones", async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json', crossShard: 'none' }); + const a = keyOnShard('cs', 0, 4); + const b = keyOnShard('cs', 1, 4); + await assert.rejects(() => db.mset([[a, 1], [b, 2]]), /spans 2 shards/); + + const sameShard = [keyOnShard('one', 0, 4), keyOnShard('one-x', 0, 4)]; + await db.mset(sameShard.map((k, i) => [k, i] as [string, number])); + assert.deepEqual(await db.mget(sameShard), [0, 1]); + await db.close(); + + // '2pc' is explicitly reserved. + await assert.rejects(() => ClusterDb.open({ dir, crossShard: '2pc' }), /2pc/); + } finally { + await rmrf(dir); + } +}); + +test('secondary index: create, findEq/findRange merged across shards, maintained on new writes, drop', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + interface U { city: string; age: number } + const db = await ClusterDb.open({ dir, shardCount: 8, valueCodec: 'json' }); + const keys = [...keysByShard('u', 120, 8).values()].flat(); + const expectedBj: string[] = []; + await db.mset( + keys.map((k, i) => { + const u = { city: i % 3 === 0 ? 'bj' : 'sh', age: 20 + (i % 40) }; + if (u.city === 'bj') expectedBj.push(k); + return [k, u] as [string, U]; + }), + ); + // Sanity: data hit several shards, so index queries must merge. + assert.ok(keysByShard('u', 120, 8).size >= 3); + + await db.createIndex('by-city', { field: 'city' }); + await db.createIndex('by-age', { field: 'age', type: 'range' }); + + const found = await db.findEq('by-city', 'bj'); + assert.deepEqual( + found.map((r) => r.key).sort(), + [...expectedBj].sort(), + ); + + const ranged = await db.findRange('by-age', { min: 25, max: 30, maxExclusive: true }); + assert.ok(ranged.length > 0); + assert.ok(ranged.every((r) => r.field >= 25 && r.field < 30)); + // Sorted by field then key across shards. + for (let i = 1; i < ranged.length; i++) assert.ok(ranged[i]!.field >= ranged[i - 1]!.field); + + // Writes after index creation are indexed too (same-shard and elsewhere). + const late = keyOnShard('late', 5, 8); + await db.set(late, { city: 'bj', age: 33 }); + const after = await db.findEq('by-city', 'bj'); + assert.ok(after.some((r) => r.key === late)); + + const defs = await db.listIndexes(); + assert.deepEqual( + defs.map((d) => d.name).sort(), + ['by-age', 'by-city'], + ); + + assert.equal(await db.dropIndex('by-city'), true); + await assert.rejects(() => db.findEq('by-city', 'bj'), /no such index/); + assert.equal(await db.dropIndex('by-city'), false); + await db.close(); + } finally { + await rmrf(dir); + } +}); + +test('text index: search merges per-shard results by score', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + interface D { title: string; body: string } + const db = await ClusterDb.open({ dir, shardCount: 8, valueCodec: 'json' }); + const docs: [string, D][] = [ + ['d:a', { title: 'rust wal', body: 'write ahead log durability in rust' }], + ['d:b', { title: 'go wal', body: 'write ahead log implementations compared' }], + ['d:c', { title: 'garden', body: 'tomatoes and basil on the balcony' }], + ['d:d', { title: 'log blog', body: 'a blog about logging and logs' }], + ['d:e', { title: 'rust garden', body: 'rust in the garden shed with tools' }], + ]; + await db.mset(docs); + // Make sure the corpus actually spans shards; otherwise the merge is untested. + assert.ok(new Set(docs.map(([k]) => db.shardOf(k))).size >= 2); + + await db.createTextIndex('txt', { fields: ['body'] }); + const hits = await db.search('txt', 'log'); + assert.ok(hits.length >= 2); + for (let i = 1; i < hits.length; i++) assert.ok(hits[i - 1]!.score >= hits[i]!.score); + const keys = hits.map((h) => h.key); + assert.ok(keys.includes('d:b') || keys.includes('d:a') || keys.includes('d:d')); + + const limited = await db.search('txt', 'rust OR log', { op: 'OR', limit: 2 }); + assert.equal(limited.length, 2); + + assert.equal(await db.dropTextIndex('txt'), true); + await assert.rejects(() => db.search('txt', 'log'), /no such text index/); + await db.close(); + } finally { + await rmrf(dir); + } +}); + +test('index definitions survive reopen (registry applied on shard open)', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + interface U { city: string } + const db = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + await db.createIndex('by-city', { field: 'city' }); + await db.close(); + + // A fresh instance writes new data; the registry must be applied to any + // shard writer it opens so the index keeps being maintained. + const db2 = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + const k = keyOnShard('reopen', 3, 4); + await db2.set(k, { city: 'gz' }); + const found = await db2.findEq('by-city', 'gz'); + assert.deepEqual(found.map((r) => r.key), [k]); + await db2.close(); + } finally { + await rmrf(dir); + } +}); + +test('compact rewrites every shard and preserves data', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + const keys = [...keysByShard('cmp', 120, 4).values()].flat(); + // Three generations per key to grow WALs. + for (let gen = 0; gen < 3; gen++) { + await db.mset(keys.map((k) => [k, gen] as [string, number])); + } + const result = await db.compact(); + assert.equal(result.skipped.length, 0); + assert.deepEqual([...result.compacted].sort((a, b) => a - b), [0, 1, 2, 3]); + + const got = await db.mget(keys); + assert.ok(got.every((v) => v === 2)); + // Still fully writable afterwards. + await db.set('cmp:after', 99); + assert.equal(await db.get('cmp:after'), 99); + await db.close(); + } finally { + await rmrf(dir); + } +}); + +test('findRange applies offset/count/reverse to the globally merged result', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + interface D { n: number } + const db = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + const byShard = keysByShard('fr', 60, 4); + assert.ok(byShard.size >= 3); + const entries: [string, D][] = []; + let i = 0; + for (const shardKeys of byShard.values()) { + for (const k of shardKeys) { + entries.push([k, { n: i % 25 }]); // Values repeat across shards. + i++; + } + } + await db.mset(entries); + await db.createIndex('by-n', { field: 'n', type: 'range' }); + + // Reference order (field asc, key asc) computed locally from the data. + const cmpKey = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0); + const rows = entries.map(([key, d]) => ({ key, n: d.n })).sort((a, b) => a.n - b.n || cmpKey(a.key, b.key)); + const keys = (rs: { key: string }[]) => rs.map((r) => r.key); + const query = async (opts: Parameters['findRange']>[1]) => keys(await db.findRange('by-n', opts)); + + assert.deepEqual(await query({}), keys(rows)); + // Count/offset are global totals, not per shard (4 shards here). + assert.deepEqual(await query({ count: 6 }), keys(rows.slice(0, 6))); + assert.deepEqual(await query({ offset: 55 }), keys(rows.slice(55))); + assert.deepEqual(await query({ offset: 12, count: 8 }), keys(rows.slice(12, 20))); + // Reverse flips the merged order, including the key tie-break. + assert.deepEqual(await query({ reverse: true }), keys([...rows].reverse())); + assert.deepEqual(await query({ reverse: true, count: 7 }), keys([...rows].reverse().slice(0, 7))); + // Bounds still apply globally on top of offset/count. + const inBounds = rows.filter((r) => r.n >= 3 && r.n < 20); + assert.deepEqual(await query({ min: 3, max: 20, maxExclusive: true, count: 5 }), keys(inBounds.slice(0, 5))); + assert.deepEqual(await query({ min: 3, max: 20, maxExclusive: true, offset: 2, count: 5 }), keys(inBounds.slice(2, 7))); + await db.close(); + } finally { + await rmrf(dir); + } +}); + +test('concurrent createIndex from two instances keeps both registry entries', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + interface D { a: number; b: number } + const shardCount = 4; + const dbA = await ClusterDb.open({ dir, shardCount, valueCodec: 'json' }); + const dbB = await ClusterDb.open({ dir, shardCount, valueCodec: 'json' }); + // Two different indexes created concurrently: neither registry write may + // clobber the other. + await Promise.all([dbA.createIndex('ia', { field: 'a' }), dbB.createIndex('ib', { field: 'b' })]); + await dbA.close(); + await dbB.close(); + + // Re-verify through a fresh instance: the registry lists both, and both + // sidecars answer on every shard. + const fresh = await ClusterDb.open({ dir, shardCount, valueCodec: 'json' }); + assert.deepEqual( + (await fresh.listIndexes()).map((d) => d.name).sort(), + ['ia', 'ib'], + ); + for (let id = 0; id < shardCount; id++) { + const k = keyOnShard('cas', id, shardCount); + await fresh.set(k, { a: id, b: id + 100 }); + assert.deepEqual((await fresh.findEq('ia', id)).map((r) => r.key), [k]); + assert.deepEqual((await fresh.findEq('ib', id + 100)).map((r) => r.key), [k]); + } + await fresh.close(); + } finally { + await rmrf(dir); + } +}); + +test('a failed unique createIndex rolls back the shards it already created on', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + interface D { u: number } + const shardCount = 4; + const db = await ClusterDb.open({ dir, shardCount, valueCodec: 'json' }); + // One u=9 doc per shard plus a second u=9 doc on the LAST shard: the + // fan-out fails there after the earlier shards already persisted the index. + for (let id = 0; id < shardCount; id++) await db.set(keyOnShard('rb', id, shardCount), { u: 9 }); + await db.set(keyOnShard('rb-dup', shardCount - 1, shardCount), { u: 9 }); + await assert.rejects(() => db.createIndex('u-idx', { field: 'u', unique: true }), /unique index "u-idx" violation/); + await db.close(); + + // Every shard sidecar must be clean (not just the registry). + for (let id = 0; id < shardCount; id++) { + const shard = await MiniDb.open({ dir: path.join(dir, shardDirName(id, shardCount)), valueCodec: 'json', readOnly: true }); + assert.deepEqual(shard.listIndexes(), []); + await shard.close(); + } + const fresh = await ClusterDb.open({ dir, shardCount, valueCodec: 'json' }); + assert.deepEqual(await fresh.listIndexes(), []); + // Writes that the phantom unique index would have rejected now succeed. + const after = Array.from({ length: shardCount }, (_, id) => keyOnShard('rb-after', id, shardCount)); + for (const k of after) await fresh.set(k, { u: 9 }); + assert.deepEqual((await fresh.mget(after)).map((d) => d?.u), after.map(() => 9)); + await fresh.close(); + } finally { + await rmrf(dir); + } +}); diff --git a/packages/minidb/test/cluster/helpers.ts b/packages/minidb/test/cluster/helpers.ts new file mode 100644 index 0000000000..730b78c7e3 --- /dev/null +++ b/packages/minidb/test/cluster/helpers.ts @@ -0,0 +1,109 @@ +// test/cluster/helpers.js +// +// Shared helpers for the cluster tests: worker-process spawning and +// deterministic key generation that lands on a chosen shard. + +import { spawn } from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { shardFor } from '../../src/cluster/utils.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +export const WORKER = path.join(__dirname, 'mp-worker.ts'); + +/** rm -rf with retry: children may still be finishing their final syscalls + * (or a cleanup may race one) when the parent starts removing the tree. */ +export async function rmrf(dir: string): Promise { + for (let attempt = 0; ; attempt++) { + try { + await fs.rm(dir, { recursive: true, force: true }); + return; + } catch (e) { + const code = (e as NodeJS.ErrnoException).code; + if (attempt >= 5 || (code !== 'ENOTEMPTY' && code !== 'EBUSY' && code !== 'EACCES' && code !== 'EPERM')) throw e; + await sleep(50 * (attempt + 1)); + } + } +} + +export interface WorkerResult { + code: number | null; + stdout: string; + stderr: string; + /** Last JSON line printed by the worker, if any. */ + json: Record | null; +} + +/** Spawn a cluster worker process and wait for it to exit. */ +export function runWorker(args: string[], opts: { timeoutMs?: number } = {}): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ['--import', 'tsx', WORKER, ...args], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (d) => (stdout += d)); + child.stderr.on('data', (d) => (stderr += d)); + const killer = + opts.timeoutMs === undefined + ? null + : setTimeout(() => { + child.kill('SIGKILL'); + reject(new Error(`worker timed out after ${opts.timeoutMs}ms\nargs: ${args.join(' ')}\nstderr: ${stderr}`)); + }, opts.timeoutMs); + child.on('error', (e) => { + if (killer) clearTimeout(killer); + reject(e); + }); + // 'close' (not 'exit'): the process is fully reaped and its stdio is + // flushed, so no lingering child can still touch the cluster directory. + child.on('close', (code) => { + if (killer) clearTimeout(killer); + const lines = stdout.trim().split('\n').filter((l) => l.startsWith('{')); + let json: Record | null = null; + if (lines.length > 0) { + try { + json = JSON.parse(lines[lines.length - 1]!) as Record; + } catch { + /* leave json null */ + } + } + resolve({ code, stdout, stderr, json }); + }); + }); +} + +/** Assert-like helper: run a worker that must exit 0, return its JSON report. */ +export async function runWorkerOk(args: string[], opts: { timeoutMs?: number } = {}): Promise> { + const r = await runWorker(args, opts); + if (r.code !== 0 || !r.json) { + throw new Error(`worker failed (code=${r.code})\nargs: ${args.join(' ')}\nstdout: ${r.stdout}\nstderr: ${r.stderr}`); + } + return r.json; +} + +/** Find a key `${seed}:${n}` that routes to the given shard. Deterministic. */ +export function keyOnShard(seed: string, shardId: number, shardCount: number): string { + for (let n = 0; ; n++) { + const key = `${seed}:${n}`; + if (shardFor(key, shardCount) === shardId) return key; + } +} + +/** Pick count keys (seed:0..) grouped by the shard they route to. */ +export function keysByShard(seed: string, count: number, shardCount: number): Map { + const out = new Map(); + for (let n = 0; n < count; n++) { + const key = `${seed}:${n}`; + const id = shardFor(key, shardCount); + const arr = out.get(id); + if (arr) arr.push(key); + else out.set(id, [key]); + } + return out; +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/packages/minidb/test/cluster/lock.test.ts b/packages/minidb/test/cluster/lock.test.ts new file mode 100644 index 0000000000..9f24a797ec --- /dev/null +++ b/packages/minidb/test/cluster/lock.test.ts @@ -0,0 +1,140 @@ +// test/cluster/lock.test.js +// +// Lock semantics inside one process: same-shard writer contention with +// acquire timeout, per-shard independence, read-only coexistence, lock lease +// renewal, and writer handoff after close. + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { ClusterDb } from '../../src/cluster/index.js'; +import { shardDirName } from '../../src/cluster/utils.js'; +import { tmpDir, rmrf } from '../e2e/helpers/tmp.js'; +import { keyOnShard, sleep } from './helpers.js'; + +test('two writers contend on the same shard; loser times out with LockError', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db1 = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + const db2 = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json', lockAcquireTimeoutMs: 150 }); + + const kOnShard0a = keyOnShard('lock', 0, 4); + const kOnShard0b = keyOnShard('lockx', 0, 4); + await db1.set(kOnShard0a, { owner: 1 }); // db1 now holds shard 0 + + const t0 = performance.now(); + await assert.rejects( + () => db2.set(kOnShard0b, { owner: 2 }), + (e: unknown) => (e as { code?: string }).code === 'ELOCKED', + ); + const waited = performance.now() - t0; + // The pool retries with backoff and gives up on the first attempt whose + // next delay would cross the deadline, so effective waits undershoot the + // configured timeout slightly. + assert.ok(waited >= 50 && waited < 5_000, `waited roughly the acquire timeout (${Math.round(waited)}ms)`); + + // A key routed to a different shard is unaffected by the contention. + const kOther = keyOnShard('other', 1, 4); + await db2.set(kOther, { owner: 2 }); + assert.deepEqual(await db2.get(kOther), { owner: 2 }); + + await db2.close(); + await db1.close(); + } finally { + await rmrf(dir); + } +}); + +test('writer handoff: after close, another instance takes the shard over', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const key = keyOnShard('handoff', 2, 4); + const db1 = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + await db1.set(key, { n: 1 }); + await db1.close(); + + const db2 = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json', lockAcquireTimeoutMs: 500 }); + assert.deepEqual(await db2.get(key), { n: 1 }); + await db2.set(key, { n: 2 }); + assert.deepEqual(await db2.get(key), { n: 2 }); + await db2.close(); + } finally { + await rmrf(dir); + } +}); + +test('read-only instance coexists with a live writer and sees its commits', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const writer = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + await writer.set('ro:k1', { v: 1 }); + + const reader = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json', readOnly: true }); + assert.deepEqual(await reader.get('ro:k1'), { v: 1 }); + + // Fresh reads observe new commits without reopening the cluster. + await writer.set('ro:k2', { v: 2 }); + assert.deepEqual(await reader.get('ro:k2'), { v: 2 }); + + // Writes on the read-only instance are rejected. + await assert.rejects(() => reader.set('ro:k3', { v: 3 }), /read-only/); + await assert.rejects( + () => reader.mset([['ro:k3', { v: 3 }]]), + (e: unknown) => e instanceof AggregateError && String(e.errors[0]).includes('read-only'), + ); + + await reader.close(); + await writer.close(); + } finally { + await rmrf(dir); + } +}); + +test('lock lease: db.lock timestamp advances while a writer is held', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json', lockRenewMs: 80, lockHoldMs: 0 }); + const key = keyOnShard('lease', 1, 4); + await db.set(key, { v: 1 }); // grabs shard 1 and starts the lease timer + + const lockPath = path.join(dir, shardDirName(1, 4), 'db.lock'); + const read = async () => JSON.parse(await fs.readFile(lockPath, 'utf8')) as { pid: number; ts: number }; + const first = await read(); + assert.equal(first.pid, process.pid); + await sleep(300); + const second = await read(); + assert.ok(second.ts > first.ts, `timestamp renewed (${first.ts} -> ${second.ts})`); + await db.close(); + } finally { + await rmrf(dir); + } +}); + +test('close releases every shard lock it holds', async () => { + const dir = await tmpDir('minidb-cluster-'); + try { + const db1 = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json' }); + await db1.mset([ + [keyOnShard('c', 0, 4), { v: 0 }], + [keyOnShard('c', 1, 4), { v: 1 }], + [keyOnShard('c', 2, 4), { v: 2 }], + [keyOnShard('c', 3, 4), { v: 3 }], + ]); + await db1.close(); + + // Nothing left locked: a fresh instance with a tiny timeout can write all shards. + const db2 = await ClusterDb.open({ dir, shardCount: 4, valueCodec: 'json', lockAcquireTimeoutMs: 100 }); + await db2.mset([ + [keyOnShard('c', 0, 4), { v: 10 }], + [keyOnShard('c', 1, 4), { v: 11 }], + [keyOnShard('c', 2, 4), { v: 12 }], + [keyOnShard('c', 3, 4), { v: 13 }], + ]); + assert.deepEqual(await db2.get(keyOnShard('c', 0, 4)), { v: 10 }); + assert.deepEqual(await db2.get(keyOnShard('c', 3, 4)), { v: 13 }); + await db2.close(); + } finally { + await rmrf(dir); + } +}); diff --git a/packages/minidb/test/cluster/mp-worker.ts b/packages/minidb/test/cluster/mp-worker.ts new file mode 100644 index 0000000000..b28d0b6eb3 --- /dev/null +++ b/packages/minidb/test/cluster/mp-worker.ts @@ -0,0 +1,233 @@ +// test/cluster/mp-worker.ts +// +// Child-process entrypoint for the multi-process cluster tests. Not a test +// file itself; spawned via `node --import tsx mp-worker.ts ...`. +// Always prints a final JSON report line and exits non-zero on failure. + +import { ClusterDb } from '../../src/cluster/index.js'; +import { shardFor } from '../../src/cluster/utils.js'; +import { LockError } from '../../src/lockfile.js'; + +const [, , mode, ...rest] = process.argv; + +function out(report: Record): void { + process.stdout.write(JSON.stringify(report) + '\n'); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function isLockError(e: unknown): boolean { + return e instanceof LockError || (e as { code?: string }).code === 'ELOCKED'; +} + +/** Retry a write op on lock contention with jittered backoff; counts retries. */ +async function withRetry(fn: () => Promise, stats: { retries: number }, deadlineMs = 60_000): Promise { + const deadline = Date.now() + deadlineMs; + for (;;) { + try { + return await fn(); + } catch (e) { + if (!isLockError(e) || Date.now() > deadline) throw e; + stats.retries++; + await sleep(20 + Math.floor(Math.random() * 60)); + } + } +} + +const pad = (n: number) => 'x'.repeat(n); + +async function main(): Promise { + const stats = { retries: 0 }; + + if (mode === 'write') { + // write [valueBytes] [fsyncPolicy] + const [dir, shardCount, prefix, n, valueBytes = '32', fsyncPolicy = 'everysec'] = rest; + const db = await ClusterDb.open({ + dir: dir!, + shardCount: Number(shardCount), + valueCodec: 'json', + fsyncPolicy: fsyncPolicy as 'always' | 'everysec' | 'no', + }); + const t0 = performance.now(); + const count = Number(n); + for (let i = 0; i < count; i++) { + const value = { p: prefix, i, pad: pad(Number(valueBytes)) }; + await withRetry(() => db.set(`${prefix}:${i}`, value), stats); + } + const ms = performance.now() - t0; + out({ ok: 1, mode, n: count, ms, retries: stats.retries }); + await db.close(); + return; + } + + if (mode === 'writekeys') { + // writekeys + const [dir, shardCount, keysCsv] = rest; + const keys = keysCsv!.split(','); + const db = await ClusterDb.open({ dir: dir!, shardCount: Number(shardCount), valueCodec: 'json' }); + const t0 = performance.now(); + for (const key of keys) { + await withRetry(() => db.set(key, { key }), stats); + } + const ms = performance.now() - t0; + out({ ok: 1, mode, n: keys.length, ms, retries: stats.retries }); + await db.close(); + return; + } + + if (mode === 'verify') { + // verify + const [dir, shardCount, prefix, n] = rest; + const db = await ClusterDb.open({ dir: dir!, shardCount: Number(shardCount), valueCodec: 'json', readOnly: true }); + const count = Number(n); + let found = 0; + for (let i = 0; i < count; i++) { + const v = (await db.get(`${prefix}:${i}`)) as { p?: string; i?: number } | undefined; + if (v === undefined || v.p !== prefix || v.i !== i) { + out({ ok: 0, mode, error: `mismatch at ${prefix}:${i}`, got: v ?? null }); + process.exit(1); + } + found++; + } + out({ ok: 1, mode, found }); + await db.close(); + return; + } + + if (mode === 'wait-read') { + // wait-read + const [dir, shardCount, key, expectedI, timeoutMs] = rest; + const db = await ClusterDb.open({ dir: dir!, shardCount: Number(shardCount), valueCodec: 'json', readOnly: true }); + const t0 = performance.now(); + const deadline = t0 + Number(timeoutMs); + for (;;) { + const v = (await db.get(key!)) as { i?: number } | undefined; + if (v !== undefined && v.i === Number(expectedI)) { + out({ ok: 1, mode, waitedMs: performance.now() - t0 }); + await db.close(); + return; + } + if (performance.now() > deadline) { + out({ ok: 0, mode, error: `timeout waiting for ${key}`, got: v ?? null }); + process.exit(1); + } + await sleep(50); + } + } + + if (mode === 'crash') { + // crash — write k{i} sequentially with fsync 'always' + // until killed; report progress every 25 keys. + const [dir, shardCount] = rest; + const db = await ClusterDb.open({ + dir: dir!, + shardCount: Number(shardCount), + valueCodec: 'json', + fsyncPolicy: 'always', + }); + for (let i = 0; ; i++) { + await db.set(`k${i}`, { i }); + if (i % 25 === 0) out({ progress: i }); + } + } + + if (mode === 'hold') { + // hold — write one key, then keep the process + // (and its shard write lock) alive until killed. + const [dir, shardCount, key] = rest; + const db = await ClusterDb.open({ dir: dir!, shardCount: Number(shardCount), valueCodec: 'json', lockHoldMs: 0 }); + await db.set(key!, { heldBy: process.pid }); + out({ ok: 1, mode, holding: key, pid: process.pid }); + setInterval(() => {}, 60_000); // stay alive; killed by the parent + return; + } + + if (mode === 'storm') { + // storm [compactThresholdBytes] [doCompact] + // Deterministic mixed write storm onto ONE shard: sets, same-key updates, + // dels, same-shard atomic batches (TYPE_BATCH frames), short/long TTL sets + // and dt changes. Reports the exact appended frame count. + const [dir, shardCount, shard, seed, n, compactThresholdBytes = '0', doCompact = '0'] = rest; + const shards = Number(shardCount); + const target = Number(shard); + const db = await ClusterDb.open({ + dir: dir!, + shardCount: shards, + valueCodec: 'json', + fsyncPolicy: 'no', + lockHoldMs: 0, + compactThresholdBytes: Number(compactThresholdBytes) > 0 ? Number(compactThresholdBytes) : undefined, + }); + const doc = (i: number) => ({ n: i, c: `c${i % 7}`, u: `${seed}-u${i}`, t: `alpha beta w${i % 13}` }); + const keys = (function* (): Generator { + for (let seq = 0; ; seq++) { + const key = `${seed}:${seq}`; + if (shardFor(key, shards) === target) yield key; + } + })(); + // Loop keys aligned with i (batch keys are NOT tracked here): updates and + // dels always target a key whose unique value only ever lives on itself. + const loopKeys: string[] = []; + let frames = 0; + const count = Number(n); + for (let i = 0; i < count; i++) { + const key = keys.next().value!; + await withRetry(() => db.set(key, doc(i), { dt: { created: 1700000000000 + i } }), stats); + frames++; + loopKeys.push(key); + if (i % 5 === 1) { + const j = i - 1; + await withRetry( + () => db.set(loopKeys[j]!, { ...doc(j), n: j * 10, t: `alpha changed w${j % 13}` }, { dt: { created: 1700000009000 + j } }), + stats, + ); + frames++; + } + if (i % 7 === 3) { + await withRetry(() => db.del(loopKeys[i - 3]!), stats); + frames++; + } + if (i % 50 === 10) { + // Atomic single-shard batch: three fresh keys plus a del of the first. + const b1 = keys.next().value!; + const b2 = keys.next().value!; + const b3 = keys.next().value!; + await withRetry( + () => + db.batch([ + { op: 'set', key: b1, value: { ...doc(i + 100000), u: `${seed}-ub${i}a` }, dt: { created: 1700001000000 + i } }, + { op: 'set', key: b2, value: { ...doc(i + 100001), u: `${seed}-ub${i}b`, t: 'alpha batch' } }, + { op: 'set', key: b3, value: { ...doc(i + 100002), u: `${seed}-ub${i}c` } }, + { op: 'del', key: b1 }, + ]), + stats, + ); + frames++; + } + if (i % 11 === 5) { + // TTL keys come from the same shard-targeted generator, so every op + // of the storm appends to this one shard's WAL. + await withRetry(() => db.set(keys.next().value!, { ...doc(i + 200000), u: `${seed}-uts${i}` }, { ttl: 50 }), stats); + frames++; + } + if (i % 11 === 6) { + await withRetry(() => db.set(keys.next().value!, { ...doc(i + 300000), u: `${seed}-utl${i}` }, { ttl: 3_600_000 }), stats); + frames++; + } + } + if (doCompact === '1') await db.compact(); + out({ ok: 1, mode, frames, retries: stats.retries }); + await db.close(); + return; + } + + out({ ok: 0, error: `unknown mode: ${mode}` }); + process.exit(1); +} + +main().catch((e) => { + out({ ok: 0, mode, error: String(e && (e as Error).stack ? (e as Error).stack : e) }); + process.exit(1); +}); diff --git a/packages/minidb/test/cluster/recovery.test.ts b/packages/minidb/test/cluster/recovery.test.ts new file mode 100644 index 0000000000..dd21829523 --- /dev/null +++ b/packages/minidb/test/cluster/recovery.test.ts @@ -0,0 +1,116 @@ +// test/cluster/recovery.test.js +// +// Crash semantics for the cluster layer: SIGKILL mid-write must not corrupt +// any shard, recovered data is a contiguous prefix of the write order, and a +// db.lock left behind by a dead process is taken over by the next writer. + +import { test } from 'vitest'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { ClusterDb } from '../../src/cluster/index.js'; +import { tmpDir } from '../e2e/helpers/tmp.js'; +import { WORKER, keyOnShard, rmrf } from './helpers.js'; + +/** Spawn the crash writer and SIGKILL it shortly after it starts writing. */ +function crashWriter(dir: string, shardCount: number): Promise { + return new Promise((resolve) => { + const child = spawn(process.execPath, ['--import', 'tsx', WORKER, 'crash', dir, String(shardCount)], { + stdio: ['ignore', 'pipe', 'ignore'], + }); + let killTimer: ReturnType | null = null; + let heartbeats = 0; + child.stdout.on('data', () => { + // Kill after several progress heartbeats (>=51 acknowledged keys), not on + // a wall-clock window: fsync 'always' throughput collapses under + // parallel-suite load, so a fixed ms window was inherently flaky. + heartbeats++; + if (!killTimer && heartbeats >= 3) killTimer = setTimeout(() => child.kill('SIGKILL'), 20); + }); + const safety = setTimeout(() => child.kill('SIGKILL'), 15_000); + child.on('exit', () => { + if (killTimer) clearTimeout(killTimer); + clearTimeout(safety); + resolve(); + }); + }); +} + +test( + 'kill -9 a writer mid-flight: every shard reopens clean and data is a contiguous prefix', + { timeout: 120_000 }, + async () => { + const runs = 3; + for (let r = 0; r < runs; r++) { + const dir = await tmpDir('minidb-cluster-crash-'); + try { + await crashWriter(dir, 4); + // The dead process left db.lock files behind; with its PID gone, the + // next writer takes over immediately. + const db = await ClusterDb.open<{ i: number }>({ + dir, + shardCount: 4, + valueCodec: 'json', + lockAcquireTimeoutMs: 3_000, + }); + let last = -1; + for (let i = 0; i < 100_000; i++) { + const v = await db.get(`k${i}`); + if (v === undefined) { + last = i - 1; + break; + } + assert.equal(v.i, i, `run${r}: value mismatch at k${i}`); + } + assert.ok(last >= 2, `run${r}: expected several durable keys, got k0..k${last}`); + // Writable after takeover. + await db.set(`post`, { i: -1 }); + assert.deepEqual(await db.get('post'), { i: -1 }); + await db.close(); + } finally { + await rmrf(dir); + } + } + }, +); + +test( + 'a db.lock left by a SIGKILLed holder is taken over by the next process', + { timeout: 60_000 }, + async () => { + const dir = await tmpDir('minidb-cluster-takeover-'); + try { + const shardCount = 4; + const key = keyOnShard('victim', 1, shardCount); + const holder = spawn(process.execPath, ['--import', 'tsx', WORKER, 'hold', dir, String(shardCount), key], { + stdio: ['ignore', 'pipe', 'inherit'], + }); + // Wait until the holder actually wrote the key (its lock is now held). + await new Promise((resolve, reject) => { + holder.stdout.on('data', (d) => { + if (String(d).includes('"holding"')) resolve(); + }); + holder.on('error', reject); + setTimeout(() => reject(new Error('holder never started')), 30_000); + }); + holder.kill('SIGKILL'); + await new Promise((r) => holder.on('exit', r)); + + const t0 = performance.now(); + const db = await ClusterDb.open<{ heldBy: number }>({ + dir, + shardCount, + valueCodec: 'json', + lockAcquireTimeoutMs: 5_000, + }); + // Same-shard write requires taking over the dead process's lock. + const sameShardKey = keyOnShard('rescuer', 1, shardCount); + await db.set(sameShardKey, { ok: 1 }); + const elapsed = performance.now() - t0; + assert.ok(elapsed < 5_000, `takeover was prompt (${Math.round(elapsed)}ms)`); + assert.equal((await db.get(sameShardKey)) !== undefined, true); + await db.close(); + } finally { + await rmrf(dir); + } + }, +); diff --git a/packages/minidb/test/compaction-fault.test.ts b/packages/minidb/test/compaction-fault.test.ts index 1ddc0315db..bea39dfa50 100644 --- a/packages/minidb/test/compaction-fault.test.ts +++ b/packages/minidb/test/compaction-fault.test.ts @@ -4,10 +4,19 @@ // node:fs/promises. These exercise branches that real filesystems essentially // never produce (a write that returns 0 bytes, a close/sync that throws). // +// The later sections cover compaction ROTATION failures end-to-end (through a +// real MiniDb on a real temp dir): a throw at wal.close(), at the WAL rename, +// or at the new WAL's open() must not wedge the database — the seal is +// one-way, so recovery swaps in a fresh WAL on db.walPath. +// // NOTE: each test resets the module registry and re-mocks node:fs/promises so a // fresh import of compaction.ts picks up that test's mocked fs. import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import type { PathLike } from 'node:fs'; import { afterEach, expect, test, vi } from 'vitest'; interface MockHandle { @@ -100,3 +109,263 @@ test('fsyncDir swallows a sync() failure and still closes the handle', async () await fsyncDir('/tmp/whatever'); assert.equal(closed, true, 'close() is called even after sync() throws'); }); + + +async function tmpDir(): Promise { + return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-rotation-fault-')); +} + +// Passthrough node:fs/promises mock for the rotation-level tests: everything +// delegates to the real module, except operations whose fault callback returns +// an error — that error is thrown instead of delegating. Lets a test fail ONE +// rotation step (a specific rename, one append-handle open) deterministically +// while the rest of MiniDb keeps using the real filesystem. +function mockFsWithFaults(faults: { + rename?: (src: string, dst: string) => Error | null; + open?: (path: string, flags: string | number | undefined) => Error | null; +}): void { + const rename = async (src: PathLike, dst: PathLike): Promise => { + const err = faults.rename?.(String(src), String(dst)); + if (err) throw err; + await fs.rename(src, dst); + }; + const open = (async (p: PathLike, flags?: string | number, mode?: string | number) => { + const err = faults.open?.(String(p), flags); + if (err) throw err; + return fs.open(p, flags as string | number | undefined, mode as never); + }) as typeof fs.open; + const mocked = { ...fs, rename, open }; + vi.doMock('node:fs/promises', () => ({ ...mocked, default: mocked })); +} + +test('wal.close() propagates a final-sync failure but still releases the file handle', async () => { + const { WAL } = await import('../src/wal.js'); + const { encodeFrame, FrameParser, TYPE_SET } = await import('../src/codec.js'); + const dir = await tmpDir(); + try { + const file = path.join(dir, 'db.wal'); + const wal = new WAL(file, { fsyncPolicy: 'no' }); // 'no': no background sync timer + await wal.open(); + await wal.append(encodeFrame({ type: TYPE_SET, key: Buffer.from('a'), value: Buffer.from('1') })); + + const origSync = wal.sync.bind(wal); + wal.sync = async () => { + wal.sync = origSync; // one-shot + throw new Error('injected fsync failure'); + }; + await assert.rejects(wal.close(), /injected fsync failure/); + // The failed close still released the handle (compaction's recovery swap + // abandons this WAL object and must not leak its fd)... + assert.equal((wal as unknown as { fh: unknown }).fh, null); + // ...the WAL stays closed... + await wal.close(); + // ...and the flushed frame stayed durable, so a fresh WAL — which is what + // rotation recovery swaps in — continues the same file at the real EOF. + const fresh = new WAL(file, { fsyncPolicy: 'no' }); + await fresh.open(); + assert.ok(fresh.size > 0); + await fresh.append(encodeFrame({ type: TYPE_SET, key: Buffer.from('b'), value: Buffer.from('2') })); + await fresh.close(); + + const frames = [...new FrameParser().feed(await fs.readFile(file))]; + assert.deepEqual( + frames.map((f) => f.key.toString()), + ['a', 'b'], + ); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('rotation: a WAL close() failure leaves the db writable and compact() retries cleanly', async () => { + const { MiniDb } = await import('../src/index.js'); + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no', compactThresholdBytes: 1 << 30 }); + const N = 200; + for (let i = 0; i < N; i++) await db.set(`k${i}`, `v${i}`); + + const wal = db.wal; + let parked: Promise | undefined; + const origClose = wal.close.bind(wal); + wal.close = async () => { + wal.close = origClose; // one-shot + // Inside the rotation the old WAL is already sealed and _rotateLock is + // held: a write issued now parks on the lock and must still succeed once + // the failed rotation has recovered. + parked = db.set('parked', 'during-failed-rotation'); + throw new Error('injected close failure'); + }; + await assert.rejects(db.compact(), /injected close failure/); + assert.equal(db.stats.compactions, 0); + assert.equal(db.stats.compactErrors, 1); + assert.match(String(db.lastCompactError), /injected close failure/); + + // The parked write and every later write hit the recovered (swapped-in) + // WAL, not the sealed/closed old one. + await parked; + await db.set('post', 'still-writable'); + assert.equal(db.get('parked'), 'during-failed-rotation'); + assert.equal(db.get('post'), 'still-writable'); + + // An explicit compact() retries cleanly after the failed rotation. + await db.compact(); + assert.equal(db.stats.compactions, 1); + assert.equal(db.stats.compactErrors, 1); + assert.equal(db.lastCompactError, null); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.size, N + 2); + assert.equal(db.get('k0'), 'v0'); + assert.equal(db.get(`k${N - 1}`), `v${N - 1}`); + assert.equal(db.get('parked'), 'during-failed-rotation'); + assert.equal(db.get('post'), 'still-writable'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('rotation: a WAL rename failure (new snapshot already in place) leaves the db writable', async () => { + let armed = true; + mockFsWithFaults({ + rename: (src, dst) => { + // Fail only the first db.wal.tmp → db.wal rename: by then the new + // snapshot has already been renamed into place, so this exercises the + // trickiest partial-rotation state (new snapshot + old full WAL). + if (armed && src.endsWith('db.wal.tmp') && dst.endsWith('db.wal')) { + armed = false; + return Object.assign(new Error('injected rename failure'), { code: 'EIO' }); + } + return null; + }, + }); + const { MiniDb } = await import('../src/index.js'); + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no', compactThresholdBytes: 1 << 30 }); + const N = 200; + for (let i = 0; i < N; i++) await db.set(`k${i}`, `v${i}`); + + await assert.rejects(db.compact(), /injected rename failure/); + assert.equal(db.stats.compactions, 0); + assert.equal(db.stats.compactErrors, 1); + + await db.set('post', 'still-writable'); + assert.equal(db.get('k0'), 'v0'); + assert.equal(db.get('post'), 'still-writable'); + + // An explicit compact() retries cleanly on top of the partial rotation. + await db.compact(); + assert.equal(db.stats.compactions, 1); + assert.equal(db.stats.compactErrors, 1); + assert.equal(db.lastCompactError, null); + await db.close(); + + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.size, N + 1); + assert.equal(db.get('k0'), 'v0'); + assert.equal(db.get(`k${N - 1}`), `v${N - 1}`); + assert.equal(db.get('post'), 'still-writable'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('rotation: a new-WAL open() failure after the renames leaves the db writable', async () => { + let canArm = true; + let armOpenFault = false; + mockFsWithFaults({ + rename: (src, dst) => { + // Arm the open fault once the FIRST WAL rename has landed, so it hits + // the first append-handle open afterwards — the fresh WAL's open() + // inside the rotation. Later compactions must re-arm nothing. + if (canArm && src.endsWith('db.wal.tmp') && dst.endsWith('db.wal')) { + canArm = false; + armOpenFault = true; + } + return null; + }, + open: (p, flags) => { + if (armOpenFault && flags === 'a' && p.endsWith('db.wal')) { + armOpenFault = false; + return Object.assign(new Error('injected open failure'), { code: 'EMFILE' }); + } + return null; + }, + }); + const { MiniDb } = await import('../src/index.js'); + const dir = await tmpDir(); + try { + const opts = { dir, valueCodec: 'string' as const, valueMode: 'disk' as const, fsyncPolicy: 'no' as const, compactThresholdBytes: 1 << 30 }; + let db = await MiniDb.open(opts); + const N = 200; + for (let i = 0; i < N; i++) await db.set(`k${i}`, `v${i}`); + + await assert.rejects(db.compact(), /injected open failure/); + assert.equal(db.stats.compactions, 0); + assert.equal(db.stats.compactErrors, 1); + + // The renames had already committed the new snapshot/WAL layout when the + // open failed, so recovery also applied the store-pointer remap. + const sawSnapshotRef = [...db.store.map.values()].some((r) => r.ref.kind === 'disk' && r.ref.loc.file === 'snapshot'); + assert.ok(sawSnapshotRef, 'store pointers were remapped to the new snapshot after the failed rotation'); + + // Disk-backed reads stay correct and the db is writable again. + assert.equal(db.get('k0'), 'v0'); + assert.equal(db.get(`k${N - 1}`), `v${N - 1}`); + await db.set('post', 'still-writable'); + assert.equal(db.get('post'), 'still-writable'); + + await db.compact(); + assert.equal(db.stats.compactions, 1); + assert.equal(db.get('k42'), 'v42'); + await db.close(); + + db = await MiniDb.open(opts); + assert.equal(db.size, N + 1); + assert.equal(db.get('k0'), 'v0'); + assert.equal(db.get(`k${N - 1}`), `v${N - 1}`); + assert.equal(db.get('post'), 'still-writable'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('a compaction whose onCompacted hook throws counts as a compactError, not a compaction', async () => { + const { MiniDb } = await import('../src/index.js'); + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no', compactThresholdBytes: 1 << 30 }); + for (let i = 0; i < 50; i++) await db.set(`k${i}`, `v${i}`); + + const hook = db.onCompacted; + let failHook = true; + db.onCompacted = () => { + if (failHook) throw new Error('injected hook failure'); + hook(); + }; + await assert.rejects(db.compact(), /injected hook failure/); + // The hook is part of the compaction: compactions counts only fully + // successful runs, even though the rotation itself had already succeeded. + assert.equal(db.stats.compactions, 0); + assert.equal(db.stats.compactErrors, 1); + assert.match(String(db.lastCompactError), /injected hook failure/); + + // The rotation succeeded, so the db keeps working normally. + await db.set('post', 'ok'); + assert.equal(db.size, 51); + + failHook = false; + await db.compact(); + assert.equal(db.stats.compactions, 1); + assert.equal(db.stats.compactErrors, 1); + assert.equal(db.lastCompactError, null); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/compaction.test.ts b/packages/minidb/test/compaction.test.ts index 01d46d0d87..c2e64480b1 100644 --- a/packages/minidb/test/compaction.test.ts +++ b/packages/minidb/test/compaction.test.ts @@ -100,7 +100,7 @@ test('del-then-compact drops tombstoned keys from the snapshot', async () => { } }); -test('concurrent SET/UPDATE/DEL during compaction survive recovery', async () => { +test('concurrent SET/UPDATE/DEL during compaction survive recovery', { timeout: 30_000 }, async () => { // Exercises the fuzzy-snapshot + WAL-tail-replay convergence: writes that // land while the snapshot is being written must all be reflected after a // reopen, with last-writer-wins semantics. @@ -110,7 +110,15 @@ test('concurrent SET/UPDATE/DEL during compaction survive recovery', async () => // 5000 keys span 2 writeSnapshot yield windows (yieldEvery=2000, src/snapshot.ts), // so the ops below genuinely race an in-progress snapshot. const N = 5000; - for (let i = 0; i < N; i++) await db.set('k' + i, 'v' + i); + for (let base = 0; base < N; base += 500) { + await db.batch( + Array.from({ length: Math.min(500, N - base) }, (_, j) => ({ + op: 'set' as const, + key: 'k' + (base + j), + value: 'v' + (base + j), + })), + ); + } // Even keys are updated, keys == 1 (mod 4) are deleted, keys == 3 (mod 4) // are left untouched, and a batch of new keys is added — all racing the @@ -143,7 +151,7 @@ test('concurrent SET/UPDATE/DEL during compaction survive recovery', async () => } finally { await fs.rm(dir, { recursive: true, force: true }); } -}, 15_000); +}); test('compaction with no concurrent writes produces an empty WAL tail', async () => { // When nothing is written during compaction, the post-fence WAL tail is empty diff --git a/packages/minidb/test/db.test.ts b/packages/minidb/test/db.test.ts index 14f4559199..8d88be7146 100644 --- a/packages/minidb/test/db.test.ts +++ b/packages/minidb/test/db.test.ts @@ -322,3 +322,86 @@ test('valueMode auto without maxMemoryBytes defaults to memory', async () => { await fs.rm(dir, { recursive: true, force: true }); } }); + +test('openOrRebuild preserves data when only a sidecar definition file is corrupt', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open>({ dir, valueCodec: 'json', fsyncPolicy: 'no' }); + for (let i = 0; i < 100; i++) await db.set(`k${i}`, { n: i }); + await db.createIndex('byN', { field: 'n' }); + await db.close(); + await fs.writeFile(path.join(dir, 'db.indexes.json'), 'corrupt{{{'); + // plain open still throws on the corrupt sidecar... + await assert.rejects(MiniDb.open({ dir, valueCodec: 'json' }), SyntaxError); + // ...but openOrRebuild drops the derived sidecars instead of wiping the data + db = await MiniDb.openOrRebuild>({ dir, valueCodec: 'json', fsyncPolicy: 'no' }); + assert.equal(db.size, 100); + assert.deepEqual(db.get('k42'), { n: 42 }); + assert.deepEqual(db.listIndexes(), []); // definitions are lost, not the data; recreate as needed + await db.createIndex('byN2', { field: 'n' }); + assert.equal(db.findEq('byN2', 42).length, 1); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('open removes stale compaction temp files', async () => { + const dir = await tmpDir(); + try { + let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); + await db.set('a', '1'); + await db.close(); + await fs.writeFile(path.join(dir, 'db.snapshot.tmp'), 'stale'); + await fs.writeFile(path.join(dir, 'db.wal.tmp'), 'stale'); + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.get('a'), '1'); + await assert.rejects(fs.stat(path.join(dir, 'db.snapshot.tmp')), /ENOENT/); + await assert.rejects(fs.stat(path.join(dir, 'db.wal.tmp')), /ENOENT/); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('query with skip/limit returns the same rows as slicing the unbounded result', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open>({ dir, valueCodec: 'json', fsyncPolicy: 'no' }); + for (let i = 0; i < 200; i++) await db.set(`k${String(i).padStart(3, '0')}`, { n: i, grp: i % 4 }); + await db.createIndex('byGrp', { field: 'grp' }); + const full = db.query({ key: { prefix: 'k1' } }); + assert.deepEqual(db.query({ key: { prefix: 'k1' }, limit: 10 }), full.slice(0, 10)); + assert.deepEqual(db.query({ key: { prefix: 'k1' }, skip: 5, limit: 3 }), full.slice(5, 8)); + const eq = db.query({ filter: { grp: 2 } }); // indexed equality path + assert.deepEqual(db.query({ filter: { grp: 2 }, limit: 7 }), eq.slice(0, 7)); + const scan = db.query({ filter: { n: { $gte: 100 } } }); // unindexed full scan path + assert.deepEqual(db.query({ filter: { n: { $gte: 100 } }, skip: 10, limit: 5 }), scan.slice(10, 15)); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('maxMemory evict-lru evicts in least-recently-used order across many victims', async () => { + const dir = await tmpDir(); + try { + const db = await MiniDb.open({ dir, valueCodec: 'string', maxMemoryBytes: 50, maxMemoryPolicy: 'evict-lru' }); + await db.set('a', '1234567890'); // ~11B per record, budget fits 4 + await db.set('b', '1234567890'); + await db.set('c', '1234567890'); + await db.set('d', '1234567890'); + assert.equal(db.get('a'), '1234567890'); // touch a: b becomes LRU + await db.set('e', '1234567890'); // exceeds budget -> evicts b + await db.set('f', '1234567890'); // exceeds budget -> evicts c + assert.equal(db.get('a'), '1234567890'); + assert.equal(db.get('b'), undefined); + assert.equal(db.get('c'), undefined); + assert.equal(db.get('d'), '1234567890'); + assert.equal(db.get('e'), '1234567890'); + assert.equal(db.get('f'), '1234567890'); + await db.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/degrade.test.ts b/packages/minidb/test/degrade.test.ts index 7420b14fc5..a9e056fc72 100644 --- a/packages/minidb/test/degrade.test.ts +++ b/packages/minidb/test/degrade.test.ts @@ -20,7 +20,7 @@ test('openOrRebuild opens a healthy db normally', async () => { await fs.rm(dir, { recursive: true, force: true }); }); -test('openOrRebuild discards a corrupt db and starts fresh', async () => { +test('openOrRebuild preserves data when only a sidecar definition file is corrupt', async () => { const dir = await tmpDir(); let db = await MiniDb.open({ dir, valueCodec: 'json' }); await db.createIndex('byX', { field: 'x' }); @@ -36,8 +36,13 @@ test('openOrRebuild discards a corrupt db and starts fresh', async () => { { onRebuild: (e) => (rebuilt = e) }, ); assert.ok(rebuilt instanceof Error, 'onRebuild called with the original error'); - assert.equal(db.size, 0, 'fresh empty db after rebuild'); - // data is gone (rebuild semantics) and the db is usable again + // the sidecar (pure derived state) is dropped, the data is NOT wiped + assert.equal(db.size, 1); + assert.deepEqual(db.get('important'), { x: 1 }); + assert.deepEqual(db.listIndexes(), []); + // indexes can be recreated and the db is usable again + await db.createIndex('byX', { field: 'x' }); + assert.equal(db.findEq('byX', 1).length, 1); await db.set('fresh', { v: 1 }); assert.deepEqual(db.get('fresh'), { v: 1 }); await db.close(); diff --git a/packages/minidb/test/e2e/boundary.test.ts b/packages/minidb/test/e2e/boundary.test.ts index 711d58d222..1e469dbaa1 100644 --- a/packages/minidb/test/e2e/boundary.test.ts +++ b/packages/minidb/test/e2e/boundary.test.ts @@ -21,7 +21,7 @@ test('boundary: key length limits', async () => { } }); -test('boundary: large value round-trips and recovers', async () => { +test('boundary: large value round-trips and recovers', { timeout: 30_000 }, async () => { const dir = await tmpDir(); const big = Buffer.alloc(5 * 1024 * 1024, 0xab); // 5 MiB big[0] = 0x01; @@ -45,7 +45,7 @@ test('boundary: large value round-trips and recovers', async () => { } }); -test('boundary: many keys survive compaction + recovery', async () => { +test('boundary: many keys survive compaction + recovery', { timeout: 30_000 }, async () => { const dir = await tmpDir(); const N = 20000; let db = await MiniDb.open({ @@ -83,7 +83,7 @@ test('boundary: empty db open/close/reopen', async () => { await rmrf(dir); }); -test('boundary: overwrite same key many times keeps size 1', async () => { +test('boundary: overwrite same key many times keeps size 1', { timeout: 30_000 }, async () => { const dir = await tmpDir(); const db = await MiniDb.open({ dir, valueCodec: 'string' }); try { diff --git a/packages/minidb/test/e2e/compaction-race.test.ts b/packages/minidb/test/e2e/compaction-race.test.ts index f6ee40f050..2bcc68aeeb 100644 --- a/packages/minidb/test/e2e/compaction-race.test.ts +++ b/packages/minidb/test/e2e/compaction-race.test.ts @@ -9,7 +9,7 @@ import assert from 'node:assert/strict'; import { MiniDb } from '../../src/index.js'; import { tmpDir, rmrf } from './helpers/tmp.js'; -test('compaction-race: concurrent writes + frequent compaction lose nothing', async () => { +test('compaction-race: concurrent writes + frequent compaction lose nothing', { timeout: 30_000 }, async () => { const dir = await tmpDir(); let db = await MiniDb.open({ dir, @@ -47,7 +47,7 @@ test('compaction-race: concurrent writes + frequent compaction lose nothing', as } }); -test('compaction-race: reads remain available during compaction', async () => { +test('compaction-race: reads remain available during compaction', { timeout: 30_000 }, async () => { const dir = await tmpDir(); const db = await MiniDb.open({ dir, @@ -70,7 +70,7 @@ test('compaction-race: reads remain available during compaction', async () => { } }); -test('compaction-race: snapshot phase does not block writes', async () => { +test('compaction-race: snapshot phase does not block writes', { timeout: 30_000 }, async () => { // A write issued while a large snapshot is being written must complete // BEFORE the whole compaction finishes — i.e. the snapshot phase is // non-blocking. writeSnapshot yields to the event loop every chunk, so the @@ -92,7 +92,20 @@ test('compaction-race: snapshot phase does not block writes', async () => { // entries = 5 explicit yields — ~5x headroom over the minimum for // "multiple yields", enough for slow CI machines without writing ~75 MB. const N = 10_000; - for (let i = 0; i < N; i++) await db.set('k' + i, { i, pad: 'x'.repeat(500) }); + { + // Prefill via batches: one WAL frame per 500 sets instead of 10k + // sequential appends — the setup phase must stay cheap on slow CI + // runners, leaving the timeout budget for the actual race below. + for (let base = 0; base < N; base += 500) { + await db.batch( + Array.from({ length: Math.min(500, N - base) }, (_, j) => ({ + op: 'set' as const, + key: 'k' + (base + j), + value: { i: base + j, pad: 'x'.repeat(500) }, + })), + ); + } + } const cp = db.compact(); const first = db.set('w0', { i: 0 }); @@ -117,9 +130,9 @@ test('compaction-race: snapshot phase does not block writes', async () => { await db.close().catch(() => {}); await rmrf(dir); } -}, 15_000); +}); -test('compaction-race: heavy writes during compaction grow a WAL tail that survives recovery', async () => { +test('compaction-race: heavy writes during compaction grow a WAL tail that survives recovery', { timeout: 30_000 }, async () => { // Sustained writes during compaction force the pre-copy loop to drain a real // WAL tail; the tail must be replayed on top of the snapshot after a reopen. const dir = await tmpDir(); @@ -133,7 +146,15 @@ test('compaction-race: heavy writes during compaction grow a WAL tail that survi // 10k keys span 5 writeSnapshot yield windows (yieldEvery=2000, src/snapshot.ts), // so compaction is still in progress while the writes below land. const N = 10_000; - for (let i = 0; i < N; i++) await db.set('k' + i, { i }); + for (let base = 0; base < N; base += 500) { + await db.batch( + Array.from({ length: Math.min(500, N - base) }, (_, j) => ({ + op: 'set' as const, + key: 'k' + (base + j), + value: { i: base + j }, + })), + ); + } const cp = db.compact(); // ~55 B/frame × 2000 ≈ 110 KB post-fence tail > SMALL_DELTA (64 KiB, @@ -155,7 +176,7 @@ test('compaction-race: heavy writes during compaction grow a WAL tail that survi } }); -test('compaction-race: valueMode disk preserves concurrent writes and remaps pointers', async () => { +test('compaction-race: valueMode disk preserves concurrent writes and remaps pointers', { timeout: 30_000 }, async () => { const dir = await tmpDir(); let db = await MiniDb.open({ dir, @@ -192,3 +213,58 @@ test('compaction-race: valueMode disk preserves concurrent writes and remaps poi await rmrf(dir); } }); + +// Regression: under sustained writes whose append rate approaches the pre-copy +// rate, auto-compactions previously never converged (stats.compactions stayed 0 +// until the storm stopped; the WAL grew unboundedly). Compaction must now give +// up pre-copying and finish via the rotation critical section. +test( + 'compaction-race: auto compaction completes during a sustained write storm', + { timeout: 60_000 }, + async () => { + const dir = await tmpDir(); + let db = await MiniDb.open({ + dir, + valueCodec: 'string', + fsyncPolicy: 'no', + compactThresholdBytes: 8 * 1024 * 1024, + }); + let stop = false; + let written = 0; + let writeError: unknown = null; + try { + const val = 'v'.repeat(1024); + const start = Date.now(); + const writers = Array.from({ length: 16 }, async () => { + while (!stop && Date.now() - start < 15_000) { + try { + await db.set(`k${written++}`, val); + } catch (e) { + writeError ??= e; + stop = true; + } + } + }); + while (!stop && db.stats.compactions < 2 && Date.now() - start < 15_000) { + await new Promise((r) => setTimeout(r, 50)); + } + stop = true; + await Promise.all(writers); + assert.equal(writeError, null, `write failed during compaction storm: ${String(writeError)}`); + assert.ok( + db.stats.compactions >= 2, + `expected auto compactions to complete during the storm, got ${db.stats.compactions} (written=${written})`, + ); + await db.close(); + + // every acknowledged write must survive compaction rotations + recovery + db = await MiniDb.open({ dir, valueCodec: 'string' }); + assert.equal(db.size, written, `size=${db.size} vs written=${written}`); + assert.equal(db.get('k0'), val); + assert.equal(db.get(`k${written - 1}`), val); + } finally { + await db.close().catch(() => {}); + await rmrf(dir); + } + }, +); diff --git a/packages/minidb/test/e2e/crash-recovery.test.ts b/packages/minidb/test/e2e/crash-recovery.test.ts index 8ff41a66a9..b59195dd6e 100644 --- a/packages/minidb/test/e2e/crash-recovery.test.ts +++ b/packages/minidb/test/e2e/crash-recovery.test.ts @@ -53,7 +53,7 @@ async function verifyContiguous(dir, label) { return last; // highest contiguous index } -test('crash-recovery: kill mid-write, recovery yields a contiguous correct prefix', async () => { +test('crash-recovery: kill mid-write, recovery yields a contiguous correct prefix', { timeout: 60_000 }, async () => { // Each run spawns a child process (the dominant cost); 5 runs with random kill // times still sample the crash window well. const runs = 5; @@ -70,7 +70,7 @@ test('crash-recovery: kill mid-write, recovery yields a contiguous correct prefi } }); -test('crash-recovery: kill during compaction, still consistent', async () => { +test('crash-recovery: kill during compaction, still consistent', { timeout: 60_000 }, async () => { const runs = 3; for (let r = 0; r < runs; r++) { const dir = await tmpDir(); diff --git a/packages/minidb/test/e2e/fuzz-model.test.ts b/packages/minidb/test/e2e/fuzz-model.test.ts index 81c116205e..aa2d91b548 100644 --- a/packages/minidb/test/e2e/fuzz-model.test.ts +++ b/packages/minidb/test/e2e/fuzz-model.test.ts @@ -58,11 +58,11 @@ async function runSeed(seed, steps) { } } -test('fuzz-model: random op sequences match a reference model (many seeds)', async () => { +test('fuzz-model: random op sequences match a reference model (many seeds)', { timeout: 60_000 }, async () => { // Mix of small and large seeds. 6 seeds × 250 steps ≈ 1.5k ops total, still // hitting every op branch (including reopen + full compare) many times per seed. const seeds = [1, 2, 3, 99999, 0xdeadbeef, 20240625]; for (const seed of seeds) { await expect(runSeed(seed, 250)).resolves.toBeUndefined(); } -}, 60_000); +}); diff --git a/packages/minidb/test/e2e/helpers/lock-racer.ts b/packages/minidb/test/e2e/helpers/lock-racer.ts new file mode 100644 index 0000000000..a2e77989f5 --- /dev/null +++ b/packages/minidb/test/e2e/helpers/lock-racer.ts @@ -0,0 +1,46 @@ +// test/e2e/helpers/lock-racer.ts +// +// Child-side counterpart of the lock-takeover stress test. Sits in a loop; +// for each round it waits for `go-` to appear in the gate directory, +// then races to acquire the lock and prints "R <0|1>". + +import fs from 'node:fs'; +import { LockFile } from '../../../src/lockfile.js'; + +const lockPath = process.argv[2]!; +const gateDir = process.argv[3]!; +const rounds = Number(process.argv[4] ?? 200); + +const sleep = (ms: number) => new Promise((res) => setTimeout(res, ms)); + +// Signal readiness; the parent plants go-0 only once every racer is parked at +// the gate. Without this, per-round "holders > 1" conflates true simultaneous +// holders with plain boot-staggered sequential acquisitions. +console.log('READY'); + +for (let r = 0; r < rounds; r++) { + const gate = `${gateDir}/go-${r}`; + for (;;) { + try { + fs.statSync(gate); + break; + } catch { + // Yield instead of spinning synchronously: a statSync busy-loop blocks + // this process's event loop and delays its stdout flushes, which + // misattributes outputs across rounds on the parent side. + await sleep(1); + } + } + const lf = new LockFile(lockPath); + const got = await lf.acquire(); + process.stdout.write(`R${r} ${process.pid} ${got ? 1 : 0}\n`); + if (got) { + // Hold the lock briefly so every loser resolves its acquire() while the + // winner still holds (an immediate release makes sequential re-acquisters + // indistinguishable from simultaneous holders). Sized generously for + // heavily loaded machines, where a racer can be descheduled for tens of + // milliseconds inside its own call. + await sleep(50); + await lf.release(); + } +} diff --git a/packages/minidb/test/e2e/index-consistency.test.ts b/packages/minidb/test/e2e/index-consistency.test.ts index b83bbbcb3c..f5cf7bd079 100644 --- a/packages/minidb/test/e2e/index-consistency.test.ts +++ b/packages/minidb/test/e2e/index-consistency.test.ts @@ -23,7 +23,7 @@ function randomDoc(rng) { }; } -test('index-consistency: indexes stay consistent with the store under random ops', async () => { +test('index-consistency: indexes stay consistent with the store under random ops', { timeout: 60_000 }, async () => { const rng = mulberry32(0x5eed1234); const dir = await tmpDir(); let db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', autoCompact: false }); diff --git a/packages/minidb/test/e2e/stress.test.ts b/packages/minidb/test/e2e/stress.test.ts new file mode 100644 index 0000000000..7a77126f2a --- /dev/null +++ b/packages/minidb/test/e2e/stress.test.ts @@ -0,0 +1,845 @@ +// test/e2e/stress.test.ts +// +// Stress tests for MiniDb. Unlike the per-component unit tests and the +// sequential fuzz/model e2e's, these hammer the database with high in-flight +// concurrency, explicit WAL-rotation storms, torn/corrupt tails, mass TTL +// expiry, memory-pressure eviction, live read-only openers, and multi-process +// lock contention — then assert the durability contract: every acknowledged +// write survives, the in-memory view and the recovered view always agree, and +// reads/queries never change behavior under load. +// +// Tests in the "bug evidence" section pin, one per discovered bug, the exact +// contract that currently breaks under stress. The "coverage" section holds +// the stress configurations that behave correctly and must keep behaving. + +import { expect, test } from 'vitest'; +import { spawn } from 'node:child_process'; +import net from 'node:net'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { MiniDb } from '../../src/index.js'; +import { startServer } from '../../src/server.js'; +import { tmpDir, rmrf } from './helpers/tmp.js'; +import { mulberry32 } from './helpers/prng.js'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +const fmtErrs = (errs: unknown[], n = 5): string => + errs + .slice(0, n) + .map((e) => String((e as Error)?.message ?? e)) + .join(' | '); + +// Write storm with an explicit compaction fired mid-run; a unique index makes +// write commits queue on the unique-conflict lock, which widens the async gap +// between the compaction gate check and the WAL append so rotations genuinely +// race in-flight writers (without it, the storm outruns every rotation). +async function driveStormWithRotation( + db: MiniDb<{ n: number; pad: string }>, + opts: { writers?: number; perWriter?: number; compactsAt?: number[] }, +): Promise<{ acked: Map; failures: unknown[] }> { + const { writers = 64, perWriter = 8 } = opts; + const compactsAt = new Set(opts.compactsAt ?? [150]); + const acked = new Map(); + const failures: unknown[] = []; + let nextN = 0; + const compacting: Promise[] = []; + await Promise.all( + Array.from({ length: writers }, async (_, w) => { + for (let i = 0; i < perWriter; i++) { + const key = `w${w}:k${i}`; + const value = { n: nextN++, pad: 'z'.repeat(400) }; + try { + await db.set(key, value); + acked.set(key, value); + if (compactsAt.has(acked.size)) compacting.push(db.compact()); + } catch (err) { + failures.push(err); + } + } + }), + ); + await Promise.all(compacting); + return { acked, failures }; +} + +// =========================================================================== +// Bug evidence: WAL compaction rotation is not atomic with respect to writers +// =========================================================================== + +// The rotation gate only parks writers that call set() AFTER _rotateLock is +// assigned; writers that already passed the check (and are suspended between +// it and wal.append) still append into the OLD WAL inside the critical +// section. Frames appended past the rotation's endOffset are lost even though +// the write returned success, so acknowledged keys vanish after reopen. +test( + 'stress: acknowledged writes must never be lost by a WAL rotation (memory mode)', + { timeout: 120_000 }, + async () => { + const dir = await tmpDir('minidb-stress-rotmem-'); + const db = await MiniDb.open<{ n: number; pad: string }>({ dir, valueCodec: 'json', fsyncPolicy: 'always' }); + await db.createIndex('n', { field: 'n', unique: true }); + const { acked, failures } = await driveStormWithRotation(db, {}); + expect(db.stats.compactions).toBeGreaterThan(0); + // No write may fail spuriously ("WAL is closed") during normal operation. + expect(failures, `spurious write failures: ${fmtErrs(failures)}`).toEqual([]); + await db.close(); + + const db2 = await MiniDb.open<{ n: number; pad: string }>({ dir, valueCodec: 'json' }); + const lost: string[] = []; + for (const [key] of acked) if (db2.get(key) === undefined) lost.push(key); + expect(lost, `${lost.length} acknowledged writes lost after rotation`).toEqual([]); + await db2.close(); + await rmrf(dir); + }, +); + +// Same rotation race in valueMode 'disk': a record whose WAL pointer is +// published against the rotated-away file points at the wrong bytes, so reads +// of acknowledged keys return other frames' payloads (or throw) — silent data +// corruption on the live database, still there after a reopen. +test( + 'stress: acknowledged writes must stay readable and exact under rotation (disk mode)', + { timeout: 120_000 }, + async () => { + const dir = await tmpDir('minidb-stress-rotdisk-'); + const opts = { dir, valueCodec: 'json' as const, valueMode: 'disk' as const, fsyncPolicy: 'always' as const }; + const db = await MiniDb.open<{ n: number; pad: string }>(opts); + await db.createIndex('n', { field: 'n', unique: true }); + const { acked, failures } = await driveStormWithRotation(db, {}); + expect(failures, `spurious write failures: ${fmtErrs(failures)}`).toEqual([]); + + const wrong: string[] = []; + for (const [key, value] of acked) { + let got: { n: number; pad: string } | undefined; + expect(() => { + got = db.get(key); + }, `read of acknowledged key ${key} threw`).not.toThrow(); + if (JSON.stringify(got) !== JSON.stringify(value)) wrong.push(key); + } + expect(wrong, `${wrong.length} acknowledged keys return corrupted values`).toEqual([]); + await db.close(); + + const db2 = await MiniDb.open<{ n: number; pad: string }>(opts); + const lost: string[] = []; + for (const [key] of acked) if (db2.get(key) === undefined) lost.push(key); + expect(lost, `${lost.length} acknowledged writes lost after rotation`).toEqual([]); + await db2.close(); + await rmrf(dir); + }, +); + +// =========================================================================== +// Bug evidence: recovery truncation desynchronizes the WAL offsets +// =========================================================================== + +// MiniDb.open opens the WAL (capturing size/nextOffset) BEFORE recovery +// truncates a torn tail; the WAL object keeps stale offsets afterwards. New +// writes in valueMode 'disk' publish value pointers shifted by the truncated +// byte count, so get() reads the NEXT frame's payload (or hits a short read), +// and compaction dies trying to read through the same stale pointers. +test( + 'stress: torn-tail recovery must not desync later writes (disk mode)', + { timeout: 60_000 }, + async () => { + const dir = await tmpDir('minidb-stress-torn-'); + const opts = { dir, valueCodec: 'buffer' as const, valueMode: 'disk' as const, fsyncPolicy: 'no' as const }; + const seedVal = Buffer.alloc(256, 0xa5); + { + const db = await MiniDb.open(opts); + for (let i = 0; i < 50; i++) await db.set(`seed${i}`, seedVal); + await db.close(); + } + // Crash-simulated torn frame at the WAL tail. + const torn = Buffer.concat([Buffer.from([0x4d, 0x44, 1, 0, 5, 0]), Buffer.alloc(97, 0xff)]); + await fs.appendFile(path.join(dir, 'db.wal'), torn); + + const db = await MiniDb.open(opts); + try { + expect(db.recoveryInfo?.truncatedWal, 'recovery must truncate the torn tail').toBe(true); + const post = (i: number): Buffer => Buffer.alloc(200 + i, (i * 13) % 251); + for (let i = 0; i < 40; i++) await db.set(`post${i}`, post(i)); + for (let i = 0; i < 40; i++) expect(db.get(`post${i}`), `get(post${i})`).toEqual(post(i)); + for (let i = 0; i < 50; i++) expect(db.get(`seed${i}`), `get(seed${i})`).toEqual(seedVal); + await db.compact(); + for (let i = 0; i < 40; i++) expect(db.get(`post${i}`), `post-compact get(post${i})`).toEqual(post(i)); + for (let i = 0; i < 50; i++) expect(db.get(`seed${i}`), `post-compact get(seed${i})`).toEqual(seedVal); + } finally { + await db.close().catch(() => {}); + } + + const db3 = await MiniDb.open(opts); + try { + for (let i = 0; i < 40; i++) { + const want = Buffer.alloc(200 + i, (i * 13) % 251); + expect(db3.get(`post${i}`), `reopened get(post${i})`).toEqual(want); + } + for (let i = 0; i < 50; i++) expect(db3.get(`seed${i}`), `reopened get(seed${i})`).toEqual(seedVal); + } finally { + await db3.close(); + await rmrf(dir); + } + }, +); + +// =========================================================================== +// Bug evidence: stale-lock takeover admits multiple owners +// =========================================================================== + +// LockFile.acquire unlinks a stale lock and retries blindly. When several +// processes race to take over a crashed owner's lock, the loser's unlink can +// delete the winner's fresh lock file (and ENOENT on readFile is also treated +// as "stale"), so many racers end up believing they hold the lock at once — +// i.e. several writers on one database directory. +test( + 'stress: a stale lock must be taken over by exactly one process', + { timeout: 180_000 }, + async () => { + const dir = await tmpDir('minidb-stress-lock-'); + const lockPath = path.join(dir, 'db.lock'); + const RACER = path.join(__dirname, 'helpers', 'lock-racer.ts'); + const DEAD_PID = 2 ** 30 - 3; + // 6-way simultaneous takeover still exposes any cascade (the historical + // failure mode grants everyone the lock), while fitting the test's process + // budget on 2-core runners (every racer is a full node+tsx child). + const RACERS = 6; + const ROUNDS = 25; + + const outputs: string[] = []; + const children = Array.from({ length: RACERS }, () => { + const child = spawn(process.execPath, ['--import', 'tsx', RACER, lockPath, dir, String(ROUNDS)], { + stdio: ['ignore', 'pipe', 'ignore'], + }); + let buf = ''; + child.stdout.on('data', (d) => { + buf += d; + let j; + while ((j = buf.indexOf('\n')) >= 0) { + outputs.push(buf.slice(0, j)); + buf = buf.slice(j + 1); + } + }); + return child; + }); + + const violations: string[] = []; + try { + // Wait for every racer to be parked at the gate, otherwise boot stagger + // (exports finish importing at very different times) makes the first + // rounds measure sequential acquisitions as exclusivity violations. + for (;;) { + if (outputs.filter((l) => l === 'READY').length >= RACERS) break; + await new Promise((res) => setTimeout(res, 5)); + } + for (let r = 0; r < ROUNDS; r++) { + await fs.writeFile(lockPath, JSON.stringify({ pid: DEAD_PID, ts: Date.now() })); + await fs.writeFile(`${dir}/go-${r}`, '1'); + const want = `R${r} `; + for (;;) { + const lines = outputs.filter((l) => l.startsWith(want)); + if (lines.length >= RACERS) { + const holders = lines.filter((l) => l.endsWith(' 1')); + if (holders.length !== 1) violations.push(`round ${r}: ${holders.length} holders (${holders.join(', ')})`); + break; + } + await new Promise((res) => setTimeout(res, 1)); + } + await fs.unlink(`${dir}/go-${r}`).catch(() => {}); + } + expect(violations, `lock takeover violated exclusivity:\n${violations.slice(0, 10).join('\n')}`).toEqual([]); + } finally { + for (const c of children) c.kill('SIGKILL'); + await rmrf(dir).catch(() => {}); + } + }, +); + +// =========================================================================== +// Bug evidence: read-only opener compacts (and destroys) a live database +// =========================================================================== + +// MiniDb.open() runs `if (db.autoCompact && shouldCompact(db)) compact(db)` +// with no readOnly guard. A read-only opener of a hot database renames +// db.snapshot/db.wal out from under the LIVE writer; the writer keeps writing +// into an unlinked inode and every subsequent acknowledged write is lost. +test( + 'stress: a read-only open must never modify the database or break a live writer', + { timeout: 60_000 }, + async () => { + const dir = await tmpDir('minidb-stress-rocompact-'); + const writer = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'always' }); + for (let i = 0; i < 20; i++) await writer.set(`before${i}`, { i, pad: 'x'.repeat(400) }); + + const reader = await MiniDb.open({ dir, valueCodec: 'json', onLockFail: 'readonly', compactThresholdBytes: 1024 }); + expect(reader.readOnly).toBe(true); + expect(reader.stats.compactions, 'read-only instance ran a compaction').toBe(0); + await reader.close(); + + for (let i = 0; i < 10; i++) await writer.set(`after${i}`, { i }); + await writer.close(); + + const check = await MiniDb.open({ dir, valueCodec: 'json' }); + let afterLive = 0; + let beforeLive = 0; + for (let i = 0; i < 10; i++) if (check.get(`after${i}`) !== undefined) afterLive++; + for (let i = 0; i < 20; i++) if (check.get(`before${i}`) !== undefined) beforeLive++; + expect(beforeLive).toBe(20); + expect(afterLive, 'writes acknowledged by the live writer after a read-only open').toBe(10); + await check.close(); + await rmrf(dir); + }, +); + +// A read-only open of an empty directory must not create files either. +test('stress: a read-only open of an empty dir creates no files', { timeout: 30_000 }, async () => { + const dir = await tmpDir('minidb-stress-rofiles-'); + const ro = await MiniDb.open({ dir, readOnly: true }); + await ro.close(); + const files = await fs.readdir(dir); + expect(files).toEqual([]); + await rmrf(dir); +}); + +// =========================================================================== +// Bug evidence: one oversized token permanently destroys a text index +// =========================================================================== + +// Postings records cap termLen at uint16. A single document carrying a +// >64KiB token makes every later postings rebuild (run by compaction) throw +// AFTER the delta + base dictionaries were already cleared — the whole text +// index is silently emptied, searches return nothing, and every compaction +// from then on fails (while stats.compactions still counts them as done). +test( + 'stress: a single oversized token must not poison the text index and compaction', + { timeout: 60_000 }, + async () => { + const dir = await tmpDir('minidb-stress-textpoison-'); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createTextIndex('docs', { fields: ['body'] }); + await db.set('normal', { body: 'hello world' }); + await db.set('poison', { body: `${'a'.repeat(70_000)} hello` }); + + // Compaction must keep working regardless of document content. + await expect(db.compact(), 'compaction failed because of a document').resolves.toBeUndefined(); + // The index must still answer instead of silently returning nothing. + expect(db.search('docs', 'hello').map((h) => h.key).sort(), 'search after compaction').toEqual([ + 'normal', + 'poison', + ]); + await db.close(); + await rmrf(dir); + }, +); + +// =========================================================================== +// Bug evidence: dt-ordered query fast path returns different rows +// =========================================================================== + +// query() takes a fast path when a single dt column is bounded and limit is +// set; the fast path ignores the dt cond's own offset/count while the general +// path honors them, so the same predicate returns different rows depending on +// whether `limit` is present. +test( + 'stress: dt-ordered fast path must agree with the general path on offset/count', + { timeout: 30_000 }, + async () => { + const dir = await tmpDir('minidb-stress-dtfast-'); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + for (let i = 0; i < 10; i++) await db.set(`d${i}`, { v: i }, { dt: { t: i * 10 } }); + const general = db.query({ dt: { t: { offset: 3, count: 4 } } }).map((r) => r.key); + const fast = db.query({ dt: { t: { offset: 3, count: 4 } }, limit: 10 }).map((r) => r.key); + expect(fast).toEqual(general); + await db.close(); + await rmrf(dir); + }, +); + +// =========================================================================== +// Bug evidence: RESP server emits out-of-order replies +// =========================================================================== + +// Each socket 'data' event spawns its own async handler, so a slow command in +// one packet (SET with fsync 'always') races socket.write()s from faster +// commands in the next packet. Replies then arrive out of order and pipelined +// clients desynchronize. +test( + 'stress: RESP server must serialize replies per connection', + { timeout: 60_000 }, + async () => { + const dir = await tmpDir('minidb-stress-resp-'); + const { port, close } = await startServer({ dir, port: 0, fsyncPolicy: 'always' }); + const SET = '*3\r\n$3\r\nSET\r\n$4\r\nkey1\r\n$1000\r\n' + 'v'.repeat(1000) + '\r\n'; + const PING = 'PING\r\n'; + + const round = (): Promise => + new Promise((resolve, reject) => { + const sock = net.createConnection(port, '127.0.0.1'); + let buf = ''; + sock.on('data', (d) => (buf += String(d))); + sock.on('error', reject); + sock.write(SET); + setTimeout(() => sock.write(PING.repeat(20)), 1); + setTimeout(() => { + sock.end(); + resolve(buf); + }, 300); + }); + + let inverted = 0; + const ROUNDS = 12; + try { + for (let r = 0; r < ROUNDS; r++) { + const reply = await round(); + const okIdx = reply.indexOf('+OK'); + const pongIdx = reply.indexOf('+PONG'); + if (pongIdx !== -1 && (okIdx === -1 || pongIdx < okIdx)) inverted++; + } + expect(inverted, `${inverted}/${ROUNDS} connections saw PINGs answered before the earlier SET`).toBe(0); + } finally { + await close(); + await rmrf(dir); + } + }, +); + +// =========================================================================== +// Coverage: torn-tail recovery in memory mode +// =========================================================================== +test( + 'stress: torn-tail recovery in memory mode stays correct across compaction', + { timeout: 60_000 }, + async () => { + const dir = await tmpDir('minidb-stress-tornmem-'); + const opts = { dir, valueCodec: 'json' as const, fsyncPolicy: 'no' as const }; + const want = new Map(); + { + const db = await MiniDb.open(opts); + for (let i = 0; i < 50; i++) { + const v = { i, pad: 's'.repeat(200) }; + await db.set(`seed${i}`, v); + want.set(`seed${i}`, v); + } + await db.close(); + } + const torn = Buffer.concat([Buffer.from([0x4d, 0x44, 1, 0, 5, 0]), Buffer.alloc(97, 0xff)]); + await fs.appendFile(path.join(dir, 'db.wal'), torn); + + const db = await MiniDb.open(opts); + try { + expect(db.recoveryInfo?.truncatedWal).toBe(true); + for (let i = 0; i < 40; i++) { + const v = { i, pad: 'p'.repeat(240) }; + await db.set(`post${i}`, v); + want.set(`post${i}`, v); + } + await db.compact(); + } finally { + await db.close().catch(() => {}); + } + const db2 = await MiniDb.open(opts); + try { + expect(db2.size).toBe(want.size); + for (const [key, value] of want) expect(db2.get(key), `recovered ${key}`).toEqual(value); + } finally { + await db2.close(); + await rmrf(dir); + } + }, +); + +// =========================================================================== +// Coverage: mass TTL expiry racing compaction and reopen +// =========================================================================== +test('stress: mass TTL expiry across compaction and reopen', { timeout: 90_000 }, async () => { + const dir = await tmpDir('minidb-stress-ttl-'); + const KEEP = 300; + const EPHEM = 700; + const db = await MiniDb.open<{ i: number; pad?: string }>({ + dir, + valueCodec: 'json', + fsyncPolicy: 'no', + compactThresholdBytes: 48 * 1024, + }); + try { + for (let i = 0; i < KEEP; i++) await db.set(`keep${i}`, { i }, { dt: { created: 1_000_000 + i } }); + for (let i = 0; i < EPHEM; i++) await db.set(`temp${i}`, { i, pad: 'z'.repeat(300) }, { ttl: 80 }); + for (let i = 0; i < 120; i++) await db.set(`churn${i}`, { i, pad: 'q'.repeat(900) }); + await new Promise((r) => setTimeout(r, 500)); + for (let i = 0; i < 120; i++) await db.del(`churn${i}`); + + expect(db.size).toBe(KEEP); + for (let i = 0; i < EPHEM; i++) expect(db.get(`temp${i}`), `temp${i} must be gone`).toBeUndefined(); + expect(db.scan().length).toBe(KEEP); + expect(db.dtRange('created').length).toBe(KEEP); + } finally { + await db.close().catch(() => {}); + } + const db2 = await MiniDb.open<{ i: number }>({ dir, valueCodec: 'json' }); + try { + expect(db2.size, 'reopened size').toBe(KEEP); + expect(db2.scan().length).toBe(KEEP); + for (let i = 0; i < KEEP; i++) expect(db2.get(`keep${i}`)).toEqual({ i }); + for (let i = 0; i < EPHEM; i++) expect(db2.has(`temp${i}`), `temp${i} resurrected`).toBe(false); + expect(db2.dtRange('created').length).toBe(KEEP); + } finally { + await db2.close(); + await rmrf(dir); + } +}); + +// =========================================================================== +// Coverage: full-text index consistency under churn, compaction, reopen +// =========================================================================== +test( + 'stress: full-text index stays consistent under churn, compaction, reopen', + { timeout: 120_000 }, + async () => { + const dir = await tmpDir('minidb-stress-text-'); + const VOCAB = Array.from({ length: 120 }, (_, i) => `tok${i}`); + const rng = mulberry32(42); + const model = new Map(); + const liveHits = (term: string): string[] => + [...model].filter(([, b]) => b.split(' ').includes(term)).map(([k]) => k).sort(); + + const db = await MiniDb.open<{ body: string }>({ dir, valueCodec: 'json', fsyncPolicy: 'no', compactThresholdBytes: 48 * 1024 }); + try { + await db.createTextIndex('docs', { fields: ['body'] }); + for (let iter = 0; iter < 1500; iter++) { + const key = `d${Math.floor(rng() * 300)}`; + if (rng() < 0.25) { + await db.del(key); + model.delete(key); + } else { + const body = Array.from({ length: 8 + Math.floor(rng() * 24) }, () => VOCAB[Math.floor(rng() * VOCAB.length)]).join(' '); + await db.set(key, { body }); + model.set(key, body); + } + if (iter % 250 === 249) { + for (const term of ['tok5', 'tok77']) { + const hits = db.search('docs', term, { limit: 10_000 }).map((h) => h.key).sort(); + expect(hits, `iter ${iter}: search("${term}")`).toEqual(liveHits(term)); + } + } + } + } finally { + await db.close().catch(() => {}); + } + const db2 = await MiniDb.open<{ body: string }>({ dir, valueCodec: 'json' }); + try { + for (const term of ['tok5', 'tok77']) { + const hits = db2.search('docs', term, { limit: 10_000 }).map((h) => h.key).sort(); + expect(hits, `after reopen: search("${term}")`).toEqual(liveHits(term)); + } + } finally { + await db2.close(); + await rmrf(dir); + } + }, +); + +// =========================================================================== +// Coverage: evict-lru under churn — budget holds, key set matches after reopen +// =========================================================================== +test( + 'stress: evict-lru under compaction churn — no resurrection, strict budget', + { timeout: 120_000 }, + async () => { + const dir = await tmpDir('minidb-stress-lru-'); + const budget = 192 * 1024; + const db = await MiniDb.open<{ i: number; pad: string }>({ + dir, + valueCodec: 'json', + fsyncPolicy: 'no', + maxMemoryBytes: budget, + maxMemoryPolicy: 'evict-lru', + compactThresholdBytes: 64 * 1024, + }); + try { + for (let i = 0; i < 2500; i++) { + await db.set(`ek${i}`, { i, pad: 'e'.repeat(1100) }); + if (i % 7 === 0) db.get(`ek${Math.max(0, i - 3)}`); + } + expect(db.stats.compactions).toBeGreaterThan(0); + expect(db.store.bytes).toBeLessThanOrEqual(Math.ceil(budget * 1.05)); + expect(db.stats.evictions).toBeGreaterThan(0); + } catch (err) { + await db.close().catch(() => {}); + await rmrf(dir).catch(() => {}); + throw err; + } + const before = new Map(db.scan().map((e) => [e.key, e.value] as const)); + await db.close(); + + const db2 = await MiniDb.open<{ i: number; pad: string }>({ dir, valueCodec: 'json' }); + try { + const after = new Map(db2.scan().map((e) => [e.key, e.value] as const)); + expect(after.size, 'reopened key set must equal pre-close key set').toBe(before.size); + for (const [key, value] of before) { + expect(after.get(key), `value of ${key} changed across reopen`).toEqual(value); + } + } finally { + await db2.close(); + await rmrf(dir); + } + }, +); + +// =========================================================================== +// Coverage: maxMemory reject leaves the db consistent and writable +// =========================================================================== +test('stress: maxMemory reject leaves the db consistent and writable', { timeout: 60_000 }, async () => { + const dir = await tmpDir('minidb-stress-reject-'); + const db = await MiniDb.open<{ i: number; pad: string }>({ + dir, + valueCodec: 'json', + maxMemoryBytes: 32 * 1024, + maxMemoryPolicy: 'reject', + }); + let ok = 0; + let rej = 0; + try { + for (let i = 0; i < 400; i++) { + try { + await db.set(`rk${i}`, { i, pad: 'r'.repeat(400) }); + ok++; + } catch (err) { + rej++; + expect(String((err as Error).message)).toMatch(/maxMemory/); + } + } + expect(ok).toBeGreaterThan(0); + expect(rej).toBeGreaterThan(0); + for (let i = 0; i < 400; i++) { + const v = db.get(`rk${i}`); + if (v !== undefined) expect(v).toEqual({ i, pad: 'r'.repeat(400) }); + } + for (let i = 0; i < 400; i += 2) await db.del(`rk${i}`); + await db.set('post-reject', { i: -1, pad: 'ok' }); + expect(db.get('post-reject')).toEqual({ i: -1, pad: 'ok' }); + } catch (err) { + await db.close().catch(() => {}); + await rmrf(dir).catch(() => {}); + throw err; + } + const before = new Map(db.scan().map((e) => [e.key, e.value] as const)); + await db.close(); + + const db2 = await MiniDb.open<{ i: number; pad: string }>({ dir, valueCodec: 'json' }); + try { + const after = new Map(db2.scan().map((e) => [e.key, e.value] as const)); + expect(after.size).toBe(before.size); + for (const [key, value] of before) expect(after.get(key), key).toEqual(value); + } finally { + await db2.close(); + await rmrf(dir); + } +}); + +// =========================================================================== +// Coverage: read-only openers alongside a live compacting writer +// =========================================================================== +test('stress: read-only openers alongside a live compacting writer', { timeout: 90_000 }, async () => { + const dir = await tmpDir('minidb-stress-ro-'); + const writer = await MiniDb.open<{ n: number; pad: string }>({ + dir, + valueCodec: 'json', + fsyncPolicy: 'no', + compactThresholdBytes: 96 * 1024, + autoCompact: true, + }); + const errors: unknown[] = []; + let n = 0; + const stop = Date.now() + 5000; + const wf = (async (): Promise => { + while (Date.now() < stop) { + await writer.set(`rk${n}`, { n, pad: 'w'.repeat(700) }); + n++; + } + })(); + try { + while (Date.now() < stop) { + try { + const ro = await MiniDb.open({ dir, valueCodec: 'json', onLockFail: 'readonly', autoCompact: false }); + const rows = ro.scan(); + for (const r of rows) expect(r.key).toMatch(/^rk\d+$/); + await ro.close(); + } catch (err) { + errors.push(err); + } + await new Promise((r) => setTimeout(r, 40)); + } + } finally { + await wf; + await writer.close(); + } + expect(errors, `read-only failures: ${fmtErrs(errors)}`).toEqual([]); + const db = await MiniDb.open<{ n: number }>({ dir, valueCodec: 'json' }); + try { + expect(db.size).toBe(n); + for (let i = 0; i < n; i++) expect(db.get(`rk${i}`)).toEqual({ n: i, pad: 'w'.repeat(700) }); + } finally { + await db.close(); + await rmrf(dir); + } +}); + +// =========================================================================== +// Coverage: batch durability under churn + explicit rotation storm +// =========================================================================== +test('stress: batch durability under compaction churn', { timeout: 120_000 }, async () => { + const dir = await tmpDir('minidb-stress-batch-'); + const db = await MiniDb.open<{ ts: string; j: number; pad: string }>({ + dir, + valueCodec: 'json', + fsyncPolicy: 'no', + compactThresholdBytes: 64 * 1024, + }); + const acked = new Map(); + const failures: unknown[] = []; + const writers = Promise.all( + Array.from({ length: 12 }, async (_, w) => { + for (let i = 0; i < 120; i++) { + const ts = `b${w}:${i}`; + const ops = Array.from({ length: 8 }, (_, j) => { + const value = { ts, j, pad: 'b'.repeat(300) }; + return { op: 'set' as const, key: `${ts}:${j}`, value }; + }); + try { + await db.batch(ops); + for (const o of ops) acked.set(o.key, o.value); + } catch (err) { + failures.push(err); + } + } + }), + ); + const rotations = (async () => { + for (let r = 0; r < 25; r++) await db.compact().catch(() => {}); + })(); + try { + await Promise.all([writers, rotations]); + expect(db.stats.compactions).toBeGreaterThan(0); + expect(failures, `spurious batch failures: ${fmtErrs(failures)}`).toEqual([]); + } finally { + await db.close().catch(() => {}); + } + const db2 = await MiniDb.open<{ ts: string; j: number; pad: string }>({ dir, valueCodec: 'json' }); + try { + expect(db2.size).toBe(acked.size); + for (const [key, value] of acked) expect(db2.get(key), key).toEqual(value); + } finally { + await db2.close(); + await rmrf(dir); + } +}); + +// =========================================================================== +// Coverage: oversized batches reject cleanly, apply nothing, db stays usable +// =========================================================================== +test('stress: a >65535-op batch rejects cleanly and applies nothing', { timeout: 60_000 }, async () => { + const dir = await tmpDir('minidb-stress-bigbatch-'); + const db = await MiniDb.open({ dir, valueCodec: 'json' }); + const ops = Array.from({ length: 65_536 }, (_, i) => ({ op: 'set' as const, key: `k${i}`, value: { i } })); + await expect(db.batch(ops)).rejects.toThrow(); + expect(db.size).toBe(0); + await db.set('alive', { ok: true }); + expect(db.size).toBe(1); + await db.close(); + const db2 = await MiniDb.open({ dir, valueCodec: 'json' }); + expect(db2.get('alive')).toEqual({ ok: true }); + await db2.close(); + await rmrf(dir); +}); + +// =========================================================================== +// Coverage: full-feature soak with churn, compaction, reopen +// =========================================================================== +test( + 'stress soak: full-feature model consistency with churn, compaction, reopen', + { timeout: 300_000 }, + async () => { + const dir = await tmpDir('minidb-stress-soak-'); + const rng = mulberry32(1337); + type Doc = { g: number; score: number; n: number; body?: string; pad?: string }; + const model = new Map(); + let nextN = 0; + let db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', compactThresholdBytes: 96 * 1024 }); + + const checkAll = async (ctx: string, dbi: MiniDb): Promise => { + const scanned = dbi.scan({}); + expect(scanned.length, `${ctx}: scan length`).toBe(model.size); + for (const e of scanned) expect(e.value, `${ctx}: value of ${e.key}`).toEqual(model.get(e.key)!.doc); + const byDt = dbi.dtRange('created').map((r) => r.key).sort(); + const modelDt = [...model].filter(([, v]) => v.dt !== null).map(([k]) => k).sort(); + expect(byDt, `${ctx}: dtRange('created')`).toEqual(modelDt); + for (const g of [0, 3, 7]) { + const hits = dbi.findEq('g', g).map((r) => r.key).sort(); + const want = [...model].filter(([, v]) => v.doc.g === g).map(([k]) => k).sort(); + expect(hits, `${ctx}: findEq(g=${g})`).toEqual(want); + } + const ranged = dbi.findRange('score', { min: 0, max: 100 }).map((r) => r.key).sort(); + const wantRanged = [...model].filter(([, v]) => v.doc.score >= 0 && v.doc.score <= 100).map(([k]) => k).sort(); + expect(ranged, `${ctx}: findRange(score 0..100)`).toEqual(wantRanged); + const hits = dbi.search('docs', 'alpha', { limit: 10_000 }).map((h) => h.key).sort(); + const wantHits = [...model].filter(([, v]) => (v.doc.body ?? '').split(' ').includes('alpha')).map(([k]) => k).sort(); + expect(hits, `${ctx}: search(alpha)`).toEqual(wantHits); + }; + + try { + await db.createIndex('g', { field: 'g' }); + await db.createIndex('score', { field: 'score', type: 'range' }); + await db.createIndex('n', { field: 'n', unique: true }); + await db.createTextIndex('docs', { fields: ['body'] }); + + const VOCAB = ['alpha', 'beta', 'gamma', 'delta', 'epsilon']; + // Sequential ops: the model is exact. (Concurrent same-key ordering with + // a model is intentionally NOT what this soak checks.) + for (let iter = 0; iter < 3000; iter++) { + const key = `s${Math.floor(rng() * 400)}`; + const roll = rng(); + if (roll < 0.5) { + const doc: Doc = { + g: Math.floor(rng() * 8), + score: Math.floor(rng() * 200) - 50, + n: nextN++, + body: Array.from({ length: 4 }, () => VOCAB[Math.floor(rng() * VOCAB.length)]).join(' '), + pad: 'p'.repeat(50 + Math.floor(rng() * 300)), + }; + const dt = rng() < 0.4 ? 1_700_000_000_000 + Math.floor(rng() * 1_000_000) : null; + await db.set(key, doc, dt ? { dt: { created: dt } } : {}); + model.set(key, { doc, dt }); + } else if (roll < 0.65) { + await db.del(key); + model.delete(key); + } else if (roll < 0.75) { + const ops = Array.from({ length: 4 }, () => { + const k = `s${Math.floor(rng() * 400)}`; + const doc: Doc = { g: Math.floor(rng() * 8), score: Math.floor(rng() * 200) - 50, n: nextN++, body: 'alpha batch' }; + return { op: 'set' as const, key: k, value: doc }; + }); + await db.batch(ops); + for (const o of ops) model.set(o.key, { doc: o.value, dt: null }); + } else { + const g = Math.floor(rng() * 8); + const hits = db.findEq('g', g); + for (const h of hits) expect(h.value?.g).toBe(g); + } + if (iter % 400 === 399) await checkAll(`iter ${iter}`, db); + if (iter % 1000 === 999 && iter < 2999) { + await db.close(); + db = await MiniDb.open({ dir, valueCodec: 'json', fsyncPolicy: 'no', compactThresholdBytes: 96 * 1024 }); + await checkAll(`post-reopen iter ${iter}`, db); + } + } + await checkAll('final', db); + await db.close(); + await rmrf(dir); + } catch (err) { + await db.close().catch(() => {}); + await rmrf(dir).catch(() => {}); + throw err; + } + }, +); + diff --git a/packages/minidb/test/recovery.test.ts b/packages/minidb/test/recovery.test.ts index 68c88c430f..44221c1381 100644 --- a/packages/minidb/test/recovery.test.ts +++ b/packages/minidb/test/recovery.test.ts @@ -109,3 +109,85 @@ test('strict mode truncates at the first bad frame', async () => { await fs.rm(dir, { recursive: true, force: true }); } }); + +// catchUpFromWal: a read-only replica applies only the WAL frames appended +// after its watermark; the result must be identical to a from-scratch replay +// (store, dt, compound, secondary, unique and text indexes alike). +for (const valueMode of ['memory', 'disk'] as const) { + test(`catchUpFromWal (${valueMode}): mixed tail applies identically to a full reopen`, { timeout: 60_000 }, async () => { + const dir = await tmpDir(); + try { + const doc = (i: number) => ({ n: i, c: `c${i % 7}`, u: `u${i}`, t: `alpha beta w${i % 13}` }); + const writer = await MiniDb.open>({ + dir, + valueCodec: 'json', + valueMode, + fsyncPolicy: 'no', + autoCompact: false, + }); + await writer.createIndex('c', { field: 'c' }); + await writer.createIndex('n', { field: 'n', type: 'range' }); + await writer.createIndex('u', { field: 'u', unique: true }); + await writer.createTextIndex('t', { fields: ['t'] }); + await writer.createCompoundIndex('cg', { groupBy: 'c', orderBy: 'n' }); + for (let i = 0; i < 2000; i++) await writer.set(`pre:${i}`, doc(i), { dt: { created: 1700000000000 + i } }); + + const reader = await MiniDb.open>({ dir, valueCodec: 'json', valueMode, readOnly: true }); + const ri = reader.recoveryInfo!; + assert.ok(ri.walScanEnd > 0); + assert.ok(ri.walIno !== 0); + + // Mixed storm: updates, new sets, dels, one BATCH frame, short/long TTL, + // dt-only change. 500 + 500 + 300 + 1 + 2 = 1303 appended frames. + for (let i = 1500; i < 2000; i++) { + await writer.set(`pre:${i}`, { ...doc(i), n: i * 10, t: `alpha changed w${i % 13}` }, { dt: { created: 1700000009000 + i } }); + } + for (let i = 0; i < 500; i++) await writer.set(`s:${i}`, { ...doc(i), u: `us${i}` }, { dt: { created: 1700001000000 + i } }); + for (let i = 0; i < 1500; i += 5) await writer.del(`pre:${i}`); + await writer.batch([ + { op: 'set', key: 's:b1', value: doc(9001), dt: { created: 1700002000000 } }, + { op: 'set', key: 's:b2', value: { ...doc(9002), t: 'alpha batch' } }, + { op: 'del', key: 's:b1' }, + ]); + await writer.set('s:short', doc(9100), { ttl: 1 }); + await writer.set('s:long', doc(9101), { ttl: 3_600_000 }); + + const res = await reader.catchUpFromWal(ri.walScanEnd); + assert.ok(res, 'clean watermark must catch up'); + assert.equal(res.appliedFrames, 1303); + assert.ok(res.offset > ri.walScanEnd); + + // Reference: a full from-scratch replay of the same files. + const ref = await MiniDb.open>({ dir, valueCodec: 'json', valueMode, readOnly: true }); + const sortByKey = (a: { key: string }, b: { key: string }): number => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0); + assert.equal(reader.size, ref.size); + assert.deepEqual(reader.scan(), ref.scan()); + assert.deepEqual(reader.dtColumns(), ref.dtColumns()); + assert.deepEqual(reader.findEq('c', 'c3').sort(sortByKey), ref.findEq('c', 'c3').sort(sortByKey)); + assert.deepEqual(reader.findEq('u', 'u1700'), ref.findEq('u', 'u1700')); + assert.deepEqual( + reader.findRange('n', { min: 100, max: 500 }).sort(sortByKey), + ref.findRange('n', { min: 100, max: 500 }).sort(sortByKey), + ); + assert.deepEqual(reader.dtRange('created', { gte: 1700000009000, limit: 20 }), ref.dtRange('created', { gte: 1700000009000, limit: 20 })); + assert.deepEqual(reader.compoundRange('cg', 'c2', { limit: 25 }), ref.compoundRange('cg', 'c2', { limit: 25 })); + assert.deepEqual(reader.search('t', 'alpha'), ref.search('t', 'alpha')); + assert.deepEqual(reader.search('t', 'changed'), ref.search('t', 'changed')); + assert.deepEqual(reader.search('t', 'batch'), ref.search('t', 'batch')); + assert.equal(reader.get('s:short'), undefined); + assert.equal(ref.get('s:short'), undefined); + assert.deepEqual(reader.get('s:long'), ref.get('s:long')); + + // No new writes: catch-up is a cheap no-op at the same offset. + assert.deepEqual(await reader.catchUpFromWal(res.offset), { offset: res.offset, appliedFrames: 0 }); + // Not a frame boundary (byte 3 of the first frame's header: flags=0). + assert.equal(await reader.catchUpFromWal(3), null); + + await ref.close(); + await reader.close(); + await writer.close(); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); +} diff --git a/packages/minidb/test/review-round2.test.ts b/packages/minidb/test/review-round2.test.ts index ff29d813ea..28e64fa0de 100644 --- a/packages/minidb/test/review-round2.test.ts +++ b/packages/minidb/test/review-round2.test.ts @@ -256,7 +256,7 @@ test('batch rejects keys longer than 128 chars', async () => { // --- #9 TTL heap does not grow unboundedly ---------------------------------- -test('repeated TTL updates on one key do not bloat the heap', async () => { +test('repeated TTL updates on one key do not bloat the heap', { timeout: 30_000 }, async () => { const dir = await tmpDir(); const db = await MiniDb.open({ dir, valueCodec: 'string', activeExpireIntervalMs: 0 }); for (let i = 0; i < 5000; i++) await db.set('k', 'v', { ttl: 1_000_000 }); @@ -344,6 +344,7 @@ test('RESP MSET sets all keys', async () => { test('openOrRebuild rebuilds on corrupt index-definition JSON', async () => { const dir = await tmpDir(); const db = await MiniDb.open({ dir, valueCodec: 'json' }); + await db.createIndex('byV', { field: 'v' }); await db.set('k', { v: 1 }); await db.close(); await fs.writeFile(path.join(dir, 'db.indexes.json'), '{ not valid json', 'utf8'); @@ -351,7 +352,9 @@ test('openOrRebuild rebuilds on corrupt index-definition JSON', async () => { let rebuilt = false; const db2 = await MiniDb.openOrRebuild({ dir, valueCodec: 'json' }, { onRebuild: () => (rebuilt = true) }); assert.ok(rebuilt, 'onRebuild should be called'); - assert.equal(db2.get('k'), undefined, 'rebuilt db is empty'); + // the corrupt sidecar (derived state) is dropped; the data is preserved + assert.deepEqual(db2.get('k'), { v: 1 }); + assert.deepEqual(db2.listIndexes(), []); await db2.close(); await fs.rm(dir, { recursive: true, force: true }); }); diff --git a/packages/minidb/test/server-extra.test.ts b/packages/minidb/test/server-extra.test.ts index 2b9298e3b1..9b336bf1c5 100644 --- a/packages/minidb/test/server-extra.test.ts +++ b/packages/minidb/test/server-extra.test.ts @@ -138,3 +138,85 @@ test('RESP: inline (non-array) command path', async () => { await fs.rm(dir, { recursive: true, force: true }); } }); + +// Accumulate reply bytes until `done` accepts them (replies may span chunks). +function collectUntil(sock: net.Socket, done: (s: string) => boolean): Promise { + return new Promise((resolve, reject) => { + let buf = ''; + const timer = setTimeout(() => reject(new Error(`timed out waiting for reply; got ${buf.length} bytes`)), 20_000); + sock.on('data', (d) => { + buf += d.toString(); + if (done(buf)) { + clearTimeout(timer); + resolve(buf); + } + }); + }); +} + +test('RESP: a client aborting mid-large-reply does not kill the server', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + sock.on('error', () => {}); // the test client itself may see the RST + const big = 'x'.repeat(4 * 1024 * 1024); + assert.equal(await send(sock, encode('SET', 'big', big)), '+OK\r\n'); + // Ask for the large value, then reset the connection without reading the + // reply: the server hits EPIPE/ECONNRESET while writing it out. + sock.write(encode('GET', 'big')); + await new Promise((r) => setTimeout(r, 20)); + sock.destroy(); + // Give the server a moment to hit the write failure, then prove a fresh + // connection is still being served. + await new Promise((r) => setTimeout(r, 100)); + const sock2 = await connect(srv.port); + assert.equal(await send(sock2, encode('PING')), '+PONG\r\n'); + sock2.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('RESP: an oversized request gets -ERR and the connection recovers', { timeout: 30_000 }, async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + // 65MB of bulk payload crosses the parser's 64MB cap. + const big = Buffer.alloc(65 * 1024 * 1024, 'x'.charCodeAt(0)); + const head = Buffer.from(`*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$${big.length}\r\n`); + sock.write(Buffer.concat([head, big, Buffer.from('\r\n')])); + // Pipelined right behind it: once the parser recovers from the -ERR this + // fresh small command must still be answered. + sock.write(encode('PING')); + const data = await collectUntil(sock, (s) => s.includes('+PONG')); + const tooLarge = data.indexOf('too large'); + const pong = data.indexOf('+PONG'); + assert.ok(tooLarge !== -1, `expected a too-large -ERR, got ${JSON.stringify(data.slice(0, 120))}`); + assert.ok(pong > tooLarge, 'PING after the oversized request must be answered'); + sock.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); + +test('RESP: one bad command does not starve its pipelined siblings', async () => { + const dir = await tmpDir(); + const srv = await startServer({ dir, port: 0, fsyncPolicy: 'no' }); + try { + const sock = await connect(srv.port); + // The 129-byte key exceeds MAX_KEY_LEN so SET throws inside the handler; + // the two PINGs in the very same chunk must still be answered, in order. + const key = 'k'.repeat(129); + sock.write(encode('SET', key, 'v') + encode('PING') + encode('PING')); + const data = await collectUntil(sock, (s) => s.endsWith('+PONG\r\n+PONG\r\n')); + assert.match(data, /^-ERR [^\r]*\r\n\+PONG\r\n\+PONG\r\n$/); + sock.end(); + } finally { + await srv.close(); + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/test/store.test.ts b/packages/minidb/test/store.test.ts index 706ae1ab73..050706e399 100644 --- a/packages/minidb/test/store.test.ts +++ b/packages/minidb/test/store.test.ts @@ -103,3 +103,68 @@ test('delete removes key from the ordered index', () => { s.del('b'); assert.deepEqual([...s.scan()].map((r) => r.key.toString()), ['a', 'c']); }); + +test('has() does not materialize disk-backed values', () => { + let reads = 0; + const s = new Store({ + activeExpireIntervalMs: 0, + readValue: () => { + reads++; + return B('disk-value'); + }, + }); + s.setRef('dk', { kind: 'disk', loc: { file: 'snapshot', off: 0, len: 10 } }); + assert.equal(reads, 0); + assert.equal(s.has('dk'), true); + assert.equal(s.has('missing'), false); + assert.equal(reads, 0); // has() must stay metadata-only (no positioned read) + assert.equal(s.get('dk').toString(), 'disk-value'); + assert.equal(reads, 1); + s.close(); +}); + +test('rawKeys yields ordered keys without materializing values', () => { + let reads = 0; + const s = new Store({ + activeExpireIntervalMs: 0, + readValue: () => { + reads++; + return B('v'); + }, + }); + s.setRef('a', { kind: 'disk', loc: { file: 'snapshot', off: 0, len: 1 } }); + s.set('b', B('2')); + s.setRef('c', { kind: 'disk', loc: { file: 'snapshot', off: 2, len: 1 } }); + s.set('d', B('4'), Date.now() - 1); // already expired + assert.deepEqual([...s.rawKeys({})], ['a', 'b', 'c']); + assert.deepEqual([...s.rawKeys({ gte: 'b' })], ['b', 'c']); + assert.equal(reads, 0); + s.close(); +}); + +test('size stays correct with and without TTL keys in play', () => { + const s = new Store({ activeExpireIntervalMs: 0 }); + for (let i = 0; i < 5; i++) s.set(`k${i}`, B('v')); + assert.equal(s.size, 5); // O(1) path (no TTL keys) + s.set('t', B('v'), Date.now() + 10_000); + assert.equal(s.size, 6); // TTL key in play -> live-count path + s.set('t', B('v2')); // overwrite without TTL + assert.equal(s.size, 6); + assert.ok(s.del('t')); + assert.equal(s.size, 5); // back to the fast path + s.set('u', B('v'), Date.now() - 1); // already expired at set + assert.equal(s.size, 5); // expired keys are not counted + s.close(); +}); + +test('active expiration drains a simultaneous-expiry storm within seconds', async () => { + // default-class settings: 100/count quota per tick + a small time budget for bursts + const s = new Store({ activeExpireIntervalMs: 100, activeExpireMaxPerTick: 100 }); + const N = 3000; + for (let i = 0; i < N; i++) s.set(`t${i}`, B('v'), Date.now() + 30); + assert.equal(s.map.size, N); + await sleep(1200); + // the old fixed ~1000/s rate would have left ~1500+ dead entries behind + assert.equal(s.map.size, 0); + s.close(); +}); diff --git a/packages/minidb/test/wal.test.ts b/packages/minidb/test/wal.test.ts index b4929369e7..11d031d657 100644 --- a/packages/minidb/test/wal.test.ts +++ b/packages/minidb/test/wal.test.ts @@ -113,3 +113,26 @@ test('recovery truncates a torn/corrupt tail at the error offset', async () => { await fs.rm(dir, { recursive: true, force: true }); } }); + +test('seal(): rejects new appends with WAL_SEALED, queued frames stay flushable', async () => { + const { dir, file } = await tmpWalPath(); + try { + const wal = new WAL(file, { fsyncPolicy: 'no' }); + await wal.open(); + await wal.append(encodeFrame({ type: TYPE_SET, key: B('a'), value: B('1') })); + wal.seal(); + wal.seal(); // idempotent + await assert.rejects(wal.append(encodeFrame({ type: TYPE_SET, key: B('b'), value: B('2') })), (err) => { + assert.equal(err.code, 'WAL_SEALED'); + return true; + }); + await wal.close(); // flushes the pre-seal frame + const frames = parseAll(await fs.readFile(file)); + assert.equal(frames.length, 1); + assert.equal(frames[0].key.toString(), 'a'); + // after close the legacy "WAL is closed" rejection is preserved + await assert.rejects(wal.append(encodeFrame({ type: TYPE_SET, key: B('c'), value: B('3') })), /WAL is closed/); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/minidb/tsdown.config.ts b/packages/minidb/tsdown.config.ts index b27e00afd7..8d04cc2dbc 100644 --- a/packages/minidb/tsdown.config.ts +++ b/packages/minidb/tsdown.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from 'tsdown'; export default defineConfig({ - entry: ['./src/index.ts'], + entry: ['./src/index.ts', './src/cluster/index.ts'], format: ['esm'], dts: false, outDir: 'dist', diff --git a/packages/minidb/vitest.config.ts b/packages/minidb/vitest.config.ts index db8268b3bb..513a55fa67 100644 --- a/packages/minidb/vitest.config.ts +++ b/packages/minidb/vitest.config.ts @@ -4,5 +4,9 @@ export default defineConfig({ test: { name: 'minidb', include: ['test/**/*.test.ts'], + // Package safety floor: process-spawning e2e's and multi-thousand-op + // tests legitimately need >5s (the vitest default) on shared CI runners + // under shard parallelism. Per-test explicit timeouts still win over this. + testTimeout: 30_000, }, });