diff --git a/package.json b/package.json index f728cdd..f9a7ffa 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@openparachute/cloud", - "version": "0.0.8-rc.127", + "version": "0.0.8-rc.128", "private": true, "description": "Open Parachute PBC's Vault Cloud \u2014 one Durable Object per vault on Cloudflare, OAuth issuer + self-serve console (accounts + vault ownership).", "license": "AGPL-3.0", diff --git a/workers/vault/src/vault-do.ts b/workers/vault/src/vault-do.ts index 7a30ad1..21e43a5 100644 --- a/workers/vault/src/vault-do.ts +++ b/workers/vault/src/vault-do.ts @@ -322,6 +322,33 @@ function liveSocketCount(sockets: WebSocket[]): number { */ const revocationTracker = new RevocationTracker(); +/** + * Delete every object under `prefix` (paginated, R2 bulk-delete caps at + * 1000/call). Returns the count deleted. Used by {@link VaultDO.purgeAttachments} + * (the import blow-away's attachments-only purge) and `handleDestroy` (the + * WHOLE `vault-/` prefix in one pass — attachments/, snapshots/, and + * exports/ together). A free function taking `bucket` explicitly, not a + * method — same reason as `pruneExportTarballs` in `export.ts`: it lets the + * cursor/chunking loop be pinned against a fake bucket in a test, without + * thousands of real R2 objects and without mutating the DO's shared `env` + * binding (which every other DO in the isolate also reads). + */ +export async function purgePrefix(bucket: R2Bucket, prefix: string): Promise { + let cursor: string | undefined; + let deleted = 0; + do { + const page = await bucket.list({ prefix, ...(cursor ? { cursor } : {}) }); + const keys = page.objects.map((o) => o.key); + for (let i = 0; i < keys.length; i += 1000) { + const batch = keys.slice(i, i + 1000); + await bucket.delete(batch); + deleted += batch.length; + } + cursor = page.truncated ? page.cursor : undefined; + } while (cursor); + return deleted; +} + export class VaultDO extends DurableObject { private shim: DatabaseShim; private store!: DoSqliteStore; @@ -358,6 +385,30 @@ export class VaultDO extends DurableObject { private r2Bytes = 0; private stateLoaded = false; + /** + * True once {@link handleDestroy} has wiped this instance's storage. + * `fetch()` checks this BEFORE `ensureState()` — see that call site's + * comment for why the ordering matters (it's what stops a later request on + * this same warm instance from re-arming alarms or re-persisting config + * into a DO whose storage was just deleted). Never persisted: it only needs + * to hold for the rest of THIS warm instance's life — a later cold wake + * (post-eviction) boots a fresh instance with `destroyed = false`, which is + * the documented residue (an `idFromName`-addressed empty-schema DO, same + * property any never-created vault name already has). + * + * Also checked, explicitly, at the top of every OTHER entry point that can + * fire on a warm instance: `webSocketMessage`, `webSocketClose`, + * `webSocketError`, `alarm()`. Today those four are safe without the guard + * too — `ensureStateForWake`'s warm fast-path never re-arms, and a non-warm + * instance reads null config from wiped storage — but that's EMERGENT + * safety, riding on other code's current shape. It would silently break if + * `ensureStateForWake` ever called `ensureState` unconditionally, or if the + * embedding provider got cached in a way that survives `deleteAll`, and no + * test would catch it. The guard makes the invariant durable instead of + * incidental, on the exact primitive the account-delete cascade extends. + */ + private destroyed = false; + // Monthly voice-minutes meter (loaded lazily with the rest of DO state). // `transcribeMinutes` is the running float of minutes used in // `transcribeMonth` (UTC "YYYY-MM"); a month rollover resets it lazily. @@ -515,6 +566,32 @@ export class VaultDO extends DurableObject { const vaultName = decodeURIComponent(m[1]!); const rest = m[2] ?? ""; + // Vault destroy (cloud#226 PR-1) — dispatched BEFORE ensureState and + // everything else, deliberately. `handleDestroy` touches only R2 + DO + // storage + WS sockets (never `this.config`/`this.store`), so it's safe + // to run this early — and it MUST run this early: ensureState's + // maybeArmEmbeddingBackfill re-arms the embedding alarm on every wake + // unless the backfill is already known-done, which would re-fire the + // very alarm `handleDestroy` just cleared. Running the destroy route + // (idempotent retries included) ahead of ensureState is what keeps a + // warm-but-destroyed instance from resurrecting any state. Auth here is + // the SAME gate every other /internal/* route uses + // (authenticateVaultRequest + internalForbidden) — neither depends on DO + // state, so hoisting them ahead of ensureState changes nothing about what + // they enforce. + if (rest === "/api/internal/destroy") { + const destroyAuth = await authenticateVaultRequest(request, this.env, vaultName); + if ("error" in destroyAuth) return destroyAuth.error; + const destroyForbidden = this.internalForbidden(destroyAuth, vaultName); + if (destroyForbidden) return destroyForbidden; + return this.handleDestroy(request, vaultName); + } + // Every OTHER route on an already-destroyed warm instance: 410, no + // writes, no ensureState (the same resurrection concern as above). + if (this.destroyed) { + return json({ error: "Gone", error_type: "vault_destroyed" }, 410); + } + await this.ensureState(vaultName); // Live-query WS binding: rebuild in-memory subscriptions from the sockets' @@ -690,6 +767,8 @@ export class VaultDO extends DurableObject { // fixes it), and a FULL vault especially needs its nightly snapshot to // land. One shared authorization gate (first-party/operator only — see // internalForbidden); the wire contract for these is cloud-runtime only. + // NOTE: /internal/destroy is NOT listed below — it's dispatched at the + // very top of `fetch()`, ahead of `ensureState()`; see that call site. if (apiPath.startsWith("/internal/")) { const forbidden = this.internalForbidden(auth, vaultName); if (forbidden) return forbidden; @@ -1404,21 +1483,93 @@ export class VaultDO extends DurableObject { }); } - /** Delete every object under the vault's attachments prefix (paginated, - * R2 bulk-delete caps at 1000/call). The R2 half of a blow-away import — - * distinct from the `exports/` + `snapshots/` prefixes, which it never - * touches. */ + /** Delete every object under the vault's attachments prefix. The R2 half of + * a blow-away import — distinct from the `exports/` + `snapshots/` + * prefixes, which it never touches (unlike {@link handleDestroy}'s + * whole-vault purge). */ private async purgeAttachments(vaultName: string): Promise { - const prefix = r2Key(vaultName, ""); - let cursor: string | undefined; - do { - const page = await this.env.ATTACHMENTS.list({ prefix, ...(cursor ? { cursor } : {}) }); - const keys = page.objects.map((o) => o.key); - for (let i = 0; i < keys.length; i += 1000) { - await this.env.ATTACHMENTS.delete(keys.slice(i, i + 1000)); - } - cursor = page.truncated ? page.cursor : undefined; - } while (cursor); + await purgePrefix(this.env.ATTACHMENTS, r2Key(vaultName, "")); + } + + /** + * POST /api/internal/destroy — irrevocably erase this vault (PR-1 of the + * vault-delete train, cloud#226). Dispatched from `fetch()` BEFORE + * `ensureState()` (see that call site's comment) — this method never + * touches `this.config`/`this.store`, only R2 + DO storage + WS sockets, so + * running it that early is safe. Body: `{"confirm":""}`, an + * exact match against the canonical (already-lowercased) vault name — + * defense-in-depth at the internal seam; the PRIMARY guard is the caller's + * auth gate (`internalForbidden`, already enforced in `fetch()` before this + * runs). + * + * Sequence (each step precedes the next), run inside a SINGLE + * `blockConcurrencyWhile` (see below for why): + * 1. Close every hibernatable WebSocket with 1001 (Going Away) — live- + * query sockets die now rather than lingering to their TTL. + * 2. Purge every R2 object under `vault-/` — ONE prefix covers all + * three families (attachments/, snapshots/, exports/), generalizing + * {@link purgeAttachments} via {@link purgePrefix}. The trailing slash + * is load-bearing: `vault-foo/` must never also match `vault-foobar/`. + * 3. `deleteAlarm()` THEN `deleteAll()` — `deleteAll()` does NOT clear a + * pending alarm, and this DO arms transcription/embedding alarms that + * must not fire against a destroyed vault. + * 4. Set `this.destroyed` LAST — `fetch()` checks this on every later + * request to this warm instance and short-circuits to 410 before + * `ensureState()` can re-persist anything. Setting it any earlier + * would let a failed purge leave a flagged instance that then SKIPS + * re-purge on retry — an R2 leak, worse than what this guards. + * + * Steps 1-4 run inside `ctx.blockConcurrencyWhile` — `purgePrefix` awaits + * multiple R2 round-trips, and DO event interleaving across awaits means a + * concurrent write request could otherwise land a new note in the gap + * between the purge finishing and `deleteAll()` clearing storage (the same + * DO-event-interleaving hazard `handleSnapshot`'s CONCURRENCY NOTE names + * above, for a far lower-stakes verb). `blockConcurrencyWhile` defers every + * other event on this instance — fetch, alarm, WS — until the callback + * resolves, closing that window. The response is still only built and + * returned AFTER the callback resolves, so this doesn't reintroduce the + * "respond before flush" problem `ctx.abort()` would cause. + * + * The `this.destroyed` short-circuit ABOVE the block (next line) still + * matters even with the lock: it's what makes a retry skip the R2 rescan + * instead of paying for a full (now-empty) `list()` pass every time. + * + * Idempotent: a second call (the identity-side cascade's retry) short- + * circuits on the flag, skips the R2 rescan, and returns the same shape + * with zero new deletions. Deliberately does NOT call `ctx.abort()` before + * responding — that would kill this reply in flight, and it's unnecessary: + * the flag alone already guarantees no further write lands. + */ + private async handleDestroy(request: Request, vaultName: string): Promise { + if (request.method !== "POST") return json({ error: "Method not allowed" }, 405); + let body: { confirm?: unknown }; + try { + body = (await request.json()) as { confirm?: unknown }; + } catch { + return json({ error: "Invalid JSON body" }, 400); + } + if (body.confirm !== vaultName) { + return json( + { error: "confirm must exactly match the vault name", error_type: "destroy_confirm_mismatch" }, + 400, + ); + } + if (this.destroyed) return json({ destroyed: true, r2_objects_deleted: 0 }); + + const deleted = await this.ctx.blockConcurrencyWhile(async () => { + for (const ws of this.ctx.getWebSockets()) this.closeWs(ws, 1001, "vault destroyed"); + + const n = await purgePrefix(this.env.ATTACHMENTS, `vault-${vaultName}/`); + + await this.ctx.storage.deleteAlarm(); + await this.ctx.storage.deleteAll(); + + this.destroyed = true; + return n; + }); + + console.log(`[destroy ${vaultName}] r2_objects_deleted=${deleted}`); + return json({ destroyed: true, r2_objects_deleted: deleted }); } private importTooLargeResponse(): Response { @@ -1646,6 +1797,10 @@ export class VaultDO extends DurableObject { this.closeWs(ws, WS_CLOSE.PROTOCOL, "vault unavailable"); return; } + // Explicit guard, not just emergent safety from `ensureStateForWake`'s + // warm fast-path never re-arming: see the field doc on `destroyed` for + // why this can't be left to fall out of other code incidentally. + if (this.destroyed) return; const vaultName = await this.ensureStateForWake(); if (!vaultName) { this.closeWs(ws, WS_CLOSE.PROTOCOL, "vault not initialized"); @@ -1682,6 +1837,7 @@ export class VaultDO extends DurableObject { async webSocketClose(ws: WebSocket, _code: number, _reason: string, _wasClean: boolean): Promise { if (this.bootError) return; + if (this.destroyed) return; await this.ensureStateForWake(); await this.ensureSubscriptionsRehydrated(); this.cleanupSocket(ws); @@ -1689,6 +1845,7 @@ export class VaultDO extends DurableObject { async webSocketError(ws: WebSocket, _error: unknown): Promise { if (this.bootError) return; + if (this.destroyed) return; await this.ensureStateForWake(); await this.ensureSubscriptionsRehydrated(); this.cleanupSocket(ws); @@ -2062,6 +2219,7 @@ export class VaultDO extends DurableObject { */ async alarm(): Promise { if (this.bootError) return; + if (this.destroyed) return; // Reentrancy guard (cloud#171 hardening), checked+set SYNCHRONOUSLY // before any `await` — closes it against a genuinely CONCURRENT second // `alarm()` invocation on this same DO instance, not just a mid-wake diff --git a/workers/vault/test/destroy.test.ts b/workers/vault/test/destroy.test.ts new file mode 100644 index 0000000..56f4fbe --- /dev/null +++ b/workers/vault/test/destroy.test.ts @@ -0,0 +1,241 @@ +/** + * Vault destroy — POST /api/internal/destroy (PR-1 of the vault-delete + * train, cloud#226). The vault worker's half of tenant-initiated deletion: + * closes live WS sockets, purges every R2 object under the vault's prefix + * (attachments/ + snapshots/ + exports/ in ONE pass), wipes DO storage + * (alarms first, `deleteAll()` doesn't clear them), and flags the warm + * instance so nothing on it can re-persist before the next eviction. + * + * Authorization is the PRE-EXISTING platform-vs-tenant gate + * (`internalForbidden`, same as every other `/api/internal/*` seam) — this + * suite pins the confirm-body defense-in-depth on top of it, plus the + * destructive sequence itself: the prefix-boundary pin (a missing trailing + * slash would over-delete a sibling vault), the warm-instance 410 with NO + * storage write landing, idempotency (the identity-side cascade retries this + * call), and `purgePrefix`'s own pagination/chunking loop at past-1000-object + * scale (a fake-bucket unit test — see the closing `describe`). + * + * NOT covered here: the `destroyed` guard added to `webSocketMessage` / + * `webSocketClose` / `webSocketError` / `alarm()`. Verified by mutation that + * it is CURRENTLY UNREACHABLE dead code — after `deleteAll()`, storage has no + * `config` key, so every one of those methods already no-ops via its OWN + * pre-existing `if (!vaultName) return` before the new guard could matter. + * That's the point: it's durability hardening against a FUTURE change to + * those methods' config-recovery path, not a behavior fixable/observable + * today — see the field doc on `VaultDO.destroyed`. A test asserting "nothing + * bad happens" here would pass identically with the guard deleted, which is + * exactly the vacuous-sentinel shape to avoid, so none is written. + */ +import { SELF, env, runInDurableObject } from "cloudflare:test"; +import { describe, expect, it } from "vitest"; +import { FIRST_PARTY_CLIENT_ID } from "../src/auth.ts"; +import { purgePrefix } from "../src/vault-do.ts"; +import { base, createNote, freshVault, mintToken, op, OP } from "./helpers.ts"; + +function firstPartyToken(vault: string): Promise { + return mintToken({ vault, scopes: `vault:${vault}:admin`, vaultScope: [vault], clientId: FIRST_PARTY_CLIENT_ID }); +} + +function destroyReq(vault: string, token: string, confirm: unknown): Promise { + return SELF.fetch(`${base(vault)}/api/internal/destroy`, { + method: "POST", + headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: JSON.stringify({ confirm }), + }); +} + +function doStub(vault: string): DurableObjectStub { + return env.VAULT.get(env.VAULT.idFromName(vault)) as unknown as DurableObjectStub; +} + +/** Upload a file via the operator token — mirrors internal-config.test.ts. */ +function upload(vault: string, size: number, name = "clip.bin"): Promise { + const fd = new FormData(); + fd.append("file", new File([new Uint8Array(size)], name, { type: "application/octet-stream" })); + return op(vault, "/api/storage/upload", { method: "POST", body: fd }); +} + +function snapshotReq(vault: string): Promise { + return SELF.fetch(`${base(vault)}/api/internal/snapshot`, { + method: "POST", + headers: { authorization: `Bearer ${OP}`, "content-type": "application/json" }, + body: JSON.stringify({ retention: { daily: 14, weekly: 8, monthly: 12 } }), + }); +} + +/** Seed a vault with a note, an attachment, a snapshot, AND an export — one + * object under each of the three R2 families the destroy purge must cover. */ +async function seedAllThreeFamilies(v: string): Promise { + await createNote(v, { content: "destroy me" }); + expect((await upload(v, 128)).status).toBe(201); + expect((await snapshotReq(v)).status).toBe(200); + const exp = await op(v, "/api/export"); + expect(exp.status).toBe(200); + await exp.arrayBuffer(); // drain the stream so the R2 write below it settles +} + +async function keysUnder(prefix: string): Promise { + const page = await env.ATTACHMENTS.list({ prefix }); + return page.objects.map((o) => o.key); +} + +describe("POST /api/internal/destroy — the erasure sequence", () => { + it("purges attachments/ + snapshots/ + exports/ in one pass and reports the count", async () => { + const v = freshVault("dx"); + await seedAllThreeFamilies(v); + const before = await keysUnder(`vault-${v}/`); + // attachment + snapshot tarball + snapshot manifest + export tarball, at least. + expect(before.length).toBeGreaterThanOrEqual(4); + expect(before.some((k) => k.startsWith(`vault-${v}/attachments/`))).toBe(true); + expect(before.some((k) => k.startsWith(`vault-${v}/snapshots/`))).toBe(true); + expect(before.some((k) => k.startsWith(`vault-${v}/exports/`))).toBe(true); + + const res = await destroyReq(v, await firstPartyToken(v), v); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body).toEqual({ destroyed: true, r2_objects_deleted: before.length }); + + expect(await keysUnder(`vault-${v}/`)).toEqual([]); + }); + + it("prefix boundary: an object under vault-x/ SURVIVES (the trailing slash is load-bearing)", async () => { + const v = freshVault("dx"); + await createNote(v, { content: "just a note, nothing fancy" }); + // A sibling vault whose name happens to be this vault's name plus a + // suffix — `vault-/` must not also match `vault-x/`. + const neighborKey = `vault-${v}x/attachments/2026-01-01/neighbor.bin`; + await env.ATTACHMENTS.put(neighborKey, new Uint8Array([1, 2, 3])); + + const res = await destroyReq(v, await firstPartyToken(v), v); + expect(res.status).toBe(200); + + expect(await keysUnder(`vault-${v}x/`)).toEqual([neighborKey]); + await env.ATTACHMENTS.delete(neighborKey); // tidy the shared bucket for later tests + }); + + it("warm-instance follow-up request → 410, and NO storage write lands", async () => { + const v = freshVault("dx"); + await createNote(v, { content: "written before destroy" }); + expect((await destroyReq(v, await firstPartyToken(v), v)).status).toBe(200); + + // Read-shaped follow-up: also 410, no exemption for reads. + const landing = await SELF.fetch(base(v)); + expect(landing.status).toBe(410); + expect(((await landing.json()) as any).error_type).toBe("vault_destroyed"); + + // Write-shaped follow-up: 410, and it must not land. + const write = await SELF.fetch(`${base(v)}/api/notes`, { + method: "POST", + headers: { authorization: `Bearer ${OP}`, "content-type": "application/json" }, + body: JSON.stringify({ content: "should never exist" }), + }); + expect(write.status).toBe(410); + + // Ground truth: DO storage is empty and no alarm got re-armed — proves + // ensureState() never ran on either follow-up (the 410 short-circuit is + // what stops maybeArmEmbeddingBackfill from re-persisting/re-arming). + await runInDurableObject(doStub(v), async (inst: any) => { + const keys = [...(await inst.ctx.storage.list()).keys()]; + expect(keys).toEqual([]); + expect(await inst.ctx.storage.getAlarm()).toBeNull(); + }); + }); + + it("a tenant-shaped OAuth admin token (non-first-party client_id) is refused — the existing platform gate", async () => { + const v = freshVault("dx"); + await createNote(v, { content: "still here" }); + const tenantAdmin = await mintToken({ + vault: v, + scopes: `vault:${v}:admin`, + vaultScope: [v], + clientId: "3f6a2e9b-1111-4222-8333-444455556666", + }); + const res = await destroyReq(v, tenantAdmin, v); + expect(res.status).toBe(403); + expect(((await res.json()) as any).error_type).toBe("internal_config_forbidden"); + + // Untouched — the refused call did nothing. + const landing = await op(v, ""); + expect(landing.status).toBe(200); + }); + + it("a confirm mismatch (or missing confirm) is refused with 400 — defense-in-depth, not the primary guard", async () => { + const v = freshVault("dx"); + const token = await firstPartyToken(v); + + const mismatch = await destroyReq(v, token, "not-the-vault-name"); + expect(mismatch.status).toBe(400); + expect(((await mismatch.json()) as any).error_type).toBe("destroy_confirm_mismatch"); + + const missing = await SELF.fetch(`${base(v)}/api/internal/destroy`, { + method: "POST", + headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, + body: JSON.stringify({}), + }); + expect(missing.status).toBe(400); + + // Untouched — neither refused call did anything. + expect((await op(v, "")).status).toBe(200); + }); + + it("a second destroy is a 200 no-op — idempotent for the identity-side cascade's retry", async () => { + const v = freshVault("dx"); + await createNote(v, { content: "one and done" }); + const token = await firstPartyToken(v); + + const first = await destroyReq(v, token, v); + expect(first.status).toBe(200); + + const second = await destroyReq(v, token, v); + expect(second.status).toBe(200); + expect((await second.json()) as any).toEqual({ destroyed: true, r2_objects_deleted: 0 }); + }); + + // "Cold idempotence" (destroy, evict, destroy again → same 200/0 shape) was + // requested on review but is SKIPPED, not just omitted: `__simulateEviction` + // (vault-do.ts, the vitest-pool-can't-evict-a-SQLite-DO workaround) only + // drops in-memory SUBSCRIPTION state — manager hooks, `wsSubs`, the + // rehydration flag. It never touches `this.destroyed`, so calling it between + // two destroys would leave `destroyed` still `true` on the SAME instance, + // making that test byte-for-byte the warm-instance idempotent-retry test + // above (a duplicate that only LOOKS like new coverage). A true cold + // instance is exactly what a fresh-`idFromName` `freshVault()` already gives + // every OTHER test in this file — there is no way, in this harness, to + // evict-and-rewake the SAME instance to get a genuinely fresh `destroyed`. +}); + +describe("purgePrefix — paginated listing (unit, fake bucket)", () => { + it("walks truncated list pages and deletes every key, chunking deletes at 1000", async () => { + // 1500 keys across 2 pages exercises the cursor loop (list's own page cap) + // AND the 1000-key delete-chunking loop, without 1500 real R2 objects — + // same convention as `pruneExportTarballs`'s fake-bucket test in + // export.test.ts. + const mk = (i: number) => `vault-x/attachments/2026-01-01/${String(i).padStart(4, "0")}.bin`; + const all = Array.from({ length: 1500 }, (_, i) => mk(i)); + const pages = [all.slice(0, 1000), all.slice(1000)]; + const deleted: string[][] = []; + let listCalls = 0; + const fake = { + list: async (opts: { prefix?: string; cursor?: string }) => { + expect(opts.prefix).toBe("vault-x/"); + const idx = opts.cursor ? Number(opts.cursor) : 0; + listCalls++; + const truncated = idx < pages.length - 1; + return { + objects: pages[idx]!.map((key) => ({ key })), + truncated, + ...(truncated ? { cursor: String(idx + 1) } : {}), + }; + }, + delete: async (keys: string | string[]) => { + deleted.push(Array.isArray(keys) ? keys : [keys]); + }, + } as unknown as R2Bucket; + + const count = await purgePrefix(fake, "vault-x/"); + expect(listCalls).toBe(2); + expect(count).toBe(1500); + for (const chunk of deleted) expect(chunk.length).toBeLessThanOrEqual(1000); + expect(deleted.flat()).toEqual(all); + }); +});