feat(vault): POST /api/internal/destroy — vault erasure (cloud#226 PR-1)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openparachute/cloud",
-  "version": "0.0.8-rc.127",
+  "version": "0.0.8-rc.128",
   "private": true,
   "description": "Open Parachute PBC's Vault Cloud \u2014 one Durable Object per vault on Cloudflare, OAuth issuer + self-serve console (accounts + vault ownership).",
   "license": "AGPL-3.0",

workers/vault/src/vault-do.ts

@@ -322,6 +322,33 @@ function liveSocketCount(sockets: WebSocket[]): number {
  */
 const revocationTracker = new RevocationTracker();
 
+/**
+ * Delete every object under `prefix` (paginated, R2 bulk-delete caps at
+ * 1000/call). Returns the count deleted. Used by {@link VaultDO.purgeAttachments}
+ * (the import blow-away's attachments-only purge) and `handleDestroy` (the
+ * WHOLE `vault-<name>/` prefix in one pass — attachments/, snapshots/, and
+ * exports/ together). A free function taking `bucket` explicitly, not a
+ * method — same reason as `pruneExportTarballs` in `export.ts`: it lets the
+ * cursor/chunking loop be pinned against a fake bucket in a test, without
+ * thousands of real R2 objects and without mutating the DO's shared `env`
+ * binding (which every other DO in the isolate also reads).
+ */
+export async function purgePrefix(bucket: R2Bucket, prefix: string): Promise<number> {
+  let cursor: string | undefined;
+  let deleted = 0;
+  do {
+    const page = await bucket.list({ prefix, ...(cursor ? { cursor } : {}) });
+    const keys = page.objects.map((o) => o.key);
+    for (let i = 0; i < keys.length; i += 1000) {
+      const batch = keys.slice(i, i + 1000);
+      await bucket.delete(batch);
+      deleted += batch.length;
+    }
+    cursor = page.truncated ? page.cursor : undefined;
+  } while (cursor);
+  return deleted;
+}
+
 export class VaultDO extends DurableObject {
   private shim: DatabaseShim;
   private store!: DoSqliteStore;
@@ -358,6 +385,30 @@ export class VaultDO extends DurableObject {
   private r2Bytes = 0;
   private stateLoaded = false;
 
+  /**
+   * True once {@link handleDestroy} has wiped this instance's storage.
+   * `fetch()` checks this BEFORE `ensureState()` — see that call site's
+   * comment for why the ordering matters (it's what stops a later request on
+   * this same warm instance from re-arming alarms or re-persisting config
+   * into a DO whose storage was just deleted). Never persisted: it only needs
+   * to hold for the rest of THIS warm instance's life — a later cold wake
+   * (post-eviction) boots a fresh instance with `destroyed = false`, which is
+   * the documented residue (an `idFromName`-addressed empty-schema DO, same
+   * property any never-created vault name already has).
+   *
+   * Also checked, explicitly, at the top of every OTHER entry point that can
+   * fire on a warm instance: `webSocketMessage`, `webSocketClose`,
+   * `webSocketError`, `alarm()`. Today those four are safe without the guard
+   * too — `ensureStateForWake`'s warm fast-path never re-arms, and a non-warm
+   * instance reads null config from wiped storage — but that's EMERGENT
+   * safety, riding on other code's current shape. It would silently break if
+   * `ensureStateForWake` ever called `ensureState` unconditionally, or if the
+   * embedding provider got cached in a way that survives `deleteAll`, and no
+   * test would catch it. The guard makes the invariant durable instead of
+   * incidental, on the exact primitive the account-delete cascade extends.
+   */
+  private destroyed = false;
+
   // Monthly voice-minutes meter (loaded lazily with the rest of DO state).
   // `transcribeMinutes` is the running float of minutes used in
   // `transcribeMonth` (UTC "YYYY-MM"); a month rollover resets it lazily.
@@ -515,6 +566,32 @@ export class VaultDO extends DurableObject {
     const vaultName = decodeURIComponent(m[1]!);
     const rest = m[2] ?? "";
 
+    // Vault destroy (cloud#226 PR-1) — dispatched BEFORE ensureState and
+    // everything else, deliberately. `handleDestroy` touches only R2 + DO
+    // storage + WS sockets (never `this.config`/`this.store`), so it's safe
+    // to run this early — and it MUST run this early: ensureState's
+    // maybeArmEmbeddingBackfill re-arms the embedding alarm on every wake
+    // unless the backfill is already known-done, which would re-fire the
+    // very alarm `handleDestroy` just cleared. Running the destroy route
+    // (idempotent retries included) ahead of ensureState is what keeps a
+    // warm-but-destroyed instance from resurrecting any state. Auth here is
+    // the SAME gate every other /internal/* route uses
+    // (authenticateVaultRequest + internalForbidden) — neither depends on DO
+    // state, so hoisting them ahead of ensureState changes nothing about what
+    // they enforce.
+    if (rest === "/api/internal/destroy") {
+      const destroyAuth = await authenticateVaultRequest(request, this.env, vaultName);
+      if ("error" in destroyAuth) return destroyAuth.error;
+      const destroyForbidden = this.internalForbidden(destroyAuth, vaultName);
+      if (destroyForbidden) return destroyForbidden;
+      return this.handleDestroy(request, vaultName);
+    }
+    // Every OTHER route on an already-destroyed warm instance: 410, no
+    // writes, no ensureState (the same resurrection concern as above).
+    if (this.destroyed) {
+      return json({ error: "Gone", error_type: "vault_destroyed" }, 410);
+    }
+
     await this.ensureState(vaultName);
 
     // Live-query WS binding: rebuild in-memory subscriptions from the sockets'
@@ -690,6 +767,8 @@ export class VaultDO extends DurableObject {
     // fixes it), and a FULL vault especially needs its nightly snapshot to
     // land. One shared authorization gate (first-party/operator only — see
     // internalForbidden); the wire contract for these is cloud-runtime only.
+    // NOTE: /internal/destroy is NOT listed below — it's dispatched at the
+    // very top of `fetch()`, ahead of `ensureState()`; see that call site.
     if (apiPath.startsWith("/internal/")) {
       const forbidden = this.internalForbidden(auth, vaultName);
       if (forbidden) return forbidden;
@@ -1404,21 +1483,93 @@ export class VaultDO extends DurableObject {
     });
   }
 
-  /** Delete every object under the vault's attachments prefix (paginated,
-   *  R2 bulk-delete caps at 1000/call). The R2 half of a blow-away import —
-   *  distinct from the `exports/` + `snapshots/` prefixes, which it never
-   *  touches. */
+  /** Delete every object under the vault's attachments prefix. The R2 half of
+   *  a blow-away import — distinct from the `exports/` + `snapshots/`
+   *  prefixes, which it never touches (unlike {@link handleDestroy}'s
+   *  whole-vault purge). */
   private async purgeAttachments(vaultName: string): Promise<void> {
-    const prefix = r2Key(vaultName, "");
-    let cursor: string | undefined;
-    do {
-      const page = await this.env.ATTACHMENTS.list({ prefix, ...(cursor ? { cursor } : {}) });
-      const keys = page.objects.map((o) => o.key);
-      for (let i = 0; i < keys.length; i += 1000) {
-        await this.env.ATTACHMENTS.delete(keys.slice(i, i + 1000));
-      }
-      cursor = page.truncated ? page.cursor : undefined;
-    } while (cursor);
+    await purgePrefix(this.env.ATTACHMENTS, r2Key(vaultName, ""));
+  }
+
+  /**
+   * POST /api/internal/destroy — irrevocably erase this vault (PR-1 of the
+   * vault-delete train, cloud#226). Dispatched from `fetch()` BEFORE
+   * `ensureState()` (see that call site's comment) — this method never
+   * touches `this.config`/`this.store`, only R2 + DO storage + WS sockets, so
+   * running it that early is safe. Body: `{"confirm":"<vault name>"}`, an
+   * exact match against the canonical (already-lowercased) vault name —
+   * defense-in-depth at the internal seam; the PRIMARY guard is the caller's
+   * auth gate (`internalForbidden`, already enforced in `fetch()` before this
+   * runs).
+   *
+   * Sequence (each step precedes the next), run inside a SINGLE
+   * `blockConcurrencyWhile` (see below for why):
+   *   1. Close every hibernatable WebSocket with 1001 (Going Away) — live-
+   *      query sockets die now rather than lingering to their TTL.
+   *   2. Purge every R2 object under `vault-<name>/` — ONE prefix covers all
+   *      three families (attachments/, snapshots/, exports/), generalizing
+   *      {@link purgeAttachments} via {@link purgePrefix}. The trailing slash
+   *      is load-bearing: `vault-foo/` must never also match `vault-foobar/`.
+   *   3. `deleteAlarm()` THEN `deleteAll()` — `deleteAll()` does NOT clear a
+   *      pending alarm, and this DO arms transcription/embedding alarms that
+   *      must not fire against a destroyed vault.
+   *   4. Set `this.destroyed` LAST — `fetch()` checks this on every later
+   *      request to this warm instance and short-circuits to 410 before
+   *      `ensureState()` can re-persist anything. Setting it any earlier
+   *      would let a failed purge leave a flagged instance that then SKIPS
+   *      re-purge on retry — an R2 leak, worse than what this guards.
+   *
+   * Steps 1-4 run inside `ctx.blockConcurrencyWhile` — `purgePrefix` awaits
+   * multiple R2 round-trips, and DO event interleaving across awaits means a
+   * concurrent write request could otherwise land a new note in the gap
+   * between the purge finishing and `deleteAll()` clearing storage (the same
+   * DO-event-interleaving hazard `handleSnapshot`'s CONCURRENCY NOTE names
+   * above, for a far lower-stakes verb). `blockConcurrencyWhile` defers every
+   * other event on this instance — fetch, alarm, WS — until the callback
+   * resolves, closing that window. The response is still only built and
+   * returned AFTER the callback resolves, so this doesn't reintroduce the
+   * "respond before flush" problem `ctx.abort()` would cause.
+   *
+   * The `this.destroyed` short-circuit ABOVE the block (next line) still
+   * matters even with the lock: it's what makes a retry skip the R2 rescan
+   * instead of paying for a full (now-empty) `list()` pass every time.
+   *
+   * Idempotent: a second call (the identity-side cascade's retry) short-
+   * circuits on the flag, skips the R2 rescan, and returns the same shape
+   * with zero new deletions. Deliberately does NOT call `ctx.abort()` before
+   * responding — that would kill this reply in flight, and it's unnecessary:
+   * the flag alone already guarantees no further write lands.
+   */
+  private async handleDestroy(request: Request, vaultName: string): Promise<Response> {
+    if (request.method !== "POST") return json({ error: "Method not allowed" }, 405);
+    let body: { confirm?: unknown };
+    try {
+      body = (await request.json()) as { confirm?: unknown };
+    } catch {
+      return json({ error: "Invalid JSON body" }, 400);
+    }
+    if (body.confirm !== vaultName) {
+      return json(
+        { error: "confirm must exactly match the vault name", error_type: "destroy_confirm_mismatch" },
+        400,
+      );
+    }
+    if (this.destroyed) return json({ destroyed: true, r2_objects_deleted: 0 });
+
+    const deleted = await this.ctx.blockConcurrencyWhile(async () => {
+      for (const ws of this.ctx.getWebSockets()) this.closeWs(ws, 1001, "vault destroyed");
+
+      const n = await purgePrefix(this.env.ATTACHMENTS, `vault-${vaultName}/`);
+
+      await this.ctx.storage.deleteAlarm();
+      await this.ctx.storage.deleteAll();
+
+      this.destroyed = true;
+      return n;
+    });
+
+    console.log(`[destroy ${vaultName}] r2_objects_deleted=${deleted}`);
+    return json({ destroyed: true, r2_objects_deleted: deleted });
   }
 
   private importTooLargeResponse(): Response {
@@ -1646,6 +1797,10 @@ export class VaultDO extends DurableObject {
       this.closeWs(ws, WS_CLOSE.PROTOCOL, "vault unavailable");
       return;
     }
+    // Explicit guard, not just emergent safety from `ensureStateForWake`'s
+    // warm fast-path never re-arming: see the field doc on `destroyed` for
+    // why this can't be left to fall out of other code incidentally.
+    if (this.destroyed) return;
     const vaultName = await this.ensureStateForWake();
     if (!vaultName) {
       this.closeWs(ws, WS_CLOSE.PROTOCOL, "vault not initialized");
@@ -1682,13 +1837,15 @@ export class VaultDO extends DurableObject {
 
   async webSocketClose(ws: WebSocket, _code: number, _reason: string, _wasClean: boolean): Promise<void> {
     if (this.bootError) return;
+    if (this.destroyed) return;
     await this.ensureStateForWake();
     await this.ensureSubscriptionsRehydrated();
     this.cleanupSocket(ws);
   }
 
   async webSocketError(ws: WebSocket, _error: unknown): Promise<void> {
     if (this.bootError) return;
+    if (this.destroyed) return;
     await this.ensureStateForWake();
     await this.ensureSubscriptionsRehydrated();
     this.cleanupSocket(ws);
@@ -2062,6 +2219,7 @@ export class VaultDO extends DurableObject {
    */
   async alarm(): Promise<void> {
     if (this.bootError) return;
+    if (this.destroyed) return;
     // Reentrancy guard (cloud#171 hardening), checked+set SYNCHRONOUSLY
     // before any `await` — closes it against a genuinely CONCURRENT second
     // `alarm()` invocation on this same DO instance, not just a mid-wake