fix(smoke): split staging gate advisory-vs-fatal + O(1) snapshot sweep

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openparachute/cloud",
-  "version": "0.0.8-rc.125",
+  "version": "0.0.8-rc.126",
   "private": true,
   "description": "Open Parachute PBC's Vault Cloud \u2014 one Durable Object per vault on Cloudflare, OAuth issuer + self-serve console (accounts + vault ownership).",
   "license": "AGPL-3.0",

scripts/smoke-report.ts

@@ -0,0 +1,76 @@
+/**
+ * smoke-report.ts — the summary/exit logic for the live smokes, extracted as a
+ * PURE module (no side effects) so the advisory-vs-fatal gate rule can be
+ * unit-tested without running the live smoke itself. Imported by
+ * scripts/smoke-staging.ts; proven by test-bun/smoke-report.test.ts.
+ *
+ * ── THE RULE: advisory vs fatal, as a CATEGORY (not a hand-maintained list) ──
+ *
+ *   FATAL — gates the deploy (exit 1). The checked thing is BROKEN: a wrong
+ *   ANSWER is the failure. Contract shapes, status codes, auth refusals, scope
+ *   boundaries, error types, note counts. EVERY assert() in the smoke is fatal.
+ *   This is the safe default: if you cannot tell which side a new check falls
+ *   on, it is fatal.
+ *
+ *   ADVISORY — loud, but does NOT gate (exit still 0). We COULDN'T VERIFY the
+ *   thing right now. The ONLY failures allowed to be advisory are a live-infra
+ *   section's couldn't-complete errors: a client-side timeout or a network
+ *   unreachable while driving real third-party / fleet infrastructure (Workers
+ *   AI, R2, a cold Durable Object, a fleet sweep). Decide with isUnverifiable();
+ *   a live section that throws anything ELSE (a TypeError, a bad-shape parse) is
+ *   a real break and stays FATAL.
+ *
+ *   Why timeouts are only "couldn't verify": a client-side timeout cannot tell
+ *   a slow-but-healthy operation from a stuck-broken one. So a timeout is never
+ *   proof the thing is broken — only proof we didn't get to check it. The
+ *   contract assertions INSIDE each live section stay fatal, so an endpoint
+ *   that answers WRONG (rather than not at all) still fails the gate.
+ *
+ * ── Three hard invariants (all pinned in smoke-report.test.ts) ──
+ *   1. Advisories never gate: exitCode depends on `fail` ALONE.
+ *   2. Advisories can't hide a fatal: any fail > 0 exits 1 and reads FAILED,
+ *      no matter how many advisories rode along.
+ *   3. Advisories are never silent: advisory > 0 always shows in the headline,
+ *      so a run carrying advisories never reads as a clean green.
+ */
+
+export interface SmokeCounts {
+  pass: number;
+  fail: number;
+  advisory: number;
+}
+
+export interface SmokeSummary {
+  exitCode: 0 | 1;
+  headline: string;
+  /** True only when nothing failed AND nothing was left unverified. */
+  clean: boolean;
+}
+
+/**
+ * The one place the gate verdict is decided. `exitCode` is a function of `fail`
+ * alone — advisories are reported but never gate (invariants 1 & 2). When any
+ * advisory is present it is always named in the headline (invariant 3).
+ */
+export function summarize(c: SmokeCounts, label = "SMOKE"): SmokeSummary {
+  const exitCode: 0 | 1 = c.fail === 0 ? 0 : 1;
+  const verdict = c.fail === 0 ? "PASSED" : "FAILED";
+  const advisoryTag = c.advisory > 0 ? `, ${c.advisory} advisory (UNVERIFIED — see below)` : "";
+  const headline = `${label} ${verdict} — ${c.pass} pass, ${c.fail} fail${advisoryTag}`;
+  return { exitCode, headline, clean: c.fail === 0 && c.advisory === 0 };
+}
+
+/**
+ * True when an error is the "couldn't reach / didn't finish in time" class — an
+ * AbortSignal.timeout (a TimeoutError), an AbortError, or a fetch network
+ * failure. These are the ONLY failures a live-infra section may downgrade to
+ * advisory; every other throw stays fatal (see THE RULE above).
+ */
+export function isUnverifiable(err: unknown): boolean {
+  const name = (err as { name?: string } | null)?.name ?? "";
+  if (name === "TimeoutError" || name === "AbortError") return true;
+  const message = String((err as { message?: string } | null)?.message ?? err);
+  return /timed out|timeout|fetch failed|network|unreachable|ECONNRESET|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket|terminated/i.test(
+    message,
+  );
+}

scripts/smoke-staging.ts

@@ -41,6 +41,7 @@ import { readFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { totpCodeAt } from "../workers/identity/src/totp.ts";
+import { isUnverifiable, summarize } from "./smoke-report.ts";
 // core resolves from the sibling parachute-vault checkout, copied into the vault
 // worker's node_modules by `bun install` (the same explicit path test-bun uses).
 import { GETTING_STARTED_PACK, welcomePack } from "../workers/vault/node_modules/@openparachute/core/src/seed-packs.js";
@@ -53,7 +54,13 @@ const REDIRECT_URI = "http://localhost:8976/callback";
 const MARKER = `smoke-${Date.now()}`;
 
 // --- tiny test harness -----------------------------------------------------
+// fail() is FATAL — it gates the deploy (exit 1). advisory() is LOUD but does
+// NOT gate — it records "we couldn't verify this right now" (a live-infra
+// timeout/unreachable, never a broken contract). The rule + the exit verdict
+// live in scripts/smoke-report.ts; see liveCatch() below for where advisories
+// come from.
 let failures = 0;
+let advisories = 0;
 const results: string[] = [];
 function ok(label: string, detail = "") {
   results.push(`  PASS  ${label}${detail ? ` — ${detail}` : ""}`);
@@ -64,9 +71,26 @@ function fail(label: string, detail = "") {
   results.push(`  FAIL  ${label}${detail ? ` — ${detail}` : ""}`);
   console.error(`\x1b[31mFAIL\x1b[0m ${label}${detail ? ` — ${detail}` : ""}`);
 }
+function advisory(label: string, detail = "") {
+  advisories++;
+  results.push(`  ADVISORY  ${label}${detail ? ` — ${detail}` : ""}`);
+  console.error(`\x1b[33mADVISORY\x1b[0m ${label}${detail ? ` — ${detail}` : ""}`);
+}
 function assert(cond: unknown, label: string, detail = "") {
   cond ? ok(label, detail) : fail(label, detail);
 }
+/**
+ * The couldn't-verify escape hatch for the live-infra sections (snapshots,
+ * voice, semantic, tickets, mock-E2E, account-MCP fan-out). A timeout/network
+ * throw while driving real third-party/fleet infra is ADVISORY (loud, does not
+ * gate — see scripts/smoke-report.ts). ANY OTHER throw is a real break and
+ * stays FATAL. The contract assert()s INSIDE each section are always fatal, so
+ * an endpoint that answers WRONG (not merely slowly) still fails the gate.
+ */
+function liveCatch(label: string, err: unknown): void {
+  if (isUnverifiable(err)) advisory(`${label} UNVERIFIED (live-infra timeout/unreachable — did not gate)`, String(err));
+  else fail(`${label} threw (unexpected — not a timeout)`, String(err));
+}
 
 // --- helpers ---------------------------------------------------------------
 function b64url(bytes: Uint8Array): string {
@@ -1363,7 +1387,7 @@ async function main() {
       );
     }
   } catch (err) {
-    fail("tier-change: live section threw (non-fatal — sections continue)", String(err));
+    liveCatch("tier-change: live section", err);
   }
 
   // 17. GFS snapshots + restore (Wave 4e). The arrival user is on the no-card
@@ -1375,22 +1399,26 @@ async function main() {
   //     notes round-trip. The live restore round-trip is wrapped so a slow DO
   //     import can't abort the sections that follow.
   //
-  //     cloud#166: this is a GENUINELY HEAVY round trip — the sweep walks
-  //     EVERY staging vault (grows with debris; see scripts/staging-sweep.ts)
-  //     and the restore itself is a full R2→new-DO tar import — and two
-  //     staging runs threw a client-side TimeoutError here as the fleet grew
-  //     past ~140 vaults. Every fetch in this section gets an EXPLICIT,
-  //     generous timeout (rather than whatever implicit default the runtime
-  //     was hitting) so a slow-but-healthy round trip has room to finish; a
-  //     genuinely stuck one still fails this section cleanly instead of
-  //     hanging the whole smoke.
+  //     cloud#166: the sweep is SCOPED to this run's own vault (?vault=), so
+  //     it is O(1), not O(fleet). Originally it walked EVERY staging vault;
+  //     as the fleet grew with smoke debris past ~140 vaults it crossed the
+  //     client timeout, and — because the sweep runs FIRST — the whole restore
+  //     section (and the deploy gate) went red for the growing fleet. Scoping
+  //     it to the one vault the test actually needs a snapshot of removes the
+  //     fleet-size dependency entirely; the nightly cron still sweeps the
+  //     whole fleet, tested unit-side (workers/identity/test/snapshots.test.ts).
+  //     The restore itself is still a real R2→new-DO tar import (O(1) per
+  //     vault); each fetch keeps an explicit, generous timeout so a
+  //     slow-but-healthy round trip has room to finish, and a genuinely stuck
+  //     one is recorded ADVISORY (couldn't verify) via liveCatch, not fatal.
   const RESTORE_ROUNDTRIP_TIMEOUT_MS = 120_000;
   try {
     const arrivalCsrf = /parachute_id_csrf=([^;]+)/.exec(arrivalCookie)?.[1] ?? "";
-    // One sweep tick via the staging-only trigger. The arrival vault is fresh
-    // this run → its snapshot (paid retention) is taken now.
+    // One sweep tick via the staging-only trigger, SCOPED to this run's fresh
+    // vault (?vault=) so it stays O(1). That vault's snapshot (paid retention)
+    // is taken now → sweep.taken === 1.
     const runSweep = async (): Promise<{ day: string; vaults: number; taken: number; skipped: number; failed: number; capped: boolean } | null> => {
-      const r = await fetch(`${IDENTITY}/__test/snapshot-run`, { method: "POST", signal: AbortSignal.timeout(RESTORE_ROUNDTRIP_TIMEOUT_MS) });
+      const r = await fetch(`${IDENTITY}/__test/snapshot-run?vault=${encodeURIComponent(arrivalVault)}`, { method: "POST", signal: AbortSignal.timeout(RESTORE_ROUNDTRIP_TIMEOUT_MS) });
       return r.status === 200 ? ((await r.json()) as { day: string; vaults: number; taken: number; skipped: number; failed: number; capped: boolean }) : null;
     };
     const sweep = await runSweep();
@@ -1461,7 +1489,7 @@ async function main() {
       }
     }
   } catch (err) {
-    fail("snapshots: live restore round-trip threw (non-fatal — sections continue)", String(err));
+    liveCatch("snapshots: live restore round-trip", err);
   }
 
   // 18. Voice transcription (cloud#56) — comp the arrival user to the PLUS
@@ -1602,7 +1630,7 @@ async function main() {
       );
     }
   } catch (err) {
-    fail("voice: live transcription section threw (non-fatal — sections continue)", String(err));
+    liveCatch("voice: live transcription section", err);
   }
 
   // 18b. Semantic search via Workers AI (C2, EXPERIMENTAL) — the one thing no
@@ -1683,7 +1711,7 @@ async function main() {
       );
     }
   } catch (err) {
-    fail("semantic: live embedding section threw (non-fatal — sections continue)", String(err));
+    liveCatch("semantic: live embedding section", err);
   }
 
   // 18c. Attachment tickets (cloud#177's DO mirror) — the pre-prod condition
@@ -1833,7 +1861,7 @@ async function main() {
       );
     }
   } catch (err) {
-    fail("tickets: live ticket round-trip section threw (non-fatal — sections continue)", String(err));
+    liveCatch("tickets: live ticket round-trip section", err);
   }
 
   // 19. MOCK-upgrade E2E (mock-payments) — the live end-to-end proof, run LAST
@@ -1948,7 +1976,7 @@ async function main() {
       }
     }
   } catch (err) {
-    fail("mock E2E: live section threw (non-fatal — sections continue)", String(err));
+    liveCatch("mock E2E: live section", err);
   }
 
   // 20. Pricing-model ENFORCEMENT E2E (the two-meter caps + the frozen floor) —
@@ -2428,13 +2456,23 @@ async function main() {
       }
     }
   } catch (err) {
-    fail("account-mcp: live section threw (non-fatal — summary follows)", String(err));
+    liveCatch("account-mcp: live section", err);
   }
 
   // --- summary ---
-  console.log(`\n${"=".repeat(60)}\nSMOKE ${failures === 0 ? "PASSED" : "FAILED"} — ${results.filter((r) => r.includes("PASS")).length} pass, ${failures} fail\n${"=".repeat(60)}`);
+  // The verdict is decided in scripts/smoke-report.ts: fatals gate (exit 1),
+  // advisories are loud but never gate and can never hide a fatal.
+  const passCount = results.filter((r) => r.startsWith("  PASS")).length;
+  const { exitCode, headline } = summarize({ pass: passCount, fail: failures, advisory: advisories });
+  console.log(`\n${"=".repeat(60)}\n${headline}\n${"=".repeat(60)}`);
   console.log(results.join("\n"));
-  process.exit(failures === 0 ? 0 : 1);
+  if (advisories > 0) {
+    // Re-surface the advisories on their own line so an "we couldn't verify
+    // this" is never buried under ~160 PASS lines and read as a clean green.
+    console.log(`\n${advisories} ADVISORY (live-infra unverified — did NOT gate the deploy; investigate if persistent):`);
+    for (const r of results.filter((r) => r.startsWith("  ADVISORY"))) console.log(r);
+  }
+  process.exit(exitCode);
 }
 
 /**

test-bun/smoke-report.test.ts

@@ -0,0 +1,86 @@
+/**
+ * The live-smoke gate rule, pinned. smoke-report.ts is what turned a chronic
+ * live-infra timeout into a hard, prod-blocking gate for nine days (cloud#166
+ * follow-up): a section labeled "non-fatal — sections continue" still called
+ * fail() and so still exit 1. These tests pin the three invariants that keep
+ * the advisory downgrade honest — advisories are loud, never gate, and can
+ * NEVER hide a real (fatal) failure.
+ *
+ * Pure module, no network — runs under the root `bun test` suite (package.json
+ * "test": "bun test src test-bun"), never imports smoke-staging.ts (which runs
+ * a live main() on import).
+ */
+import { describe, expect, it } from "bun:test";
+import { isUnverifiable, summarize } from "../scripts/smoke-report.ts";
+
+describe("summarize — the gate verdict", () => {
+  it("clean run: no fail, no advisory → exit 0, PASSED, reads clean", () => {
+    const s = summarize({ pass: 163, fail: 0, advisory: 0 });
+    expect(s.exitCode).toBe(0);
+    expect(s.clean).toBe(true);
+    expect(s.headline).toBe("SMOKE PASSED — 163 pass, 0 fail");
+    expect(s.headline).not.toMatch(/advisory/i);
+  });
+
+  it("INVARIANT 1 — advisories never gate: fail 0 + advisory > 0 → exit 0", () => {
+    const s = summarize({ pass: 163, fail: 0, advisory: 2 });
+    expect(s.exitCode).toBe(0); // the whole point: an unverified live section does NOT block prod
+    expect(s.headline).toMatch(/PASSED/);
+  });
+
+  it("INVARIANT 3 — advisories are never silent: advisory > 0 shows in the headline, run is not clean", () => {
+    const s = summarize({ pass: 163, fail: 0, advisory: 1 });
+    expect(s.clean).toBe(false); // a run with advisories must never read as a clean green
+    expect(s.headline).toContain("1 advisory");
+    expect(s.headline).toContain("UNVERIFIED");
+  });
+
+  it("a fatal alone → exit 1, FAILED", () => {
+    const s = summarize({ pass: 100, fail: 1, advisory: 0 });
+    expect(s.exitCode).toBe(1);
+    expect(s.headline).toMatch(/FAILED/);
+    expect(s.headline).toContain("1 fail");
+  });
+
+  it("INVARIANT 2 — advisories can't hide a fatal: fail > 0 exits 1 even with advisories piled on", () => {
+    // The failure mode we are guarding against: a real contract break exiting 0
+    // because some live section also went advisory in the same run.
+    for (const advisory of [0, 1, 5, 99]) {
+      const s = summarize({ pass: 120, fail: 1, advisory });
+      expect(s.exitCode).toBe(1);
+      expect(s.headline).toMatch(/FAILED/);
+    }
+    // And many fatals with many advisories is still — unambiguously — a fail.
+    const s = summarize({ pass: 120, fail: 4, advisory: 3 });
+    expect(s.exitCode).toBe(1);
+    expect(s.headline).toContain("4 fail");
+    expect(s.headline).toContain("3 advisory");
+  });
+
+  it("honors a custom label (so smoke-prod could share this verdict logic)", () => {
+    expect(summarize({ pass: 5, fail: 0, advisory: 0 }, "PROD SMOKE").headline).toMatch(/^PROD SMOKE PASSED/);
+  });
+});
+
+describe("isUnverifiable — only timeouts/unreachable may downgrade to advisory", () => {
+  it("an AbortSignal.timeout (TimeoutError) is unverifiable → advisory-eligible", () => {
+    const err = Object.assign(new Error("The operation timed out."), { name: "TimeoutError" });
+    expect(isUnverifiable(err)).toBe(true);
+  });
+
+  it("an AbortError is unverifiable", () => {
+    expect(isUnverifiable(Object.assign(new Error("aborted"), { name: "AbortError" }))).toBe(true);
+  });
+
+  it("network-class fetch failures are unverifiable", () => {
+    expect(isUnverifiable(new Error("fetch failed"))).toBe(true);
+    expect(isUnverifiable(new Error("network connection lost"))).toBe(true);
+    expect(isUnverifiable(new Error("connect ETIMEDOUT 1.2.3.4:443"))).toBe(true);
+  });
+
+  it("a real break stays FATAL — a TypeError / bad-shape parse is NOT downgradeable", () => {
+    expect(isUnverifiable(new TypeError("undefined is not an object (reading 'id')"))).toBe(false);
+    expect(isUnverifiable(new Error("expected 7 notes, got 3"))).toBe(false);
+    expect(isUnverifiable(new SyntaxError("Unexpected token < in JSON"))).toBe(false);
+  });
+});

workers/identity/src/index.ts

@@ -372,7 +372,12 @@ app.post("/__test/usage-run", async (c) => {
 app.post("/__test/snapshot-run", async (c) => {
   const deps = depsFor(c.env);
   if (!deps.exposeDevLinks) return c.notFound();
-  return c.json(await runSnapshotSweep(c.env, deps));
+  // Optional single-vault scope: smoke-staging §17 passes ?vault=<its own fresh
+  // vault> so its restore round-trip stays O(1) instead of sweeping the whole
+  // staging fleet (which outgrew the smoke's client timeout — cloud#166). No
+  // param → the full fleet sweep, unchanged.
+  const onlyVault = c.req.query("vault") || undefined;
+  return c.json(await runSnapshotSweep(c.env, deps, onlyVault ? { onlyVault } : {}));
 });
 
 /**

workers/identity/src/snapshots.ts

@@ -75,21 +75,33 @@ export interface SnapshotSweepSummary {
 /**
  * One nightly tick: snapshot every vault under its owner's plan policy and
  * mirror each returned manifest into D1. `opts.runCap` exists only for tests.
+ *
+ * `opts.onlyVault` scopes the run to a SINGLE vault instead of the whole fleet.
+ * The nightly cron never sets it; the staging-only /__test/snapshot-run trigger
+ * passes it so the live smoke's restore round-trip (smoke-staging.ts §17) stays
+ * O(1) rather than O(fleet) — a fleet-wide sweep outgrew the smoke's client
+ * timeout as staging debris accumulated (cloud#166).
  */
 export async function runSnapshotSweep(
   env: Env,
   deps: OAuthDeps,
-  opts: { runCap?: number } = {},
+  opts: { runCap?: number; onlyVault?: string } = {},
 ): Promise<SnapshotSweepSummary> {
   const runCap = opts.runCap ?? SNAPSHOT_RUN_CAP;
   const now = deps.now?.() ?? new Date();
   const day = now.toISOString().slice(0, 10);
 
-  const res = await env.DB.prepare(
-    `SELECT v.name, v.owner_user_id, u.plan FROM vaults v JOIN users u ON u.id = v.owner_user_id ORDER BY v.name LIMIT ?`,
-  )
-    .bind(runCap + 1)
-    .all<{ name: string; owner_user_id: string; plan: string }>();
+  const res = opts.onlyVault
+    ? await env.DB.prepare(
+        `SELECT v.name, v.owner_user_id, u.plan FROM vaults v JOIN users u ON u.id = v.owner_user_id WHERE v.name = ? LIMIT 1`,
+      )
+        .bind(opts.onlyVault)
+        .all<{ name: string; owner_user_id: string; plan: string }>()
+    : await env.DB.prepare(
+        `SELECT v.name, v.owner_user_id, u.plan FROM vaults v JOIN users u ON u.id = v.owner_user_id ORDER BY v.name LIMIT ?`,
+      )
+        .bind(runCap + 1)
+        .all<{ name: string; owner_user_id: string; plan: string }>();
   const rows = res.results ?? [];
   const capped = rows.length > runCap;
   const vaults = capped ? rows.slice(0, runCap) : rows;