diff --git a/package.json b/package.json index ec7b482..f038869 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@openparachute/cloud", - "version": "0.0.8-rc.125", + "version": "0.0.8-rc.126", "private": true, "description": "Open Parachute PBC's Vault Cloud \u2014 one Durable Object per vault on Cloudflare, OAuth issuer + self-serve console (accounts + vault ownership).", "license": "AGPL-3.0", diff --git a/scripts/smoke-report.ts b/scripts/smoke-report.ts new file mode 100644 index 0000000..66c3a35 --- /dev/null +++ b/scripts/smoke-report.ts @@ -0,0 +1,76 @@ +/** + * smoke-report.ts — the summary/exit logic for the live smokes, extracted as a + * PURE module (no side effects) so the advisory-vs-fatal gate rule can be + * unit-tested without running the live smoke itself. Imported by + * scripts/smoke-staging.ts; proven by test-bun/smoke-report.test.ts. + * + * ── THE RULE: advisory vs fatal, as a CATEGORY (not a hand-maintained list) ── + * + * FATAL — gates the deploy (exit 1). The checked thing is BROKEN: a wrong + * ANSWER is the failure. Contract shapes, status codes, auth refusals, scope + * boundaries, error types, note counts. EVERY assert() in the smoke is fatal. + * This is the safe default: if you cannot tell which side a new check falls + * on, it is fatal. + * + * ADVISORY — loud, but does NOT gate (exit still 0). We COULDN'T VERIFY the + * thing right now. The ONLY failures allowed to be advisory are a live-infra + * section's couldn't-complete errors: a client-side timeout or a network + * unreachable while driving real third-party / fleet infrastructure (Workers + * AI, R2, a cold Durable Object, a fleet sweep). Decide with isUnverifiable(); + * a live section that throws anything ELSE (a TypeError, a bad-shape parse) is + * a real break and stays FATAL. + * + * Why timeouts are only "couldn't verify": a client-side timeout cannot tell + * a slow-but-healthy operation from a stuck-broken one. So a timeout is never + * proof the thing is broken — only proof we didn't get to check it. The + * contract assertions INSIDE each live section stay fatal, so an endpoint + * that answers WRONG (rather than not at all) still fails the gate. + * + * ── Three hard invariants (all pinned in smoke-report.test.ts) ── + * 1. Advisories never gate: exitCode depends on `fail` ALONE. + * 2. Advisories can't hide a fatal: any fail > 0 exits 1 and reads FAILED, + * no matter how many advisories rode along. + * 3. Advisories are never silent: advisory > 0 always shows in the headline, + * so a run carrying advisories never reads as a clean green. + */ + +export interface SmokeCounts { + pass: number; + fail: number; + advisory: number; +} + +export interface SmokeSummary { + exitCode: 0 | 1; + headline: string; + /** True only when nothing failed AND nothing was left unverified. */ + clean: boolean; +} + +/** + * The one place the gate verdict is decided. `exitCode` is a function of `fail` + * alone — advisories are reported but never gate (invariants 1 & 2). When any + * advisory is present it is always named in the headline (invariant 3). + */ +export function summarize(c: SmokeCounts, label = "SMOKE"): SmokeSummary { + const exitCode: 0 | 1 = c.fail === 0 ? 0 : 1; + const verdict = c.fail === 0 ? "PASSED" : "FAILED"; + const advisoryTag = c.advisory > 0 ? `, ${c.advisory} advisory (UNVERIFIED — see below)` : ""; + const headline = `${label} ${verdict} — ${c.pass} pass, ${c.fail} fail${advisoryTag}`; + return { exitCode, headline, clean: c.fail === 0 && c.advisory === 0 }; +} + +/** + * True when an error is the "couldn't reach / didn't finish in time" class — an + * AbortSignal.timeout (a TimeoutError), an AbortError, or a fetch network + * failure. These are the ONLY failures a live-infra section may downgrade to + * advisory; every other throw stays fatal (see THE RULE above). + */ +export function isUnverifiable(err: unknown): boolean { + const name = (err as { name?: string } | null)?.name ?? ""; + if (name === "TimeoutError" || name === "AbortError") return true; + const message = String((err as { message?: string } | null)?.message ?? err); + return /timed out|timeout|fetch failed|network|unreachable|ECONNRESET|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket|terminated/i.test( + message, + ); +} diff --git a/scripts/smoke-staging.ts b/scripts/smoke-staging.ts index 4efea41..4e83b05 100644 --- a/scripts/smoke-staging.ts +++ b/scripts/smoke-staging.ts @@ -41,6 +41,7 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { totpCodeAt } from "../workers/identity/src/totp.ts"; +import { isUnverifiable, summarize } from "./smoke-report.ts"; // core resolves from the sibling parachute-vault checkout, copied into the vault // worker's node_modules by `bun install` (the same explicit path test-bun uses). import { GETTING_STARTED_PACK, welcomePack } from "../workers/vault/node_modules/@openparachute/core/src/seed-packs.js"; @@ -53,7 +54,13 @@ const REDIRECT_URI = "http://localhost:8976/callback"; const MARKER = `smoke-${Date.now()}`; // --- tiny test harness ----------------------------------------------------- +// fail() is FATAL — it gates the deploy (exit 1). advisory() is LOUD but does +// NOT gate — it records "we couldn't verify this right now" (a live-infra +// timeout/unreachable, never a broken contract). The rule + the exit verdict +// live in scripts/smoke-report.ts; see liveCatch() below for where advisories +// come from. let failures = 0; +let advisories = 0; const results: string[] = []; function ok(label: string, detail = "") { results.push(` PASS ${label}${detail ? ` — ${detail}` : ""}`); @@ -64,9 +71,26 @@ function fail(label: string, detail = "") { results.push(` FAIL ${label}${detail ? ` — ${detail}` : ""}`); console.error(`\x1b[31mFAIL\x1b[0m ${label}${detail ? ` — ${detail}` : ""}`); } +function advisory(label: string, detail = "") { + advisories++; + results.push(` ADVISORY ${label}${detail ? ` — ${detail}` : ""}`); + console.error(`\x1b[33mADVISORY\x1b[0m ${label}${detail ? ` — ${detail}` : ""}`); +} function assert(cond: unknown, label: string, detail = "") { cond ? ok(label, detail) : fail(label, detail); } +/** + * The couldn't-verify escape hatch for the live-infra sections (snapshots, + * voice, semantic, tickets, mock-E2E, account-MCP fan-out). A timeout/network + * throw while driving real third-party/fleet infra is ADVISORY (loud, does not + * gate — see scripts/smoke-report.ts). ANY OTHER throw is a real break and + * stays FATAL. The contract assert()s INSIDE each section are always fatal, so + * an endpoint that answers WRONG (not merely slowly) still fails the gate. + */ +function liveCatch(label: string, err: unknown): void { + if (isUnverifiable(err)) advisory(`${label} UNVERIFIED (live-infra timeout/unreachable — did not gate)`, String(err)); + else fail(`${label} threw (unexpected — not a timeout)`, String(err)); +} // --- helpers --------------------------------------------------------------- function b64url(bytes: Uint8Array): string { @@ -1363,7 +1387,7 @@ async function main() { ); } } catch (err) { - fail("tier-change: live section threw (non-fatal — sections continue)", String(err)); + liveCatch("tier-change: live section", err); } // 17. GFS snapshots + restore (Wave 4e). The arrival user is on the no-card @@ -1375,22 +1399,26 @@ async function main() { // notes round-trip. The live restore round-trip is wrapped so a slow DO // import can't abort the sections that follow. // - // cloud#166: this is a GENUINELY HEAVY round trip — the sweep walks - // EVERY staging vault (grows with debris; see scripts/staging-sweep.ts) - // and the restore itself is a full R2→new-DO tar import — and two - // staging runs threw a client-side TimeoutError here as the fleet grew - // past ~140 vaults. Every fetch in this section gets an EXPLICIT, - // generous timeout (rather than whatever implicit default the runtime - // was hitting) so a slow-but-healthy round trip has room to finish; a - // genuinely stuck one still fails this section cleanly instead of - // hanging the whole smoke. + // cloud#166: the sweep is SCOPED to this run's own vault (?vault=), so + // it is O(1), not O(fleet). Originally it walked EVERY staging vault; + // as the fleet grew with smoke debris past ~140 vaults it crossed the + // client timeout, and — because the sweep runs FIRST — the whole restore + // section (and the deploy gate) went red for the growing fleet. Scoping + // it to the one vault the test actually needs a snapshot of removes the + // fleet-size dependency entirely; the nightly cron still sweeps the + // whole fleet, tested unit-side (workers/identity/test/snapshots.test.ts). + // The restore itself is still a real R2→new-DO tar import (O(1) per + // vault); each fetch keeps an explicit, generous timeout so a + // slow-but-healthy round trip has room to finish, and a genuinely stuck + // one is recorded ADVISORY (couldn't verify) via liveCatch, not fatal. const RESTORE_ROUNDTRIP_TIMEOUT_MS = 120_000; try { const arrivalCsrf = /parachute_id_csrf=([^;]+)/.exec(arrivalCookie)?.[1] ?? ""; - // One sweep tick via the staging-only trigger. The arrival vault is fresh - // this run → its snapshot (paid retention) is taken now. + // One sweep tick via the staging-only trigger, SCOPED to this run's fresh + // vault (?vault=) so it stays O(1). That vault's snapshot (paid retention) + // is taken now → sweep.taken === 1. const runSweep = async (): Promise<{ day: string; vaults: number; taken: number; skipped: number; failed: number; capped: boolean } | null> => { - const r = await fetch(`${IDENTITY}/__test/snapshot-run`, { method: "POST", signal: AbortSignal.timeout(RESTORE_ROUNDTRIP_TIMEOUT_MS) }); + const r = await fetch(`${IDENTITY}/__test/snapshot-run?vault=${encodeURIComponent(arrivalVault)}`, { method: "POST", signal: AbortSignal.timeout(RESTORE_ROUNDTRIP_TIMEOUT_MS) }); return r.status === 200 ? ((await r.json()) as { day: string; vaults: number; taken: number; skipped: number; failed: number; capped: boolean }) : null; }; const sweep = await runSweep(); @@ -1461,7 +1489,7 @@ async function main() { } } } catch (err) { - fail("snapshots: live restore round-trip threw (non-fatal — sections continue)", String(err)); + liveCatch("snapshots: live restore round-trip", err); } // 18. Voice transcription (cloud#56) — comp the arrival user to the PLUS @@ -1602,7 +1630,7 @@ async function main() { ); } } catch (err) { - fail("voice: live transcription section threw (non-fatal — sections continue)", String(err)); + liveCatch("voice: live transcription section", err); } // 18b. Semantic search via Workers AI (C2, EXPERIMENTAL) — the one thing no @@ -1683,7 +1711,7 @@ async function main() { ); } } catch (err) { - fail("semantic: live embedding section threw (non-fatal — sections continue)", String(err)); + liveCatch("semantic: live embedding section", err); } // 18c. Attachment tickets (cloud#177's DO mirror) — the pre-prod condition @@ -1833,7 +1861,7 @@ async function main() { ); } } catch (err) { - fail("tickets: live ticket round-trip section threw (non-fatal — sections continue)", String(err)); + liveCatch("tickets: live ticket round-trip section", err); } // 19. MOCK-upgrade E2E (mock-payments) — the live end-to-end proof, run LAST @@ -1948,7 +1976,7 @@ async function main() { } } } catch (err) { - fail("mock E2E: live section threw (non-fatal — sections continue)", String(err)); + liveCatch("mock E2E: live section", err); } // 20. Pricing-model ENFORCEMENT E2E (the two-meter caps + the frozen floor) — @@ -2428,13 +2456,23 @@ async function main() { } } } catch (err) { - fail("account-mcp: live section threw (non-fatal — summary follows)", String(err)); + liveCatch("account-mcp: live section", err); } // --- summary --- - console.log(`\n${"=".repeat(60)}\nSMOKE ${failures === 0 ? "PASSED" : "FAILED"} — ${results.filter((r) => r.includes("PASS")).length} pass, ${failures} fail\n${"=".repeat(60)}`); + // The verdict is decided in scripts/smoke-report.ts: fatals gate (exit 1), + // advisories are loud but never gate and can never hide a fatal. + const passCount = results.filter((r) => r.startsWith(" PASS")).length; + const { exitCode, headline } = summarize({ pass: passCount, fail: failures, advisory: advisories }); + console.log(`\n${"=".repeat(60)}\n${headline}\n${"=".repeat(60)}`); console.log(results.join("\n")); - process.exit(failures === 0 ? 0 : 1); + if (advisories > 0) { + // Re-surface the advisories on their own line so an "we couldn't verify + // this" is never buried under ~160 PASS lines and read as a clean green. + console.log(`\n${advisories} ADVISORY (live-infra unverified — did NOT gate the deploy; investigate if persistent):`); + for (const r of results.filter((r) => r.startsWith(" ADVISORY"))) console.log(r); + } + process.exit(exitCode); } /** diff --git a/test-bun/smoke-report.test.ts b/test-bun/smoke-report.test.ts new file mode 100644 index 0000000..c15c02c --- /dev/null +++ b/test-bun/smoke-report.test.ts @@ -0,0 +1,86 @@ +/** + * The live-smoke gate rule, pinned. smoke-report.ts is what turned a chronic + * live-infra timeout into a hard, prod-blocking gate for nine days (cloud#166 + * follow-up): a section labeled "non-fatal — sections continue" still called + * fail() and so still exit 1. These tests pin the three invariants that keep + * the advisory downgrade honest — advisories are loud, never gate, and can + * NEVER hide a real (fatal) failure. + * + * Pure module, no network — runs under the root `bun test` suite (package.json + * "test": "bun test src test-bun"), never imports smoke-staging.ts (which runs + * a live main() on import). + */ +import { describe, expect, it } from "bun:test"; +import { isUnverifiable, summarize } from "../scripts/smoke-report.ts"; + +describe("summarize — the gate verdict", () => { + it("clean run: no fail, no advisory → exit 0, PASSED, reads clean", () => { + const s = summarize({ pass: 163, fail: 0, advisory: 0 }); + expect(s.exitCode).toBe(0); + expect(s.clean).toBe(true); + expect(s.headline).toBe("SMOKE PASSED — 163 pass, 0 fail"); + expect(s.headline).not.toMatch(/advisory/i); + }); + + it("INVARIANT 1 — advisories never gate: fail 0 + advisory > 0 → exit 0", () => { + const s = summarize({ pass: 163, fail: 0, advisory: 2 }); + expect(s.exitCode).toBe(0); // the whole point: an unverified live section does NOT block prod + expect(s.headline).toMatch(/PASSED/); + }); + + it("INVARIANT 3 — advisories are never silent: advisory > 0 shows in the headline, run is not clean", () => { + const s = summarize({ pass: 163, fail: 0, advisory: 1 }); + expect(s.clean).toBe(false); // a run with advisories must never read as a clean green + expect(s.headline).toContain("1 advisory"); + expect(s.headline).toContain("UNVERIFIED"); + }); + + it("a fatal alone → exit 1, FAILED", () => { + const s = summarize({ pass: 100, fail: 1, advisory: 0 }); + expect(s.exitCode).toBe(1); + expect(s.headline).toMatch(/FAILED/); + expect(s.headline).toContain("1 fail"); + }); + + it("INVARIANT 2 — advisories can't hide a fatal: fail > 0 exits 1 even with advisories piled on", () => { + // The failure mode we are guarding against: a real contract break exiting 0 + // because some live section also went advisory in the same run. + for (const advisory of [0, 1, 5, 99]) { + const s = summarize({ pass: 120, fail: 1, advisory }); + expect(s.exitCode).toBe(1); + expect(s.headline).toMatch(/FAILED/); + } + // And many fatals with many advisories is still — unambiguously — a fail. + const s = summarize({ pass: 120, fail: 4, advisory: 3 }); + expect(s.exitCode).toBe(1); + expect(s.headline).toContain("4 fail"); + expect(s.headline).toContain("3 advisory"); + }); + + it("honors a custom label (so smoke-prod could share this verdict logic)", () => { + expect(summarize({ pass: 5, fail: 0, advisory: 0 }, "PROD SMOKE").headline).toMatch(/^PROD SMOKE PASSED/); + }); +}); + +describe("isUnverifiable — only timeouts/unreachable may downgrade to advisory", () => { + it("an AbortSignal.timeout (TimeoutError) is unverifiable → advisory-eligible", () => { + const err = Object.assign(new Error("The operation timed out."), { name: "TimeoutError" }); + expect(isUnverifiable(err)).toBe(true); + }); + + it("an AbortError is unverifiable", () => { + expect(isUnverifiable(Object.assign(new Error("aborted"), { name: "AbortError" }))).toBe(true); + }); + + it("network-class fetch failures are unverifiable", () => { + expect(isUnverifiable(new Error("fetch failed"))).toBe(true); + expect(isUnverifiable(new Error("network connection lost"))).toBe(true); + expect(isUnverifiable(new Error("connect ETIMEDOUT 1.2.3.4:443"))).toBe(true); + }); + + it("a real break stays FATAL — a TypeError / bad-shape parse is NOT downgradeable", () => { + expect(isUnverifiable(new TypeError("undefined is not an object (reading 'id')"))).toBe(false); + expect(isUnverifiable(new Error("expected 7 notes, got 3"))).toBe(false); + expect(isUnverifiable(new SyntaxError("Unexpected token < in JSON"))).toBe(false); + }); +}); diff --git a/workers/identity/src/index.ts b/workers/identity/src/index.ts index 6b644f5..46e7df2 100644 --- a/workers/identity/src/index.ts +++ b/workers/identity/src/index.ts @@ -372,7 +372,12 @@ app.post("/__test/usage-run", async (c) => { app.post("/__test/snapshot-run", async (c) => { const deps = depsFor(c.env); if (!deps.exposeDevLinks) return c.notFound(); - return c.json(await runSnapshotSweep(c.env, deps)); + // Optional single-vault scope: smoke-staging §17 passes ?vault= so its restore round-trip stays O(1) instead of sweeping the whole + // staging fleet (which outgrew the smoke's client timeout — cloud#166). No + // param → the full fleet sweep, unchanged. + const onlyVault = c.req.query("vault") || undefined; + return c.json(await runSnapshotSweep(c.env, deps, onlyVault ? { onlyVault } : {})); }); /** diff --git a/workers/identity/src/snapshots.ts b/workers/identity/src/snapshots.ts index 686c258..c105ad8 100644 --- a/workers/identity/src/snapshots.ts +++ b/workers/identity/src/snapshots.ts @@ -75,21 +75,33 @@ export interface SnapshotSweepSummary { /** * One nightly tick: snapshot every vault under its owner's plan policy and * mirror each returned manifest into D1. `opts.runCap` exists only for tests. + * + * `opts.onlyVault` scopes the run to a SINGLE vault instead of the whole fleet. + * The nightly cron never sets it; the staging-only /__test/snapshot-run trigger + * passes it so the live smoke's restore round-trip (smoke-staging.ts §17) stays + * O(1) rather than O(fleet) — a fleet-wide sweep outgrew the smoke's client + * timeout as staging debris accumulated (cloud#166). */ export async function runSnapshotSweep( env: Env, deps: OAuthDeps, - opts: { runCap?: number } = {}, + opts: { runCap?: number; onlyVault?: string } = {}, ): Promise { const runCap = opts.runCap ?? SNAPSHOT_RUN_CAP; const now = deps.now?.() ?? new Date(); const day = now.toISOString().slice(0, 10); - const res = await env.DB.prepare( - `SELECT v.name, v.owner_user_id, u.plan FROM vaults v JOIN users u ON u.id = v.owner_user_id ORDER BY v.name LIMIT ?`, - ) - .bind(runCap + 1) - .all<{ name: string; owner_user_id: string; plan: string }>(); + const res = opts.onlyVault + ? await env.DB.prepare( + `SELECT v.name, v.owner_user_id, u.plan FROM vaults v JOIN users u ON u.id = v.owner_user_id WHERE v.name = ? LIMIT 1`, + ) + .bind(opts.onlyVault) + .all<{ name: string; owner_user_id: string; plan: string }>() + : await env.DB.prepare( + `SELECT v.name, v.owner_user_id, u.plan FROM vaults v JOIN users u ON u.id = v.owner_user_id ORDER BY v.name LIMIT ?`, + ) + .bind(runCap + 1) + .all<{ name: string; owner_user_id: string; plan: string }>(); const rows = res.results ?? []; const capped = rows.length > runCap; const vaults = capped ? rows.slice(0, runCap) : rows; diff --git a/workers/identity/test/snapshots.test.ts b/workers/identity/test/snapshots.test.ts index 4b34858..cfdba13 100644 --- a/workers/identity/test/snapshots.test.ts +++ b/workers/identity/test/snapshots.test.ts @@ -323,6 +323,30 @@ describe("runSnapshotSweep", () => { expect(summary).toEqual({ day: TODAY, vaults: 2, taken: 2, skipped: 0, failed: 0, capped: true }); expect(SNAPSHOT_RUN_CAP).toBe(500); }); + + test("onlyVault scopes the run to a SINGLE vault — the rest of the fleet is untouched (O(1), not O(fleet))", async () => { + const { id: owner } = await seedUser("sweep-scope@example.com"); + await env.DB.prepare("UPDATE users SET plan = 'standard' WHERE id = ?").bind(owner).run(); + await seedVault("scope-mine", owner); + await seedVault("scope-other", owner); + // ONLY the scoped vault's DO is intercepted. The other vault must not be + // called at all — an unexpected call would trip disableNetConnect, and + // assertNoPendingInterceptors (afterEach) proves the scoped interceptor was + // the sole fetch. + const manifest = [wireEntry("scope-mine", "2026-07-03T04:00:00.000Z")]; + interceptSnapshotPost("scope-mine", { skipped: false, manifest }); + + const summary = await quietly(() => runSnapshotSweep(env, sweepDeps(), { onlyVault: "scope-mine" })); + expect(summary).toEqual({ day: TODAY, vaults: 1, taken: 1, skipped: 0, failed: 0, capped: false }); + // Only the scoped vault's mirror row exists — scope-other was never swept. + expect((await mirrorRows()).map((r) => r.vault_name)).toEqual(["scope-mine"]); + }); + + test("onlyVault for an unknown name → an empty, clean run (no vault calls)", async () => { + const summary = await quietly(() => runSnapshotSweep(env, sweepDeps(), { onlyVault: "does-not-exist" })); + expect(summary).toEqual({ day: TODAY, vaults: 0, taken: 0, skipped: 0, failed: 0, capped: false }); + expect(await mirrorRows()).toEqual([]); + }); }); // --- listSnapshotsForVaults --------------------------------------------------------- @@ -617,4 +641,22 @@ describe("staging snapshot trigger", () => { const prod = await worker.fetch(req(), { ...env, ENVIRONMENT: "production" }); expect(prod.status).toBe(404); }); + + test("?vault= scopes the trigger to that one vault (smoke §17's O(1) restore setup)", async () => { + const { id: owner } = await seedUser("trigger-scope@example.com"); + await seedVault("trig-scope-a", owner); + await seedVault("trig-scope-b", owner); + const manifest = [wireEntry("trig-scope-a", "2026-06-29T04:00:00.000Z")]; + interceptSnapshotPost("trig-scope-a", { skipped: false, manifest }); + + const res = await quietly(async () => + worker.fetch(new Request(`${ISSUER}/__test/snapshot-run?vault=trig-scope-a`, { method: "POST" }), env), + ); + expect(res.status).toBe(200); + const summary = (await res.json()) as { vaults: number; taken: number }; + expect(summary.vaults).toBe(1); // only the scoped vault, not the fleet + expect(summary.taken).toBe(1); + // trig-scope-b was never snapshotted — no interceptor + assertNoPendingInterceptors. + expect((await mirrorRows()).map((r) => r.vault_name)).toEqual(["trig-scope-a"]); + }); });