diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0978b6d..e5db841 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,9 @@ jobs: - name: Typecheck run: mise run typecheck + - name: Validate canonical proof bundles + run: mise run validate-bundles + - name: Build run: mise run build diff --git a/dogfood/20260325-week8-contract-locks/manifest.json b/dogfood/20260325-week8-contract-locks/manifest.json index d512cec..766b165 100644 --- a/dogfood/20260325-week8-contract-locks/manifest.json +++ b/dogfood/20260325-week8-contract-locks/manifest.json @@ -14,31 +14,45 @@ "artifacts": [ { "path": "logs/01-golden-envelopes.json", - "description": "Pretty-printed Vitest JSON output for the Week 8 golden-envelope suite" + "description": "Pretty-printed Vitest JSON output for the Week 8 golden-envelope suite", + "sha256": "985c613652551c4e395b444c0256db6c5dd8e17ea952b3ea6dcf148f36af1a2a", + "bytes": 21655 }, { "path": "logs/01-golden-envelopes.stderr.txt", - "description": "Captured stderr for the JSON-reporter Vitest run" + "description": "Captured stderr for the JSON-reporter Vitest run", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "bytes": 0 }, { "path": "logs/02-golden-envelopes.txt", - "description": "Human-readable Vitest output showing the full Week 8 golden-envelope suite passing" + "description": "Human-readable Vitest output showing the full Week 8 golden-envelope suite passing", + "sha256": "1331dd870574ac6a647882e376bc4baffa32cbdc90e6507a1e0908d4f16c3fa2", + "bytes": 504 }, { "path": "logs/02-golden-envelopes.stderr.txt", - "description": "Captured stderr for the human-readable Vitest run" + "description": "Captured stderr for the human-readable Vitest run", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "bytes": 0 }, { "path": "snapshots/01-golden-envelopes.json", - "description": "Bundle-local JSON mirror used by the review and validation tooling" + "description": "Bundle-local JSON mirror used by the review and validation tooling", + "sha256": "985c613652551c4e395b444c0256db6c5dd8e17ea952b3ea6dcf148f36af1a2a", + "bytes": 21655 }, { "path": "screenshots/01-review-page.png", - "description": "Screenshot of the generated local review page" + "description": "Screenshot of the generated local review page", + "sha256": "74b1e45cb7c73056ed4082537a45b98281e98cc64341ca2ab7480fdde24f7650", + "bytes": 3866344 }, { "path": "command-status.tsv", - "description": "Per-step exit-code ledger for capture, review-page generation, validation, and screenshot creation" + "description": "Per-step exit-code ledger for capture, review-page generation, validation, and screenshot creation", + "sha256": "a9ac6f0c136b672582067a912f299cb52a314cdf7b2796d18d5b180251b2186a", + "bytes": 1362 } ] } diff --git a/dogfood/20260326-week9-release-readiness/manifest.json b/dogfood/20260326-week9-release-readiness/manifest.json index 7390c8c..55036b7 100644 --- a/dogfood/20260326-week9-release-readiness/manifest.json +++ b/dogfood/20260326-week9-release-readiness/manifest.json @@ -23,95 +23,141 @@ "artifacts": [ { "path": "notes.md", - "description": "Reviewer-facing narrative explaining what Week 9 functionality was proven and what limitations were observed" + "description": "Reviewer-facing narrative explaining what Week 9 functionality was proven and what limitations were observed", + "sha256": "773200f78c0c31b649834f74edd53a68cf6c569c75129584007dfcb03ba3cac0", + "bytes": 5159 }, { "path": "commands.sh", - "description": "Reproducible capture script for recreating the Week 9 release-readiness bundle" + "description": "Reproducible capture script for recreating the Week 9 release-readiness bundle", + "sha256": "d37691e848c7073166b9a2cc8bf3bdddd2f315b428667f4270ebb828ef050292", + "bytes": 11047 }, { "path": "command-status.tsv", - "description": "Step-by-step exit-code ledger for capture, artifact copies, review-page generation, validation, and formatting checks" + "description": "Step-by-step exit-code ledger for capture, artifact copies, review-page generation, validation, and formatting checks", + "sha256": "ef1f471884714a5c6934d5c9d13fb542c8b68487e9cd321decffe53f75f3678d", + "bytes": 3014 }, { "path": "logs/01-doctor.json", - "description": "Doctor JSON envelope showing home isolation and browser cache accessibility checks under an isolated home" + "description": "Doctor JSON envelope showing home isolation and browser cache accessibility checks under an isolated home", + "sha256": "8e39a2b7eec4eb11ebe61cc0f550360f71808f8ebed175669cecaf3a9ccd7aa6", + "bytes": 3979 }, { "path": "snapshots/01-doctor.json", - "description": "Bundle-local JSON mirror of the doctor output used by validator/review tooling" + "description": "Bundle-local JSON mirror of the doctor output used by validator/review tooling", + "sha256": "8e39a2b7eec4eb11ebe61cc0f550360f71808f8ebed175669cecaf3a9ccd7aa6", + "bytes": 3979 }, { "path": "logs/03-create-inspect.json", - "description": "Initial inspect output for the isolated session immediately after creation" + "description": "Initial inspect output for the isolated session immediately after creation", + "sha256": "5b2ef102511fcf4932f1b8027b5105b933b99288a77ff27e8b441cb1b94f5f36", + "bytes": 1041 }, { "path": "logs/04-run-echo.json", - "description": "Run-command JSON envelope for the Week 9 proof banner command" + "description": "Run-command JSON envelope for the Week 9 proof banner command", + "sha256": "e6b6728ac88145f64836ff169db9d23fd793e3a07fe6c3c2a0f98de6b46e892c", + "bytes": 267 }, { "path": "logs/05-run-sysinfo.json", - "description": "Run-command JSON envelope for the in-session system-information command" + "description": "Run-command JSON envelope for the in-session system-information command", + "sha256": "18bcec421a0f54dbc6e0373b1f18287d5086a1699294e167daa79bf26101ca15", + "bytes": 267 }, { "path": "logs/06-wait-stable.json", - "description": "Renderer wait result proving the terminal frame was stable before capture" + "description": "Renderer wait result proving the terminal frame was stable before capture", + "sha256": "c90eaf0ae00e714ee418f1a9e7f67b82c9c07f8fb2dd76529750f7417b320838", + "bytes": 207 }, { "path": "logs/07-screenshot.json", - "description": "Screenshot JSON envelope for the copied PNG artifact" + "description": "Screenshot JSON envelope for the copied PNG artifact", + "sha256": "26ace7e437f72673ff415a5ae7c38477e28fa2e40e57657fdd3221a34d338173", + "bytes": 699 }, { "path": "screenshots/01-after-run.png", - "description": "Renderer screenshot captured after the two run commands completed" + "description": "Renderer screenshot captured after the two run commands completed", + "sha256": "6b07d07c3a58035b4d432d4b449fc2e5423b02be422c2aaeb4588e36ef230d1d", + "bytes": 22396 }, { "path": "logs/08-snapshot.json", - "description": "Structured snapshot JSON envelope for the same post-run terminal frame" + "description": "Structured snapshot JSON envelope for the same post-run terminal frame", + "sha256": "f18e2577dbdd4f1e1ba2d4851dbc43bc03adb50ef8f1d94f1b4ff1cb92fd94bc", + "bytes": 1969 }, { "path": "snapshots/02-post-run-structured.json", - "description": "Bundle-local JSON mirror of the structured post-run snapshot" + "description": "Bundle-local JSON mirror of the structured post-run snapshot", + "sha256": "f18e2577dbdd4f1e1ba2d4851dbc43bc03adb50ef8f1d94f1b4ff1cb92fd94bc", + "bytes": 1969 }, { "path": "logs/09-export-asciicast.json", - "description": "Asciicast export JSON envelope" + "description": "Asciicast export JSON envelope", + "sha256": "621e71afbd92115e63fd36c99007d2ea38e1334bdea6485479fdd5df5d0fa905", + "bytes": 686 }, { "path": "recordings/week9.cast", - "description": "Exported asciicast recording for the isolated Week 9 proof session" + "description": "Exported asciicast recording for the isolated Week 9 proof session", + "sha256": "073533533b48cddf4ca7c4b97de1da8e116fd0d54157c1492ea90bf9df2f1f79", + "bytes": 1207 }, { "path": "logs/10-export-webm.json", - "description": "WebM export JSON envelope" + "description": "WebM export JSON envelope", + "sha256": "342641327bfd5f89cdb074f9faff1f195b56c5ee3aa329c3c2afc6fb54911178", + "bytes": 746 }, { "path": "videos/week9.webm", - "description": "Exported WebM replay for the isolated Week 9 proof session" + "description": "Exported WebM replay for the isolated Week 9 proof session", + "sha256": "3677c43c2a4a76e9d93209ac466569f711c4d6469f3cfaab6136b02ed8d613ef", + "bytes": 54201 }, { "path": "logs/11-final-inspect.json", - "description": "Final inspect output confirming healthy artifacts and live renderer runtime before teardown" + "description": "Final inspect output confirming healthy artifacts and live renderer runtime before teardown", + "sha256": "a720a4bee3b2d0ea39bd73d8421fb509fcd4d19e98ae51d4b639aee3d3ae012d", + "bytes": 1135 }, { "path": "snapshots/03-final-inspect.json", - "description": "Bundle-local JSON mirror of the final inspect output" + "description": "Bundle-local JSON mirror of the final inspect output", + "sha256": "a720a4bee3b2d0ea39bd73d8421fb509fcd4d19e98ae51d4b639aee3d3ae012d", + "bytes": 1135 }, { "path": "logs/12-destroy.json", - "description": "Destroy output confirming the isolated session was cleaned up" + "description": "Destroy output confirming the isolated session was cleaned up", + "sha256": "740964f59859ec137d5a1c72a4a6cb97701d618805c993444127e802851f35e0", + "bytes": 172 }, { "path": "logs/13-review-bundle.txt", - "description": "Review-bundle output showing the static reviewer page generation path" + "description": "Review-bundle output showing the static reviewer page generation path", + "sha256": "5f11b2e1ae34c872d7d40aeabba57e875c956433669977334d883fc7b389ac5e", + "bytes": 110 }, { "path": "logs/14-validate-bundle.txt", - "description": "Bundle validation stdout for the interactive-renderer profile" + "description": "Bundle validation stdout for the interactive-renderer profile", + "sha256": "d455b4edd3dc16e741030547ad133e36da6649ef56640d76bad5708cd23cc989", + "bytes": 1052 }, { "path": "index.html", - "description": "Generated static review page for the complete Week 9 proof bundle" + "description": "Generated static review page for the complete Week 9 proof bundle", + "sha256": "d715ed55733a39d9a547ae94d91d5486b68858e5e300bbd53b80c1ef93c13524", + "bytes": 77959 } ] } diff --git a/dogfood/agent-uses-agent-tty/manifest.json b/dogfood/agent-uses-agent-tty/manifest.json new file mode 100644 index 0000000..4f3c50e --- /dev/null +++ b/dogfood/agent-uses-agent-tty/manifest.json @@ -0,0 +1,273 @@ +{ + "bundle": "agent-uses-agent-tty", + "title": "Agents drive agent-tty: Codex and Claude TUIs through agent-tty", + "description": "Evergreen demo capturing Codex and Claude TUIs each driving nvim --clean inside an agent-tty session. Records outer and inner WebMs, asciicasts, transcripts, thumbnails, and a reproduce.sh script.", + "createdAt": "2026-04-24T00:00:00Z", + "scenario": "agent-uses-agent-tty", + "result": "pass", + "commands": [ + "./dogfood/agent-uses-agent-tty/reproduce.sh", + "./node_modules/.bin/tsx src/cli/main.ts create --json --home -- codex ...", + "./node_modules/.bin/tsx src/cli/main.ts create --json --home -- claude ...", + "./node_modules/.bin/tsx src/cli/main.ts record export --format webm --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts record export --format asciicast --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts snapshot --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts screenshot --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts wait --exit --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts destroy --json --home " + ], + "artifacts": [ + { + "path": "README.md", + "description": "Reviewer-facing demo narrative explaining the Codex and Claude TUI runs through agent-tty", + "sha256": "a864270255d7d55e093164a37db2a3bf5e7489c1a4bbb4f78d23d415dca62ac6", + "bytes": 6121 + }, + { + "path": "reproduce.sh", + "description": "Self-contained generator script that captures fresh outer and inner artifacts", + "sha256": "2c00466058d56014e24ab0422a27f37bc4c159a0416c3ed7f986a83f5073a21c", + "bytes": 31969 + }, + { + "path": "environment.txt", + "description": "Captured tool versions and runtime environment for the demo", + "sha256": "9de3d4b7820ea3a4be914fe31b2d5dc7469f9db74a6550fd1af86e53b552862c", + "bytes": 5428 + }, + { + "path": "prompts/template.md", + "description": "Shared inner-agent prompt template used by both Codex and Claude runs", + "sha256": "ee54dc471a4de94b69e8833b6fcbb2558be9b8b6f6139fb1ce2ccc56d201a3d5", + "bytes": 853 + }, + { + "path": "claude-outer-create.json", + "description": "create envelope for the outer agent-tty session driving Claude", + "sha256": "37bf1f75d51714dbb2d1c139efa5175809433d462c398b158bf0bb2512f276d8", + "bytes": 2598 + }, + { + "path": "claude-outer-destroy.json", + "description": "destroy envelope for the outer Claude session", + "sha256": "c59ac1b6b8e118280f736ca6e3c266925188f7b42388da5b8f052a3e2d85f80f", + "bytes": 172 + }, + { + "path": "claude-outer-record-cast.json", + "description": "record export envelope for the outer Claude asciicast", + "sha256": "84f93d7dc6eeb8acf54c1311a69db353bfe94691ada349c4891ce88fdc738adb", + "bytes": 675 + }, + { + "path": "claude-outer-record-webm.json", + "description": "record export envelope for the outer Claude WebM recording", + "sha256": "b15189385fc78691057e04792d003a7952c95e6612d0948503ba994b6d21faca", + "bytes": 783 + }, + { + "path": "claude-outer-screenshot.json", + "description": "screenshot envelope for the outer Claude session", + "sha256": "681c7c93ea8c457c60eda49455980a3f2231b5273018b748a839911a605f508e", + "bytes": 781 + }, + { + "path": "claude-outer-snapshot.json", + "description": "snapshot envelope for the outer Claude session", + "sha256": "1154292a7692336013c39baf7923938bc489207036e7fce42ab3de124cbf2fe8", + "bytes": 13041 + }, + { + "path": "claude-outer-wait-exit.json", + "description": "wait envelope for the outer Claude session exit", + "sha256": "6933fa9d8d84bdba959cf93ce1d5a48d419ef24e3046168d3a4841e7fba791a1", + "bytes": 141 + }, + { + "path": "codex-outer-create.json", + "description": "create envelope for the outer agent-tty session driving Codex", + "sha256": "e4d8829f23ef89bade3c55b6671988058755f2a4d098afa15b5c444ed4907843", + "bytes": 2597 + }, + { + "path": "codex-outer-destroy.json", + "description": "destroy envelope for the outer Codex session", + "sha256": "eae666b9ac04df46e544fba1b24853dcaa83618212377be4af880f0898b3504b", + "bytes": 172 + }, + { + "path": "codex-outer-record-cast.json", + "description": "record export envelope for the outer Codex asciicast", + "sha256": "9c2b81d346566e72be3b6340a3881121b7a80e12abb6dc9bd071233bda9de943", + "bytes": 677 + }, + { + "path": "codex-outer-record-webm.json", + "description": "record export envelope for the outer Codex WebM recording", + "sha256": "0d767af1e890d0a3568c376f8debb7ac36e2de038dedd84e88dd6691a4a3e096", + "bytes": 784 + }, + { + "path": "codex-outer-screenshot.json", + "description": "screenshot envelope for the outer Codex session", + "sha256": "3efa2c4608826b5b50172347dcad174169a0894b3a510f4fb130745b54805e3f", + "bytes": 783 + }, + { + "path": "codex-outer-snapshot.json", + "description": "snapshot envelope for the outer Codex session", + "sha256": "3debc5db124454a2da9a8d003193070840d42de4a391d2b62288c17b41a7e3f6", + "bytes": 4981 + }, + { + "path": "codex-outer-wait-exit.json", + "description": "wait envelope for the outer Codex session exit", + "sha256": "c0e9672799b724148e0b6c2280e2bdb66e6709e583af4ce6242b61430a76c8f8", + "bytes": 141 + }, + { + "path": "artifacts/claude-agent-transcript.txt", + "description": "Inner Claude agent transcript captured during the run", + "sha256": "03ecb9043fc1d012002971b0db8ed3e4380aa7ac5091665a4e13304ef0f2153f", + "bytes": 12169 + }, + { + "path": "artifacts/claude-demo-note.txt", + "description": "Inner Claude demo note produced by the agent", + "sha256": "818c0d5f147a79968105195e1ff627588add18fa6a9bacf8ec4ea4f3696dc2e3", + "bytes": 55 + }, + { + "path": "artifacts/claude-final-file-proof.txt", + "description": "Proof of the final file written by Claude inside the inner nvim", + "sha256": "2dcbfad2baa1ae9389f5715357488ea8926ec5d1b2516e5b687c08197d487031", + "bytes": 452 + }, + { + "path": "artifacts/claude-inner-nvim.cast", + "description": "Inner asciicast of Claude driving nvim --clean", + "sha256": "a75b1d85fe13dc3e58aa8aa673336ab9262afbc0cac40a7e91316b00713dd86a", + "bytes": 14067 + }, + { + "path": "artifacts/claude-inner-nvim.webm", + "description": "Inner WebM of Claude driving nvim --clean", + "sha256": "5816cda8e4293547075ce8031b0e041e062db025b52765229f5756d5cffb4cd4", + "bytes": 318282 + }, + { + "path": "artifacts/claude-outer.cast", + "description": "Outer asciicast of agent-tty driving Claude", + "sha256": "fb72cf32c6e81a1d81ce5f264c344dff6c33bd02466584a940e9d81000147bd0", + "bytes": 88486 + }, + { + "path": "artifacts/claude-outer.webm", + "description": "Outer WebM of agent-tty driving Claude", + "sha256": "d4cdcca13c03a0ba6f727165aa87442847044398e35473713c4bb4d77ffdab19", + "bytes": 960568 + }, + { + "path": "artifacts/claude-outer-full.webm", + "description": "Full outer WebM of agent-tty driving Claude including setup", + "sha256": "c2e660bf1ecc999c0c2ef7d9a0b9005067b7679627ddba7e6635a7f279805034", + "bytes": 1776039 + }, + { + "path": "artifacts/claude-outer-snapshot.txt", + "description": "Text snapshot of the outer Claude session", + "sha256": "03ecb9043fc1d012002971b0db8ed3e4380aa7ac5091665a4e13304ef0f2153f", + "bytes": 12169 + }, + { + "path": "artifacts/claude-prompt.md", + "description": "Resolved prompt used by the Claude inner agent", + "sha256": "e802e58efde89381ac0cd52e4ee7cd937fad15dd5659b6158788e1db0323031d", + "bytes": 1420 + }, + { + "path": "artifacts/claude-recording-summary.txt", + "description": "Reviewer summary of the Claude recording set", + "sha256": "b3c176d1f5d982f7fbd123de3bedb29b3c2c886b7cdb26cfd4c3806fe2abf7dd", + "bytes": 134 + }, + { + "path": "artifacts/claude-thumbnail.png", + "description": "Thumbnail still frame for the Claude recording", + "sha256": "7d3d04a2827d386f287ad506615139614e25c199fd73e2a8a06c440a37bcdb41", + "bytes": 87418 + }, + { + "path": "artifacts/codex-agent-transcript.txt", + "description": "Inner Codex agent transcript captured during the run", + "sha256": "ec218396afcac7c2218ba6a93d72f205291282c5e92ce41b3ed083219be350eb", + "bytes": 4605 + }, + { + "path": "artifacts/codex-demo-note.txt", + "description": "Inner Codex demo note produced by the agent", + "sha256": "818c0d5f147a79968105195e1ff627588add18fa6a9bacf8ec4ea4f3696dc2e3", + "bytes": 55 + }, + { + "path": "artifacts/codex-final-file-proof.txt", + "description": "Proof of the final file written by Codex inside the inner nvim", + "sha256": "340f9b4ce59bc36e878b3a378e54be5ecc07c07eb8bc90cf442f16616c9cea1c", + "bytes": 449 + }, + { + "path": "artifacts/codex-inner-nvim.cast", + "description": "Inner asciicast of Codex driving nvim --clean", + "sha256": "4ed823ae9b1b61452f6c2ed9102f3b56214588f88f24b1690a8084434acce290", + "bytes": 14774 + }, + { + "path": "artifacts/codex-inner-nvim.webm", + "description": "Inner WebM of Codex driving nvim --clean", + "sha256": "544426727ca466115b5bafe5fd17d70ba3c5f1e9904d568e6c4b57ecb4b03304", + "bytes": 318019 + }, + { + "path": "artifacts/codex-outer.cast", + "description": "Outer asciicast of agent-tty driving Codex", + "sha256": "53aac241471c1a9a81826c5eb335d02afe17495174cee80868f967174387b988", + "bytes": 390553 + }, + { + "path": "artifacts/codex-outer.webm", + "description": "Outer WebM of agent-tty driving Codex", + "sha256": "0e2a6b39634a54efbdbcb1f81a637896beeb65384f4564748bb067e88d9084ed", + "bytes": 315697 + }, + { + "path": "artifacts/codex-outer-full.webm", + "description": "Full outer WebM of agent-tty driving Codex including setup", + "sha256": "591448b68cf14480119c61b8a630069f2ebf562636df5f0f20498d4dcf9544d1", + "bytes": 2133094 + }, + { + "path": "artifacts/codex-outer-snapshot.txt", + "description": "Text snapshot of the outer Codex session", + "sha256": "ec218396afcac7c2218ba6a93d72f205291282c5e92ce41b3ed083219be350eb", + "bytes": 4605 + }, + { + "path": "artifacts/codex-prompt.md", + "description": "Resolved prompt used by the Codex inner agent", + "sha256": "5e0db2cd1eefcc8e332bce4690a4b9c741f72be5e32c01c760e7c36ac4d4be85", + "bytes": 1416 + }, + { + "path": "artifacts/codex-recording-summary.txt", + "description": "Reviewer summary of the Codex recording set", + "sha256": "20c6aa049ad6528577c01a68a876167c3dbb418de046e58e43397db9f2e5ff51", + "bytes": 130 + }, + { + "path": "artifacts/codex-thumbnail.png", + "description": "Thumbnail still frame for the Codex recording", + "sha256": "cb35e7261a0bcb762031e01a01acf125740823e6e0cd1bdfa4d25afea35e738a", + "bytes": 108115 + } + ] +} diff --git a/dogfood/run-command/manifest.json b/dogfood/run-command/manifest.json new file mode 100644 index 0000000..8727e46 --- /dev/null +++ b/dogfood/run-command/manifest.json @@ -0,0 +1,143 @@ +{ + "bundle": "run-command", + "title": "Run command end-to-end proof", + "description": "End-to-end dogfood pass for the first-class run command: waited success, no-wait acceptance, timeout contract, run event-log evidence, and reviewable renderer artifacts.", + "createdAt": "2026-03-26T20:54:41Z", + "scenario": "run-command", + "result": "pass", + "commands": [ + "./node_modules/.bin/tsx src/cli/main.ts create --json --home -- /bin/bash", + "./node_modules/.bin/tsx src/cli/main.ts run 'echo hello-dogfood' --timeout 15000 --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts run 'echo async-dogfood' --no-wait --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts create --json --home -- /bin/sh -c 'stty -echo; exec sleep 60'", + "./node_modules/.bin/tsx src/cli/main.ts run 'echo will-timeout' --timeout 2000 --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts snapshot --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts screenshot --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts record export --format asciicast --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts record export --format webm --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts destroy --json --home ", + "./node_modules/.bin/tsx src/cli/main.ts destroy --json --home " + ], + "artifacts": [ + { + "path": "README.md", + "description": "Reviewer-facing narrative explaining the run-command end-to-end proof", + "sha256": "e56a309b9031bec6905a8ebe80f43948822209c19edf8681b2a0dba76630299b", + "bytes": 2540 + }, + { + "path": "commands.sh", + "description": "Reproducible CLI flow used to generate the bundle", + "sha256": "26ebcd24a33d0cbf5e965b4a34a685c8fe4c9b2ec5455d6cdc4aa391ed9e8a6e", + "bytes": 1177 + }, + { + "path": "bash-session-id.txt", + "description": "Session identifier captured for the interactive bash proof", + "sha256": "346676fbab0803bdf7fbfce5b81410b9d692d24fce2149f9843e91d0dee0f220", + "bytes": 27 + }, + { + "path": "timeout-session-id.txt", + "description": "Session identifier captured for the timeout proof", + "sha256": "6eaf71b45cdb10218ee2c54f357a03cee2f6e68738223edb01d6822e3f8777ef", + "bytes": 27 + }, + { + "path": "isolated-home.txt", + "description": "Recorded AGENT_TERMINAL_HOME used during capture", + "sha256": "04afe6e09020e5aaec605bd14844b1a3491b325999746158c20a9db51aa000b6", + "bytes": 20 + }, + { + "path": "create-bash.json", + "description": "create envelope for the interactive bash proof session", + "sha256": "d11c2c685a122373d0609cfa00302645f72bf8f319f4dbb23e9a2c93ecdc5294", + "bytes": 251 + }, + { + "path": "create-timeout.json", + "description": "create envelope for the dedicated timeout proof session", + "sha256": "0869d1e8f804cf039c0e52140cbbb4e04881750268548bc8bb83e49e021af1f2", + "bytes": 251 + }, + { + "path": "destroy-bash.json", + "description": "destroy envelope for the interactive bash proof session", + "sha256": "95a75e55c54d4624e9d1dabff9523ad96067d90472fe45262a06fc146548b0ce", + "bytes": 172 + }, + { + "path": "destroy-timeout.json", + "description": "destroy envelope for the dedicated timeout proof session", + "sha256": "e307d0d3a1fc6f7a8381c4348ea7243828fcecf4f694e1939b3628c621cd492c", + "bytes": 172 + }, + { + "path": "waited-success.json", + "description": "run envelope showing waited success completion (seq 3)", + "sha256": "b5db52a528e6579bff4edc7f5e2866579d78bf2de4f1f52e49de599ecf42acef", + "bytes": 267 + }, + { + "path": "no-wait.json", + "description": "run envelope showing no-wait acceptance with no completion fields", + "sha256": "811bf5970a1ef330ce8d6de04023b75b7e58d5dd6bee09be7f05e30766cce39e", + "bytes": 134 + }, + { + "path": "timeout.json", + "description": "run envelope showing timeout contract: accepted true, completed false, timedOut true", + "sha256": "ea6a62a1a40b9c6eb1ce2987ea5d492cd5865cee343be00dae0b08b43c1fd99c", + "bytes": 268 + }, + { + "path": "input-run-events.jsonl", + "description": "Raw input_run event-log entries copied from both sessions", + "sha256": "0151861f608b04771237e7c491c66869c26d5d8d9173343a4c39c2a3a6f8b90f", + "bytes": 466 + }, + { + "path": "snapshot.json", + "description": "snapshot envelope capturing post-run terminal state", + "sha256": "e954495aa5fdae4cb433c6c0ca349b4d0e97653ad212ea384d576894c147d493", + "bytes": 1889 + }, + { + "path": "screenshot-result.json", + "description": "screenshot envelope referencing the renderer PNG output", + "sha256": "b5c104a589258a6cc89d7766c03012255a9e0c4434c113d20d22ce9a4c78a168", + "bytes": 681 + }, + { + "path": "screenshot.png", + "description": "Renderer screenshot of the interactive bash session", + "sha256": "aea6b8bc32d3d0907c48f645051c0889cc73f2c3644701c0dd8b07f7507e94a7", + "bytes": 17487 + }, + { + "path": "record-export-asciicast.json", + "description": "record export envelope for the asciicast recording", + "sha256": "74fbd7d84c1d3f2e83da3b84bc19a5ed0a988293f3e5dc56a4f796fe2c6ef732", + "bytes": 668 + }, + { + "path": "record-export-webm.json", + "description": "record export envelope for the WebM recording", + "sha256": "d442f3f2666c0c0c7e1c4d0dba53304a7e84df2bb3e8b306223dcbd2a25d410d", + "bytes": 728 + }, + { + "path": "run-command.cast", + "description": "Exported asciicast recording for review playback", + "sha256": "49b7f58cc7ef1888bcd2e10b238bbe3b95bd3460ab8415934a59bdc7b1738d36", + "bytes": 1063 + }, + { + "path": "run-command.webm", + "description": "Exported WebM recording for review playback", + "sha256": "49e0e3c0a6b86661eacbca9765d7a310af092f2b1b4c9344487fcea58ccb89c7", + "bytes": 43456 + } + ] +} diff --git a/mise.toml b/mise.toml index cfd30a2..5406ca9 100644 --- a/mise.toml +++ b/mise.toml @@ -35,6 +35,20 @@ outputs = ["dist/**/*.js", "dist/**/*.d.ts", "dist/**/*.map"] description = "Validate the tarball route and check the current git-install caveat" run = "npm run smoke:install -- --skip-build" +[tasks.validate-bundles] +description = "Validate canonical proof bundles against the canonical schema" +run = "npm run validate-bundle:canonical" +sources = [ + "dogfood/CATALOG.md", + "dogfood/20260326-week9-release-readiness/**", + "dogfood/20260325-week8-contract-locks/**", + "dogfood/run-command/**", + "dogfood/agent-uses-agent-tty/**", + "src/tools/validate-bundle.ts", + "src/tools/validate-canonical-bundles.ts", + "src/tools/bundleManifestSchema.ts", +] + [tasks.typecheck] description = "Typecheck" run = "npm run typecheck" diff --git a/package.json b/package.json index e2f06e7..056ce1b 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "test:watch": "vitest", "typecheck": "tsc -p tsconfig.json --noEmit", "validate-bundle": "tsx src/tools/validate-bundle.ts", + "validate-bundle:canonical": "tsx src/tools/validate-canonical-bundles.ts", "verify": "npm run format:check && npm run lint && npm run typecheck && npm run test && npm run build && npm run smoke:install -- --skip-build", "version:json": "tsx src/cli/main.ts version --json" }, diff --git a/scripts/seed-canonical-manifest.mjs b/scripts/seed-canonical-manifest.mjs new file mode 100644 index 0000000..ecf98c2 --- /dev/null +++ b/scripts/seed-canonical-manifest.mjs @@ -0,0 +1,87 @@ +#!/usr/bin/env node +/** + * One-time helper: given a bundle root and a list of artifact paths (relative + * to the bundle root), compute sha256 + bytes for each and emit a JSON object + * with the `artifacts: [...]` block ready to paste into a canonical + * manifest.json. + * + * Usage: + * node ./scripts/seed-canonical-manifest.mjs < artifact-list.txt + * Where each line of stdin is `\t`. + */ + +import { createHash } from 'node:crypto'; +import { createReadStream } from 'node:fs'; +import { stat } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import process from 'node:process'; + +async function hashFile(filePath) { + const hash = createHash('sha256'); + return await new Promise((resolvePromise, rejectPromise) => { + const stream = createReadStream(filePath); + stream.on('data', (chunk) => hash.update(chunk)); + stream.on('end', () => { + resolvePromise(hash.digest('hex')); + }); + stream.on('error', (error) => { + stream.destroy(); + rejectPromise(error); + }); + }); +} + +async function main() { + const bundleDir = process.argv[2]; + if (bundleDir === undefined) { + process.stderr.write( + 'Usage: node ./scripts/seed-canonical-manifest.mjs < artifact-list.txt\n', + ); + process.exit(1); + } + const root = resolve(bundleDir); + + const stdinText = await new Promise((resolvePromise, rejectPromise) => { + let buffer = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk) => { + buffer += chunk; + }); + process.stdin.on('end', () => { + resolvePromise(buffer); + }); + process.stdin.on('error', rejectPromise); + }); + + const artifacts = []; + for (const rawLine of stdinText.split('\n')) { + const line = rawLine.trimEnd(); + if (line.length === 0 || line.startsWith('#')) { + continue; + } + const tabIndex = line.indexOf('\t'); + if (tabIndex === -1) { + process.stderr.write(`skipping malformed line: ${line}\n`); + continue; + } + const path = line.slice(0, tabIndex).trim(); + const description = line.slice(tabIndex + 1).trim(); + const filePath = join(root, path); + const stats = await stat(filePath); + if (!stats.isFile()) { + process.stderr.write(`not a regular file: ${path}\n`); + process.exit(1); + } + const sha256 = await hashFile(filePath); + artifacts.push({ + path, + description, + sha256, + bytes: stats.size, + }); + } + + process.stdout.write(`${JSON.stringify({ artifacts }, null, 2)}\n`); +} + +await main(); diff --git a/src/cli/commands/gc.ts b/src/cli/commands/gc.ts index 74b5c21..3fa60b1 100644 --- a/src/cli/commands/gc.ts +++ b/src/cli/commands/gc.ts @@ -14,10 +14,7 @@ import type { SessionRecord } from '../../protocol/schemas.js'; import { readManifestIfExists } from '../../storage/manifests.js'; import { manifestPath, sessionDir } from '../../storage/sessionPaths.js'; import { invariant } from '../../util/assert.js'; - -interface NodeError extends Error { - code?: string; -} +import { hasErrorCode } from '../../util/hasErrorCode.js'; export interface GcResult { removedSessions: string[]; @@ -67,10 +64,6 @@ const defaultDependencies: GcDependencies = { now: () => new Date(), }; -function hasErrorCode(error: unknown, code: string): boolean { - return error instanceof Error && (error as NodeError).code === code; -} - function getErrorMessage(error: unknown): string { if (error instanceof Error && error.message.length > 0) { return error.message; diff --git a/src/cli/commands/inspect.ts b/src/cli/commands/inspect.ts index e236b53..23c3c1a 100644 --- a/src/cli/commands/inspect.ts +++ b/src/cli/commands/inspect.ts @@ -1,6 +1,8 @@ import { HostInspectResultSchema, type ArtifactHealthSummary, + type HostInspectResult, + type HostInfo, type InspectResult, type RendererRuntimeSummary, } from '../../protocol/messages.js'; @@ -9,7 +11,10 @@ import type { SessionRecord, SessionStatus } from '../../protocol/schemas.js'; import { CliError } from '../errors.js'; import type { CommandContext } from '../context.js'; -import { countEventLogEntries } from '../../host/eventLog.js'; +import { + countEventLogEntries, + statEventLogBytes, +} from '../../host/eventLog.js'; import { reconcileSession } from '../../host/lifecycle.js'; import { sendRpc } from '../../host/rpcClient.js'; import { @@ -64,6 +69,7 @@ const RENDERER_BACKEND = 'ghostty-web'; function deriveRendererRuntimeSummary(options: { usedOfflineReplay: boolean; sessionStatus: SessionStatus; + hostInfo?: HostInspectResult; }): RendererRuntimeSummary { if (options.usedOfflineReplay) { return { @@ -83,18 +89,39 @@ function deriveRendererRuntimeSummary(options: { }; } + const hostInfo = options.hostInfo; return { backend: RENDERER_BACKEND, mode: 'live-host', status: 'healthy', + ...(hostInfo?.rendererProfile !== undefined + ? { profile: hostInfo.rendererProfile } + : {}), + ...(hostInfo?.rendererBooted !== undefined + ? { booted: hostInfo.rendererBooted } + : {}), + ...(hostInfo?.rendererBootInFlight !== undefined + ? { bootInFlight: hostInfo.rendererBootInFlight } + : {}), }; } function formatRendererRuntime(summary: RendererRuntimeSummary): string { const reasonSuffix = summary.reason === undefined ? '' : ` — ${summary.reason}`; + const extras: string[] = []; + if (summary.profile !== undefined) { + extras.push(`profile: ${summary.profile}`); + } + if (summary.booted !== undefined) { + extras.push(`booted: ${summary.booted ? 'yes' : 'no'}`); + } + if (summary.bootInFlight === true) { + extras.push('boot-in-flight'); + } + const extrasSuffix = extras.length > 0 ? ` [${extras.join(', ')}]` : ''; - return `${summary.backend} (${summary.mode}, ${summary.status}${reasonSuffix})`; + return `${summary.backend} (${summary.mode}, ${summary.status}${reasonSuffix})${extrasSuffix}`; } function formatSessionLines(result: InspectResult): string[] { @@ -116,8 +143,19 @@ function formatSessionLines(result: InspectResult): string[] { lines.push(`Last Event Seq: ${String(result.lastEventSeq)}`); } + if (result.eventLogBytes !== undefined) { + lines.push(`Event Log Bytes: ${String(result.eventLogBytes)}`); + } + lines.push(`Uptime: ${String(uptime)}ms`); + if (result.host !== undefined) { + if (result.host.cliVersion !== undefined) { + lines.push(`Host CLI Version: ${result.host.cliVersion}`); + } + lines.push(`RPC Socket: ${result.host.rpcSocketPath}`); + } + if (result.artifacts !== undefined) { lines.push( `Artifacts: ${String(result.artifacts.total)} total (${formatArtifactKinds(result.artifacts.byKind)}), health: ${result.artifacts.health}`, @@ -171,6 +209,7 @@ export async function runInspectCommand( } const isLiveHostEligible = isLiveHostEligibleSessionStatus(session.status); + let hostInfo: HostInspectResult | undefined; if (isLiveHostEligible) { try { const rawResult: unknown = await sendRpc( @@ -185,6 +224,7 @@ export async function runInspectCommand( }); } session = parsedResult.data.session; + hostInfo = parsedResult.data; } catch (error) { if ( error instanceof CliError && @@ -193,13 +233,16 @@ export async function runInspectCommand( await reconcileSession(sessionDirectory); session = await readManifest(manifestFile); usedOfflineReplay = true; + hostInfo = undefined; } else { throw error; } } } - const eventCount = await countEventLogEntries(eventLogPath(sessionDirectory)); + const eventLogFile = eventLogPath(sessionDirectory); + const eventCount = await countEventLogEntries(eventLogFile); + const eventLogBytes = await statEventLogBytes(eventLogFile); const uptime = computeUptime(session); let artifacts: ArtifactHealthSummary | undefined; try { @@ -213,7 +256,17 @@ export async function runInspectCommand( const rendererRuntime = deriveRendererRuntimeSummary({ usedOfflineReplay, sessionStatus: session.status, + ...(hostInfo !== undefined ? { hostInfo } : {}), }); + const host: HostInfo | undefined = + hostInfo !== undefined && hostInfo.rpcSocketPath !== undefined + ? { + ...(hostInfo.cliVersion !== undefined + ? { cliVersion: hostInfo.cliVersion } + : {}), + rpcSocketPath: hostInfo.rpcSocketPath, + } + : undefined; const result: InspectResult = { session, eventCount, @@ -223,6 +276,8 @@ export async function runInspectCommand( artifacts, usedOfflineReplay, rendererRuntime, + ...(host !== undefined ? { host } : {}), + ...(eventLogBytes !== undefined ? { eventLogBytes } : {}), }; emitSuccess({ diff --git a/src/cli/commands/record-export.ts b/src/cli/commands/record-export.ts index 3d9068b..77c278d 100644 --- a/src/cli/commands/record-export.ts +++ b/src/cli/commands/record-export.ts @@ -46,7 +46,7 @@ import { sessionDir, } from '../../storage/sessionPaths.js'; import { invariant } from '../../util/assert.js'; -import { loadPackageMetadata } from './version.js'; +import { loadPackageMetadata } from '../../util/packageMetadata.js'; const RecordExportFormatSchema = z.enum(['asciicast', 'webm']); diff --git a/src/cli/commands/version.ts b/src/cli/commands/version.ts index e05029c..952fe42 100644 --- a/src/cli/commands/version.ts +++ b/src/cli/commands/version.ts @@ -1,20 +1,14 @@ -import { readFile } from 'node:fs/promises'; import process from 'node:process'; import { emitSuccess } from '../output.js'; import type { CapabilityEntry } from '../../renderer/capabilities.js'; import { discoverCapabilities } from '../../renderer/capabilities.js'; -import { assertString } from '../../util/assert.js'; +import { loadPackageMetadata } from '../../util/packageMetadata.js'; const COMMAND_NAME = 'version'; const PROTOCOL_VERSION = '0.1.0'; -interface PackageMetadata { - name: string; - version: string; -} - export interface VersionResult { cliVersion: string; protocolVersion: string; @@ -27,25 +21,6 @@ export interface VersionResult { capabilities?: CapabilityEntry[]; } -export async function loadPackageMetadata(): Promise { - const packageJsonUrl = new URL('../../../package.json', import.meta.url); - const rawPackageJson = await readFile(packageJsonUrl, 'utf8'); - const parsedPackageJson = JSON.parse(rawPackageJson) as Record< - string, - unknown - >; - const packageName = parsedPackageJson.name; - const packageVersion = parsedPackageJson.version; - - assertString(packageName, 'package.json name must be a string'); - assertString(packageVersion, 'package.json version must be a string'); - - return { - name: packageName, - version: packageVersion, - }; -} - export async function buildVersionResult(options?: { includeCapabilities?: boolean; }): Promise { diff --git a/src/host/eventLog.ts b/src/host/eventLog.ts index db6f83e..82cd4b0 100644 --- a/src/host/eventLog.ts +++ b/src/host/eventLog.ts @@ -1,5 +1,5 @@ import { createReadStream } from 'node:fs'; -import { open, readFile } from 'node:fs/promises'; +import { open, readFile, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { createInterface } from 'node:readline'; @@ -20,6 +20,7 @@ import { parseEventLogContent, } from '../storage/eventLogCodec.js'; import { invariant } from '../util/assert.js'; +import { hasErrorCode } from '../util/hasErrorCode.js'; const OutputEventPayloadSchema = z .object({ @@ -182,6 +183,27 @@ function deriveNextSeq(records: readonly EventRecord[]): number { return lastRecord.seq + 1; } +/** + * Returns the event log file size in bytes, or `undefined` when the file + * does not exist (ENOENT). Non-ENOENT errors propagate to the caller. + */ +export async function statEventLogBytes( + filePath: string, +): Promise { + assertFilePath(filePath); + + try { + const stats = await stat(filePath); + return stats.size; + } catch (error: unknown) { + if (hasErrorCode(error, 'ENOENT')) { + return undefined; + } + + throw error; + } +} + export async function countEventLogEntries(filePath: string): Promise { assertFilePath(filePath); @@ -199,12 +221,7 @@ export async function countEventLogEntries(filePath: string): Promise { } } } catch (error: unknown) { - if ( - typeof error === 'object' && - error !== null && - 'code' in error && - error.code === 'ENOENT' - ) { + if (hasErrorCode(error, 'ENOENT')) { return 0; } diff --git a/src/host/hostMain.ts b/src/host/hostMain.ts index c4c9385..5b28a36 100644 --- a/src/host/hostMain.ts +++ b/src/host/hostMain.ts @@ -2,6 +2,7 @@ import { mkdir } from 'node:fs/promises'; import { dirname } from 'node:path'; import process from 'node:process'; +import { loadPackageMetadata } from '../util/packageMetadata.js'; import { matchRenderWaitSnapshot, prepareRenderWaitCondition, @@ -153,6 +154,19 @@ export async function runHost(sessionId: string): Promise { const eventLog = await EventLog.open(ePath); + // Lazy + tolerant: `cliVersion` is an optional inspect field, so a missing + // or unreadable package.json must not crash the host. Cache the first + // attempt for the lifetime of the host process. + let packageMetadataPromise: + | Promise<{ version: string } | undefined> + | undefined; + const getPackageMetadata = (): Promise<{ version: string } | undefined> => { + if (packageMetadataPromise === undefined) { + packageMetadataPromise = loadPackageMetadata().catch(() => undefined); + } + return packageMetadataPromise; + }; + const rendererManager = new HostRendererManager({ sessionId, sessionDir: sessDir, @@ -410,7 +424,29 @@ export async function runHost(sessionId: string): Promise { }; const handlers: Record = { - inspect: () => Promise.resolve({ session: state.snapshot() }), + inspect: async () => { + // Capture every observable in a single synchronous tick before the + // `await` below yields the event loop. Concurrent RPC handlers can + // otherwise mutate renderer state and the session snapshot between + // reads, producing a result that does not correspond to any real + // point in time. + const sessionSnapshot = state.snapshot(); + const rendererProfile = rendererManager.getCurrentProfileName(); + const rendererBooted = rendererManager.isBooted(); + const rendererBootInFlight = rendererManager.isBootInFlight(); + + const packageMetadata = await getPackageMetadata(); + return { + session: sessionSnapshot, + ...(packageMetadata !== undefined + ? { cliVersion: packageMetadata.version } + : {}), + rpcSocketPath: sPath, + ...(rendererProfile !== null ? { rendererProfile } : {}), + rendererBooted, + rendererBootInFlight, + }; + }, snapshot: async (params: unknown) => { const { format: requestedFormat, diff --git a/src/host/lifecycle.ts b/src/host/lifecycle.ts index 8d8e70b..f876d5f 100644 --- a/src/host/lifecycle.ts +++ b/src/host/lifecycle.ts @@ -27,6 +27,7 @@ import { } from '../storage/sessionPaths.js'; import { makeAbortError, throwIfAborted } from '../util/abort.js'; import { invariant } from '../util/assert.js'; +import { hasErrorCode } from '../util/hasErrorCode.js'; import { sendRpc } from './rpcClient.js'; const DESTROY_POLL_INTERVAL_MS = 100; @@ -60,10 +61,6 @@ async function pollDelay( } } -interface NodeError extends Error { - code?: string; -} - export interface AllocateConfig { home: string; command: string[]; @@ -98,14 +95,6 @@ export interface SessionSummary { pid: number | null; } -function isNodeError(error: unknown): error is NodeError { - return error instanceof Error; -} - -function hasErrorCode(error: unknown, code: string): boolean { - return isNodeError(error) && error.code === code; -} - function makeInvalidDimensionError( label: 'cols' | 'rows', value: unknown, diff --git a/src/host/renderer.ts b/src/host/renderer.ts index 66796b2..582feaf 100644 --- a/src/host/renderer.ts +++ b/src/host/renderer.ts @@ -39,6 +39,7 @@ export class HostRendererManager { private currentBackend: RendererBackend | null = null; private currentBackendKey: string | null = null; + private currentProfileName: string | null = null; private cachedInitialCols: number | null = null; private cachedInitialRows: number | null = null; private bootPromise: Promise | null = null; @@ -137,6 +138,18 @@ export class HostRendererManager { }); } + isBooted(): boolean { + return this.currentBackend !== null && this.currentBackend.isBooted; + } + + isBootInFlight(): boolean { + return this.bootPromise !== null; + } + + getCurrentProfileName(): string | null { + return this.currentProfileName; + } + private async ensureBackend( rendererName: RendererName, profile: RenderProfileConfig, @@ -158,6 +171,7 @@ export class HostRendererManager { this.currentBackend = backend; this.currentBackendKey = backendKey; + this.currentProfileName = profile.name; } invariant(this.currentBackend !== null, 'current backend must exist'); @@ -203,6 +217,7 @@ export class HostRendererManager { this.currentBackend = null; this.currentBackendKey = null; + this.currentProfileName = null; this.cachedInitialCols = null; this.cachedInitialRows = null; this.bootPromise = null; diff --git a/src/protocol/messages.ts b/src/protocol/messages.ts index f3a2180..349f903 100644 --- a/src/protocol/messages.ts +++ b/src/protocol/messages.ts @@ -103,10 +103,36 @@ export type InspectParams = z.infer; export const HostInspectResultSchema = z .object({ session: SessionRecordSchema, + // Best-effort: the host reads `package.json` lazily on first inspect + // call and falls back to `undefined` when the file is missing or + // unreadable, rather than failing the RPC. + cliVersion: z.string().min(1).optional(), + // Always populated by the live host (the socket path it accepts + // RPC connections on). Optional in the schema only so older hosts + // that predate this field still parse. + rpcSocketPath: z.string().min(1).optional(), + // Renderer state captured atomically in the same synchronous tick. + // Populated only when the host has reached the inspect handler. An + // older host or one whose renderer has never bootstrapped omits + // these and the CLI surfaces them as absent on `rendererRuntime`. + rendererProfile: z.string().min(1).optional(), + rendererBooted: z.boolean().optional(), + rendererBootInFlight: z.boolean().optional(), }) .strict(); export type HostInspectResult = z.infer; +export const HostInfoSchema = z + .object({ + // `cliVersion` is best-effort: when `package.json` is unreadable the + // host omits it rather than failing the inspect RPC. `rpcSocketPath` + // is always populated for live-host inspects. + cliVersion: z.string().min(1).optional(), + rpcSocketPath: z.string().min(1), + }) + .strict(); +export type HostInfo = z.infer; + export const TerminationCategorySchema = z.enum([ 'running', 'clean-exit', @@ -157,6 +183,15 @@ export const InspectResultSchema = z artifacts: ArtifactHealthSummarySchema.optional(), usedOfflineReplay: z.boolean().optional(), rendererRuntime: RendererRuntimeSummarySchema, + // Populated only when the inspect call reached a live host (i.e. + // `rendererRuntime.mode === 'live-host'`). Absent in offline-replay + // mode (`usedOfflineReplay === true` or the session is not live-host + // eligible). + host: HostInfoSchema.optional(), + // Populated in both live and offline-replay modes from a `stat()` on the + // session's `events.jsonl`. Absent only when the event log file is + // missing on disk (e.g. a session that crashed before its first write). + eventLogBytes: z.number().int().nonnegative().optional(), }) .strict(); export type InspectResult = z.infer; diff --git a/src/renderer/capabilities.ts b/src/renderer/capabilities.ts index 7018e11..b155ae9 100644 --- a/src/renderer/capabilities.ts +++ b/src/renderer/capabilities.ts @@ -54,6 +54,14 @@ export const RendererRuntimeSummarySchema = z mode: RendererRuntimeModeSchema, status: RendererRuntimeStatusSchema, reason: z.string().optional(), + // `profile`, `booted`, and `bootInFlight` are only populated when + // `mode === 'live-host'` and the host has reached the inspect RPC + // handler. All three stay absent in offline-replay mode and when the + // host runs an older protocol version that does not surface them. + // `min(1)` matches the producer-side `HostInspectResultSchema`. + profile: z.string().min(1).optional(), + booted: z.boolean().optional(), + bootInFlight: z.boolean().optional(), }) .strict(); export type RendererRuntimeSummary = z.infer< diff --git a/src/tools/bundleManifestSchema.ts b/src/tools/bundleManifestSchema.ts new file mode 100644 index 0000000..c94fd55 --- /dev/null +++ b/src/tools/bundleManifestSchema.ts @@ -0,0 +1,47 @@ +/** + * Strict Zod schema for the `manifest.json` files that govern the canonical + * proof bundles. `RELEASE.md` names the three release-signoff bundles + * (`dogfood/20260326-week9-release-readiness/`, + * `dogfood/20260325-week8-contract-locks/`, and `dogfood/run-command/`). + * `dogfood/agent-uses-agent-tty/` is the evergreen agent demo bundle + * (surfaced in the README and `CHANGELOG.md`), locked here on the same + * schema so CI catches drift in the same place. + * + * Required `sha256` + `bytes` per artifact let `validate-bundle.ts --profile + * canonical` recompute and compare each digest, catching any byte-level drift + * in a canonical bundle. Historical bundles use the permissive + * `BundleManifestSchema` in `review-bundle.ts` instead. + */ + +import { z } from 'zod'; + +const SHA256_HEX_REGEX = /^[0-9a-f]{64}$/; + +export const CanonicalBundleArtifactSchema = z + .object({ + path: z.string().min(1), + description: z.string().min(1), + sha256: z.string().regex(SHA256_HEX_REGEX), + bytes: z.number().int().nonnegative(), + }) + .strict(); +export type CanonicalBundleArtifact = z.infer< + typeof CanonicalBundleArtifactSchema +>; + +export const CanonicalBundleManifestSchema = z + .object({ + bundle: z.string().min(1), + title: z.string().min(1), + description: z.string().min(1), + createdAt: z.iso.datetime({ offset: true }), + result: z.enum(['pass', 'fail', 'partial']), + commands: z.array(z.string().min(1)).min(1), + artifacts: z.array(CanonicalBundleArtifactSchema).min(1), + week: z.number().int().optional(), + scenario: z.string().min(1).optional(), + }) + .strict(); +export type CanonicalBundleManifest = z.infer< + typeof CanonicalBundleManifestSchema +>; diff --git a/src/tools/review-bundle.ts b/src/tools/review-bundle.ts index 7527987..8c42be9 100644 --- a/src/tools/review-bundle.ts +++ b/src/tools/review-bundle.ts @@ -17,11 +17,12 @@ import { sep, } from 'node:path'; import process from 'node:process'; -import { pathToFileURL } from 'node:url'; import { z } from 'zod'; import { assertString, invariant } from '../util/assert.js'; +import { isDirectExecution } from '../util/isDirectExecution.js'; +import { isWithinRoot } from '../util/isWithinRoot.js'; export type ArtifactKind = | 'screenshot' @@ -160,11 +161,6 @@ function normalizeSeparators(value: string): string { return value.split(sep).join('/'); } -function isWithinRoot(rootPath: string, candidatePath: string): boolean { - const relPath = relative(rootPath, candidatePath); - return relPath === '' || (!relPath.startsWith('..') && !isAbsolute(relPath)); -} - function htmlEscape(value: string): string { return value .replaceAll('&', '&') @@ -1320,15 +1316,7 @@ export async function runReviewBundleCli( } } -function isDirectExecution(): boolean { - const entryPoint = process.argv[1]; - if (entryPoint === undefined) { - return false; - } - return import.meta.url === pathToFileURL(entryPoint).href; -} - -if (isDirectExecution()) { +if (isDirectExecution(import.meta.url)) { const exitCode = await runReviewBundleCli(process.argv.slice(2)); process.exitCode = exitCode; } diff --git a/src/tools/validate-bundle.ts b/src/tools/validate-bundle.ts index f8906ab..dbb8cc2 100644 --- a/src/tools/validate-bundle.ts +++ b/src/tools/validate-bundle.ts @@ -4,22 +4,28 @@ * Usage: npm run validate-bundle -- [--profile ] */ +import { createHash } from 'node:crypto'; +import { createReadStream } from 'node:fs'; import { readFile, realpath, stat } from 'node:fs/promises'; -import { join, resolve } from 'node:path'; +import { join, relative, resolve } from 'node:path'; import process from 'node:process'; -import { pathToFileURL } from 'node:url'; +import { pipeline } from 'node:stream/promises'; +import { CanonicalBundleManifestSchema } from './bundleManifestSchema.js'; import { scanBundleArtifacts } from './review-bundle.js'; import { assertString, invariant } from '../util/assert.js'; +import { hasErrorCode } from '../util/hasErrorCode.js'; +import { isDirectExecution } from '../util/isDirectExecution.js'; +import { isWithinRoot } from '../util/isWithinRoot.js'; const BUNDLE_VALIDATION_PROFILES = [ 'contract-reporting', 'interactive-renderer', + 'canonical', ] as const; export type BundleValidationProfile = - | 'contract-reporting' - | 'interactive-renderer'; + (typeof BUNDLE_VALIDATION_PROFILES)[number]; export interface BundleValidationResult { bundleDir: string; @@ -70,8 +76,11 @@ export const MAX_JSON_FILE_BYTES = 50 * 1024 * 1024; async function isFile(filePath: string): Promise { try { return (await stat(filePath)).isFile(); - } catch { - return false; + } catch (error) { + if (hasErrorCode(error, 'ENOENT')) { + return false; + } + throw error; } } @@ -137,6 +146,314 @@ function summarizeValidation(result: BundleValidationResult): string[] { ]; } +async function hashFile(filePath: string): Promise { + const hash = createHash('sha256'); + await pipeline(createReadStream(filePath), hash); + return hash.digest('hex'); +} + +async function runCanonicalChecks( + bundleRoot: string, +): Promise { + const checks: BundleValidationCheck[] = []; + + const manifestPath = join(bundleRoot, 'manifest.json'); + let manifestText: string; + try { + manifestText = await readFile(manifestPath, 'utf8'); + } catch (error) { + checks.push( + buildCheck( + 'manifest-exists', + false, + `Could not read manifest.json: ${String(error)}`, + ), + ); + return checks; + } + checks.push( + buildCheck( + 'manifest-exists', + true, + `Read manifest.json (${String(Buffer.byteLength(manifestText, 'utf8'))} bytes).`, + ), + ); + + let manifestJson: unknown; + try { + manifestJson = JSON.parse(manifestText); + } catch (error) { + checks.push( + buildCheck( + 'manifest-parses', + false, + `manifest.json is not valid JSON: ${String(error)}`, + ), + ); + return checks; + } + + const parseResult = CanonicalBundleManifestSchema.safeParse(manifestJson); + if (!parseResult.success) { + const issues = parseResult.error.issues + .map((issue) => `${issue.path.join('.')}: ${issue.message}`) + .join('; '); + checks.push( + buildCheck( + 'manifest-parses', + false, + `manifest.json does not match CanonicalBundleManifestSchema: ${issues}`, + ), + ); + return checks; + } + const manifest = parseResult.data; + checks.push( + buildCheck( + 'manifest-parses', + true, + `Manifest matches schema (${String(manifest.artifacts.length)} artifact entr${manifest.artifacts.length === 1 ? 'y' : 'ies'}).`, + ), + ); + + const escapedArtifacts: string[] = []; + const missingArtifacts: string[] = []; + const statErrors: string[] = []; + const sizeMismatches: string[] = []; + const hashMismatches: string[] = []; + let bytesCheckedCount = 0; + let hashedCount = 0; + + for (const artifact of manifest.artifacts) { + const artifactPath = join(bundleRoot, artifact.path); + if (!isWithinRoot(bundleRoot, artifactPath)) { + escapedArtifacts.push(artifact.path); + continue; + } + let stats; + try { + stats = await stat(artifactPath); + } catch (error) { + if (hasErrorCode(error, 'ENOENT')) { + missingArtifacts.push(artifact.path); + } else { + statErrors.push(`${artifact.path}: ${String(error)}`); + } + continue; + } + if (!stats.isFile()) { + missingArtifacts.push(artifact.path); + continue; + } + bytesCheckedCount += 1; + if (stats.size !== artifact.bytes) { + sizeMismatches.push( + `${artifact.path} (manifest: ${String(artifact.bytes)}, on-disk: ${String(stats.size)})`, + ); + continue; + } + hashedCount += 1; + const actualHash = await hashFile(artifactPath); + if (actualHash !== artifact.sha256) { + hashMismatches.push( + `${artifact.path} (manifest: ${artifact.sha256}, on-disk: ${actualHash})`, + ); + } + } + + const totalArtifacts = manifest.artifacts.length; + const presentOk = + missingArtifacts.length === 0 && + escapedArtifacts.length === 0 && + statErrors.length === 0; + const presentParts: string[] = []; + if (missingArtifacts.length > 0) { + presentParts.push(`Missing: ${missingArtifacts.join(', ')}`); + } + if (escapedArtifacts.length > 0) { + presentParts.push( + `Paths escape bundle root: ${escapedArtifacts.join(', ')}`, + ); + } + if (statErrors.length > 0) { + presentParts.push(`stat() errors: ${statErrors.join('; ')}`); + } + checks.push( + buildCheck( + 'artifacts-present', + presentOk, + presentOk + ? `All ${String(totalArtifacts)} artifact(s) present as regular files.` + : presentParts.join(' | '), + ), + ); + + const bytesOk = sizeMismatches.length === 0 && bytesCheckedCount > 0; + checks.push( + buildCheck( + 'artifacts-bytes-match', + bytesOk, + sizeMismatches.length > 0 + ? `Byte-size mismatches: ${sizeMismatches.join('; ')}` + : bytesCheckedCount === 0 + ? `No artifacts available to verify (${String(totalArtifacts)} skipped).` + : `${String(bytesCheckedCount)} of ${String(totalArtifacts)} artifact byte sizes match the manifest${bytesCheckedCount === totalArtifacts ? '.' : ` (${String(totalArtifacts - bytesCheckedCount)} skipped).`}`, + ), + ); + + const hashOk = hashMismatches.length === 0 && hashedCount > 0; + checks.push( + buildCheck( + 'artifacts-sha256-match', + hashOk, + hashMismatches.length > 0 + ? `sha256 mismatches: ${hashMismatches.join('; ')}` + : hashedCount === 0 + ? `No artifacts available to verify (${String(totalArtifacts)} skipped).` + : `${String(hashedCount)} of ${String(totalArtifacts)} artifact sha256 digests match the manifest${hashedCount === totalArtifacts ? '.' : ` (${String(totalArtifacts - hashedCount)} skipped).`}`, + ), + ); + + const commandsShPath = join(bundleRoot, 'commands.sh'); + const reproduceShPath = join(bundleRoot, 'reproduce.sh'); + const hasReproduceScript = + (await isFile(commandsShPath)) || (await isFile(reproduceShPath)); + checks.push( + buildCheck( + 'reproduce-script-exists', + hasReproduceScript, + hasReproduceScript + ? 'Found commands.sh or reproduce.sh.' + : 'Expected commands.sh or reproduce.sh in the bundle root.', + ), + ); + + if (manifest.result === 'pass') { + const tsvPath = join(bundleRoot, 'command-status.tsv'); + const tsvExists = await isFile(tsvPath); + if (tsvExists) { + const tsvContent = await readFile(tsvPath, 'utf8'); + const lines = tsvContent.split('\n').filter((line) => line.length > 0); + const headerColumns = lines[0]?.split('\t') ?? []; + const hasHeader = headerColumns.length > 1; + // Locate the `status` column by header name; fall back to column index 1 + // (the historical TSV layout) so existing bundles still validate. + const statusColumnIndex = (() => { + const fromHeader = headerColumns.findIndex( + (column) => column.trim().toLowerCase() === 'status', + ); + return fromHeader >= 0 ? fromHeader : 1; + })(); + const dataLines = lines.slice(1); + const failingRows = dataLines.filter((line) => { + const columns = line.split('\t'); + return columns[statusColumnIndex]?.trim().toLowerCase() === 'fail'; + }); + checks.push( + buildCheck( + 'command-status-tsv-clean-if-pass', + hasHeader && failingRows.length === 0, + hasHeader && failingRows.length === 0 + ? `command-status.tsv has a header and no failing rows (${String(dataLines.length)} data row(s)).` + : !hasHeader + ? 'command-status.tsv is missing a header row.' + : `command-status.tsv has ${String(failingRows.length)} failing row(s) in the status column.`, + ), + ); + } else { + checks.push( + buildCheck( + 'command-status-tsv-clean-if-pass', + true, + 'command-status.tsv not present (scenario bundle); skipping.', + ), + ); + } + } + + const hasNotesOrReadme = + (await isFile(join(bundleRoot, 'notes.md'))) || + (await isFile(join(bundleRoot, 'README.md'))); + checks.push( + buildCheck( + 'notes-or-readme-present', + hasNotesOrReadme, + hasNotesOrReadme + ? 'Found notes.md or README.md.' + : 'Expected notes.md or README.md in the bundle root.', + ), + ); + + return checks; +} + +export interface CatalogParityResult { + ok: boolean; + missing: string[]; +} + +/** + * Confirms that every `//...` path mentioned in + * the catalog markdown resolves to an existing directory under + * `dogfoodRoot`. Glob-shaped historical entries (e.g. + * `dogfood/20260319-*`) are deliberately skipped; the regex requires a + * literal trailing path component so truncated globs do not register as + * real directories. + * + * Returns `{ ok: true, missing: [] }` on success. On failure, `missing` + * lists the bundle names whose directory could not be `stat()`d (or whose + * stat returned a non-directory). Non-ENOENT stat errors propagate to the + * caller via `hasErrorCode`. + */ +export async function checkCatalogParity( + catalogPath: string, + dogfoodRoot: string, +): Promise { + const catalogText = await readFile(catalogPath, 'utf8'); + const dogfoodRelativeName = relative(resolve(dogfoodRoot, '..'), dogfoodRoot); + const escapedPrefix = dogfoodRelativeName.replace( + /[.*+?^${}()|[\]\\]/g, + '\\$&', + ); + // Match `dogfood//...` paths in CATALOG.md only when they include a + // trailing path component (i.e. end with `/` after the bundle name). This + // skips glob-shaped entries like `dogfood/20260319-*` which truncate at the + // `*` and would otherwise look like real directories. + const pathRegex = new RegExp( + `${escapedPrefix}\\/([A-Za-z0-9._][A-Za-z0-9._-]*)\\/`, + 'g', + ); + const seen = new Set(); + const missing: string[] = []; + for (const match of catalogText.matchAll(pathRegex)) { + const firstSegment = match[1]; + if (firstSegment === undefined || firstSegment.length === 0) { + continue; + } + if (firstSegment.endsWith('-')) { + continue; + } + if (seen.has(firstSegment)) { + continue; + } + seen.add(firstSegment); + const dirPath = join(dogfoodRoot, firstSegment); + try { + const stats = await stat(dirPath); + if (!stats.isDirectory()) { + missing.push(firstSegment); + } + } catch (error) { + if (hasErrorCode(error, 'ENOENT')) { + missing.push(firstSegment); + } else { + throw error; + } + } + } + return { ok: missing.length === 0, missing }; +} + export async function validateBundle( bundleDir: string, profile: BundleValidationProfile, @@ -193,6 +510,17 @@ export async function validateBundle( ), ); + if (profile === 'canonical') { + const canonicalChecks = await runCanonicalChecks(bundleRoot); + checks.push(...canonicalChecks); + return { + bundleDir: bundleRoot, + profile, + ok: checks.every((check) => check.ok), + checks, + }; + } + const artifacts = await scanBundleArtifacts(bundleRoot); const jsonArtifacts = artifacts.filter( (artifact) => artifact.kind === 'json', @@ -354,15 +682,7 @@ export async function runValidateBundleCli( return result.ok ? 0 : 1; } -function isDirectExecution(): boolean { - const entryPoint = process.argv[1]; - if (entryPoint === undefined) { - return false; - } - return import.meta.url === pathToFileURL(entryPoint).href; -} - -if (isDirectExecution()) { +if (isDirectExecution(import.meta.url)) { const exitCode = await runValidateBundleCli(process.argv.slice(2)); process.exitCode = exitCode; } diff --git a/src/tools/validate-canonical-bundles.ts b/src/tools/validate-canonical-bundles.ts new file mode 100644 index 0000000..96342f2 --- /dev/null +++ b/src/tools/validate-canonical-bundles.ts @@ -0,0 +1,109 @@ +/** + * Validates the four canonical proof bundles against the strict `canonical` + * profile, plus a CATALOG.md parity check. + * + * RELEASE.md's validation section names the three release-signoff bundles + * (`20260326-week9-release-readiness`, `20260325-week8-contract-locks`, + * `run-command`). `agent-uses-agent-tty` is the evergreen agent demo bundle + * (surfaced in the README and `CHANGELOG.md` rather than `RELEASE.md`), + * locked here on the same schema so CI catches drift in the same place. + * + * Run via `npm run validate-bundle:canonical` or `mise run validate-bundles`. + */ + +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; + +import { + checkCatalogParity, + validateBundle, + type BundleValidationResult, +} from './validate-bundle.js'; +import { isDirectExecution } from '../util/isDirectExecution.js'; + +const CANONICAL_BUNDLES = [ + 'dogfood/20260326-week9-release-readiness', + 'dogfood/20260325-week8-contract-locks', + 'dogfood/run-command', + 'dogfood/agent-uses-agent-tty', +] as const; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); + +function synthesizeCrashResult( + bundlePath: string, + error: unknown, +): BundleValidationResult { + return { + bundleDir: bundlePath, + profile: 'canonical', + ok: false, + checks: [ + { + name: 'validation-error', + ok: false, + message: `Bundle validation crashed: ${String(error)}`, + }, + ], + }; +} + +export async function validateCanonicalBundles(): Promise { + const bundleResults = await Promise.all( + CANONICAL_BUNDLES.map(async (relativeBundle) => { + const bundlePath = resolve(REPO_ROOT, relativeBundle); + try { + const result = await validateBundle(bundlePath, 'canonical'); + return { relativeBundle, result }; + } catch (error) { + return { + relativeBundle, + result: synthesizeCrashResult(bundlePath, error), + }; + } + }), + ); + + let allOk = true; + for (const { relativeBundle, result } of bundleResults) { + const status = result.ok ? 'PASS' : 'FAIL'; + process.stderr.write( + `validate-bundle ${status} canonical: ${relativeBundle}\n`, + ); + for (const check of result.checks) { + const mark = check.ok ? '✓' : '✗'; + process.stderr.write(` ${mark} ${check.name}: ${check.message}\n`); + } + if (!result.ok) { + allOk = false; + } + } + + try { + const catalogResult = await checkCatalogParity( + resolve(REPO_ROOT, 'dogfood/CATALOG.md'), + resolve(REPO_ROOT, 'dogfood'), + ); + if (catalogResult.ok) { + process.stderr.write( + 'catalog-parity PASS: every CATALOG.md entry resolves to a directory\n', + ); + } else { + process.stderr.write( + `catalog-parity FAIL: ${String(catalogResult.missing.length)} CATALOG.md entr${catalogResult.missing.length === 1 ? 'y' : 'ies'} did not resolve: ${catalogResult.missing.join(', ')}\n`, + ); + allOk = false; + } + } catch (error) { + process.stderr.write(`catalog-parity ERROR: ${String(error)}\n`); + allOk = false; + } + + return allOk ? 0 : 1; +} + +if (isDirectExecution(import.meta.url)) { + const exitCode = await validateCanonicalBundles(); + process.exitCode = exitCode; +} diff --git a/src/util/hasErrorCode.ts b/src/util/hasErrorCode.ts new file mode 100644 index 0000000..da3ddde --- /dev/null +++ b/src/util/hasErrorCode.ts @@ -0,0 +1,11 @@ +/** + * Returns `true` when the thrown value is a Node-style error whose `code` + * matches the given identifier (e.g. `'ENOENT'`, `'EACCES'`, `'ESRCH'`). + * + * Accepts any unknown value and falls back to `false` when the value is not + * an Error or lacks a `code` property, so callers can use this in catch + * blocks without further type guards. + */ +export function hasErrorCode(error: unknown, code: string): boolean { + return error instanceof Error && (error as { code?: unknown }).code === code; +} diff --git a/src/util/isDirectExecution.ts b/src/util/isDirectExecution.ts new file mode 100644 index 0000000..a44d450 --- /dev/null +++ b/src/util/isDirectExecution.ts @@ -0,0 +1,16 @@ +import process from 'node:process'; +import { pathToFileURL } from 'node:url'; + +/** + * Returns `true` when the calling module is the script being executed directly + * (e.g. `node path/to/file.mjs` or `tsx path/to/file.ts`), rather than + * imported. Pass `import.meta.url` from the caller; it cannot be resolved + * from here because every module has its own. + */ +export function isDirectExecution(importMetaUrl: string): boolean { + const entryPoint = process.argv[1]; + if (entryPoint === undefined) { + return false; + } + return importMetaUrl === pathToFileURL(entryPoint).href; +} diff --git a/src/util/isWithinRoot.ts b/src/util/isWithinRoot.ts new file mode 100644 index 0000000..6402741 --- /dev/null +++ b/src/util/isWithinRoot.ts @@ -0,0 +1,20 @@ +import { isAbsolute, relative, resolve, sep } from 'node:path'; + +/** + * Returns `true` when `candidatePath` resolves to a location at or beneath + * `rootPath`. The candidate may be relative; it is `resolve()`d before the + * containment check. The root itself counts as inside (relative path is the + * empty string). + * + * Used to guard manifest-driven file access from path-traversal entries like + * `../../.git/config`. + */ +export function isWithinRoot(rootPath: string, candidatePath: string): boolean { + const relPath = relative(rootPath, resolve(candidatePath)); + return ( + relPath === '' || + (relPath !== '..' && + !relPath.startsWith(`..${sep}`) && + !isAbsolute(relPath)) + ); +} diff --git a/src/util/packageMetadata.ts b/src/util/packageMetadata.ts new file mode 100644 index 0000000..84bc436 --- /dev/null +++ b/src/util/packageMetadata.ts @@ -0,0 +1,34 @@ +import { readFile } from 'node:fs/promises'; + +import { assertString } from './assert.js'; + +export interface PackageMetadata { + name: string; + version: string; +} + +/** + * Reads the root `package.json` (resolved relative to this module's disk + * location, not the caller's) and returns the package name and version. + * Throws when the file is missing or when `name` / `version` is absent or + * non-string. Callers that need a tolerant fallback should wrap the call in + * `.catch()`. + */ +export async function loadPackageMetadata(): Promise { + const packageJsonUrl = new URL('../../package.json', import.meta.url); + const rawPackageJson = await readFile(packageJsonUrl, 'utf8'); + const parsedPackageJson = JSON.parse(rawPackageJson) as Record< + string, + unknown + >; + const packageName = parsedPackageJson.name; + const packageVersion = parsedPackageJson.version; + + assertString(packageName, 'package.json name must be a string'); + assertString(packageVersion, 'package.json version must be a string'); + + return { + name: packageName, + version: packageVersion, + }; +} diff --git a/test/e2e/hello-prompt.test.ts b/test/e2e/hello-prompt.test.ts index 41db4fa..16fe7a7 100644 --- a/test/e2e/hello-prompt.test.ts +++ b/test/e2e/hello-prompt.test.ts @@ -138,11 +138,13 @@ describe('hello-prompt e2e', { timeout: 30_000 }, () => { expect(inspectRunning.command).toBe('inspect'); expect(inspectRunning.result.session.status).toBe('running'); expect(inspectRunning.result.session.exitCode).toBeNull(); - expect(inspectRunning.result.rendererRuntime).toEqual({ - backend: 'ghostty-web', - mode: 'live-host', - status: 'healthy', - }); + expect(inspectRunning.result.rendererRuntime).toEqual( + expect.objectContaining({ + backend: 'ghostty-web', + mode: 'live-host', + status: 'healthy', + }), + ); const typeExitEnvelope = runCliJson>>( ['type', sessionId, 'exit'], diff --git a/test/integration/lifecycle.test.ts b/test/integration/lifecycle.test.ts index 6f16acc..ed1ffa3 100644 --- a/test/integration/lifecycle.test.ts +++ b/test/integration/lifecycle.test.ts @@ -123,11 +123,13 @@ describe('lifecycle integration', { timeout: 30000 }, () => { expect(inspectEnvelope.result.session.status).toBe('running'); expect(inspectEnvelope.result.session.hostPid).toBeTypeOf('number'); expect(inspectEnvelope.result.session.childPid).toBeTypeOf('number'); - expect(inspectEnvelope.result.rendererRuntime).toEqual({ - backend: 'ghostty-web', - mode: 'live-host', - status: 'healthy', - }); + expect(inspectEnvelope.result.rendererRuntime).toEqual( + expect.objectContaining({ + backend: 'ghostty-web', + mode: 'live-host', + status: 'healthy', + }), + ); expect(listedSession?.pid).toBe(inspectEnvelope.result.session.childPid); const destroyResult = runCli(['destroy', sessionId, '--json'], { diff --git a/test/unit/commands/inspect.test.ts b/test/unit/commands/inspect.test.ts index 0ffab92..f8670e3 100644 --- a/test/unit/commands/inspect.test.ts +++ b/test/unit/commands/inspect.test.ts @@ -5,6 +5,7 @@ import { ERROR_CODES, makeCliError } from '../../../src/protocol/errors.js'; const mocks = vi.hoisted(() => ({ computeArtifactHealth: vi.fn(), countEventLogEntries: vi.fn(), + statEventLogBytes: vi.fn(), deriveTerminationCategory: vi.fn(), emitSuccess: vi.fn(), reconcileSession: vi.fn(), @@ -23,6 +24,7 @@ vi.mock('../../../src/cli/output.js', () => ({ vi.mock('../../../src/host/eventLog.js', () => ({ countEventLogEntries: mocks.countEventLogEntries, + statEventLogBytes: mocks.statEventLogBytes, })); vi.mock('../../../src/host/lifecycle.js', () => ({ @@ -128,6 +130,7 @@ describe('inspect command', () => { ); mocks.computeArtifactHealth.mockResolvedValue(DEFAULT_ARTIFACT_HEALTH); mocks.countEventLogEntries.mockResolvedValue(2); + mocks.statEventLogBytes.mockResolvedValue(undefined); mocks.deriveTerminationCategory.mockReturnValue('running'); mocks.readManifestIfExists.mockResolvedValue( createSessionRecord('running'), @@ -182,23 +185,21 @@ describe('inspect command', () => { lines: string[]; }; - expect(emitted).toEqual( + expect(emitted.command).toBe('inspect'); + expect(emitted.json).toBe(false); + expect(emitted.result).toEqual( expect.objectContaining({ - command: 'inspect', - json: false, - result: { - session: liveSession, - eventCount: 2, - uptime: 5000, - lastEventSeq: 1, - terminationCategory: 'running', - artifacts: DEFAULT_ARTIFACT_HEALTH, - usedOfflineReplay: false, - rendererRuntime: { - backend: 'ghostty-web', - mode: 'live-host', - status: 'healthy', - }, + session: liveSession, + eventCount: 2, + uptime: 5000, + lastEventSeq: 1, + terminationCategory: 'running', + artifacts: DEFAULT_ARTIFACT_HEALTH, + usedOfflineReplay: false, + rendererRuntime: { + backend: 'ghostty-web', + mode: 'live-host', + status: 'healthy', }, }), ); @@ -379,24 +380,22 @@ describe('inspect command', () => { lines: string[]; }; - expect(emitted).toEqual( + expect(emitted.command).toBe('inspect'); + expect(emitted.json).toBe(true); + expect(emitted.result).toEqual( expect.objectContaining({ - command: 'inspect', - json: true, - result: { - session: reconciledSession, - eventCount: 2, - uptime: 1000, - lastEventSeq: 1, - terminationCategory: 'clean-exit', - artifacts: DEFAULT_ARTIFACT_HEALTH, - usedOfflineReplay: true, - rendererRuntime: { - backend: 'ghostty-web', - mode: 'offline-replay', - status: 'fallback', - reason: 'host-unreachable', - }, + session: reconciledSession, + eventCount: 2, + uptime: 1000, + lastEventSeq: 1, + terminationCategory: 'clean-exit', + artifacts: DEFAULT_ARTIFACT_HEALTH, + usedOfflineReplay: true, + rendererRuntime: { + backend: 'ghostty-web', + mode: 'offline-replay', + status: 'fallback', + reason: 'host-unreachable', }, }), ); @@ -513,4 +512,169 @@ describe('inspect command', () => { expect.stringMatching(/^Last Event Seq:/), ); }); + + it('surfaces host info and renderer extensions in live mode', async () => { + const liveSession = createSessionRecord('running'); + mocks.sendRpc.mockResolvedValue({ + session: liveSession, + cliVersion: '0.2.1', + rpcSocketPath: '/tmp/agent-tty/sessions/session-01/rpc.sock', + rendererProfile: 'reference-dark', + rendererBooted: true, + rendererBootInFlight: false, + }); + + await runInspectCommand({ + context: TEST_CONTEXT, + json: true, + sessionId: 'session-01', + }); + + const emitted = getLastEmitSuccessPayload() as { + result: { + host?: { cliVersion: string; rpcSocketPath: string }; + rendererRuntime: { + backend: string; + mode: string; + status: string; + profile?: string; + booted?: boolean; + bootInFlight?: boolean; + }; + }; + lines: string[]; + }; + + expect(emitted.result.host).toEqual({ + cliVersion: '0.2.1', + rpcSocketPath: '/tmp/agent-tty/sessions/session-01/rpc.sock', + }); + expect(emitted.result.rendererRuntime).toEqual({ + backend: 'ghostty-web', + mode: 'live-host', + status: 'healthy', + profile: 'reference-dark', + booted: true, + bootInFlight: false, + }); + expect(emitted.lines).toEqual( + expect.arrayContaining([ + 'Host CLI Version: 0.2.1', + 'RPC Socket: /tmp/agent-tty/sessions/session-01/rpc.sock', + 'Renderer: ghostty-web (live-host, healthy) [profile: reference-dark, booted: yes]', + ]), + ); + }); + + it('surfaces host.rpcSocketPath even when cliVersion is unavailable', async () => { + // Exercises the DEREM-24 fix: when `loadPackageMetadata` fails on the + // host, `cliVersion` is omitted from the RPC response but + // `rpcSocketPath` is still populated. The CLI must surface the socket + // path instead of dropping the entire `host` block. + const liveSession = createSessionRecord('running'); + mocks.sendRpc.mockResolvedValue({ + session: liveSession, + rpcSocketPath: '/tmp/agent-tty/sessions/session-01/rpc.sock', + rendererBooted: false, + rendererBootInFlight: false, + }); + + await runInspectCommand({ + context: TEST_CONTEXT, + json: true, + sessionId: 'session-01', + }); + + const emitted = getLastEmitSuccessPayload() as { + result: { + host?: { cliVersion?: string; rpcSocketPath: string }; + }; + lines: string[]; + }; + + expect(emitted.result.host).toEqual({ + rpcSocketPath: '/tmp/agent-tty/sessions/session-01/rpc.sock', + }); + expect(emitted.result.host?.cliVersion).toBeUndefined(); + expect(emitted.lines).toContain( + 'RPC Socket: /tmp/agent-tty/sessions/session-01/rpc.sock', + ); + expect(emitted.lines).not.toEqual( + expect.arrayContaining([expect.stringMatching(/^Host CLI Version:/)]), + ); + }); + + it('omits host info and renderer extensions in offline-replay mode', async () => { + const reconciledSession = createSessionRecord('exited'); + mocks.sendRpc.mockRejectedValue( + makeCliError(ERROR_CODES.HOST_UNREACHABLE, { + message: 'Session host is unreachable.', + }), + ); + mocks.readManifest.mockResolvedValue(reconciledSession); + mocks.deriveTerminationCategory.mockReturnValue('clean-exit'); + + await runInspectCommand({ + context: TEST_CONTEXT, + json: true, + sessionId: 'session-01', + }); + + const emitted = getLastEmitSuccessPayload() as { + result: { + host?: unknown; + rendererRuntime: { + backend: string; + mode: string; + profile?: string; + booted?: boolean; + bootInFlight?: boolean; + }; + }; + }; + + expect(emitted.result.host).toBeUndefined(); + expect(emitted.result.rendererRuntime.profile).toBeUndefined(); + expect(emitted.result.rendererRuntime.booted).toBeUndefined(); + expect(emitted.result.rendererRuntime.bootInFlight).toBeUndefined(); + }); + + it('surfaces eventLogBytes in both live and offline modes', async () => { + const liveSession = createSessionRecord('running'); + mocks.sendRpc.mockResolvedValue({ session: liveSession }); + mocks.statEventLogBytes.mockResolvedValue(4096); + + await runInspectCommand({ + context: TEST_CONTEXT, + json: true, + sessionId: 'session-01', + }); + + const liveEmitted = getLastEmitSuccessPayload() as { + result: { eventLogBytes?: number }; + lines: string[]; + }; + expect(liveEmitted.result.eventLogBytes).toBe(4096); + expect(liveEmitted.lines).toContain('Event Log Bytes: 4096'); + + const reconciledSession = createSessionRecord('exited'); + mocks.sendRpc.mockRejectedValue( + makeCliError(ERROR_CODES.HOST_UNREACHABLE, { + message: 'Session host is unreachable.', + }), + ); + mocks.readManifest.mockResolvedValue(reconciledSession); + mocks.statEventLogBytes.mockResolvedValue(8192); + + await runInspectCommand({ + context: TEST_CONTEXT, + json: true, + sessionId: 'session-01', + }); + + const offlineEmitted = getLastEmitSuccessPayload() as { + result: { eventLogBytes?: number }; + }; + expect(offlineEmitted.result.eventLogBytes).toBe(8192); + }); }); diff --git a/test/unit/commands/record-export.test.ts b/test/unit/commands/record-export.test.ts index f6900c8..f734149 100644 --- a/test/unit/commands/record-export.test.ts +++ b/test/unit/commands/record-export.test.ts @@ -74,7 +74,7 @@ vi.mock('../../../src/storage/artifactPaths.js', () => ({ recordingFilename: mocks.recordingFilename, })); -vi.mock('../../../src/cli/commands/version.js', () => ({ +vi.mock('../../../src/util/packageMetadata.js', () => ({ loadPackageMetadata: mocks.loadPackageMetadata, })); diff --git a/test/unit/commands/version.test.ts b/test/unit/commands/version.test.ts index 1a28b52..e0bdb25 100644 --- a/test/unit/commands/version.test.ts +++ b/test/unit/commands/version.test.ts @@ -1,10 +1,8 @@ import { describe, expect, it, vi } from 'vitest'; import * as capabilitiesModule from '../../../src/renderer/capabilities.js'; -import { - buildVersionResult, - loadPackageMetadata, -} from '../../../src/cli/commands/version.js'; +import { buildVersionResult } from '../../../src/cli/commands/version.js'; +import { loadPackageMetadata } from '../../../src/util/packageMetadata.js'; const SEMVER_WITH_OPTIONAL_PRERELEASE = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/; diff --git a/test/unit/host/eventLog.test.ts b/test/unit/host/eventLog.test.ts index 398cb73..aba578c 100644 --- a/test/unit/host/eventLog.test.ts +++ b/test/unit/host/eventLog.test.ts @@ -15,6 +15,7 @@ import { countEventLogEntries, EventLog, MAX_EVENT_BUFFER_ENTRIES, + statEventLogBytes, } from '../../../src/host/eventLog.js'; import { MAX_EVENT_LOG_SIZE } from '../../../src/storage/eventLogCodec.js'; @@ -60,6 +61,55 @@ describe('countEventLogEntries', () => { }); }); +describe('statEventLogBytes', () => { + beforeEach(async () => { + // oxfmt-ignore + tempDir = await realpath(await mkdtemp(join(tmpdir(), 'agent-tty-event-log-'))); + eventLogPath = join(tempDir, 'events.jsonl'); + }); + + afterEach(async () => { + if (tempDir.length > 0) { + await rm(tempDir, { recursive: true, force: true }); + } + }); + + it('returns undefined when the event log file is missing', async () => { + await expect(statEventLogBytes(eventLogPath)).resolves.toBeUndefined(); + }); + + it('returns 0 for an empty event log file', async () => { + await writeFile(eventLogPath, '', 'utf8'); + + await expect(statEventLogBytes(eventLogPath)).resolves.toBe(0); + }); + + it('returns the byte size of a populated event log file', async () => { + const payload = `${JSON.stringify({ seq: 0, type: 'output' })}\n`; + await writeFile(eventLogPath, payload, 'utf8'); + + await expect(statEventLogBytes(eventLogPath)).resolves.toBe( + Buffer.byteLength(payload, 'utf8'), + ); + }); + + it('rejects empty file paths', async () => { + await expect(statEventLogBytes('')).rejects.toThrow(); + }); + + it('rethrows non-ENOENT errors so callers see them', async () => { + // A path whose parent is a regular file rather than a directory triggers + // ENOTDIR from stat(), which must propagate (only ENOENT may be swallowed). + const blocker = join(tempDir, 'blocker'); + await writeFile(blocker, 'x', 'utf8'); + const childOfFile = join(blocker, 'events.jsonl'); + + await expect(statEventLogBytes(childOfFile)).rejects.toMatchObject({ + code: 'ENOTDIR', + }); + }); +}); + describe('EventLog', () => { beforeEach(async () => { // oxfmt-ignore diff --git a/test/unit/host/renderer.test.ts b/test/unit/host/renderer.test.ts index 2c15bd0..a506015 100644 --- a/test/unit/host/renderer.test.ts +++ b/test/unit/host/renderer.test.ts @@ -328,6 +328,65 @@ describe('HostRendererManager', () => { expect(backendFactory).not.toHaveBeenCalled(); }); + it('exposes booted, bootInFlight, and current profile name getters', async () => { + const manager = new HostRendererManager({ + sessionId: 'session-01', + sessionDir, + backendFactory, + }); + + expect(manager.isBooted()).toBe(false); + expect(manager.isBootInFlight()).toBe(false); + expect(manager.getCurrentProfileName()).toBeNull(); + + await manager.getBackend( + 'ghostty-web', + createProfile('reference-dark'), + null, + ); + + expect(manager.isBooted()).toBe(true); + expect(manager.isBootInFlight()).toBe(false); + expect(manager.getCurrentProfileName()).toBe('reference-dark'); + + await manager.dispose(); + + expect(manager.isBooted()).toBe(false); + expect(manager.getCurrentProfileName()).toBeNull(); + }); + + it('reports isBootInFlight as true while a boot is awaiting', async () => { + const manager = new HostRendererManager({ + sessionId: 'session-01', + sessionDir, + backendFactory, + }); + const bootDeferred = createDeferred(); + + backendFactory.mockImplementationOnce(() => { + const backend = createFakeBackend({ + bootImplementation: () => + bootDeferred.promise.then(() => { + backend.setBooted(true); + }), + }); + backends.push(backend); + return backend; + }); + + const inflight = manager.getBackend('ghostty-web', createProfile(), null); + await flushAsyncQueue(); + + expect(manager.isBootInFlight()).toBe(true); + expect(manager.isBooted()).toBe(false); + + bootDeferred.resolve(undefined); + await inflight; + + expect(manager.isBootInFlight()).toBe(false); + expect(manager.isBooted()).toBe(true); + }); + it('allocates screenshot paths inside the session screenshots directory', async () => { const manager = new HostRendererManager({ sessionId: 'session-01', diff --git a/test/unit/tools/validate-bundle.test.ts b/test/unit/tools/validate-bundle.test.ts index 942a2e4..690cf12 100644 --- a/test/unit/tools/validate-bundle.test.ts +++ b/test/unit/tools/validate-bundle.test.ts @@ -11,7 +11,10 @@ import { dirname, join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; +import { createHash } from 'node:crypto'; + import { + checkCatalogParity, MAX_JSON_FILE_BYTES, runValidateBundleCli, validateBundle, @@ -235,3 +238,578 @@ describe('validate-bundle', () => { expect(findCheck(result, 'bundle-exists').ok).toBe(false); }); }); + +interface CanonicalArtifactFixture { + path: string; + description: string; + content: string; +} + +interface CanonicalManifestOptions { + commands?: string[]; + result?: 'pass' | 'fail' | 'partial'; + scenario?: string; + week?: number; + extraFields?: Record; +} + +function sha256Hex(text: string): string { + return createHash('sha256').update(text).digest('hex'); +} + +async function writeCanonicalBundle( + artifacts: CanonicalArtifactFixture[], + options: CanonicalManifestOptions = {}, +): Promise { + const bundleRoot = await createTempDir(); + for (const artifact of artifacts) { + await writeFixtureFile(bundleRoot, artifact.path, artifact.content); + } + const manifest = { + bundle: 'fixture-bundle', + title: 'Fixture bundle', + description: 'Fixture canonical bundle for validate-bundle tests', + createdAt: '2026-05-14T00:00:00Z', + scenario: options.scenario ?? 'fixture-bundle', + ...(options.week !== undefined ? { week: options.week } : {}), + result: options.result ?? 'pass', + commands: options.commands ?? ['echo test'], + artifacts: artifacts.map((artifact) => ({ + path: artifact.path, + description: artifact.description, + sha256: sha256Hex(artifact.content), + bytes: Buffer.byteLength(artifact.content, 'utf8'), + })), + ...options.extraFields, + }; + await writeFixtureFile( + bundleRoot, + 'manifest.json', + `${JSON.stringify(manifest, null, 2)}\n`, + ); + return bundleRoot; +} + +describe('validate-bundle canonical profile', () => { + it('passes a well-formed canonical bundle', async () => { + const bundleRoot = await writeCanonicalBundle([ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\necho test\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{"ok":true}\n', + }, + ]); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(true); + expect(result.profile).toBe('canonical'); + expect(findCheck(result, 'manifest-exists').ok).toBe(true); + expect(findCheck(result, 'manifest-parses').ok).toBe(true); + expect(findCheck(result, 'artifacts-present').ok).toBe(true); + expect(findCheck(result, 'artifacts-bytes-match').ok).toBe(true); + expect(findCheck(result, 'artifacts-sha256-match').ok).toBe(true); + expect(findCheck(result, 'reproduce-script-exists').ok).toBe(true); + expect(findCheck(result, 'notes-or-readme-present').ok).toBe(true); + }); + + it('accepts reproduce.sh in place of commands.sh', async () => { + const bundleRoot = await writeCanonicalBundle([ + { path: 'README.md', description: 'narrative', content: '# Bundle\n' }, + { + path: 'reproduce.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ]); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(true); + expect(findCheck(result, 'reproduce-script-exists').ok).toBe(true); + expect(findCheck(result, 'notes-or-readme-present').ok).toBe(true); + }); + + it('fails fast when manifest.json is missing', async () => { + const bundleRoot = await createTempDir(); + await writeFixtureFile(bundleRoot, 'notes.md', '# Notes\n'); + await writeFixtureFile(bundleRoot, 'commands.sh', '#!/bin/sh\n'); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect(findCheck(result, 'manifest-exists').ok).toBe(false); + expect( + result.checks.some((check) => check.name === 'manifest-parses'), + ).toBe(false); + }); + + it('fails when manifest.json is not valid JSON', async () => { + const bundleRoot = await createTempDir(); + await writeFixtureFile(bundleRoot, 'manifest.json', '{not json}\n'); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect(findCheck(result, 'manifest-exists').ok).toBe(true); + expect(findCheck(result, 'manifest-parses').ok).toBe(false); + expect(findCheck(result, 'manifest-parses').message).toContain( + 'not valid JSON', + ); + }); + + it('fails when manifest.json does not match the canonical schema', async () => { + const bundleRoot = await createTempDir(); + await writeFixtureFile( + bundleRoot, + 'manifest.json', + JSON.stringify({ bundle: 'incomplete' }), + ); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect(findCheck(result, 'manifest-parses').ok).toBe(false); + expect(findCheck(result, 'manifest-parses').message).toContain( + 'CanonicalBundleManifestSchema', + ); + }); + + it('fails when a manifest artifact is missing on disk', async () => { + const bundleRoot = await writeCanonicalBundle([ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ]); + await rm(join(bundleRoot, 'envelope.json')); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect(findCheck(result, 'artifacts-present').ok).toBe(false); + expect(findCheck(result, 'artifacts-present').message).toContain( + 'envelope.json', + ); + }); + + it('fails when an artifact byte size disagrees with the manifest', async () => { + const bundleRoot = await writeCanonicalBundle([ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ]); + await writeFile( + join(bundleRoot, 'envelope.json'), + '{"changed":true}\n', + 'utf8', + ); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect(findCheck(result, 'artifacts-bytes-match').ok).toBe(false); + expect(findCheck(result, 'artifacts-bytes-match').message).toContain( + 'envelope.json', + ); + }); + + it('fails when an artifact sha256 disagrees with the manifest', async () => { + const bundleRoot = await writeCanonicalBundle([ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + // Original content is exactly 3 bytes ('{}\n'); replacement is also 3 + // bytes so size matches but sha256 differs. + content: '{}\n', + }, + ]); + await writeFile(join(bundleRoot, 'envelope.json'), 'xx\n', 'utf8'); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect(findCheck(result, 'artifacts-bytes-match').ok).toBe(true); + expect(findCheck(result, 'artifacts-sha256-match').ok).toBe(false); + expect(findCheck(result, 'artifacts-sha256-match').message).toContain( + 'envelope.json', + ); + }); + + it('fails when neither commands.sh nor reproduce.sh is present', async () => { + const bundleRoot = await writeCanonicalBundle([ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ]); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect(findCheck(result, 'reproduce-script-exists').ok).toBe(false); + }); + + it('fails when neither notes.md nor README.md is present', async () => { + const bundleRoot = await writeCanonicalBundle([ + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ]); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect(findCheck(result, 'notes-or-readme-present').ok).toBe(false); + }); + + it('skips command-status.tsv check when the file is absent on a passing bundle', async () => { + const bundleRoot = await writeCanonicalBundle( + [ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ], + { result: 'pass' }, + ); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(true); + expect( + findCheck(result, 'command-status-tsv-clean-if-pass').message, + ).toContain('not present'); + }); + + it('fails when command-status.tsv has a failing row on a passing bundle', async () => { + const bundleRoot = await writeCanonicalBundle( + [ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ], + { result: 'pass' }, + ); + await writeFixtureFile( + bundleRoot, + 'command-status.tsv', + 'step\tstatus\n01-create\tpass\n02-inspect\tfail\n', + ); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect(findCheck(result, 'command-status-tsv-clean-if-pass').ok).toBe( + false, + ); + expect( + findCheck(result, 'command-status-tsv-clean-if-pass').message, + ).toContain('1 failing row'); + }); + + it('fails when command-status.tsv is missing a header on a passing bundle', async () => { + const bundleRoot = await writeCanonicalBundle( + [ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ], + { result: 'pass' }, + ); + await writeFixtureFile( + bundleRoot, + 'command-status.tsv', + 'one-column-only\n', + ); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect( + findCheck(result, 'command-status-tsv-clean-if-pass').message, + ).toContain('header row'); + }); + + it('skips command-status.tsv check entirely for non-pass canonical bundles', async () => { + const bundleRoot = await writeCanonicalBundle( + [ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ], + { result: 'fail' }, + ); + await writeFixtureFile( + bundleRoot, + 'command-status.tsv', + 'step\tstatus\nbroken\tfail\n', + ); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect( + result.checks.some( + (check) => check.name === 'command-status-tsv-clean-if-pass', + ), + ).toBe(false); + }); + + it('ignores cells containing "fail" outside the status column', async () => { + const bundleRoot = await writeCanonicalBundle( + [ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ], + { result: 'pass' }, + ); + // Notes column mentions "fail"; must not trip the check. + await writeFixtureFile( + bundleRoot, + 'command-status.tsv', + 'step\tstatus\tnotes\n01-create\tpass\ttested fail path\n', + ); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(true); + expect(findCheck(result, 'command-status-tsv-clean-if-pass').ok).toBe(true); + }); + + it('rejects artifacts whose paths escape the bundle root', async () => { + const bundleRoot = await writeCanonicalBundle([ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ]); + // Hand-craft a manifest with a path-traversal entry, overwriting the + // helper-generated one. + const escapingManifest = { + bundle: 'fixture-bundle', + title: 'Escaping fixture', + description: 'Manifest with a path that resolves outside the bundle', + createdAt: '2026-05-14T00:00:00Z', + scenario: 'fixture-bundle', + result: 'pass', + commands: ['echo test'], + artifacts: [ + { + path: '../escape-target', + description: 'evil', + sha256: + '0000000000000000000000000000000000000000000000000000000000000000', + bytes: 0, + }, + ], + }; + await writeFile( + join(bundleRoot, 'manifest.json'), + `${JSON.stringify(escapingManifest, null, 2)}\n`, + 'utf8', + ); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect(findCheck(result, 'artifacts-present').ok).toBe(false); + expect(findCheck(result, 'artifacts-present').message).toContain( + 'escape bundle root', + ); + expect(findCheck(result, 'artifacts-present').message).toContain( + '../escape-target', + ); + }); + + it('reports byte-match counts that reflect skipped artifacts', async () => { + const bundleRoot = await writeCanonicalBundle([ + { path: 'notes.md', description: 'narrative', content: '# Notes\n' }, + { + path: 'commands.sh', + description: 'reproduce', + content: '#!/bin/sh\n', + }, + { + path: 'envelope.json', + description: 'cli envelope', + content: '{}\n', + }, + ]); + await rm(join(bundleRoot, 'envelope.json')); + + const result = await validateBundle(bundleRoot, 'canonical'); + + expect(result.ok).toBe(false); + expect(findCheck(result, 'artifacts-bytes-match').message).toContain( + '2 of 3', + ); + expect(findCheck(result, 'artifacts-sha256-match').message).toContain( + '2 of 3', + ); + }); +}); + +describe('checkCatalogParity', () => { + async function writeCatalogFixture( + entries: string[], + realDirectories: string[], + ): Promise<{ catalogPath: string; dogfoodRoot: string }> { + const repoRoot = await createTempDir(); + const dogfoodRoot = join(repoRoot, 'dogfood'); + await mkdir(dogfoodRoot, { recursive: true }); + for (const directory of realDirectories) { + await mkdir(join(dogfoodRoot, directory), { recursive: true }); + } + const catalogPath = join(dogfoodRoot, 'CATALOG.md'); + await writeFile(catalogPath, `${entries.join('\n')}\n`, 'utf8'); + return { catalogPath, dogfoodRoot }; + } + + it('passes when every listed bundle resolves to a real directory', async () => { + const { catalogPath, dogfoodRoot } = await writeCatalogFixture( + [ + '| `dogfood/scenario-one/` | First scenario |', + '| `dogfood/scenario-two/` | Second scenario |', + ], + ['scenario-one', 'scenario-two'], + ); + + const result = await checkCatalogParity(catalogPath, dogfoodRoot); + + expect(result.ok).toBe(true); + expect(result.missing).toEqual([]); + }); + + it('reports missing directories listed in the catalog', async () => { + const { catalogPath, dogfoodRoot } = await writeCatalogFixture( + [ + '| `dogfood/scenario-one/` | First scenario |', + '| `dogfood/missing-scenario/` | Stale entry |', + ], + ['scenario-one'], + ); + + const result = await checkCatalogParity(catalogPath, dogfoodRoot); + + expect(result.ok).toBe(false); + expect(result.missing).toEqual(['missing-scenario']); + }); + + it('skips glob-shaped historical entries that truncate at the asterisk', async () => { + const { catalogPath, dogfoodRoot } = await writeCatalogFixture( + [ + '| `dogfood/scenario-one/` | First scenario |', + 'Historical: `dogfood/20260319-*`, `dogfood/20260321-*`', + ], + ['scenario-one'], + ); + + const result = await checkCatalogParity(catalogPath, dogfoodRoot); + + expect(result.ok).toBe(true); + expect(result.missing).toEqual([]); + }); + + it('deduplicates repeated bundle references', async () => { + const { catalogPath, dogfoodRoot } = await writeCatalogFixture( + [ + '| `dogfood/scenario-one/` | First scenario |', + '`dogfood/scenario-one/foo` is also referenced', + ], + ['scenario-one'], + ); + + const result = await checkCatalogParity(catalogPath, dogfoodRoot); + + expect(result.ok).toBe(true); + expect(result.missing).toEqual([]); + }); +}); diff --git a/test/unit/tools/validate-canonical-bundles.test.ts b/test/unit/tools/validate-canonical-bundles.test.ts new file mode 100644 index 0000000..ce03051 --- /dev/null +++ b/test/unit/tools/validate-canonical-bundles.test.ts @@ -0,0 +1,165 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { BundleValidationResult } from '../../../src/tools/validate-bundle.js'; + +const mocks = vi.hoisted(() => ({ + validateBundle: vi.fn(), + checkCatalogParity: vi.fn(), +})); + +vi.mock('../../../src/tools/validate-bundle.js', () => ({ + validateBundle: mocks.validateBundle, + checkCatalogParity: mocks.checkCatalogParity, +})); + +import { validateCanonicalBundles } from '../../../src/tools/validate-canonical-bundles.js'; + +const CANONICAL_BUNDLE_COUNT = 4; + +function passResult(bundlePath: string): BundleValidationResult { + return { + bundleDir: bundlePath, + profile: 'canonical', + ok: true, + checks: [ + { + name: 'manifest-exists', + ok: true, + message: 'Read manifest.json (123 bytes).', + }, + ], + }; +} + +function failResult( + bundlePath: string, + reason: string, +): BundleValidationResult { + return { + bundleDir: bundlePath, + profile: 'canonical', + ok: false, + checks: [ + { + name: 'artifacts-sha256-match', + ok: false, + message: reason, + }, + ], + }; +} + +let stderrChunks: string[]; + +beforeEach(() => { + stderrChunks = []; + vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => { + stderrChunks.push( + typeof chunk === 'string' + ? chunk + : Buffer.from(chunk as Uint8Array).toString('utf8'), + ); + return true; + }) as typeof process.stderr.write); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('validateCanonicalBundles orchestrator', () => { + it('returns 0 when every bundle and the catalog all pass', async () => { + mocks.validateBundle.mockImplementation((bundlePath: string) => + Promise.resolve(passResult(bundlePath)), + ); + mocks.checkCatalogParity.mockResolvedValue({ ok: true, missing: [] }); + + const exitCode = await validateCanonicalBundles(); + + expect(exitCode).toBe(0); + expect(mocks.validateBundle).toHaveBeenCalledTimes(CANONICAL_BUNDLE_COUNT); + expect(mocks.checkCatalogParity).toHaveBeenCalledTimes(1); + const output = stderrChunks.join(''); + expect(output).toContain('validate-bundle PASS canonical:'); + expect(output).toContain( + 'catalog-parity PASS: every CATALOG.md entry resolves to a directory', + ); + }); + + it('returns 1 when a bundle reports ok: false', async () => { + mocks.validateBundle.mockImplementation((bundlePath: string) => { + if (bundlePath.endsWith('run-command')) { + return Promise.resolve(failResult(bundlePath, 'sha256 mismatch: foo')); + } + return Promise.resolve(passResult(bundlePath)); + }); + mocks.checkCatalogParity.mockResolvedValue({ ok: true, missing: [] }); + + const exitCode = await validateCanonicalBundles(); + + expect(exitCode).toBe(1); + const output = stderrChunks.join(''); + expect(output).toContain('validate-bundle FAIL canonical:'); + expect(output).toContain('run-command'); + expect(output).toContain('sha256 mismatch: foo'); + }); + + it('synthesizes a validation-error check when a bundle validation throws', async () => { + mocks.validateBundle.mockImplementation((bundlePath: string) => { + if (bundlePath.endsWith('run-command')) { + return Promise.reject(new Error('boom: stream destroyed')); + } + return Promise.resolve(passResult(bundlePath)); + }); + mocks.checkCatalogParity.mockResolvedValue({ ok: true, missing: [] }); + + const exitCode = await validateCanonicalBundles(); + + expect(exitCode).toBe(1); + const output = stderrChunks.join(''); + expect(output).toContain( + 'validate-bundle FAIL canonical: dogfood/run-command', + ); + expect(output).toContain('validation-error'); + expect(output).toContain('Bundle validation crashed'); + expect(output).toContain('boom: stream destroyed'); + // The other three bundles still produced their PASS lines; a single + // crash does not abort the whole batch. + expect(output.match(/validate-bundle PASS canonical:/g)).toHaveLength(3); + }); + + it('returns 1 when catalog parity reports missing entries', async () => { + mocks.validateBundle.mockImplementation((bundlePath: string) => + Promise.resolve(passResult(bundlePath)), + ); + mocks.checkCatalogParity.mockResolvedValue({ + ok: false, + missing: ['stale-bundle'], + }); + + const exitCode = await validateCanonicalBundles(); + + expect(exitCode).toBe(1); + const output = stderrChunks.join(''); + expect(output).toContain( + 'catalog-parity FAIL: 1 CATALOG.md entry did not resolve', + ); + expect(output).toContain('stale-bundle'); + }); + + it('returns 1 and logs catalog-parity ERROR when checkCatalogParity throws', async () => { + mocks.validateBundle.mockImplementation((bundlePath: string) => + Promise.resolve(passResult(bundlePath)), + ); + mocks.checkCatalogParity.mockRejectedValue( + new Error('EACCES: permission denied'), + ); + + const exitCode = await validateCanonicalBundles(); + + expect(exitCode).toBe(1); + const output = stderrChunks.join(''); + expect(output).toContain('catalog-parity ERROR'); + expect(output).toContain('EACCES: permission denied'); + }); +});