From 4dbf7d834a023cff071dbdd66257129a5382404c Mon Sep 17 00:00:00 2001 From: Beta-Devin AI <248786709+beta-devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 11:42:53 +0000 Subject: [PATCH] computer, examples/worker, docs: split shell bundle into feature groups The worker backend embeds the whole just-bash shell in SHELL_MODULES, and a handful of optional commands dominate the upload: curl carries undici (~620 KB) and html-to-markdown carries domino (~555 KB), with python, sqlite, and js-exec behind them. Every consumer paid for all of them because the modules shipped as one frozen record with no seam to drop a command. build-bundle.mjs now partitions the emitted modules into a core group plus one group per optional command and writes them under generated/ as separate modules. A chunk lands in a feature group only when that feature is its sole reacher and core cannot reach it, so shared chunks and anything a kept command needs stay in core and dropping one feature never breaks another. undici is attributed to curl by hand because just-bash loads it from the shell body rather than through the curl command chunk. shell-modules.ts assembles SHELL_MODULES from the groups, importing each through its @cloudflare/computer/shell/ subpath so a consumer can swap one for @cloudflare/computer/empty with a wrangler alias. That drops the feature's exclusive chunks, heavy dependency included, from the uploaded Worker. Leaving the alias out ships the whole shell, byte-for-byte the single-file bundle. The command still parses; invoking it after its chunk is gone fails at runtime with a module-not-found. The worker example drops curl and html-to-markdown, taking its upload from ~5,956 KiB to ~4,774 KiB raw. The package and worker docs describe the mechanism and the available features. --- .gitignore | 9 +- docs/12_worker_backend.md | 10 +- examples/worker/README.md | 31 +++ examples/worker/wrangler.jsonc | 16 ++ packages/computer/README.md | 23 ++ packages/computer/package.json | 28 +++ packages/computer/rolldown.config.ts | 19 ++ .../computer/src/backends/worker/empty.ts | 6 + .../backends/worker/generated-bundle.test.ts | 58 ----- .../computer/src/backends/worker/index.ts | 2 +- .../backends/worker/script/build-bundle.mjs | 232 +++++++++++++++--- .../src/backends/worker/shell-modules.test.ts | 87 +++++++ .../src/backends/worker/shell-modules.ts | 36 +++ .../computer/src/backends/worker/worker.ts | 2 +- .../test-helpers/shell-module-aliases.ts | 24 ++ packages/computer/tsconfig.json | 13 +- packages/computer/vitest.config.ts | 9 + .../computer/vitest.config.worker-backend.ts | 9 + 18 files changed, 510 insertions(+), 104 deletions(-) create mode 100644 packages/computer/src/backends/worker/empty.ts delete mode 100644 packages/computer/src/backends/worker/generated-bundle.test.ts create mode 100644 packages/computer/src/backends/worker/shell-modules.test.ts create mode 100644 packages/computer/src/backends/worker/shell-modules.ts create mode 100644 packages/computer/test-helpers/shell-module-aliases.ts diff --git a/.gitignore b/.gitignore index 8bf20f8a..2d35ca09 100644 --- a/.gitignore +++ b/.gitignore @@ -6,10 +6,11 @@ PLAN.md # tracked code and must not be caught by this rule. /artifacts/ -# Generated bundle for the worker backend's ShellWorker. Built -# by packages/computer/src/backends/worker/script/build-bundle.mjs -# on prepare / pretest / pretypecheck. -packages/computer/src/backends/worker/generated-bundle.ts +# Generated shell-module groups for the worker backend's +# ShellWorker (core plus one file per optional feature). Built by +# packages/computer/src/backends/worker/script/build-bundle.mjs on +# prepare / pretest / pretypecheck. +packages/computer/src/backends/worker/generated/ # SEA binary destinations populated at publish time from # artifacts/computerd/ via the build-bin step. The @cloudflare/computer diff --git a/docs/12_worker_backend.md b/docs/12_worker_backend.md index a511d221..46b59a60 100644 --- a/docs/12_worker_backend.md +++ b/docs/12_worker_backend.md @@ -254,7 +254,15 @@ network-bound `git` subcommands do. See `@cloudflare/computer/backends/worker` as `SHELL_MODULES` (a record of module name → source covering the entry plus every code-split chunk); the backend hands the whole record - to the Loader callback itself. + to the Loader callback itself. `SHELL_MODULES` is assembled + from per-feature groups — a core group plus one per optional + command (`curl`, `html-to-markdown`, `python`, `sqlite`, + `js-exec`), each published on its own + `@cloudflare/computer/shell/` subpath. Aliasing a + subpath to `@cloudflare/computer/empty` in `wrangler.jsonc` + drops that feature's chunks — and its heavy dependency (undici + for `curl`, domino for `html-to-markdown`) — from the uploaded + Worker. The DO's backend wiring fits in three lines: diff --git a/examples/worker/README.md b/examples/worker/README.md index 5d8dee73..5cae5f74 100644 --- a/examples/worker/README.md +++ b/examples/worker/README.md @@ -129,6 +129,37 @@ curl -X POST http://127.0.0.1:8787/c/demo/exec \ -d '{"command":"cat hello.txt && wc -l hello.txt","encoding":"utf8"}' ``` +## Trimming the shell bundle + +`SHELL_MODULES` carries every just-bash command, and a handful of +optional ones dominate the upload: `curl` drags in undici +(~620 KB) and `html-to-markdown` drags in domino (~555 KB), with +`python`, `sqlite`, and `js-exec` behind them. Each optional +command's chunks live behind their own +`@cloudflare/computer/shell/` subpath, so a Worker that +doesn't need one can drop it at build time with a wrangler +`alias`: + +```jsonc +"alias": { + "@cloudflare/computer/shell/curl": "@cloudflare/computer/empty", + "@cloudflare/computer/shell/html-to-markdown": "@cloudflare/computer/empty" +} +``` + +Aliasing a feature to `@cloudflare/computer/empty` swaps its +module group for an empty table, so its chunks — the heavy +dependency included — never enter the uploaded Worker. The +available features are `curl`, `html-to-markdown`, `python`, +`sqlite`, and `js-exec`. The command still parses; invoking one +after its chunk is gone fails at runtime with a "module not +found". Leaving the alias out ships the whole shell. + +This example drops `curl` and `html-to-markdown`, which takes the +Worker upload from ~5,956 KiB to ~4,774 KiB raw (~1,344 KiB to +~1,071 KiB gzip). Remove the `alias` entries in +[`wrangler.jsonc`](wrangler.jsonc) to ship them. + ## Layout ``` diff --git a/examples/worker/wrangler.jsonc b/examples/worker/wrangler.jsonc index 6f2b0274..e9e3d4e5 100644 --- a/examples/worker/wrangler.jsonc +++ b/examples/worker/wrangler.jsonc @@ -11,6 +11,22 @@ "compatibility_date": "2026-05-26", "compatibility_flags": ["nodejs_compat", "experimental"], + // Drop optional just-bash commands this example doesn't use. + // @cloudflare/computer bundles each optional command's chunks + // (curl's undici, html-to-markdown's domino, python, sqlite, + // js-exec) behind its own @cloudflare/computer/shell/ + // subpath. Aliasing a feature to @cloudflare/computer/empty + // swaps it for an empty module table, so its chunks — including + // the heavy dependency — never enter the uploaded Worker. + // Dropping curl and html-to-markdown alone removes ~1.2 MB raw + // (~620 KB undici + ~555 KB domino). The commands still parse; + // invoking one fails at runtime once its chunk is gone. Remove + // an entry to ship that feature. + "alias": { + "@cloudflare/computer/shell/curl": "@cloudflare/computer/empty", + "@cloudflare/computer/shell/html-to-markdown": "@cloudflare/computer/empty" + }, + // Worker Loader binding. The DO calls env.LOADER.get(id, ...) // to mint a Dynamic Worker that the WorkerBackend then // dispatches shell.exec into. diff --git a/packages/computer/README.md b/packages/computer/README.md index 0b864726..ff2b0c6c 100644 --- a/packages/computer/README.md +++ b/packages/computer/README.md @@ -128,6 +128,29 @@ export class ContainerExample extends withWorkspace( ) {} ``` +The worker backend bundles every just-bash command into +`SHELL_MODULES`, and a few optional ones dominate the upload — +`curl` carries undici (~620 KB) and `html-to-markdown` carries +domino (~555 KB). Each optional command's chunks sit behind their +own `@cloudflare/computer/shell/` subpath, so a Worker +that doesn't need one drops it at build time by aliasing the +subpath to `@cloudflare/computer/empty` in `wrangler.jsonc`: + +```jsonc +"alias": { + "@cloudflare/computer/shell/curl": "@cloudflare/computer/empty", + "@cloudflare/computer/shell/html-to-markdown": "@cloudflare/computer/empty" +} +``` + +That swaps the feature's module group for an empty table, so its +chunks — the heavy dependency included — never reach the uploaded +Worker; dropping both above removes ~1.2 MB raw. The features are +`curl`, `html-to-markdown`, `python`, `sqlite`, and `js-exec`. +The command still parses, but invoking one after its chunk is +gone fails at runtime with a "module not found". Leaving the +alias out ships the whole shell. See `examples/worker/`. + Filesystem only — no backend, no shell: ```ts diff --git a/packages/computer/package.json b/packages/computer/package.json index 3f1373a0..360645fe 100644 --- a/packages/computer/package.json +++ b/packages/computer/package.json @@ -39,6 +39,34 @@ "types": "./dist/backends/worker/index.d.ts", "import": "./dist/backends/worker/index.js" }, + "./shell/core": { + "types": "./dist/backends/worker/shell/core.d.ts", + "default": "./dist/backends/worker/shell/core.js" + }, + "./shell/curl": { + "types": "./dist/backends/worker/shell/curl.d.ts", + "default": "./dist/backends/worker/shell/curl.js" + }, + "./shell/html-to-markdown": { + "types": "./dist/backends/worker/shell/html-to-markdown.d.ts", + "default": "./dist/backends/worker/shell/html-to-markdown.js" + }, + "./shell/python": { + "types": "./dist/backends/worker/shell/python.d.ts", + "default": "./dist/backends/worker/shell/python.js" + }, + "./shell/sqlite": { + "types": "./dist/backends/worker/shell/sqlite.d.ts", + "default": "./dist/backends/worker/shell/sqlite.js" + }, + "./shell/js-exec": { + "types": "./dist/backends/worker/shell/js-exec.d.ts", + "default": "./dist/backends/worker/shell/js-exec.js" + }, + "./empty": { + "types": "./dist/backends/worker/empty.d.ts", + "default": "./dist/backends/worker/empty.js" + }, "./observe/cloudflare": { "types": "./dist/observe/cloudflare.d.ts", "import": "./dist/observe/cloudflare.js" diff --git a/packages/computer/rolldown.config.ts b/packages/computer/rolldown.config.ts index db90774f..4ddcee2d 100644 --- a/packages/computer/rolldown.config.ts +++ b/packages/computer/rolldown.config.ts @@ -30,6 +30,18 @@ export default defineConfig({ "tools/index": "src/tools/index.ts", "backends/container/index": "src/backends/container/index.ts", "backends/worker/index": "src/backends/worker/index.ts", + // The shell-module groups build-bundle.mjs emits, plus the + // empty stub. Each is its own entry so it lands at the dist + // path the ./shell/* and ./empty package exports point at; + // shell-modules.ts imports them by subpath (kept external + // below) so a consumer can alias any of them out. + "backends/worker/shell/core": "src/backends/worker/generated/core.ts", + "backends/worker/shell/curl": "src/backends/worker/generated/curl.ts", + "backends/worker/shell/html-to-markdown": "src/backends/worker/generated/html-to-markdown.ts", + "backends/worker/shell/python": "src/backends/worker/generated/python.ts", + "backends/worker/shell/sqlite": "src/backends/worker/generated/sqlite.ts", + "backends/worker/shell/js-exec": "src/backends/worker/generated/js-exec.ts", + "backends/worker/empty": "src/backends/worker/empty.ts", "observe/cloudflare": "src/observe/cloudflare.ts", }, external: [ @@ -41,6 +53,13 @@ export default defineConfig({ "isomorphic-git", /^isomorphic-git\//, "just-bash", + // shell-modules.ts imports the generated groups by their + // published subpath so a consumer's wrangler `alias` can swap + // any of them for @cloudflare/computer/empty. Keep the + // specifiers intact in the emitted bundle rather than inlining + // the group here; each group is built as its own entry above. + /^@cloudflare\/computer\/shell\//, + "@cloudflare/computer/empty", "node:crypto", "node:events", ], diff --git a/packages/computer/src/backends/worker/empty.ts b/packages/computer/src/backends/worker/empty.ts new file mode 100644 index 00000000..b8dcd648 --- /dev/null +++ b/packages/computer/src/backends/worker/empty.ts @@ -0,0 +1,6 @@ +// Empty shell-module group. Alias a +// @cloudflare/computer/shell/ subpath to this module +// (published as @cloudflare/computer/empty) in wrangler.jsonc to +// drop that feature's chunks from the uploaded Worker. + +export default Object.freeze({}) as Readonly>; diff --git a/packages/computer/src/backends/worker/generated-bundle.test.ts b/packages/computer/src/backends/worker/generated-bundle.test.ts deleted file mode 100644 index c18f2235..00000000 --- a/packages/computer/src/backends/worker/generated-bundle.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -// Tests the shape build-bundle.mjs produces in generated-bundle.ts. -// -// The bundle used to be a single ~3 MB JS string assigned to -// SHELL_BUNDLE. esbuild was inlining every dynamic import() -// just-bash makes (python3, js-exec, sqlite3, html-to-markdown, -// curl, …) into the one string, so workerd's Worker Loader -// parsed all of it on every cold start even though the default -// ShellWorker disables python, javascript, and network. -// -// build-bundle.mjs now runs esbuild with splitting: true and -// emits a record of module name → source. The host Worker -// spreads the whole record into the Loader callback's modules -// table; workerd parses each chunk on first import, so the -// dynamic ones stay cold until a script actually reaches for -// them. These tests are the contract. - -import { describe, expect, it } from "vitest"; - -import { SHELL_MODULES } from "./generated-bundle.js"; - -describe("SHELL_MODULES", () => { - it("exposes shell.js as the main module", () => { - expect(SHELL_MODULES["shell.js"]).toBeDefined(); - expect(typeof SHELL_MODULES["shell.js"].js).toBe("string"); - expect(SHELL_MODULES["shell.js"].js.length).toBeGreaterThan(0); - }); - - it("keeps the main module under 1 MB so cold start parses ~650 KB, not 3 MB", () => { - // Static-reachable set from entrypoint.ts measured at ~651 KB. - // Anything materially above that means esbuild stopped - // splitting and went back to inlining dynamic imports. - const mainBytes = SHELL_MODULES["shell.js"].js.length; - expect(mainBytes).toBeLessThan(1_000_000); - }); - - it("splits dynamic just-bash chunks into separate modules", () => { - // The whole point of (2): the bundle is no longer one blob. - // At least one chunk besides shell.js should be present. - const names = Object.keys(SHELL_MODULES); - expect(names.length).toBeGreaterThan(1); - expect(names).toContain("shell.js"); - }); - - it("emits chunk module names with a .js extension", () => { - // workerd's Worker Loader rejects extensionless module names - // for bare-string modules; chunks must keep their .js suffix. - for (const name of Object.keys(SHELL_MODULES)) { - expect(name.endsWith(".js")).toBe(true); - } - }); - - it("every module entry has a js source string", () => { - for (const [name, mod] of Object.entries(SHELL_MODULES)) { - expect(typeof mod.js, `module ${name}`).toBe("string"); - expect(mod.js.length, `module ${name} non-empty`).toBeGreaterThan(0); - } - }); -}); diff --git a/packages/computer/src/backends/worker/index.ts b/packages/computer/src/backends/worker/index.ts index 1dd9cf23..a0bf2132 100644 --- a/packages/computer/src/backends/worker/index.ts +++ b/packages/computer/src/backends/worker/index.ts @@ -25,7 +25,7 @@ export { type WorkspaceFs, WorkspaceFsAdapter } from "./adapter.js"; export { type ArtifactsCommandHost, defineArtifactsCommand } from "./artifacts-command.js"; export { type AssetsCommandHost, defineAssetsCommand } from "./assets-command.js"; export { type ExecInput, ShellWorker, type ShellWorkerOptions } from "./entrypoint.js"; -export { SHELL_MODULES } from "./generated-bundle.js"; export { defineGitCommand, type GitCommandHost } from "./git-command.js"; export { SHELL_RUNTIME_MODULES } from "./runtime-modules.js"; +export { SHELL_MODULES } from "./shell-modules.js"; export { WorkerBackend, type WorkerBackendOptions, type WorkerShellFetcher } from "./worker.js"; diff --git a/packages/computer/src/backends/worker/script/build-bundle.mjs b/packages/computer/src/backends/worker/script/build-bundle.mjs index c749c349..01fa394f 100644 --- a/packages/computer/src/backends/worker/script/build-bundle.mjs +++ b/packages/computer/src/backends/worker/script/build-bundle.mjs @@ -1,8 +1,8 @@ -// Bundle entrypoint.ts into a record of module-name → source -// string and write it to generated-bundle.ts as a TypeScript -// module exporting that record (SHELL_MODULES). Consumers -// spread SHELL_MODULES into the Worker Loader callback's -// `modules` field without running a build of their own. +// Bundle entrypoint.ts into a set of module-name → source +// string records and write them under generated/ as TypeScript +// modules, each default-exporting one record. Consumers spread +// the records into the Worker Loader callback's `modules` field +// without running a build of their own. // // Why a record and not a single string? esbuild's bundle: true // inlines dynamic import() targets too, which dragged the full @@ -15,7 +15,19 @@ // it, so the cold-start cost matches the static-reachable set // (~650 KB) and optional features stay free until used. // -// The output file is gitignored. It's regenerated by the +// Why several files and not one? Each optional command (curl, +// html-to-markdown, python, sqlite, js-exec) carries chunks that +// are only reachable through that command — curl's undici +// (~620 KB) and html-to-markdown's domino (~555 KB) dominate the +// bundle. Splitting them into their own generated module lets a +// consumer alias the feature out at build time +// (@cloudflare/computer/shell/ → @cloudflare/computer/empty), +// dropping the feature's exclusive chunks — and its heavy +// dependency — from the uploaded Worker. shell-modules.ts spreads +// every group back together, so the default build is byte-for-byte +// what the single-file bundle produced. +// +// The generated/ output is gitignored. It's regenerated by the // computer package's prepare / pretest / pretypecheck scripts // so any install from this repo or downstream consumer sees a // fresh bundle matching the source. @@ -30,15 +42,31 @@ const here = dirname(fileURLToPath(import.meta.url)); // Script lives at .../backends/worker/script/build-bundle.mjs; // the bundle target is one level up at .../backends/worker/. const root = resolve(here, ".."); -const out = resolve(root, "generated-bundle.ts"); +const outDir = resolve(root, "generated"); + +// Optional commands whose chunks are worth splitting out so a +// consumer can drop them. Keyed by the feature name used in the +// package subpath (@cloudflare/computer/shell/); the +// value lists the just-bash command names that pull the feature +// in. A chunk lands in a feature group only when that feature is +// the sole optional owner of it and core can't reach it; anything +// shared with core or with a second feature stays in core so +// dropping one feature never breaks another. +const OPTIONAL_FEATURES = { + curl: ["curl"], + "html-to-markdown": ["html-to-markdown"], + python: ["python3", "python"], + sqlite: ["sqlite3"], + "js-exec": ["js-exec", "node"], +}; -await mkdir(dirname(out), { recursive: true }); +await mkdir(outDir, { recursive: true }); // esbuild's code-splitting requires an outdir to compute chunk // paths against. Use a scratch directory so the build output is // transient; we read result.outputFiles and never touch disk // for the .js fragments themselves. -const outdir = await mkdtemp(resolve(tmpdir(), "shell-bundle-")); +const scratch = await mkdtemp(resolve(tmpdir(), "shell-bundle-")); let result; try { @@ -47,7 +75,7 @@ try { bundle: true, write: false, splitting: true, - outdir, + outdir: scratch, format: "esm", target: "es2022", platform: "neutral", @@ -112,27 +140,20 @@ try { // The scratch outdir is only used to anchor relative paths. // No files were written (write: false), so removing it is // best-effort cleanup of the empty directory. - await rm(outdir, { recursive: true, force: true }); + await rm(scratch, { recursive: true, force: true }); } if (!result.outputFiles || result.outputFiles.length === 0) { throw new Error("build-bundle: esbuild returned no output"); } -// Collect each emitted file into the modules record keyed by -// its outdir-relative path. esbuild writes the entry as -// shell.js and chunks as chunk-.js per entryNames / -// chunkNames above. Worker Loader's `{ js: "..." }` module -// shape lets the keys carry the .js extension cleanly. -// -// Order is stable: sort by name so the generated file diffs -// predictably across builds when content hashes don't change. +// Collect each emitted file into a modules record keyed by its +// scratch-relative path. esbuild writes the entry as shell.js and +// chunks as chunk-.js per entryNames / chunkNames above. const modules = {}; -let totalBytes = 0; for (const file of result.outputFiles) { - const name = relative(outdir, file.path).split(/[\\/]/).join("/"); - modules[name] = { js: file.text }; - totalBytes += file.text.length; + const name = relative(scratch, file.path).split(/[\\/]/).join("/"); + modules[name] = file.text; } if (!modules["shell.js"]) { @@ -141,22 +162,157 @@ if (!modules["shell.js"]) { ); } -const ordered = {}; -for (const name of Object.keys(modules).sort()) { - ordered[name] = modules[name]; +const partition = partitionModules(modules); + +// Emit one generated file per group. shell-modules.ts imports +// each by its @cloudflare/computer/shell/ subpath and +// spreads them back together. +const groupNames = ["core", ...Object.keys(OPTIONAL_FEATURES)]; +let totalBytes = 0; +for (const group of groupNames) { + const names = (partition[group] ?? []).sort(); + const record = {}; + for (const name of names) { + record[name] = { js: modules[name] }; + totalBytes += modules[name].length; + } + const header = + `// Generated by script/build-bundle.mjs — do not edit.\n` + + `// Shell modules exclusive to the "${group}" feature group.\n`; + const body = `export default Object.freeze(${JSON.stringify( + record, + null, + 2, + )}) as Readonly>;\n`; + await writeFile(resolve(outDir, `${group}.ts`), `${header}\n${body}`); } -const header = "// Generated by script/build-bundle.mjs — do not edit.\n"; -const body = `export const SHELL_MODULES: Readonly> = Object.freeze(${JSON.stringify( - ordered, - null, - 2, -)});\n`; -const source = `${header}\n${body}`; - -await writeFile(out, source); -const chunkCount = Object.keys(ordered).length; -const mainBytes = ordered["shell.js"].js.length; +const coreCount = (partition.core ?? []).length; +const mainBytes = modules["shell.js"].length; +const featureSummary = Object.keys(OPTIONAL_FEATURES) + .map((f) => `${f} ${(partition[f] ?? []).length}`) + .join(", "); console.log( - `Wrote ${out} (${chunkCount} modules, shell.js ${mainBytes} bytes, total ${totalBytes} bytes)`, + `Wrote ${outDir} (core ${coreCount} modules, shell.js ${mainBytes} bytes, ` + + `features: ${featureSummary}, total ${totalBytes} bytes)`, ); + +// Assign every emitted module to exactly one group: "core" or one +// of the OPTIONAL_FEATURES keys. A module belongs to a feature +// only when that feature is its sole reacher and core can't reach +// it; everything else — shared chunks, chunks reachable from a +// kept command, the shell.js entry itself — stays in core. +function partitionModules(mods) { + const names = Object.keys(mods); + const staticEdges = new Map(); + const dynamicEdges = new Map(); + for (const name of names) { + staticEdges.set(name, moduleEdges(mods[name], /* dynamic */ false)); + dynamicEdges.set(name, moduleEdges(mods[name], /* dynamic */ true)); + } + + const closure = (starts, followDynamic) => { + const seen = new Set(); + const stack = [...starts]; + while (stack.length > 0) { + const cur = stack.pop(); + if (seen.has(cur) || !mods[cur]) continue; + seen.add(cur); + for (const next of staticEdges.get(cur) ?? []) stack.push(next); + if (followDynamic) for (const next of dynamicEdges.get(cur) ?? []) stack.push(next); + } + return seen; + }; + + const registry = parseCommandChunks(mods["shell.js"]); + const undici = parseNetworkChunk(mods["shell.js"]); + + const optionalCommands = new Set(Object.values(OPTIONAL_FEATURES).flat()); + + // Core reach: everything statically pulled by shell.js (the + // always-parsed entry) plus the full closure of every command + // that isn't optional. Dynamic edges out of shell.js are the + // per-command import() fan-out — following them would drag every + // optional chunk into core, so core uses shell.js's static edges + // only. + const coreReach = closure(["shell.js"], /* dynamic */ false); + for (const [command, chunk] of Object.entries(registry)) { + if (!optionalCommands.has(command)) { + for (const m of closure([chunk], /* dynamic */ true)) coreReach.add(m); + } + } + + // Each feature reaches the full closure of its command entry + // chunks. curl also owns the network agent chunk (undici), which + // shell.js imports lazily from its own body rather than through + // the curl command chunk. + const featureReach = new Map(); + for (const [feature, commands] of Object.entries(OPTIONAL_FEATURES)) { + const roots = commands.map((c) => registry[c]).filter(Boolean); + if (feature === "curl" && undici) roots.push(undici); + featureReach.set(feature, closure(roots, /* dynamic */ true)); + } + + const partition = { core: [] }; + for (const feature of Object.keys(OPTIONAL_FEATURES)) partition[feature] = []; + for (const name of names) { + const owners = []; + if (coreReach.has(name)) owners.push("core"); + for (const feature of Object.keys(OPTIONAL_FEATURES)) { + if (featureReach.get(feature).has(name)) owners.push(feature); + } + const optionalOwners = owners.filter((o) => o !== "core"); + if (!owners.includes("core") && optionalOwners.length === 1) { + partition[optionalOwners[0]].push(name); + } else { + partition.core.push(name); + } + } + return partition; +} + +// Import specifiers a module references. Static edges are the +// top-level `import`/`export … from` and bare side-effect +// imports; dynamic edges are `import(...)` calls. Only relative +// chunk specifiers matter — externals resolve at runtime. +function moduleEdges(source, dynamic) { + const targets = new Set(); + if (dynamic) { + for (const m of source.matchAll(/import\("(\.\/[^"]+)"\)/g)) { + targets.add(m[1].replace(/^\.\//, "")); + } + return targets; + } + for (const m of source.matchAll(/(?:import|export)[^;]*?from\s*"(\.\/[^"]+)"/g)) { + targets.add(m[1].replace(/^\.\//, "")); + } + for (const m of source.matchAll(/import\s*"(\.\/[^"]+)"/g)) { + targets.add(m[1].replace(/^\.\//, "")); + } + return targets; +} + +// Map each just-bash command to the chunk its lazy loader +// imports. The registry entries look like +// { name: "curl", load: async () => (await import("./chunk-…js")).curlCommand } +function parseCommandChunks(shellSource) { + const registry = {}; + const re = + /\{\s*name:\s*"([^"]+)",\s*load:\s*async\s*\(\)\s*=>\s*\(await import\("(\.\/chunk-[^"]+)"\)\)/g; + for (const m of shellSource.matchAll(re)) { + registry[m[1]] = m[2].replace(/^\.\//, ""); + } + return registry; +} + +// Find the chunk shell.js lazily imports to build its network +// Agent. just-bash loads undici on demand from its own body +// (`await import("./chunk-…"), new X.Agent(`), not through the +// curl command chunk, so it needs to be attributed to curl by +// hand. +function parseNetworkChunk(shellSource) { + const m = shellSource.match( + /await import\("(\.\/chunk-[^"]+)"\),\s*\w+\s*=\s*new\s+\w+\.Agent\(/, + ); + return m ? m[1].replace(/^\.\//, "") : null; +} diff --git a/packages/computer/src/backends/worker/shell-modules.test.ts b/packages/computer/src/backends/worker/shell-modules.test.ts new file mode 100644 index 00000000..1c432e30 --- /dev/null +++ b/packages/computer/src/backends/worker/shell-modules.test.ts @@ -0,0 +1,87 @@ +// Tests the shape shell-modules.ts assembles from the per-feature +// groups build-bundle.mjs emits under generated/. +// +// The bundle used to be a single ~3 MB JS string. esbuild was +// inlining every dynamic import() just-bash makes (python3, +// js-exec, sqlite3, html-to-markdown, curl, …) into the one +// string, so workerd's Worker Loader parsed all of it on every +// cold start even though the default ShellWorker disables python, +// javascript, and network. +// +// build-bundle.mjs now runs esbuild with splitting: true and +// partitions the emitted modules into a core group plus one group +// per optional command. shell-modules.ts imports each group by +// its @cloudflare/computer/shell/* subpath and spreads them into +// SHELL_MODULES; the host Worker hands that to the Loader callback, +// and workerd parses each chunk on first import. A consumer can +// alias a group's subpath to @cloudflare/computer/empty to drop +// the feature's chunks from the upload. These tests are the +// contract. + +import curlModules from "@cloudflare/computer/shell/curl"; +import htmlToMarkdownModules from "@cloudflare/computer/shell/html-to-markdown"; +import { describe, expect, it } from "vitest"; +import { SHELL_MODULES } from "./shell-modules.js"; + +describe("SHELL_MODULES", () => { + it("exposes shell.js as the main module", () => { + expect(SHELL_MODULES["shell.js"]).toBeDefined(); + expect(typeof SHELL_MODULES["shell.js"].js).toBe("string"); + expect(SHELL_MODULES["shell.js"].js.length).toBeGreaterThan(0); + }); + + it("keeps the main module under 1 MB so cold start parses ~650 KB, not 3 MB", () => { + // Static-reachable set from entrypoint.ts measured at ~651 KB. + // Anything materially above that means esbuild stopped + // splitting and went back to inlining dynamic imports. + const mainBytes = SHELL_MODULES["shell.js"].js.length; + expect(mainBytes).toBeLessThan(1_000_000); + }); + + it("splits dynamic just-bash chunks into separate modules", () => { + // The whole point of splitting: the bundle is no longer one + // blob. At least one chunk besides shell.js should be present. + const names = Object.keys(SHELL_MODULES); + expect(names.length).toBeGreaterThan(1); + expect(names).toContain("shell.js"); + }); + + it("emits chunk module names with a .js extension", () => { + // workerd's Worker Loader rejects extensionless module names + // for bare-string modules; chunks must keep their .js suffix. + for (const name of Object.keys(SHELL_MODULES)) { + expect(name.endsWith(".js")).toBe(true); + } + }); + + it("every module entry has a js source string", () => { + for (const [name, mod] of Object.entries(SHELL_MODULES)) { + expect(typeof mod.js, `module ${name}`).toBe("string"); + expect(mod.js.length, `module ${name} non-empty`).toBeGreaterThan(0); + } + }); + + it("carries each optional feature's chunks in its own group", () => { + // curl and html-to-markdown own the two heaviest dependency + // chunks (undici, domino). They must be assembled into + // SHELL_MODULES by default, and live in their own group so a + // consumer can alias the group out. + expect(Object.keys(curlModules).length).toBeGreaterThan(0); + expect(Object.keys(htmlToMarkdownModules).length).toBeGreaterThan(0); + for (const name of Object.keys(curlModules)) { + expect(SHELL_MODULES[name]).toBeDefined(); + } + for (const name of Object.keys(htmlToMarkdownModules)) { + expect(SHELL_MODULES[name]).toBeDefined(); + } + }); + + it("keeps feature groups disjoint from each other", () => { + // A chunk owned by curl must not also appear in the + // html-to-markdown group; a shared chunk belongs in core. + const curlNames = new Set(Object.keys(curlModules)); + for (const name of Object.keys(htmlToMarkdownModules)) { + expect(curlNames.has(name), `${name} in both groups`).toBe(false); + } + }); +}); diff --git a/packages/computer/src/backends/worker/shell-modules.ts b/packages/computer/src/backends/worker/shell-modules.ts new file mode 100644 index 00000000..0c686a27 --- /dev/null +++ b/packages/computer/src/backends/worker/shell-modules.ts @@ -0,0 +1,36 @@ +// Assembles SHELL_MODULES from the per-feature module groups +// build-bundle.mjs emits under generated/. +// +// Each group is imported through its @cloudflare/computer/shell/* +// subpath rather than a relative path so a consumer can drop a +// feature at build time without patching this package. Aliasing +// the subpath to @cloudflare/computer/empty in wrangler.jsonc: +// +// "alias": { +// "@cloudflare/computer/shell/curl": "@cloudflare/computer/empty", +// "@cloudflare/computer/shell/html-to-markdown": "@cloudflare/computer/empty" +// } +// +// swaps that group for an empty record, and the feature's +// exclusive chunks — curl's undici (~620 KB), html-to-markdown's +// domino (~555 KB), and the like — never enter the uploaded +// Worker. The command still parses; invoking it fails at runtime +// with a "module not found" once its chunk is gone. Leaving the +// alias out ships every group, byte-for-byte the single-file +// bundle. + +import coreModules from "@cloudflare/computer/shell/core"; +import curlModules from "@cloudflare/computer/shell/curl"; +import htmlToMarkdownModules from "@cloudflare/computer/shell/html-to-markdown"; +import jsExecModules from "@cloudflare/computer/shell/js-exec"; +import pythonModules from "@cloudflare/computer/shell/python"; +import sqliteModules from "@cloudflare/computer/shell/sqlite"; + +export const SHELL_MODULES: Readonly> = Object.freeze({ + ...coreModules, + ...curlModules, + ...htmlToMarkdownModules, + ...pythonModules, + ...sqliteModules, + ...jsExecModules, +}); diff --git a/packages/computer/src/backends/worker/worker.ts b/packages/computer/src/backends/worker/worker.ts index c0ebf291..2975988e 100644 --- a/packages/computer/src/backends/worker/worker.ts +++ b/packages/computer/src/backends/worker/worker.ts @@ -26,8 +26,8 @@ import type { ExecEvent, ShellRPC, SyncRPC, WorkspaceRPC } from "@cloudflare/com import type { BackendHandle, WorkspaceBackend } from "../../backend.js"; import type { WorkspaceServiceProxyProps } from "../../proxy.js"; -import { SHELL_MODULES } from "./generated-bundle.js"; import { SHELL_RUNTIME_MODULES } from "./runtime-modules.js"; +import { SHELL_MODULES } from "./shell-modules.js"; // The shape the loaded ShellWorker exposes. The host-side // implementation lives in ./entrypoint.ts; the backend consumes diff --git a/packages/computer/test-helpers/shell-module-aliases.ts b/packages/computer/test-helpers/shell-module-aliases.ts new file mode 100644 index 00000000..c38eb270 --- /dev/null +++ b/packages/computer/test-helpers/shell-module-aliases.ts @@ -0,0 +1,24 @@ +// Resolve the published @cloudflare/computer/shell/* subpaths and +// @cloudflare/computer/empty to their source files for the test +// runners. shell-modules.ts imports the shell-module groups by +// subpath so a consumer's wrangler `alias` can drop a feature; +// those subpaths resolve through the package's dist exports, +// which a src-based test run doesn't build. Point them at the +// generated src files (and the empty stub) instead. + +import { resolve } from "node:path"; + +const src = resolve(import.meta.dirname, "..", "src", "backends", "worker"); + +const groups = ["core", "curl", "html-to-markdown", "python", "sqlite", "js-exec"] as const; + +export const shellModuleAliases = [ + ...groups.map((group) => ({ + find: `@cloudflare/computer/shell/${group}`, + replacement: resolve(src, "generated", `${group}.ts`), + })), + { + find: "@cloudflare/computer/empty", + replacement: resolve(src, "empty.ts"), + }, +]; diff --git a/packages/computer/tsconfig.json b/packages/computer/tsconfig.json index 0d9b0289..b344497f 100644 --- a/packages/computer/tsconfig.json +++ b/packages/computer/tsconfig.json @@ -10,7 +10,18 @@ "esModuleInterop": true, "resolveJsonModule": true, "isolatedModules": true, - "types": ["@cloudflare/workers-types", "vitest/globals", "node"] + "types": ["@cloudflare/workers-types", "vitest/globals", "node"], + "paths": { + "@cloudflare/computer/shell/core": ["./src/backends/worker/generated/core.ts"], + "@cloudflare/computer/shell/curl": ["./src/backends/worker/generated/curl.ts"], + "@cloudflare/computer/shell/html-to-markdown": [ + "./src/backends/worker/generated/html-to-markdown.ts" + ], + "@cloudflare/computer/shell/python": ["./src/backends/worker/generated/python.ts"], + "@cloudflare/computer/shell/sqlite": ["./src/backends/worker/generated/sqlite.ts"], + "@cloudflare/computer/shell/js-exec": ["./src/backends/worker/generated/js-exec.ts"], + "@cloudflare/computer/empty": ["./src/backends/worker/empty.ts"] + } }, "include": ["src/**/*.ts", "*.ts"] } diff --git a/packages/computer/vitest.config.ts b/packages/computer/vitest.config.ts index 1c0687d9..6e1e12d0 100644 --- a/packages/computer/vitest.config.ts +++ b/packages/computer/vitest.config.ts @@ -2,6 +2,8 @@ import { resolve } from "node:path"; import { defineConfig } from "vitest/config"; +import { shellModuleAliases } from "./test-helpers/shell-module-aliases.js"; + export default defineConfig({ resolve: { alias: [ @@ -21,6 +23,13 @@ export default defineConfig({ find: "cloudflare:workers", replacement: resolve(__dirname, "test-helpers/cloudflare-workers-stub.ts"), }, + // shell-modules.ts imports the generated groups by their + // published @cloudflare/computer/shell/* subpath so a + // consumer can alias them out. Those subpaths resolve + // through the package's dist exports, which don't exist + // under the src test runner — point them at the generated + // src files instead. + ...shellModuleAliases, ], }, test: { diff --git a/packages/computer/vitest.config.worker-backend.ts b/packages/computer/vitest.config.worker-backend.ts index 935cb625..f0c9b087 100644 --- a/packages/computer/vitest.config.worker-backend.ts +++ b/packages/computer/vitest.config.worker-backend.ts @@ -8,12 +8,21 @@ import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; import { defineConfig } from "vitest/config"; +import { shellModuleAliases } from "./test-helpers/shell-module-aliases.js"; + export default defineConfig({ plugins: [ cloudflareTest({ wrangler: { configPath: "./tests/wrangler.worker-backend.jsonc" }, }), ], + resolve: { + // The WorkerBackend pulls SHELL_MODULES, which imports the + // shell-module groups by their @cloudflare/computer/shell/* + // subpath. Those resolve through dist exports that this src + // test run doesn't build; map them to the generated src files. + alias: shellModuleAliases, + }, test: { globals: true, include: ["tests/worker-backend.test.ts"],