Add s3 storage

.mise/config.coverage.toml

@@ -248,9 +248,29 @@ cargo clean -p et-ws-wasm-agent --target wasm32-unknown-unknown
 driver="${CHROMEDRIVER:-$(mise which chromedriver)}"
 
 # Build + run cov-server itself instrumented so the launcher is not reported as untested.
-# It uses native -Cinstrument-coverage (with nightly branch coverage, matching the rest of the pipeline) and
-# flushes its counters to this .profraw on the clean exit that removing the marker below triggers.
-RUSTFLAGS="-Cinstrument-coverage -Zcoverage-options=branch" cargo build -q -p et-ws-test-server --bin cov-server
+# It uses native -Cinstrument-coverage and flushes its counters to this .profraw on the clean exit that removing
+# the marker below triggers.
+#
+# Deliberately WITHOUT `-Zcoverage-options=branch`, unlike the rest of the pipeline: branch regions over async
+# code make llvm-cov segfault while reading this binary's coverage map, so the `-object target/debug/cov-server`
+# export below aborts the whole task with
+#     bash: line 98: 49175 Segmentation fault: 11  "$llbin/llvm-cov" export --format=lcov
+#       --instr-profile "$csp.profdata" -object target/debug/cov-server > "$csp.lcov"
+# and no LLVM stack trace. Under lldb the frame is
+# `llvm::coverage::CoverageMapping::getInstantiationGroups(llvm::StringRef) const + 412`, an EXC_BAD_ACCESS on a
+# near-null address -- the open upstream bug https://github.com/llvm/llvm-project/issues/189169 ("Crash in
+# llvm-cov for rust with async code and --branch option", minimal repro at
+# https://github.com/mscharley/llvm-cov-crash). It reproduces identically under every llvm-cov on hand (the
+# nightly, stable and previous-stable toolchains' copies), is unaffected by `-num-threads=1` or an 8x larger
+# stack, and follows the object rather than the profile.
+# cov-server crossed the threshold when it gained the storage service's object_store dependency tree (285 crates,
+# a 10.4 MB __llvm_covfun), which is why the coverage lane passed on commit d8d42f9c and failed from
+# 6e80f4decd7d7d78ec6bd770d52105f4bd5db15a onward -- e.g.
+# https://github.com/edge-toolkit/core/actions/runs/30988267419/job/92248023234.
+# Dropping branch regions for this one binary keeps its line coverage and costs only its own BRDA records; every
+# other crate still gets branch coverage via `a_wasm_cov_instr` / the show-env wrapper. Restore the flag here
+# once #189169 is fixed and the toolchain ships it.
+RUSTFLAGS="-Cinstrument-coverage" cargo build -q -p et-ws-test-server --bin cov-server
 LLVM_PROFILE_FILE="{{ config_root }}/$covdir/cov-server-%p.profraw" ./target/debug/cov-server "$covdir/server-ready" &
 server_pid=$!
 trap 'kill "$server_pid" 2>/dev/null || true' EXIT

.mise/config.toml

@@ -61,7 +61,7 @@ cargo-binstall = "latest"
 cmake = "latest"
 "conda:openssl" = "3"
 conftest = "latest"
-"github:nextest-rs/nextest" = "latest"
+"github:nextest-rs/nextest" = { version = "latest", version_prefix = "cargo-nextest-" }
 taplo = "latest"
 watchexec = "latest"
 # uutils coreutils: the Rust multicall binary, invoked as `coreutils <util>` in tasks.
@@ -97,6 +97,12 @@ ripgrep = "latest"
 "github:benhoyt/goawk" = "latest"
 "github:caldempsey/parfit" = "latest"
 "github:grok-rs/waitup" = "latest"
+# The macos/x64 half of rustfs, the one platform upstream ships no prebuilt for.
+# A git spec rather than a bare `cargo:rustfs`: the crates.io crate is a stale 0.0.2 placeholder while real
+# releases are 1.0.0-beta.x on GitHub only, so installing the crate would give this platform different software
+# than every other one. `tag:` builds the exact source the prebuilt is cut from, keeping them aligned -- bump it
+# with the `rustfs` entry. A source build is acceptable here because macos/x64 is second-tier.
+"cargo:rustfs/rustfs" = { version = "tag:1.0.0-beta.12", os = ["macos/x64"] }
 # ONNX Runtime prebuilt -- ort-sys 2.0.0-rc.10 binds to ORT 1.22.x.
 "cargo:ryl" = { version = "latest", os = ["macos/x64"] }
 "github:microsoft/onnxruntime" = "1.22.0"
@@ -139,6 +145,12 @@ protoc = "latest"
 # exact `3.x.y` dir. Bump in lockstep with py3_unix/py3_win/pylib.
 python = "3.13.14"
 rclone = "latest"
+# rustfs is the S3-compatible server the et-storage-service S3-backend test runs against.
+# Version-pinned rather than `latest` because upstream has published only prereleases so far, which mise's
+# version list filters out -- `latest` resolves to nothing. Upstream ships prebuilts for linux (x64+arm64,
+# gnu+musl), macos/arm64 and windows/x64 but NOT macos/x64, so that one platform builds from source via the
+# cargo entry below. Bump both in lockstep.
+rustfs = { version = "1.0.0-beta.12", os = ["linux", "macos/arm64", "windows"] }
 typos = "latest"
 uv = "0.11.8"
 wasm-tools = "latest"

CLAUDE.md

@@ -126,7 +126,7 @@ languages this repo uses:
 - **Dockerfile `RUN` block**: switch to BuildKit's HEREDOC form
   (`RUN bash <<'EOF'` ... `EOF`) -- each shell command sits on its own line
   with no continuation needed. Three rules, all enforced by
-  `config/conftest/policy/dockerfile.rego` + the matching semgrep rule
+  `config/conftest/policy/dockerfile/dockerfile.rego` + the matching semgrep rule
   under `config/semgrep/`: (1) interpreter is **`bash`, placed BEFORE the
   `<<TAG`** (default `/bin/sh` on Debian/Ubuntu is dash, which rejects
   `set -euo pipefail`; the inverse `RUN <<EOF bash` form is silently
@@ -599,7 +599,7 @@ Five supported platforms in two tiers. **Main tier:** macOS arm64, Linux x64, Wi
 source-build isn't acceptable. **Second tier:** Linux arm64, macOS x64 -- every tool must still install and run,
 but slower install mechanisms are allowed because release authors often skip prebuilts for these arches: a
 `cargo:` source-build (or alternate backend) `os`-scoped to a second-tier-only platform is fine. The conftest
-mise policy (`config/conftest/policy/mise.rego`) enforces both rules, including the narrow per-name allowlist for
+mise policy (`config/conftest/policy/mise/mise.rego`) enforces both rules, including the narrow per-name allowlist for
 tools that have no prebuilt at any triple.
 
 Skipping a tool entirely with `MISE_DISABLE_TOOLS` is reserved for `Dockerfile.nanoserver`, where trimming the
@@ -727,7 +727,7 @@ pattern is the same for every cache entry; copy the `rustpython` / `augeas` / `d
    for ad-hoc rebuilds (e.g. version bump). No `push:` trigger -- we
    don't want a `main`-merge to rebuild assets.
 
-Required side-effect: an `etc-` entry in `config/conftest/policy/mise.rego`'s allowlist of `http:` tools that have
+Required side-effect: an `etc-` entry in `config/conftest/policy/mise/mise.rego`'s allowlist of `http:` tools that have
 no prebuilt at any triple, if applicable. (Skip if the tool is OS-scoped and the allowlist already covers it.)
 
 ## Fetch resilience: prefer upstream-cache, not retry wrappers
@@ -798,14 +798,10 @@ When the question is "can install-action install X?", check the TOOLS.md first;
 cargo-quickinstall's release tags for `X-<ver>`. A hit in either tier means install-action will fetch a prebuilt;
 absence in both means CI would pay a source-build cost.
 
-Even a manifest hit isn't a guarantee -- install-action's resolver is strict about the binary names it expects to
-find inside the prebuilt archive, and upstream renames break it silently. Concrete cautionary tale: `aube` IS in
-install-action's manifest, but the manifest expects an `aubr` binary
-(`When resolving aube bin aubr is not found. This binary is not optional so it must be included in the archive`),
-which recent aube releases don't ship. install-action then falls through to a real `cargo install` source build,
-which then flakes on crates.io. The mise-managed `setup-aube` task (npm-backed, `continue-on-error`) was the
-reliable path here; reach for install-action only when the prebuilt actually exists for the target triple _and_
-the binary names still match.
+Even a manifest hit isn't a guarantee -- install-action's resolver insists on the exact binary names its manifest
+expects to find inside the prebuilt archive, so an upstream rename makes it fall through to a real `cargo install`
+source build (which then flakes on crates.io) with no signal beyond a `bin <name> is not found` line. Reach for
+install-action only when the prebuilt actually exists for the target triple _and_ the binary names still match.
 
 ## Linting
 
@@ -1087,7 +1083,7 @@ first, and only bring the question to the user if neither is workable -- don't a
 ## Clippy lints
 
 **Never weaken or disable a lint to make code pass -- not the workspace lint config (`[workspace.lints.*]`,
-`.clippy.toml` thresholds, the ast-grep / taplo rules) -- without explicit operator permission.** Setting a
+`config/clippy.toml` thresholds, the ast-grep / taplo rules) -- without explicit operator permission.** Setting a
 denied lint to `allow` or raising a threshold is a project-policy change, not a fix. If a lint is in the way, fix
 the code (or justify it with a scoped `#[expect(..., reason = "...")]`); if you believe the lint itself is wrong,
 stop and ask.

Cargo.toml

@@ -106,6 +106,11 @@ local-ip-address = "0.6"
 log = "0.4"
 minicov = "0.3"
 onnx-extractor = "0.3"
+# Storage backend abstraction: local disk by default, any object_store backend via a URL.
+# `aws` pulls the S3 client (which also covers S3-compatible servers such as the rustfs mise tool the backend
+# test runs against). Only `aws` is enabled -- adding `gcp`/`azure`/`http` is a one-word change if a consumer
+# ever needs them, and each drags its own SDK surface, so they stay off until asked for.
+object_store = { version = "0.14", default-features = false, features = ["aws", "fs"] }
 openapiv3 = "2"
 opentelemetry = "0.31"
 opentelemetry-appender-tracing = "0.31"
@@ -182,6 +187,9 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 tracing-wasm = "0.2"
 tree-sitter = "0.25"
 tree-sitter-zig = "1"
+# Already resolved transitively (object_store, reqwest, ...), so this adds no new crate or license to review.
+# object_store's parse_url_opts takes a `&Url` and does not re-export the type, so a backend-URL config needs it.
+url = "2"
 utoipa = { version = "5", features = ["actix_extras", "yaml"] }
 uuid = { version = "1", features = ["serde", "v4", "v7"] }
 wasm-bindgen = "0.2"

config/ast-grep/rules/doc-summary-ends-with-period.yaml

@@ -38,6 +38,7 @@ files:
   - services/storage/src/tty_image.rs
   - services/storage/tests/image_logging.rs
   - services/storage/tests/put.rs
+  - services/storage/tests/s3_backend.rs
   - services/websockify/src/lib.rs
   - services/websockify/tests/relay.rs
   - services/ws-modules/except1/src/lib.rs

config/conftest/policy/mise/mise.rego

@@ -163,6 +163,11 @@ allowed_os_scoped_tool := {
 	# windows_exporter is a Windows-only host/GPU Prometheus exporter, so the o2-winmetrics task scopes it to Windows.
 	"github:prometheus-community/windows_exporter",
 	"cargo:findutils",
+	# rustfs is the S3 server the storage backend test runs against.
+	# Upstream ships no macos/x64 prebuilt, so that one platform builds from the same git tag via cargo while the
+	# others take the github release. Between the two entries every platform is covered.
+	"rustfs",
+	"cargo:rustfs/rustfs",
 	"cargo:ryl",
 	# http:et-rp is os-scoped to only those platforms whose tarball is already in the rp-v<N> release.
 	# Add a platform by dispatching the upstream-cache.yaml workflow on that host.

config/lychee.toml

@@ -36,4 +36,9 @@ exclude_path = ["data", "generated", "node_modules", "target"]
 #     [404] https://github.com/edge-toolkit/core/actions/runs/28698136445/job/85111320237 (at 447:1)
 # from CLAUDE.md's attestation-flake note on commit e401831a98197068f978ef760134caff882e2615 while the same
 # URL returned 200 to a local curl minutes later. Checking them is all noise, no signal.
-exclude = ['\{|%7B', '^https://github\.com/[^/]+/[^/]+/actions/runs/', '^https?://host[:/]']
+#
+# The `file:` entry is the container-internal storage mount the compose generator emits as a `STORAGE_URL`
+# value; it names a path inside the ws-server image, so it is never resolvable from the host doing the check.
+# Scoped to that exact URL rather than the whole `file:` scheme, because relative links in markdown resolve to
+# `file:` URIs too and those are real links worth checking.
+exclude = ['\{|%7B', '^file:///app/storage$', '^https://github\.com/[^/]+/[^/]+/actions/runs/', '^https?://host[:/]']

config/semgrep/module-description-length.yaml

@@ -3,10 +3,10 @@ rules:
     languages: [generic]
     paths:
       include:
-        - "services/ws-modules/*/Cargo.toml"
-        - "services/ws-modules/*/pyproject.toml"
-        - "services/ws-modules/*/build.zig.zon"
-        - "services/ws-modules/*/pkg/package.json"
+        - "/services/ws-modules/*/Cargo.toml"
+        - "/services/ws-modules/*/pyproject.toml"
+        - "/services/ws-modules/*/build.zig.zon"
+        - "/services/ws-modules/*/pkg/package.json"
     # Enforce ws-module description length at the source manifests, not by trimming labels in the browser.
     # A ws-module description is the module-dropdown label on the ws-server index page. Two regex alternatives
     # cover the syntaxes the description travels through: `description = "..."` (Cargo.toml / pyproject.toml,

config/taplo/mise-cargo-backend-allowlist.schema.json

@@ -16,6 +16,7 @@
               "cargo:dart-typegen",
               "cargo:findutils",
               "cargo:open",
+              "cargo:rustfs/rustfs",
               "cargo:ryl",
               "cargo:wasm-opt"
             ]