From f13c49ee26c668e5d64a9ed2b9266a8954ccf619 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 7 Aug 2026 21:45:08 +0000 Subject: [PATCH 01/11] docs(release): claim ENG-RELEASE-BINARIES spike Inventory the downloadable server-bundle capability and materialize the records-only helper claim. FOLLOWING_AGENTS_PROTOCOL Assisted-by: Codex:gpt-5 [Codex] --- .agents/coordination.md | 16 ++++++++++++++++ .agents/engine-matrix.md | 5 +++-- docs/BENCHMARKS.md | 1 + docs/STATUS.md | 2 +- scripts/check-agent-record.py | 8 +++++++- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.agents/coordination.md b/.agents/coordination.md index c256ee1ea..3b7db6d75 100644 --- a/.agents/coordination.md +++ b/.agents/coordination.md @@ -118,6 +118,22 @@ without the selected contention proof for their entire run are discarded. ## Active claims +**Server binary release-matrix spike (`ENG-RELEASE-BINARIES`, 2026-08-07, +`CLAIM-ENG-RELEASE-BINARIES-SPIKE`).** Codex (GPT-5), isolated worktree +`/home/mudler/_git/vllm.cpp-release-binaries`, branch +`row/ENG-RELEASE-BINARIES`, base `origin/main` `9cb799cb`. Records/spec-only +claim for issue #117 and the user-approved hybrid stable/preview downloadable +server-bundle matrix, including the fully-static feasibility boundary. Owns +only NEW `.agents/specs/release-binary-matrix.md`, the new +`ENG-RELEASE-BINARIES` engine-matrix row, its single roadmap portfolio link, +this claim, the required current-state checkpoint in `docs/STATUS.md` and +`docs/BENCHMARKS.md`, the engine inventory count pin in +`scripts/check-agent-record.py`, `.agents/NOW.md`, and the append-only +`.agents/state.md`. +No release workflow, CMake/package implementation, source, test, model, kernel, +generated fixture, download, build, GPU, or service change is in scope. Draft +PR is the claim; exact PR number is added after GitHub allocates it. + **CPU grouped keep-quant GEMM activation-dtype P0 (`QUANT-GGUF-CIQ-GEMM`, 2026-08-06, `CLAIM-QUANT-GGUF-CIQ-GROUPED-DTYPE`).** Claude Code (claude-opus-5), isolated worktree `/home/mudler/_git/vllmcpp-ciq-grouped`, diff --git a/.agents/engine-matrix.md b/.agents/engine-matrix.md index 36d0b8d46..03e1ecd19 100644 --- a/.agents/engine-matrix.md +++ b/.agents/engine-matrix.md @@ -42,11 +42,11 @@ forensics: roadmap_v1.md and the parity ledger. | Sampling and generation | 15 | 4 | 2 | 0 | 0 | 3 | 0 | 1 | 5 | | Structured output and tools | 7 | 0 | 3 | 0 | 0 | 2 | 0 | 0 | 2 | | Speculative decoding | 21 | 0 | 0 | 1 | 0 | 4 | 0 | 4 | 11 | -| Serving, API, CLI, library | 24 | 10 | 2 | 1 | 0 | 4 | 2 | 1 | 4 | +| Serving, API, CLI, library | 25 | 10 | 2 | 1 | 0 | 4 | 2 | 1 | 5 | | LoRA and adapters | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | | Long context and attention | 10 | 5 | 0 | 0 | 1 | 0 | 1 | 0 | 3 | | Loading, tokenizer, config | 9 | 3 | 3 | 0 | 0 | 0 | 1 | 1 | 1 | -| **Total** | **142** | **36** | **16** | **4** | **7** | **20** | **8** | **9** | **41** | +| **Total** | **143** | **36** | **16** | **4** | **7** | **20** | **8** | **9** | **42** | ## Engine core and scheduling @@ -207,6 +207,7 @@ claims it. | `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; `examples/bench/main.cpp:40,109`; `examples/bench/bench_core.h:96,468` | server help contract `examples/CMakeLists.txt:34`; benchmark `tests/examples/test_bench.cpp:15,48` | `planned: specs/cli-serve-bench.md` | `PARTIAL` | - | | `SERVE-GATE-ONLINE` | Same-corpus online correctness, TTFT/TPOT/ITL, throughput and peak-memory gate vs vLLM v0.25.0 | T0 | `vllm/benchmarks/serve.py:1,581-615`; [v0.25 audit](sync/2026-07-12-702f481.md); `tests/benchmarks/test_serve_cli.py:1` | Schema-v5 harness plus [trace controller](../include/vt/cuda/cuda_profiler_control.h#L13), [production component driver](../scripts/dgx-gdn-packed-component.sh), and fail-closed [component finalizer](../tools/bench/gdn_packed_component.py) | **BINDING `9ecd9d0`: 114/124** (async default ON; mem 4/4, c1 20/20, c2 20/20, c16 19/20, c4 & c32 18/20, c8 15/20; `benchmark_binding` refers here, superseding `3f256ab` 55/124 and `246a23c` 49/124, both retained immutable). Two-grid totality with `f0fb727` (111/124) is 115/124 effective parity vs vLLM 0.25.0 (27B). Async CLOSED the c16/c32 ITL tails (ours now BEATS vLLM: c16 p99 1.055, c32 p90 1.034/p99 1.078) and leaves a stable c8 `p99_itl` ~0.86 residual, ROOT-CAUSED (2026-07-18, `CLAIM-C8-P99-TAIL-1`, [spec](specs/c8-p99-itl-tail-2026-07-18.md)) as IRREDUCIBLE-AS-MIRRORED: our deterministic synchronous forward keeps co-admitted c8 requests in byte-identical lockstep where vLLM's async-future jitter de-phases them; the c16/c32 INVERSION proves this is the trailing edge of the per-step determinism that wins c16/c32 + throughput, not a capability gap (scheduler + async placeholder byte-identical, `tests/vllm/v1/test_scheduler_wave.cpp:265`, [tail spec](specs/tail-stall-analysis-2026-07-16.md)). Full grid + per-binding forensics: roadmap_v1.md + parity ledger; no packed speed credit | [online serving gate](specs/cuda-online-serving-gate.md); [merged GDN projections](specs/gdn-merged-input-projections.md); [packed decode](specs/gdn-packed-decode.md) | `ANCHOR-BACKFILL` | CLAIM-SERVE-GATE-1 | | `SERVE-E2E-NIGHTLY` | Server conformance and real-model nightly suites for all release gates | T0 | `tests/entrypoints/openai/`; `tests/v1/e2e/`; `.buildkite/test-pipeline.yaml` | current unit/conformance tests only; no scheduled DGX suite | `tests/vllm/entrypoints/openai/test_conformance.cpp:1`; `tests/parity/test_qwen36_paged_engine.cpp:78`; `tests/parity/test_qwen27_paged_engine.cpp:110` | `planned: specs/server-e2e-nightly.md` | `INVENTORIED` | - | +| `ENG-RELEASE-BINARIES` | Downloadable, backend-specific `vllm-server` release bundles and fully-static feasibility boundary | T0 | vLLM release lanes `.buildkite/release-pipeline.yaml:1-18,34-170` @ `555967922`; release-image dependency boundary `docker/Dockerfile.cpu:262-290` | server target only `examples/CMakeLists.txt:54-64`; no install, archive, manifest, provenance, or release-publish implementation | help smoke only `examples/CMakeLists.txt:59-63`; issue `#117` requests binary releases | `planned: specs/release-binary-matrix.md` | `INVENTORIED` | - | | `SERVE-CLI-CHAT` | Interactive `chat` and `complete` commands against a running OpenAI-compatible server, plus preservation of the existing local-model completion invocation | T1 | registration `vllm/entrypoints/cli/main.py:17-37,73-98`; client/model resolution + stream shaping `vllm/entrypoints/cli/openai.py:30-100`; chat `:155-234`; complete `:237-312` at `5559679229` | current in-process completion only `examples/cli/main.cpp:1-207`; remote command implementation absent | C-ABI stream baseline `tests/capi/test_capi.cpp:567-711`; chat-template baseline `tests/capi/test_chat_prompt.cpp:37-89`; command/fake-server tests absent | [cli-chat-complete.md](specs/cli-chat-complete.md) | `ANCHOR-BACKFILL` | `CLAIM-SERVE-CLI-CHAT-SPIKE` | | `SERVE-POOLING-ENDPOINTS` | Embeddings, pooling, score, rerank, classify HTTP surface (`/v1/embeddings`, `/pooling`, `/score`, `/rerank`, `/classify`). **SPIKED 2026-07-28 (`CLAIM-POOLING`):** the whole pooling task class is scoped in [pooling-task-class.md](specs/pooling-task-class.md) — endpoints depend on the pooling RUNNER (`ENG-POOLING-RUNNER`, W3 LANDED 2026-07-29 — `PoolingRunner` returns pooled data via the model `Pooler`) and a concrete pooling model. Endpoint protocol + handler port is the W4 brick | T2 | `vllm/entrypoints/pooling/embed/api_router.py:28`; `vllm/entrypoints/pooling/scoring/api_router.py:37,71`; `vllm/entrypoints/pooling/classify/api_router.py:26` | - | - | [pooling-task-class.md](specs/pooling-task-class.md) | `SPIKE` | `CLAIM-POOLING` | | `ENG-POOLER-SEQ` | The non-generative POOLER OP — turn hidden states into a pooled embedding/logit row instead of a sampled token. **W1 LANDED + CPU-GATED 2026-07-28 (`CLAIM-POOLING`, NOT pushed):** the sequence pooling methods `CLSPool`/`LastPool`/`MeanPool` (+ `GetSeqPoolingMethod` factory) over a packed `[num_tokens, hidden]` CPU buffer keyed by a minimal `PoolingCursor` (CLS/MEAN reject partial prefill, LAST allows it, MeanPool upcasts to float32) and the activation heads `PoolerIdentity`/`PoolerNormalize` (L2 `F.normalize`)/`PoolerMultiLabelClassify` (sigmoid)/`PoolerClassify` (sigmoid if `num_labels<2` else `softmax`). Unit-gated vs DOUBLE-PRECISION references, RED-first. **W2 LANDED + CPU-GATED 2026-07-29 (`CLAIM-POOLING`, NOT pushed):** the pooler HEADS composite (`EmbeddingPoolerHead` = projector→matryoshka→normalize; `ClassifierPoolerHead` = classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` (method∩head task intersection) + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` groupby-task routing (`ForEmbedding`/`ForSeqCls` + a mixed embed+classify batch + ctor task-support validation), and the `PoolerConfig`/`PoolingParams`/`PoolingParamsUpdate` structs; `test_pooler_heads` 27/27 (240 asserts) vs double-precision refs, RED-first (disable matryoshka slice + logit_mean calibration → 8 cases / 50 asserts fail). RESIDUALS (named, spec §Work breakdown): the endpoints (W4), tokwise `AllPool`/`StepPool` (W5), a concrete pooling MODEL + real-oracle cosine gate (W3-model — see `ENG-POOLING-RUNNER`) | T2 | `vllm/model_executor/layers/pooler/seqwise/methods.py:35-121`; `vllm/model_executor/layers/pooler/activations.py:106-158`; `vllm/model_executor/layers/pooler/seqwise/heads.py:19-196`; `vllm/model_executor/layers/pooler/seqwise/poolers.py:41-138`; `vllm/model_executor/layers/pooler/special.py:23-140`; `vllm/model_executor/layers/pooler/common.py:12-30`; `vllm/pooling_params.py:35-70`; `vllm/config/pooler.py:16-90`; `vllm/v1/pool/metadata.py:13-71`; `tests/model_executor/layers/test_pooler_methods.py`, `tests/model_executor/layers/test_pooler_activations.py`, `tests/model_executor/layers/test_pooler_heads.py` | `include/vllm/model_executor/layers/pooler/{methods,activations,pooling_metadata,common,pooling_params,pooler_config,heads,poolers,dispatch_pooler}.h` + `src/vllm/model_executor/layers/pooler/{methods,activations,heads,poolers,dispatch_pooler}.cpp` — anchor `src/vllm/model_executor/layers/pooler/dispatch_pooler.cpp:13` | `tests/vllm/model_executor/layers/pooler/test_pooler.cpp` (CLS/LAST/MEAN + factory + activations, 50 asserts) + `test_pooler_heads.cpp` (Embedding/Classifier heads + SequencePooler + DispatchPooler, 240 asserts) — anchor `tests/vllm/model_executor/layers/pooler/test_pooler.cpp:81` | [pooling-task-class.md](specs/pooling-task-class.md) | `ACTIVE` | `CLAIM-POOLING` | diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index f70a66e96..222431ae7 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -4,6 +4,7 @@ | Reference | Workload | Headline | Tokens | |---|---|---|---| +| **Binary release matrix (inventoried)** | `ENG-RELEASE-BINARIES`: backend-specific `vllm-server` archives plus fully-static feasibility | **PENDING:** spike in progress; no archive, runtime validation, or release-performance result exists yet | n/a | | **Developer agent entry point (implemented)** | `DOCS-AGENT-PROTOCOL-ENTRYPOINT`: public contribution guide + synchronized, mutation-gated pre-claim intake rule | Rebased documentation/protocol only; benchmark void | n/a | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **Record repair 2026-08-07** | `main` was red on `check-agent-record` + `check-env-doc`, blocking every PR. Dangling `kda-chunk-aot/` link and two undocumented env vars. No behaviour change | diff --git a/docs/STATUS.md b/docs/STATUS.md index 06e445fe4..77027d136 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -112,7 +112,7 @@ token-for-token correctness against the pinned oracle. | Tool-call parsing | 36 parser families / 40 accepted names, streaming | Every vLLM tool parser at the pin except the three Rust/Harmony-backed ones: pure-text parsers ported 1:1, the six engine-backed families reimplemented from their wire formats, all held to the upstream test suites. Selection via `--tool-call-parser` (server), `tool_parser` (C ABI), or template auto-detection; native-syntax forced tool_choice where expressible. Tables: docs/BENCHMARKS.md | | Reasoning parsing (`SAMPLE-REASONING`, ACTIVE, partial coverage) | 9 parsers, streaming | think_auto (auto-detect default: content unless markers appear), deepseek_r1, deepseek_v3 (passthrough) / holo2 (thinking→R1), mistral ([THINK]), minimax_m2 (+append_think), step3, olmo3 - reasoning split engine-side BEFORE tool parsing, streamed as `reasoning` deltas in the chat chunks. Coverage: 9 of upstream's ~28 registered names (remaining text families + engine-backed adapters tracked as W2/W3 in specs/reasoning-parsers.md); each ported parser doctest-gated vs its tests/reasoning case | | Unified streaming parser engine | Core, assembly, serving-SSE dispatch landed, gated; all 10 engine-backed families ported (family parity closed); JSON-schema tool-arg type coercion landed | The vLLM 0.26 declarative `parser/engine/` (shared state machine plus all 10 configs: qwen3, seed_oss, kimi_k2, minimax_m2, glm47_moe, deepseek_v4/v32, nemotron_v3, gemma4, inkling) and assembly layer, gated field-for-field vs vLLM 0.26. An engine-backed `--tool-call-parser` name drives the live chat SSE chunks, off by default. When a request's tools declare typed parameters, the assembled tool-call arguments are coerced to the declared JSON types (int/number/bool/string/array/null) 1:1 with vLLM `_fix_arg_types`, in both streaming and one-shot; no schema means the arguments pass through as strings unchanged. Details: .agents/specs/parser-assembly-c8.md | -| OpenAI server | Supported (subset) | `/v1/completions`, `/v1/chat/completions`, streaming SSE, `/v1/models`, `/health`, `/version`, `/ping`, `/metrics`, `/tokenize` (raw-`prompt` and chat-`messages`), `/detokenize`, `/tokenizer_info`, `/server_info`, `/reset_prefix_cache`, `/abort_requests`; `/v1/videos` in OpenAI's Sora shape + `GET /v1/videos/{id}/content`, conditioned on an `input_reference` image or the `metadata` video/audio references. `/tokenizer_info` and `/abort_requests` are flag-gated; `/metrics` and `/reset_prefix_cache` have handlers but no live backing on the async path. Endpoint list and flags: docs/USAGE.md. Depth-2 async serving no longer corrupts the host heap under `ignore_eos` | +| OpenAI server | Supported subset; archives INVENTORIED | `/v1/completions`, `/v1/chat/completions`, streaming SSE, `/v1/models`, `/health`, `/version`, `/ping`, `/metrics`, `/tokenize` (raw-`prompt` and chat-`messages`), `/detokenize`, `/tokenizer_info`, `/server_info`, `/reset_prefix_cache`, `/abort_requests`; `/v1/videos` in OpenAI's Sora shape + `GET /v1/videos/{id}/content`, conditioned on an `input_reference` image or the `metadata` video/audio references. `/tokenizer_info` and `/abort_requests` are flag-gated; `/metrics` and `/reset_prefix_cache` have handlers but no live backing on the async path. Endpoint list and flags: docs/USAGE.md. Depth-2 async serving no longer corrupts the host heap under `ignore_eos` | | Pooling task class (embeddings / classify / score / rerank) | Spiked; pooler op + heads composite + pooling runner path landed (CPU), not yet servable end-to-end (no concrete model / endpoints) | The non-generative task class. W0 spike over the whole vLLM pooling surface (`.agents/specs/pooling-task-class.md`, `CLAIM-POOLING`). W1 landed the pooler OP (CLS/LAST/MEAN + Identity/Normalize/MultiLabelClassify/Classify activations, double-precision-gated). **W2 landed the pooler HEADS composite** — `EmbeddingPoolerHead` (projector→matryoshka→normalize), `ClassifierPoolerHead` (classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` task routing, and the `PoolerConfig`/`PoolingParams` structs (`test_pooler_heads` 27/27, 240 asserts, RED-first). **W3 landed the pooling RUNNER path** — `PoolingRunner` applies the model's `Pooler` to the last hidden state and returns pooled embeddings instead of sampled tokens, gated by a STRUCTURAL cosine-parity check vs a double-precision LAST+normalize reference (`test_pooling_runner` 5/5, 14 asserts, RED-first). NOT yet servable / honest residuals: a concrete pooling MODEL forward + the REAL-model oracle cosine gate (`vllm.LLM(task="embed").encode`) — no cosine-vs-oracle number is fabricated (W3-model); the `/v1/embeddings` + score/rerank/classify endpoints (W4); tokwise AllPool/StepPool (W5). See docs/BENCHMARKS.md | | Plugin system (out-of-core registration) | Spiked; first CPU brick landed, not yet wired into any production path | The extensibility-first discovery layer. W0 spike over vLLM's plugin surface (general / platform / io_processor / endpoint groups, the `register_model` an out-of-tree plugin calls, the invocation seams) is committed (`.agents/specs/plugin-system.md`, `ENG-PLUGIN-SYSTEM` ACTIVE, `CLAIM-PLUGIN-SYSTEM`). W1 landed `vllm::plugins::LoadGeneralPlugins()` + the out-of-core general-plugin registration seam (`RegisterGeneralPlugin` / `REGISTER_VLLM_GENERAL_PLUGIN`) over the existing `REGISTER_VLLM_MODEL`-style registries (the in-tree factory `MODEL-FACTORY-registry` is record-repaired `DONE` 2026-08-05: 28 self-registering TUs, dgx debt paid by the 2026-07-23 seven-gate run): a 1:1 mirror of `load_general_plugins` (load-once idempotence, the `VLLM_PLUGINS` allowlist, per-plugin failure isolation). Proven by an out-of-core toy-model plugin that registers a toy architecture through the public `RegisterModel` seam — unit-gated RED-first (`test_plugin_system` 1 case / 29 assertions: the toy arch resolves ONLY after LoadGeneralPlugins runs it, and not under `VLLM_PLUGINS=""`). Python entry points have no C++20 analogue, so discovery is the project's static-init/`dlopen` registration idiom (recorded porting-inventory §9). NOT yet wired: real shared-object `dlopen` + the C-ABI `vllm_plugin_register` entry (W2), the engine/CLI `--load-plugins` wiring that calls LoadGeneralPlugins from the construction paths (W3), the platform/quant plugin kinds (W4), and the io_processor/stat_logger/endpoint groups (W5) are named residuals. See docs/BENCHMARKS.md | | Offline Batch API (JSONL file runner) | Spiked; first CPU brick landed, not yet exposed as a CLI | The offline OpenAI Batch API: read a JSONL of OpenAI-format requests, run each through the engine, write a JSONL of responses. W0 spike over vLLM's `run_batch.py` (schema, endpoint dispatch, run loop, file I/O) is committed (`.agents/specs/batch-api.md`, `SERVE-BATCH-API` ACTIVE, `CLAIM-BATCH-API`). W1 landed `RunBatch` (`RunLine`/`RunLines`/`Run`) + `RunBatchFile` — a pure orchestrator over the existing `OpenAIServingChat::create_chat_completion` (NO reimplemented generation), 1:1 with vLLM's endpoint_registry url→handler map: `/v1/chat/completions` dispatch, the `BatchResponseData`/`BatchRequestOutput` schema (`vllm-` ids, custom_id echoed), the `run_request` AllResponse/ErrorResponse/stream branches, and the unsupported-endpoint/url error rows. Unit-gated RED-first (`test_openai_run_batch` 7 cases / 80 assertions over the synthetic serving engine: ordered rows + custom_id echo + per-line BatchRequestOutput schema round-trip, a malformed line isolated into an error row so the batch continues, dispatch + 404 error rows; dropping the custom_id echo fails 9 assertions). Recorded deviation: a malformed line is isolated (batch continues) where upstream aborts the job. NOT yet exposed: the `vllm run-batch` CLI + `BatchFrontendArgs` (W2), embeddings/score/rerank dispatch (W3, rides pooling endpoints), audio transcription/translation + media fetch (W4), and http(s)/data-URL file I/O + metrics + overlapped `AsyncLLM` submission (W5) are named residuals. See docs/BENCHMARKS.md | diff --git a/scripts/check-agent-record.py b/scripts/check-agent-record.py index 862789b86..80f22a501 100644 --- a/scripts/check-agent-record.py +++ b/scripts/check-agent-record.py @@ -292,7 +292,13 @@ # over HTTP at all. CPU-landed + gated, `CLAIM-SERVE-VIDEOS-REFS`, spec # `specs/minimax-h3.md` §10. # Bumped for a real new row, never to make a failing state transition pass. -ENGINE_ROWS = 142 +# 143 since 2026-08-07: +`ENG-RELEASE-BINARIES` (downloadable, backend-specific +# server bundles and their static/runtime dependency contract) — a real +# distribution capability requested in issue #117, not a restatement of the +# server implementation. Inventoried while its release-matrix spike is written; +# no packaging support is claimed by the count bump. +# Bumped for a real new row, never to make a failing state transition pass. +ENGINE_ROWS = 143 MATRIX_PATHS = [ENGINE_MATRIX, *(path for path, _ in MATRICES.values())] REQUIRED = [ From 95674a0b68448379818992c5d6a51f155ef44b6a Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 7 Aug 2026 22:07:13 +0000 Subject: [PATCH 02/11] docs(release): spike downloadable server bundle matrix Define the stable/runtime-gated and preview/build-only artifact matrix, per-SM CUDA and AOT contract, staged archive gates, supply-chain evidence, and helper-sized implementation sequence. FOLLOWING_AGENTS_PROTOCOL Assisted-by: Codex:gpt-5 [Codex] --- .agents/NOW.md | 1 + .agents/coordination.md | 5 +- .agents/engine-matrix.md | 6 +- .agents/roadmap_v1.md | 1 + .agents/specs/release-binary-matrix.md | 318 +++++++++++++++++++++++++ .agents/state.md | 26 ++ docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- 8 files changed, 355 insertions(+), 6 deletions(-) create mode 100644 .agents/specs/release-binary-matrix.md diff --git a/.agents/NOW.md b/.agents/NOW.md index e237ef1f9..fba32e735 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -28,6 +28,7 @@ Working head: `row/backend-rocm-w0` (#41). Prior: benchmark checkpoint | Supported-models list | **LANDED**: FEATURES arch table CI-bound (33 archs) | — | | `/v1/videos` OpenAI shape | **MERGED** (#71): Sora `model`/`size`/`seconds` + `GET /{id}/content` | `row/SERVE-VIDEOS-REFS` PR open: reference conditioning | | `BACKEND-ROCM` W0 | Skeleton in; **HIP never compiled** (no AMD HW) | #41 contributors build it; a compile error IS the deliverable | +| Release binaries | **SPIKE** (#129) | Fresh review; then W1 install/package target | | Surface coverage (`ARCH-ONE-SURFACE`) | **ROW 1 LANDED (#121)**: Parakeet ASR on the surface (`vllm_transcribe` v11, `/v1/audio/transcriptions`, ratchet 12→11) | Next fold row | In-flight (default-OFF, not pushed): `laguna-fp4proj-prod`, laguna diff --git a/.agents/coordination.md b/.agents/coordination.md index 3b7db6d75..074e25f0f 100644 --- a/.agents/coordination.md +++ b/.agents/coordination.md @@ -132,7 +132,9 @@ this claim, the required current-state checkpoint in `docs/STATUS.md` and `.agents/state.md`. No release workflow, CMake/package implementation, source, test, model, kernel, generated fixture, download, build, GPU, or service change is in scope. Draft -PR is the claim; exact PR number is added after GitHub allocates it. +PR [#129](https://github.com/mudler/vllm.cpp/pull/129) is the claim. The spike +contract is complete on the branch and awaits fresh review; the row stays +`SPIKE`. **CPU grouped keep-quant GEMM activation-dtype P0 (`QUANT-GGUF-CIQ-GEMM`, 2026-08-06, `CLAIM-QUANT-GGUF-CIQ-GROUPED-DTYPE`).** Claude Code @@ -1411,6 +1413,7 @@ table, tests, CMake. Details in the state-log entry of the same date. | Claim | Row IDs | Agent | Worktree / remote dir | Branch | Owned scope | State | Last update | |---|---|---|---|---|---|---|---| +| `CLAIM-ENG-RELEASE-BINARIES-SPIKE` | `ENG-RELEASE-BINARIES` | Codex (GPT-5) | `/home/mudler/_git/vllm.cpp-release-binaries`; records/spec only, no build/GPU/download/service work | `row/ENG-RELEASE-BINARIES`, base `9cb799cb`, draft PR #129 | Accepted release-bundle spike plus its matrix/roadmap/coordination/state/NOW and public-doc lifecycle checkpoint; no CMake, workflow, source, test, or artifact implementation | `ACTIVE` | 2026-08-07 — spike contract complete; row stays `SPIKE`; awaiting fresh review | | `CLAIM-KIMI-LINEAR-W0` | `MODEL-TEXT-kimi-linear-kimi-linear-for-causal-lm` (stays SPIKE — dedicated W0 spike) | Claude Code (opus-4-8) | isolated worktree `.claude/worktrees/agent-a771cc029e6843be0`; CPU-only, records-only — NO build, NO GPU, NO download (two GPU jobs queued ahead; the W0 GPU golden capture is a SEPARATE later step) | branch `worktree-agent-a771cc029e6843be0`, base `origin/main` HEAD `10dd23ee` | The FULL dedicated W0 spike for `KimiLinearForCausalLM` (Kimi-Linear-48B-A3B) per the spike-first protocol, so W1 implementation can start immediately. Owns ONLY: NEW `.agents/specs/kimi-linear.md`, the `MODEL-TEXT-kimi-linear-*` matrix row (Spike link → the dedicated spec + note; STAYS `SPIKE` because it is actively claimed — protocol forbids a claimed row from being `READY`; checklist mark stays `📋`), this claim row + the narrative block below, the roadmap breadth note, `docs/STATUS.md`/`docs/BENCHMARKS.md`/`docs/FEATURES.md` one-liners, the `.agents/NOW.md` live-claim row + stamp, and one `.agents/state.md` entry. **NON-COLLISION:** records-only — touches NO model/kernel/registry/loader source, NO CMake, NO test. Co-owns the Kimi-Linear row with `CLAIM-MLA-DEEPSEEK` (MLA half) + `CLAIM-KDA-KERNEL` (KDA host refs), exactly as the K3 row is co-owned by `CLAIM-KIMI-K3-SCOPE`/`CLAIM-KIMI-K3-W2-W5`. | `DONE` | 2026-08-05 — **W0 DEDICATED SPIKE LANDED (records-only, NOT pushed).** Authoritative `config.json` fetched (27 layers = 20 KDA + 7 NoPE-MLA; 256e/top-8/1-shared sigmoid `noaux_tc` `routed_scaling=2.446` `first_k_dense_replace=1`; `num_nextn_predict_layers=0` ⇒ no MTP); reuse-vs-new map with our `file:line`; HW-fit (FITS one GB10, 91.5 GiB / 0.77× pool — real e2e SACRED gate reachable); W0-W7 breakdown + GPU golden-capture recipe. NET-NEW = KDA device kernel (host refs landed, `CLAIM-KDA-KERNEL`), NoPE-MLA branch, hybrid schedule/het-KV, loader name-map. Record checkers green (`check-model-checklist`, `check-agent-record`, `check-doc-checkpoint`, `check-now-current`). | | `CLAIM-KIMI-LINEAR-W1` | `MODEL-TEXT-kimi-linear-kimi-linear-for-causal-lm` (stays SPIKE — W1 registry/config/loader scaffolding; forward REFUSES-by-name) | Claude Code (opus-4-8) | isolated worktree `.claude/worktrees/agent-a11ae231a71e8694b`; CPU-only `build-cpu` (`-DVLLM_CPP_CUDA=OFF` Release); NO GPU, NO download — additive TU + one REGISTER line, forward is a `VT_CHECK(false)` stub | branch `worktree-agent-a11ae231a71e8694b`, base `origin/main` HEAD `053116df` | Kimi-Linear-48B-A3B W1 — registry + config + loader + KV-spec scaffolding so the W3-W6 forward can start. Owns ONLY: NEW `include/vllm/model_executor/models/kimi_linear.h`, NEW `src/vllm/model_executor/models/{kimi_linear_registry,kimi_linear_weights,kimi_linear}.cpp`, NEW `tests/vllm/models/test_kimi_linear_scaffold.cpp`, its two CMake registration lines (`CMakeLists.txt` source list + `tests/CMakeLists.txt`), the `KimiLinearForCausalLM` sorted-set + error-message + model-property inserts in `tests/vllm/models/test_model_registry.cpp`, the `MODEL-TEXT-kimi-linear-*` row cells + checklist Status (STAYS `SPIKE`/`📋` — the forward refuses, rollup UNCHANGED), this claim, `.agents/NOW.md`, `docs/STATUS.md`/`docs/BENCHMARKS.md`/`docs/FEATURES.md` one-liners, and one `.agents/state.md` entry. **NON-COLLISION:** additive TU + one REGISTER line ⇒ ZERO edit to any shared array; the forward is a `VT_CHECK(false)` refuse stub so NO production/device path changes; co-owns the Kimi-Linear row with `CLAIM-MLA-DEEPSEEK` + `CLAIM-KDA-KERNEL` + `CLAIM-KIMI-LINEAR-W0`; MUST NOT touch README, Metal/SACRED/apex/darwin, or any other model/kernel source. | `DONE` | 2026-08-05 — **W1 CPU SCAFFOLDING LANDED (foreground, NOT pushed).** Additive registry (`kimi_linear`/`KimiLinearForCausalLM`, `is_hybrid`, `supports_multimodal=false`) + `ParseKimiLinearParams` (20 KDA + 7 NoPE-MLA schedule, MLA 512/128/64/128 q_lora-null, 256e/top-8/1-shared sigmoid `noaux_tc` 2.446, `first_k_dense_replace=1`; asserts `mla_use_nope`/`q_lora==null`) + `EnumerateKimiLinearTensors` name-map VERIFIED vs the real HF safetensors index (MoE = `block_sparse_moe.*`, the correction of the DERIVED K3 `mlp.*`) + loader THROWS BY NAME on missing/mis-shaped tensors + het KV spec (MLA latent-576 + KDA mamba `12288×3`/`32×128×128`) + REFUSE-by-name forward. `test_kimi_linear_scaffold` **9/9·83** + `test_model_registry` **24/24** + `test_kimi_k3_scaffold` 6/6 unchanged; clean CPU Release build; `check-fusion-consistency`/`check-runner-routing-consistency` green (refuse stub skipped); record checkers rc=0. RESIDUAL (W3-W6): KDA device kernel + NoPE-MLA route + sigmoid-noaux MoE + het-KV born-on-runner forward + the e2e SACRED gate (spec §8 recipe). | | `CLAIM-KIMI-LINEAR-W2` | `MODEL-TEXT-kimi-linear-kimi-linear-for-causal-lm` (stays SPIKE — W2-W6 CPU REFERENCE forward; the DEVICE runner forward stays refuse-by-name) | Claude Code (opus-4-8) | isolated worktree `.claude/worktrees/agent-aec13af094cd4692f`; CPU-only `build` (`-DVLLM_CPP_CUDA=OFF` RelWithDebInfo); NO GPU, NO download — additive forward TU + a host-materialization pass in the existing loader + additive test | branch `laguna-s21-w7-speed-profile` (worktree), base `origin/main` HEAD `0988db48` | Kimi-Linear-48B-A3B W2-W6 — the CPU REFERENCE forward, per-op gated, replacing the refuse-by-name host `KimiLinearModel::Forward`. Owns ONLY: NEW `src/vllm/model_executor/models/kimi_linear_forward.cpp`, NEW `tests/vllm/models/test_kimi_linear_forward.cpp`, its two CMake lines (`CMakeLists.txt` source list + `tests/CMakeLists.txt`), ADDITIVE host-weight structs + per-op decls in `include/vllm/model_executor/models/kimi_linear.h`, the host-materialization pass in `src/vllm/model_executor/models/kimi_linear_weights.cpp`, the device-refuse doc/rename in `src/vllm/model_executor/models/kimi_linear.cpp`, the `MODEL-TEXT-kimi-linear-*` row cells + checklist Status (STAYS `SPIKE`/`📋` — the DEVICE forward refuses; rollup UNCHANGED), this claim, `.agents/NOW.md`, `docs/STATUS.md`/`docs/BENCHMARKS.md` one-liners, and one `.agents/state.md` entry. **NON-COLLISION:** additive TU + additive header structs; the host `Forward` becomes real but `ForwardDevice` (the DEFAULT gather_logits runner path) STAYS `VT_CHECK(false)` so the born-on-runner CI guard skips it (NO allowlist) and NO production/device path changes; co-owns the Kimi-Linear row with `CLAIM-MLA-DEEPSEEK` + `CLAIM-KDA-KERNEL` + `CLAIM-KIMI-LINEAR-W0/W1`; MUST NOT touch README, Metal/SACRED/apex/darwin, or any other model/kernel source. | `DONE` | 2026-08-05 — **W2-W6 CPU REFERENCE FORWARD LANDED (foreground, NOT pushed).** Real host `KimiLinearModel::Forward` composes the whole 27-layer hybrid from the landed primitives: KDA layer (`vllm::kimi_kda` refs + the gated-delta recurrence `fused_recurrent.py:122-149`), NoPE-MLA (materialized-MHA reference, scaling `qk_head**-0.5`, no RoPE), sigmoid `noaux_tc` MoE (bias-select/unbiased-weight, renormalize, `routed_scaling=2.446`) + shared expert, dense layer-0 SwiGLU; loader now MATERIALIZES the host float weights (bf16/f32→f32). `test_kimi_linear_forward` **6/6·246** (KDA==kimi_kda composition; NoPE-MLA==materialized-MHA ref; router+block==hand-computed; loader materializes; whole 2-layer forward finite/coherent; greedy-decodes 5 tokens, context advances) + `test_kimi_linear_scaffold` 9/9·83 + `test_kimi_kda` 14/14 UNCHANGED; clean CPU build; `check-fusion-consistency`/`check-runner-routing-consistency` green (DEVICE forward refuse-skipped, no allowlist). RESIDUAL: the DEVICE born-on-runner forward + the W0/W7 e2e SACRED token golden on GB10 (spec §8 recipe) + speed. | diff --git a/.agents/engine-matrix.md b/.agents/engine-matrix.md index 03e1ecd19..2f9809c1d 100644 --- a/.agents/engine-matrix.md +++ b/.agents/engine-matrix.md @@ -42,11 +42,11 @@ forensics: roadmap_v1.md and the parity ledger. | Sampling and generation | 15 | 4 | 2 | 0 | 0 | 3 | 0 | 1 | 5 | | Structured output and tools | 7 | 0 | 3 | 0 | 0 | 2 | 0 | 0 | 2 | | Speculative decoding | 21 | 0 | 0 | 1 | 0 | 4 | 0 | 4 | 11 | -| Serving, API, CLI, library | 25 | 10 | 2 | 1 | 0 | 4 | 2 | 1 | 5 | +| Serving, API, CLI, library | 25 | 10 | 2 | 2 | 0 | 4 | 2 | 1 | 4 | | LoRA and adapters | 2 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | | Long context and attention | 10 | 5 | 0 | 0 | 1 | 0 | 1 | 0 | 3 | | Loading, tokenizer, config | 9 | 3 | 3 | 0 | 0 | 0 | 1 | 1 | 1 | -| **Total** | **143** | **36** | **16** | **4** | **7** | **20** | **8** | **9** | **42** | +| **Total** | **143** | **36** | **16** | **5** | **7** | **20** | **8** | **9** | **41** | ## Engine core and scheduling @@ -207,7 +207,7 @@ claims it. | `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; `examples/bench/main.cpp:40,109`; `examples/bench/bench_core.h:96,468` | server help contract `examples/CMakeLists.txt:34`; benchmark `tests/examples/test_bench.cpp:15,48` | `planned: specs/cli-serve-bench.md` | `PARTIAL` | - | | `SERVE-GATE-ONLINE` | Same-corpus online correctness, TTFT/TPOT/ITL, throughput and peak-memory gate vs vLLM v0.25.0 | T0 | `vllm/benchmarks/serve.py:1,581-615`; [v0.25 audit](sync/2026-07-12-702f481.md); `tests/benchmarks/test_serve_cli.py:1` | Schema-v5 harness plus [trace controller](../include/vt/cuda/cuda_profiler_control.h#L13), [production component driver](../scripts/dgx-gdn-packed-component.sh), and fail-closed [component finalizer](../tools/bench/gdn_packed_component.py) | **BINDING `9ecd9d0`: 114/124** (async default ON; mem 4/4, c1 20/20, c2 20/20, c16 19/20, c4 & c32 18/20, c8 15/20; `benchmark_binding` refers here, superseding `3f256ab` 55/124 and `246a23c` 49/124, both retained immutable). Two-grid totality with `f0fb727` (111/124) is 115/124 effective parity vs vLLM 0.25.0 (27B). Async CLOSED the c16/c32 ITL tails (ours now BEATS vLLM: c16 p99 1.055, c32 p90 1.034/p99 1.078) and leaves a stable c8 `p99_itl` ~0.86 residual, ROOT-CAUSED (2026-07-18, `CLAIM-C8-P99-TAIL-1`, [spec](specs/c8-p99-itl-tail-2026-07-18.md)) as IRREDUCIBLE-AS-MIRRORED: our deterministic synchronous forward keeps co-admitted c8 requests in byte-identical lockstep where vLLM's async-future jitter de-phases them; the c16/c32 INVERSION proves this is the trailing edge of the per-step determinism that wins c16/c32 + throughput, not a capability gap (scheduler + async placeholder byte-identical, `tests/vllm/v1/test_scheduler_wave.cpp:265`, [tail spec](specs/tail-stall-analysis-2026-07-16.md)). Full grid + per-binding forensics: roadmap_v1.md + parity ledger; no packed speed credit | [online serving gate](specs/cuda-online-serving-gate.md); [merged GDN projections](specs/gdn-merged-input-projections.md); [packed decode](specs/gdn-packed-decode.md) | `ANCHOR-BACKFILL` | CLAIM-SERVE-GATE-1 | | `SERVE-E2E-NIGHTLY` | Server conformance and real-model nightly suites for all release gates | T0 | `tests/entrypoints/openai/`; `tests/v1/e2e/`; `.buildkite/test-pipeline.yaml` | current unit/conformance tests only; no scheduled DGX suite | `tests/vllm/entrypoints/openai/test_conformance.cpp:1`; `tests/parity/test_qwen36_paged_engine.cpp:78`; `tests/parity/test_qwen27_paged_engine.cpp:110` | `planned: specs/server-e2e-nightly.md` | `INVENTORIED` | - | -| `ENG-RELEASE-BINARIES` | Downloadable, backend-specific `vllm-server` release bundles and fully-static feasibility boundary | T0 | vLLM release lanes `.buildkite/release-pipeline.yaml:1-18,34-170` @ `555967922`; release-image dependency boundary `docker/Dockerfile.cpu:262-290` | server target only `examples/CMakeLists.txt:54-64`; no install, archive, manifest, provenance, or release-publish implementation | help smoke only `examples/CMakeLists.txt:59-63`; issue `#117` requests binary releases | `planned: specs/release-binary-matrix.md` | `INVENTORIED` | - | +| `ENG-RELEASE-BINARIES` | Downloadable, backend-specific `vllm-server` release bundles and fully-static feasibility boundary | T0 | vLLM release lanes `.buildkite/release-pipeline.yaml:1-18,34-170` @ `555967922`; release-image dependency boundary `docker/Dockerfile.cpu:262-290` | server target only `examples/CMakeLists.txt:54-64`; no install, archive, manifest, provenance, or release-publish implementation | help smoke only `examples/CMakeLists.txt:59-63`; issue `#117`; accepted matrix/package/gate contract in [release-binary-matrix.md](specs/release-binary-matrix.md) | [release-binary-matrix.md](specs/release-binary-matrix.md) | `SPIKE` | `CLAIM-ENG-RELEASE-BINARIES-SPIKE` | | `SERVE-CLI-CHAT` | Interactive `chat` and `complete` commands against a running OpenAI-compatible server, plus preservation of the existing local-model completion invocation | T1 | registration `vllm/entrypoints/cli/main.py:17-37,73-98`; client/model resolution + stream shaping `vllm/entrypoints/cli/openai.py:30-100`; chat `:155-234`; complete `:237-312` at `5559679229` | current in-process completion only `examples/cli/main.cpp:1-207`; remote command implementation absent | C-ABI stream baseline `tests/capi/test_capi.cpp:567-711`; chat-template baseline `tests/capi/test_chat_prompt.cpp:37-89`; command/fake-server tests absent | [cli-chat-complete.md](specs/cli-chat-complete.md) | `ANCHOR-BACKFILL` | `CLAIM-SERVE-CLI-CHAT-SPIKE` | | `SERVE-POOLING-ENDPOINTS` | Embeddings, pooling, score, rerank, classify HTTP surface (`/v1/embeddings`, `/pooling`, `/score`, `/rerank`, `/classify`). **SPIKED 2026-07-28 (`CLAIM-POOLING`):** the whole pooling task class is scoped in [pooling-task-class.md](specs/pooling-task-class.md) — endpoints depend on the pooling RUNNER (`ENG-POOLING-RUNNER`, W3 LANDED 2026-07-29 — `PoolingRunner` returns pooled data via the model `Pooler`) and a concrete pooling model. Endpoint protocol + handler port is the W4 brick | T2 | `vllm/entrypoints/pooling/embed/api_router.py:28`; `vllm/entrypoints/pooling/scoring/api_router.py:37,71`; `vllm/entrypoints/pooling/classify/api_router.py:26` | - | - | [pooling-task-class.md](specs/pooling-task-class.md) | `SPIKE` | `CLAIM-POOLING` | | `ENG-POOLER-SEQ` | The non-generative POOLER OP — turn hidden states into a pooled embedding/logit row instead of a sampled token. **W1 LANDED + CPU-GATED 2026-07-28 (`CLAIM-POOLING`, NOT pushed):** the sequence pooling methods `CLSPool`/`LastPool`/`MeanPool` (+ `GetSeqPoolingMethod` factory) over a packed `[num_tokens, hidden]` CPU buffer keyed by a minimal `PoolingCursor` (CLS/MEAN reject partial prefill, LAST allows it, MeanPool upcasts to float32) and the activation heads `PoolerIdentity`/`PoolerNormalize` (L2 `F.normalize`)/`PoolerMultiLabelClassify` (sigmoid)/`PoolerClassify` (sigmoid if `num_labels<2` else `softmax`). Unit-gated vs DOUBLE-PRECISION references, RED-first. **W2 LANDED + CPU-GATED 2026-07-29 (`CLAIM-POOLING`, NOT pushed):** the pooler HEADS composite (`EmbeddingPoolerHead` = projector→matryoshka→normalize; `ClassifierPoolerHead` = classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` (method∩head task intersection) + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` groupby-task routing (`ForEmbedding`/`ForSeqCls` + a mixed embed+classify batch + ctor task-support validation), and the `PoolerConfig`/`PoolingParams`/`PoolingParamsUpdate` structs; `test_pooler_heads` 27/27 (240 asserts) vs double-precision refs, RED-first (disable matryoshka slice + logit_mean calibration → 8 cases / 50 asserts fail). RESIDUALS (named, spec §Work breakdown): the endpoints (W4), tokwise `AllPool`/`StepPool` (W5), a concrete pooling MODEL + real-oracle cosine gate (W3-model — see `ENG-POOLING-RUNNER`) | T2 | `vllm/model_executor/layers/pooler/seqwise/methods.py:35-121`; `vllm/model_executor/layers/pooler/activations.py:106-158`; `vllm/model_executor/layers/pooler/seqwise/heads.py:19-196`; `vllm/model_executor/layers/pooler/seqwise/poolers.py:41-138`; `vllm/model_executor/layers/pooler/special.py:23-140`; `vllm/model_executor/layers/pooler/common.py:12-30`; `vllm/pooling_params.py:35-70`; `vllm/config/pooler.py:16-90`; `vllm/v1/pool/metadata.py:13-71`; `tests/model_executor/layers/test_pooler_methods.py`, `tests/model_executor/layers/test_pooler_activations.py`, `tests/model_executor/layers/test_pooler_heads.py` | `include/vllm/model_executor/layers/pooler/{methods,activations,pooling_metadata,common,pooling_params,pooler_config,heads,poolers,dispatch_pooler}.h` + `src/vllm/model_executor/layers/pooler/{methods,activations,heads,poolers,dispatch_pooler}.cpp` — anchor `src/vllm/model_executor/layers/pooler/dispatch_pooler.cpp:13` | `tests/vllm/model_executor/layers/pooler/test_pooler.cpp` (CLS/LAST/MEAN + factory + activations, 50 asserts) + `test_pooler_heads.cpp` (Embedding/Classifier heads + SequencePooler + DispatchPooler, 240 asserts) — anchor `tests/vllm/model_executor/layers/pooler/test_pooler.cpp:81` | [pooling-task-class.md](specs/pooling-task-class.md) | `ACTIVE` | `CLAIM-POOLING` | diff --git a/.agents/roadmap_v1.md b/.agents/roadmap_v1.md index 3dcb855b9..10f779d73 100644 --- a/.agents/roadmap_v1.md +++ b/.agents/roadmap_v1.md @@ -80,6 +80,7 @@ models we already ship + benchmark. Full seam map + M0–M5 W-plan: | 14 | `ROAD-V1-D5` | LoRA, local KV/weight offload, expert streaming, wider model zoo | [engine matrix](engine-matrix.md), [model matrix](model-matrix.md) | corrected expert-streaming spike accepted (`ENG-EXPERT-STREAM` READY): bank-only safetensors→Marlin bank, fixed contiguous cache slots matching Marlin dense strides, logical→slot remap after explicit router D2H, chunked C` working everywhere `--dit ` did; gated CPU-only (72/72, 54497 post-rebase) on index/name mapping and on the REAL 535-tensor geometry read from a sparse 13-shard release. The DEVICE streamer landed 2026-08-07 (`row/H3-BF16-SHARDED-STREAM`, spec §8.14): one tensor at a time, zero host buffer for the bulk, bit-exact vs the non-streamed reference (73/73, 55203). **ENCODER + THE NUMBER 2026-08-07 (`row/H3-ENC-BF16-COND-DIFF`, spec §8.15)**: the 14-shard bf16 text encoder streams too and `--encoder-only` runs it alone; measured over 233 tokens, Q4_K_M-vs-bf16 conditioning is cos 0.99745 mean / 6.85% rel RMS excl. sink / 3.5 deg median rotation — as much as a one-word prompt edit, but DIFFUSE. Whether the RENDER changes is NOT established (75/75, 55609). This unblocks the bf16-vs-quantized quality A/B; no render or speed number is claimed. Spec §8.13. **W-FP4a LANDED (CPU) 2026-08-06 (`row/H3-FP4-SPEED`)**: the NVFP4 DiT projections now keep FP4 PACKED and route through the shared `dense_nvfp4::MatmulNvfp4W4A16D` (Marlin W4A16 — vLLM's own forced-a16 selection; SAME kernel as Laguna/dense-Qwen3 NVFP4; no new quant code); fp4-vs-bf16 WIRING gate GREEN (62/62·30039, W4A16 dispatcher runs all 11 quantized GEMMs). **GB10 leg LANDED 2026-08-06 (`row/H3-FP4-GPU-E2E`, PR #64):** Marlin W4A16 RAN on sm_121a (`dense_gemms==11` default / `marlin_gemms==11` VT_MARLIN_DENSE=0, `fallback_gemms==0`), fp4-vs-bf16 BYTE-EXACT; fp4 is a MEMORY win (~16 vs ~66 GB), ~0.79–0.83× the bf16 arm per diffusion forward (compute-bound large M; 3.47× faster at small decode-like M). Real-checkpoint fp4-resident t2va e2e RUNS end-to-end (real 18.75 GB NVFP4 DiT + VAEs + GGUF Qwen3-VL-32B encoder → valid mp4/wav; DiT s/step 5.45/20.0/209 s @512/768/REF-768×1344-209f) but frames are a non-scene patch-grid at 12/20/50 steps → OPEN render-coherence bug (device VAE decode / denoise), separate from the fp4 speed work. vLLM-Omni serves NO quantized H3 (BF16-only) -> HW/loader-forced-indirect (4×B300 209f render 86.964 s vs 1×GB10 209 s/forward). | | 15 | `ROAD-V1-D6` | **llama.cpp device breadth folded into scope (user-directed 2026-08-05):** the 11 ggml backends vLLM has no platform for — cann, musa, opencl, openvino, rpc, webgpu, zdnn, zendnn, hexagon, blas, virtgpu — inventoried as `BACKEND-GGML-*`. **SPIKES FIRST:** no implementation before each row's `.agents/specs/.md` clears the spike contract, per the standing directive. vLLM stays the mirror source; llama.cpp is the breadth reference. | [backend matrix](backend-matrix.md) | ☐ per-row spike required | `INVENTORIED` | first spike accepted | +| REL | `ROAD-V1-RELEASE` | Downloadable backend/host-specific static-core `vllm-server` bundles, with stable channels gated by matching runtime evidence and build-only targets labeled preview | [`ENG-RELEASE-BINARIES`](engine-matrix.md) | [release binary matrix](specs/release-binary-matrix.md) | `SPIKE` | Fresh review of PR #129, then claim W1: server install component and staged package target; no archive exists yet | An area row cannot enter `READY` without a real spike under `specs/`, and cannot enter `DONE` without exact code and test/evidence anchors. Closed execution diff --git a/.agents/specs/release-binary-matrix.md b/.agents/specs/release-binary-matrix.md new file mode 100644 index 000000000..b718b4651 --- /dev/null +++ b/.agents/specs/release-binary-matrix.md @@ -0,0 +1,318 @@ +# Downloadable server binary release matrix + +Status: accepted spike for `ENG-RELEASE-BINARIES`. This document defines the +release contract and helper-sized implementation order. It does not claim that +an archive, release workflow, or runtime result exists. + +Pins: vLLM parity source `555967922`; vllm.cpp spike baseline `f13c49ee`; +request [#117](https://github.com/mudler/vllm.cpp/issues/117); claim +`CLAIM-ENG-RELEASE-BINARIES-SPIKE` in draft PR +[#129](https://github.com/mudler/vllm.cpp/pull/129). + +## Scope and product contract + +The deliverable is a downloadable, backend-specific `vllm-server` bundle. The +server links the vllm.cpp core statically, while unavoidable platform runtimes +remain explicit dependencies. A release bundle is an installed staging tree, +not a copy from a build directory. It contains at least: + +- `bin/vllm-server`, linked to the static `vllm` core; +- `VERSION`, a machine-readable release manifest, SHA256 checksum, SPDX JSON + SBOM, build-provenance attestation, and third-party license notices; +- runtime files that the selected lane is licensed and designed to redistribute + (for example MLX's dylib/metallib in the MLX preview lane); and +- no model weights, tokenizer assets, Python, PyTorch, Triton runtime, compiler, + or source/build directory. + +The bundle name includes project version, backend, target, OS, host ABI, and +archive format. CPU architecture, CUDA SM, libc floor, macOS deployment target, +and optional provider are never implicit. A future server install component and +package target must stage this exact tree. The current CMake installs +`libvllm.{a,so}` and `vllm.h` but does not install the server +(`CMakeLists.txt:1712-1783`; `examples/CMakeLists.txt:54-64`), so the install and +package targets are implementation work, not present-tense capability. + +Out of scope for this spike: implementing CMake or CI, publishing a release, +bundling models, promising cross-libc portability, and widening any backend, +model, quantization, or kernel support claim. + +## Upstream chain + +Pinned vLLM's structural release reference is +`.buildkite/release-pipeline.yaml:1-18,34-170`; its CPU release-image dependency +boundary is `docker/Dockerfile.cpu:262-290`. Those lanes establish that release +construction, validation, and publication are separate concerns. vllm.cpp does +not copy Python wheels or containers: it applies the same separation to native +installed server archives while vLLM remains the runtime-behavior oracle. + +## Our baseline + +The current server is the CMake target `server`, linked to the static `vllm` +target and gated only by a help smoke (`examples/CMakeLists.txt:54-64`). The +library install rules package `libvllm.a`, the shared C ABI library, and +`include/vllm.h` (`CMakeLists.txt:1712-1783`). There is no server install rule, +archive layout, release manifest, staged dependency audit, provenance, or +publish workflow. Existing CUDA AOT trees are exactly the six directories named +below; ROCm's opt-in skeleton has never compiled on AMD hardware. + +## Evidence classes and publication policy + +The channel is evidence-driven, not backend-driven: + +| Channel | Required evidence | User-facing promise | +|---|---|---| +| `stable` | clean build, staged-archive validation, matching-hardware runtime smoke, representative correctness gate, and release-tag rerun all pass for the exact target tuple | supported downloadable bundle for the named tuple | +| `preview` | clean build and staged-archive validation pass; runtime, correctness, or performance evidence is absent or incomplete | build-only/testing-welcome artifact; no runtime-support or performance claim | +| `blocked` | the backend does not yet compile or lacks the minimum packaging/runtime contract | no artifact is published | + +Build success never sets runtime evidence true. Runtime smoke never sets +correctness or performance evidence true. The release manifest carries +independent booleans and evidence URLs for `build_verified`, +`archive_smoke_verified`, `dependency_audit_verified`, `runtime_verified`, +`correctness_verified`, and `performance_verified`. It also records the exact +commit, clean-tree status, compiler/toolchain, CMake cache options, target +architecture, host ABI, dependency versions, and test commands. Stable +publication fails closed unless every stable-required boolean is true; +preview publication preserves false values rather than deriving or omitting +them. + +Performance evidence is informational in an archive manifest until the normal +same-box, same-workload vLLM/competitor gate has run. This release row cannot +turn a build result into a throughput claim. + +## Release matrix + +The initial matrix is deliberately hybrid. Runtime-gated tuples may graduate +to stable; build-only tuples remain downloadable previews so users can test +hardware the project does not own. + +| Artifact tuple | Initial channel | Backend flags and evidence boundary | +|---|---|---| +| `linux-x86_64-glibc-cpu` | stable after matching x86_64 runtime gate | CPU core; CUDA, Metal, MLX, Vulkan, and HIP explicitly off; record glibc and libstdc++ floors | +| `linux-aarch64-glibc-cpu` | stable after matching arm64 runtime gate | same CPU contract, independent arm64 runtime evidence; never inferred from x86_64 | +| `macos-arm64-metal` | stable after M-series runtime gate | native Metal, MLX off; record deployment target and required system frameworks | +| `macos-arm64-metal-mlx` | preview until its exact bundled MLX tuple is runtime/correctness-gated | Metal plus opt-in MLX provider and redistribution/license audit; record MLX dylib and metallib versions | +| `linux-x86_64-glibc-vulkan` | preview | Vulkan explicitly on; loader/device/driver remain external; only the Vulkan-supported model/quant surface is declared | +| `linux-x86_64-musl-cpu-static` | experimental preview | literal-static feasibility lane; CPU only; see the static boundary below | +| ROCm/HIP | blocked | `VLLM_CPP_HIP` skeleton has never compiled on AMD hardware; no archive until compile, staged smoke, and matching-hardware gates exist | + +`AUTO` is forbidden in release presets. Every backend/provider is selected +explicitly from the canonical CMake surface: `VLLM_CPP_CUDA`, +`VLLM_CPP_CUDA_ARCHITECTURES`, `VLLM_CPP_METAL`, `VLLM_CPP_MLX`, `MLX_ROOT`, +`VLLM_CPP_VULKAN`, `VLLM_CPP_HIP`, `VLLM_CPP_HIP_ARCHITECTURES`, +`VLLM_CPP_SERVER`, `VLLM_CPP_BUILD_EXAMPLES`, `VLLM_CPP_BUILD_TESTS`, and +`VLLM_CPP_TRITON` (`CMakeLists.txt:24-120`; `cmake/TritonAOT.cmake:52-87`). +The manifest records resolved feature-table output; it does not say “all CUDA +features” or “all models.” + +### CUDA artifacts: one SM and one host ABI each + +There is no generic CUDA archive and no cross-SM runtime claim. Every supported +release target is a separate artifact: + +| CUDA SM | Host ABI | AOT | Initial channel and honest boundary | +|---|---|---:|---| +| `80` | Linux x86_64, glibc | on | preview; derived/build-verified, no matching runtime board | +| `86` | Linux x86_64, glibc | on | preview; same-major derived/build evidence, no matching runtime board | +| `87` | Linux aarch64, glibc/Jetson | off | preview; portable synchronous path runtime-verified, default async path remains a known bug | +| `89` | Linux x86_64, glibc | on | preview; derived/build-verified, no matching runtime board | +| `90a` | Linux x86_64, glibc | on | preview; derived/build-verified, no matching runtime board | +| `100a` | Linux x86_64, glibc | on | preview; derived/build-verified, no matching runtime board | +| `103a` | Linux x86_64, glibc | off | preview; portable build-only target | +| `110` | Linux aarch64, glibc/Thor | off | preview until the archive itself repeats the existing portable runtime proof | +| `120a` | Linux x86_64, glibc | off | preview; build-supported, no matching runtime board | +| `121a` | Linux aarch64, glibc/GB10 | on | stable candidate; must rerun archive smoke, both gate-model correctness, and release performance on GB10 | + +The AOT split is exact: `80`, `86`, `89`, `90a`, `100a`, and `121a` consume +the complete vendored trees under `src/vt/cuda/triton_aot_vendored/`; +`87`, `103a`, `110`, and `120a` set `VLLM_CPP_TRITON=OFF`. No lane fabricates +or borrows another SM's cubin. AOT artifacts are single-architecture, so a +fat CUDA build with AOT is forbidden by both this matrix and +`cmake/TritonAOT.cmake:94-126`. This release program does not publish a +fat+AOT archive. + +The host ABI is part of the tuple because a CUDA cubin does not make its ELF +host executable portable. Preview labels remain even when SASS exists if the +exact archive has not run on matching silicon. The matrix makes no blanket +claim about fast-path coverage: each manifest must preserve the resolved +`VT_CUDA_FEATURE_TABLE` cells from `cmake/CudaArchFeatures.cmake:213-357`. + +## Static and external-runtime boundary + +The normal bundles are **static-core**, not “one literal static executable.” +`vllm-server` contains the static project core but may dynamically depend on the +host C/C++ runtime, pthreads, platform frameworks, or a selected accelerator +runtime. Those dependencies must be enumerated and audited from the staged +binary. + +The one literal-static experiment is +`linux-x86_64-musl-cpu-static`. It is CPU-only and passes only when `file` +identifies a static executable, `ldd` reports no dynamic interpreter, the server +help and loopback health smoke pass in a minimal container, and DNS/thread/file +loading behavior is exercised. It stays experimental preview even when green; +the result decides whether a second arm64 musl lane is warranted. The archive +must not silently disable server functionality to obtain a static link. + +Accelerator drivers are honest external boundaries. NVIDIA's kernel driver and +CUDA driver ABI, the Vulkan loader/ICD and device driver, macOS Metal system +frameworks, and the ROCm kernel/user runtime cannot be made portable by +statically linking the vllm.cpp core. The archive manifest names the minimum +tested driver/runtime; it never claims to bundle a GPU driver. MLX is an +opt-in preview exception whose redistributable dylib/metallib may be carried +only with its exact license and version. + +`ffmpeg` is also external. The server's video path defaults to the `ffmpeg` +executable and spawns it from the example boundary +(`examples/server/main.cpp:215,349-350,741-758,839,1090-1093`). Bundles do not +silently vendor it. The manifest and README say that text serving needs no +ffmpeg and video generation requires a compatible executable on `PATH` or an +explicit `--video-ffmpeg` path. Models, tokenizer data, certificates, and GPU +drivers are runtime inputs, not archive payloads. + +## Gates: staged archive and supply chain + +Every CI lane builds, installs into an empty staging prefix, creates the archive, +extracts it into a second empty directory, and validates only that extracted +tree. A build-tree smoke is not release evidence. + +Required gates: + +1. **Package contents:** exact allowlist, executable bit, no absolute build + paths, no source/object files, no credentials, and version output matching + the tag, commit, manifest, and C ABI version. +2. **Server smoke:** `vllm-server --help`; bind loopback on an ephemeral port; + `/health`, `/version`, and clean shutdown; then a small representative model + request on lanes with matching runtime hardware. +3. **Dependency audit:** Linux `readelf` plus `ldd`/`lddtree`, macOS `otool`, and + platform equivalents reject an undeclared shared object, build-directory + RPATH/RUNPATH, absolute developer path, or missing library. ELF RPATH must be + absent or relative to the extracted bundle; Mach-O install names must use + system paths, `@rpath`, or `@loader_path` as declared. +4. **Architecture audit:** `file`/ELF/Mach-O headers match the host ABI; CUDA + `cuobjdump` shows only the named SM; an AOT-on artifact's manifest and cubins + match that same SM; AOT-off artifacts contain no Triton AOT payload. +5. **Correctness:** CPU unit/conformance tests before packaging; after + extraction, representative endpoint and model checks. Stable `sm_121a` owes + both project gate models and the normal oracle comparison. Other stable + lanes owe the row's available backend/model correctness contract. +6. **Supply chain:** archive SHA256, SPDX JSON SBOM, source/dependency/license + inventory, third-party notices, immutable build provenance, and a `VERSION` + record containing tag, commit, clean-tree bit, compiler, backend, target, + host ABI, and C ABI version. The checksum and provenance refer to the final + archive bytes, not the staging directory. + +The release manifest schema and its checker are versioned together. Missing +evidence is `false` with a reason; command failure cannot collapse into “not +applicable.” + +## Port map + +| Responsibility | Local destination | Contract | +|---|---|---| +| server install/package component | top-level and `examples/CMakeLists.txt` | stage the existing static-core server under its canonical output name without changing server behavior | +| release tuple/preset | focused release CMake presets or matrix data | explicit backend/provider/host ABI; no `AUTO`, wildcard SM, or blanket feature claim | +| manifest and supply-chain metadata | release scripts plus a versioned schema | independent evidence values, final-archive SHA256, SPDX SBOM, provenance, version and licenses | +| staged archive validator | release checker tests/scripts | validate extracted bytes, dependencies, RPATH/install names, host architecture and AOT SM | +| dry-run/tag automation | release workflow | build, verify, attest and publish as isolated least-privilege stages | +| release index | generated public release notes/index | derive channel and limitations from verified manifests, never handwritten assumptions | + +## Tests to port + +The pinned upstream release pipeline is the executable structural spec; its +separate build/test/publish stages are mirrored by W4 rather than porting a +Python runtime test. Existing local executable specs remain binding: + +- `test_server_help` (`examples/CMakeLists.txt:59-63`) runs from the extracted + archive, not only the build tree; +- endpoint conformance under `tests/vllm/entrypoints/openai/` supplies the + server protocol smoke before a lane can become stable; +- each backend row's existing correctness suite and matching-hardware model + gate supplies runtime evidence; cross-builds record runtime false; and +- new manifest/archive fixtures mutate every required evidence field, RPATH, + dependency allowlist, architecture and AOT association to prove the checker + fails for the named reason. + +## Least-privilege CI and release flow + +The same build definition serves pull-request dry runs and tags, but authority +is separated by stage: + +1. **Plan:** read-only checkout computes the matrix and validates the tag/version + relationship. A pull request or manual dry run cannot create a release. +2. **Build:** per-tuple jobs have `contents: read`, no token write scope, no OIDC, + and upload temporary workflow artifacts only. Cross builds produce build + evidence, never runtime evidence. +3. **Verify:** fresh jobs download and extract archives, run the staged checks, + and emit the independent evidence manifest. Hardware jobs receive only the + exact tuple assigned to that runner. +4. **Attest:** only the provenance job receives `id-token: write`; it signs the + verified archive digest, not arbitrary workspace content. +5. **Publish:** a protected tag/environment job alone receives + `contents: write`. It downloads verified immutable archives, checks their + digests and evidence channel, and attaches them to the matching release. + +Fork pull requests never receive release secrets. Tag names are untrusted input +until the version gate passes. Artifact names are allowlisted, publish uses no +wildcards, and a failed lane cannot be replaced by an older workflow artifact. + +## Dependencies + +- the existing server and library install boundaries; +- canonical CMake backend flags and CUDA feature-table resolution; +- complete per-SM Triton AOT trees where AOT is enabled; +- matching runtime hosts for any stable tuple and the normal GPU contention + protocol for correctness/performance gates; +- redistribution-compatible licenses for every bundled runtime file; +- platform dependency-inspection tools plus an SPDX SBOM/provenance generator; + and +- protected release environments for attestation and publication authority. + +## Work breakdown: helper-sized implementation plan + +Each work unit is a separate claim with its own red-first checker change and +fresh review. No unit advances this row beyond `ACTIVE` until its gates pass; +the present checkpoint stays `SPIKE`. + +| Work | Deliverable | Exit gate | +|---|---|---| +| W1 | canonical `vllm-server` output name, install component, and staging/package target for the existing static-core server | install into empty prefix; extracted help smoke; existing library install unchanged | +| W2 | versioned release-manifest generator and schema with independent evidence booleans | fixture tests distinguish absent, false, failed, and true evidence; mutation of each required field is red | +| W3 | staged archive validator: content allowlist, dependency/RPATH audit, architecture/AOT audit, SHA256, VERSION, licenses, SPDX SBOM | Linux fixture/archive tests red-first; no build-tree paths accepted | +| W4 | least-privilege dry-run/tag workflow skeleton, immutable artifact handoff, provenance and protected publish stages | permissions checker plus dry run proves no release is created and publish cannot consume unverified bytes | +| W5 | Linux glibc CPU x86_64 and arm64 bundles plus experimental x86_64 musl literal-static lane | both glibc tuples runtime-smoked on matching hosts before stable; musl static/loopback checks remain preview | +| W6 | macOS arm64 native-Metal bundle | extracted archive model smoke on M-series hardware; dependency/install-name audit; MLX absent | +| W7 | macOS arm64 MLX preview bundle | exact MLX dylib/metallib/license inventory and extracted runtime/correctness smoke; remains preview until all stable gates exist | +| W8 | Linux x86_64 Vulkan preview bundle | extracted archive runtime smoke on a Vulkan runner; loader/ICD stay external and declared | +| W9a | CUDA previews for `80`, `86`, `87`, `89` | four separate host/SM artifacts; exact AOT split; build/archive evidence never upgrades missing runtime booleans | +| W9b | CUDA previews for `90a`, `100a`, `103a`, `110` | same contract; `110` repeats matching-hardware archive smoke before any channel change | +| W9c | CUDA `120a` preview and `121a` stable candidate | separate artifacts; `121a` extracted archive reruns both gate models and release-performance gate on GB10 | +| W10 | release index/docs and retention policy generated from manifests | every link, checksum, channel, host ABI, driver boundary, and known limitation matches published bytes | + +ROCm remains blocked outside these work units until its backend row first +compiles on AMD hardware. A new lane is added by changing this matrix and its +tests before workflow expansion, never by a wildcard build. + +## Risks and decisions + +- “Static” without the `static-core` qualifier is rejected for normal GPU and + platform bundles; only the musl CPU experiment may say literal-static. +- Stable is a property of an exact artifact tuple and evidence set, not of a + backend family. A later toolchain or dependency change reruns the gates. +- Cross-compilation can prove bytes and architecture, not execution. Preview is + the honest publication channel for community hardware coverage. +- CUDA fast-path availability differs by SM. The feature-table output and AOT + state are manifest data; archive names do not imply feature parity. +- ffmpeg and accelerator drivers are external operational dependencies. Their + absence must fail the relevant feature actionably, not corrupt a general + server smoke. +- Reproducibility means a recorded clean recipe plus immutable provenance first; + byte-for-byte rebuild reproducibility is a separate evidence field until + demonstrated. + +## Spike verdict + +The release program is feasible as backend-specific static-core bundles with a +hybrid stable/preview channel. Literal-static scope is limited to the +experimental musl CPU lane. ROCm is blocked. The first implementation slice is +W1, followed by the manifest/validator substrate before any publishing workflow. +`ENG-RELEASE-BINARIES` is `SPIKE`, not `READY`, `ACTIVE`, or `DONE`. diff --git a/.agents/state.md b/.agents/state.md index 884da1e48..1c3e73ca4 100644 --- a/.agents/state.md +++ b/.agents/state.md @@ -42227,3 +42227,29 @@ Voxtral remain off-registry (fold #9/#10 of the audit). Next levers, unstarted: wider per-lane GEMV loads, and Tier-1 vocabulary for the `qkv_split -> rope -> reshape_and_cache` chain (672 dispatches -> 224). + +## 2026-08-07 — Downloadable static-core server release matrix spiked (PR #129) + + +`ENG-RELEASE-BINARIES` moves `INVENTORIED` -> `SPIKE`; no implementation or +artifact is claimed. The accepted contract is +`.agents/specs/release-binary-matrix.md`: backend/host-specific +`vllm-server` archives, stable only after matching-hardware archive runtime and +correctness gates, and preview for derived/build-only tuples. + +The initial matrix covers CPU x86_64+arm64, macOS arm64 Metal and opt-in MLX, +Vulkan preview, an experimental literal-static x86_64-musl CPU feasibility +lane, and ten separate CUDA SM artifacts (`80,86,87,89,90a,100a,103a,110,120a,121a`) +with explicit host ABIs. Triton AOT is ON only for the six complete vendored +trees (`80,86,89,90a,100a,121a`) and OFF for `87,103a,110,120a`; fat+AOT is +forbidden. ROCm stays blocked because the HIP skeleton has never compiled on +AMD hardware. + +The package contract adds a future server install/component target, extracted +archive smoke, dependency and RPATH audits, independent evidence booleans, +SHA256, SPDX SBOM, provenance, version and license records, and a tag/dry-run +least-privilege workflow. The normal promise is static project core with honest +external ffmpeg/platform/driver boundaries, not a magically static GPU stack. +First implementation after fresh review is W1: install and package the existing +static-core server; manifest, validator, CI, platform, and CUDA bricks follow as +separate claims. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 222431ae7..aff75e569 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -4,7 +4,7 @@ | Reference | Workload | Headline | Tokens | |---|---|---|---| -| **Binary release matrix (inventoried)** | `ENG-RELEASE-BINARIES`: backend-specific `vllm-server` archives plus fully-static feasibility | **PENDING:** spike in progress; no archive, runtime validation, or release-performance result exists yet | n/a | +| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: backend-specific static-core `vllm-server` archives; literal-static limited to an experimental musl CPU lane | **PENDING:** release contract accepted; no archive, staged smoke, runtime validation, or release-performance result exists yet | n/a | | **Developer agent entry point (implemented)** | `DOCS-AGENT-PROTOCOL-ENTRYPOINT`: public contribution guide + synchronized, mutation-gated pre-claim intake rule | Rebased documentation/protocol only; benchmark void | n/a | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **Record repair 2026-08-07** | `main` was red on `check-agent-record` + `check-env-doc`, blocking every PR. Dangling `kda-chunk-aot/` link and two undocumented env vars. No behaviour change | diff --git a/docs/STATUS.md b/docs/STATUS.md index 77027d136..749c4ff29 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -112,7 +112,7 @@ token-for-token correctness against the pinned oracle. | Tool-call parsing | 36 parser families / 40 accepted names, streaming | Every vLLM tool parser at the pin except the three Rust/Harmony-backed ones: pure-text parsers ported 1:1, the six engine-backed families reimplemented from their wire formats, all held to the upstream test suites. Selection via `--tool-call-parser` (server), `tool_parser` (C ABI), or template auto-detection; native-syntax forced tool_choice where expressible. Tables: docs/BENCHMARKS.md | | Reasoning parsing (`SAMPLE-REASONING`, ACTIVE, partial coverage) | 9 parsers, streaming | think_auto (auto-detect default: content unless markers appear), deepseek_r1, deepseek_v3 (passthrough) / holo2 (thinking→R1), mistral ([THINK]), minimax_m2 (+append_think), step3, olmo3 - reasoning split engine-side BEFORE tool parsing, streamed as `reasoning` deltas in the chat chunks. Coverage: 9 of upstream's ~28 registered names (remaining text families + engine-backed adapters tracked as W2/W3 in specs/reasoning-parsers.md); each ported parser doctest-gated vs its tests/reasoning case | | Unified streaming parser engine | Core, assembly, serving-SSE dispatch landed, gated; all 10 engine-backed families ported (family parity closed); JSON-schema tool-arg type coercion landed | The vLLM 0.26 declarative `parser/engine/` (shared state machine plus all 10 configs: qwen3, seed_oss, kimi_k2, minimax_m2, glm47_moe, deepseek_v4/v32, nemotron_v3, gemma4, inkling) and assembly layer, gated field-for-field vs vLLM 0.26. An engine-backed `--tool-call-parser` name drives the live chat SSE chunks, off by default. When a request's tools declare typed parameters, the assembled tool-call arguments are coerced to the declared JSON types (int/number/bool/string/array/null) 1:1 with vLLM `_fix_arg_types`, in both streaming and one-shot; no schema means the arguments pass through as strings unchanged. Details: .agents/specs/parser-assembly-c8.md | -| OpenAI server | Supported subset; archives INVENTORIED | `/v1/completions`, `/v1/chat/completions`, streaming SSE, `/v1/models`, `/health`, `/version`, `/ping`, `/metrics`, `/tokenize` (raw-`prompt` and chat-`messages`), `/detokenize`, `/tokenizer_info`, `/server_info`, `/reset_prefix_cache`, `/abort_requests`; `/v1/videos` in OpenAI's Sora shape + `GET /v1/videos/{id}/content`, conditioned on an `input_reference` image or the `metadata` video/audio references. `/tokenizer_info` and `/abort_requests` are flag-gated; `/metrics` and `/reset_prefix_cache` have handlers but no live backing on the async path. Endpoint list and flags: docs/USAGE.md. Depth-2 async serving no longer corrupts the host heap under `ignore_eos` | +| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): static-core backend/host bundles; runtime-gated stable vs build-only preview; no archive exists | | Pooling task class (embeddings / classify / score / rerank) | Spiked; pooler op + heads composite + pooling runner path landed (CPU), not yet servable end-to-end (no concrete model / endpoints) | The non-generative task class. W0 spike over the whole vLLM pooling surface (`.agents/specs/pooling-task-class.md`, `CLAIM-POOLING`). W1 landed the pooler OP (CLS/LAST/MEAN + Identity/Normalize/MultiLabelClassify/Classify activations, double-precision-gated). **W2 landed the pooler HEADS composite** — `EmbeddingPoolerHead` (projector→matryoshka→normalize), `ClassifierPoolerHead` (classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` task routing, and the `PoolerConfig`/`PoolingParams` structs (`test_pooler_heads` 27/27, 240 asserts, RED-first). **W3 landed the pooling RUNNER path** — `PoolingRunner` applies the model's `Pooler` to the last hidden state and returns pooled embeddings instead of sampled tokens, gated by a STRUCTURAL cosine-parity check vs a double-precision LAST+normalize reference (`test_pooling_runner` 5/5, 14 asserts, RED-first). NOT yet servable / honest residuals: a concrete pooling MODEL forward + the REAL-model oracle cosine gate (`vllm.LLM(task="embed").encode`) — no cosine-vs-oracle number is fabricated (W3-model); the `/v1/embeddings` + score/rerank/classify endpoints (W4); tokwise AllPool/StepPool (W5). See docs/BENCHMARKS.md | | Plugin system (out-of-core registration) | Spiked; first CPU brick landed, not yet wired into any production path | The extensibility-first discovery layer. W0 spike over vLLM's plugin surface (general / platform / io_processor / endpoint groups, the `register_model` an out-of-tree plugin calls, the invocation seams) is committed (`.agents/specs/plugin-system.md`, `ENG-PLUGIN-SYSTEM` ACTIVE, `CLAIM-PLUGIN-SYSTEM`). W1 landed `vllm::plugins::LoadGeneralPlugins()` + the out-of-core general-plugin registration seam (`RegisterGeneralPlugin` / `REGISTER_VLLM_GENERAL_PLUGIN`) over the existing `REGISTER_VLLM_MODEL`-style registries (the in-tree factory `MODEL-FACTORY-registry` is record-repaired `DONE` 2026-08-05: 28 self-registering TUs, dgx debt paid by the 2026-07-23 seven-gate run): a 1:1 mirror of `load_general_plugins` (load-once idempotence, the `VLLM_PLUGINS` allowlist, per-plugin failure isolation). Proven by an out-of-core toy-model plugin that registers a toy architecture through the public `RegisterModel` seam — unit-gated RED-first (`test_plugin_system` 1 case / 29 assertions: the toy arch resolves ONLY after LoadGeneralPlugins runs it, and not under `VLLM_PLUGINS=""`). Python entry points have no C++20 analogue, so discovery is the project's static-init/`dlopen` registration idiom (recorded porting-inventory §9). NOT yet wired: real shared-object `dlopen` + the C-ABI `vllm_plugin_register` entry (W2), the engine/CLI `--load-plugins` wiring that calls LoadGeneralPlugins from the construction paths (W3), the platform/quant plugin kinds (W4), and the io_processor/stat_logger/endpoint groups (W5) are named residuals. See docs/BENCHMARKS.md | | Offline Batch API (JSONL file runner) | Spiked; first CPU brick landed, not yet exposed as a CLI | The offline OpenAI Batch API: read a JSONL of OpenAI-format requests, run each through the engine, write a JSONL of responses. W0 spike over vLLM's `run_batch.py` (schema, endpoint dispatch, run loop, file I/O) is committed (`.agents/specs/batch-api.md`, `SERVE-BATCH-API` ACTIVE, `CLAIM-BATCH-API`). W1 landed `RunBatch` (`RunLine`/`RunLines`/`Run`) + `RunBatchFile` — a pure orchestrator over the existing `OpenAIServingChat::create_chat_completion` (NO reimplemented generation), 1:1 with vLLM's endpoint_registry url→handler map: `/v1/chat/completions` dispatch, the `BatchResponseData`/`BatchRequestOutput` schema (`vllm-` ids, custom_id echoed), the `run_request` AllResponse/ErrorResponse/stream branches, and the unsupported-endpoint/url error rows. Unit-gated RED-first (`test_openai_run_batch` 7 cases / 80 assertions over the synthetic serving engine: ordered rows + custom_id echo + per-line BatchRequestOutput schema round-trip, a malformed line isolated into an error row so the batch continues, dispatch + 404 error rows; dropping the custom_id echo fails 9 assertions). Recorded deviation: a malformed line is isolated (batch continues) where upstream aborts the job. NOT yet exposed: the `vllm run-batch` CLI + `BatchFrontendArgs` (W2), embeddings/score/rerank dispatch (W3, rides pooling endpoints), audio transcription/translation + media fetch (W4), and http(s)/data-URL file I/O + metrics + overlapped `AsyncLLM` submission (W5) are named residuals. See docs/BENCHMARKS.md | From 88c21f0e40e0d982423e3047c6234286feec04e7 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 7 Aug 2026 22:33:12 +0000 Subject: [PATCH 03/11] docs(release): revise primary CPU and CUDA artifacts Make host-ABI fat CUDA and adaptive CPU binaries the primary downloads, move cross-family gencode, multi-SM AOT dispatch, and CPU ISA audits ahead of packaging, and retain per-target evidence plus optional per-SM diagnostics. FOLLOWING_AGENTS_PROTOCOL Assisted-by: Codex:gpt-5 [Codex] --- .agents/NOW.md | 2 +- .agents/coordination.md | 6 +- .agents/engine-matrix.md | 2 +- .agents/roadmap_v1.md | 2 +- .agents/specs/release-binary-matrix.md | 257 ++++++++++++++++++------- .agents/state.md | 26 +++ docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- 8 files changed, 220 insertions(+), 79 deletions(-) diff --git a/.agents/NOW.md b/.agents/NOW.md index fba32e735..cf00f19ab 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -28,7 +28,7 @@ Working head: `row/backend-rocm-w0` (#41). Prior: benchmark checkpoint | Supported-models list | **LANDED**: FEATURES arch table CI-bound (33 archs) | — | | `/v1/videos` OpenAI shape | **MERGED** (#71): Sora `model`/`size`/`seconds` + `GET /{id}/content` | `row/SERVE-VIDEOS-REFS` PR open: reference conditioning | | `BACKEND-ROCM` W0 | Skeleton in; **HIP never compiled** (no AMD HW) | #41 contributors build it; a compile error IS the deliverable | -| Release binaries | **SPIKE** (#129) | Fresh review; then W1 install/package target | +| Release binaries | **SPIKE** fat CUDA/adaptive CPU (#129) | W1 per-source gencode | | Surface coverage (`ARCH-ONE-SURFACE`) | **ROW 1 LANDED (#121)**: Parakeet ASR on the surface (`vllm_transcribe` v11, `/v1/audio/transcriptions`, ratchet 12→11) | Next fold row | In-flight (default-OFF, not pushed): `laguna-fp4proj-prod`, laguna diff --git a/.agents/coordination.md b/.agents/coordination.md index 074e25f0f..c5649f1d2 100644 --- a/.agents/coordination.md +++ b/.agents/coordination.md @@ -133,8 +133,8 @@ this claim, the required current-state checkpoint in `docs/STATUS.md` and No release workflow, CMake/package implementation, source, test, model, kernel, generated fixture, download, build, GPU, or service change is in scope. Draft PR [#129](https://github.com/mudler/vllm.cpp/pull/129) is the claim. The spike -contract is complete on the branch and awaits fresh review; the row stays -`SPIKE`. +contract incorporates the user-reviewed fat-CUDA/adaptive-CPU primary-artifact +revision and awaits fresh review; the row stays `SPIKE`. **CPU grouped keep-quant GEMM activation-dtype P0 (`QUANT-GGUF-CIQ-GEMM`, 2026-08-06, `CLAIM-QUANT-GGUF-CIQ-GROUPED-DTYPE`).** Claude Code @@ -1413,7 +1413,7 @@ table, tests, CMake. Details in the state-log entry of the same date. | Claim | Row IDs | Agent | Worktree / remote dir | Branch | Owned scope | State | Last update | |---|---|---|---|---|---|---|---| -| `CLAIM-ENG-RELEASE-BINARIES-SPIKE` | `ENG-RELEASE-BINARIES` | Codex (GPT-5) | `/home/mudler/_git/vllm.cpp-release-binaries`; records/spec only, no build/GPU/download/service work | `row/ENG-RELEASE-BINARIES`, base `9cb799cb`, draft PR #129 | Accepted release-bundle spike plus its matrix/roadmap/coordination/state/NOW and public-doc lifecycle checkpoint; no CMake, workflow, source, test, or artifact implementation | `ACTIVE` | 2026-08-07 — spike contract complete; row stays `SPIKE`; awaiting fresh review | +| `CLAIM-ENG-RELEASE-BINARIES-SPIKE` | `ENG-RELEASE-BINARIES` | Codex (GPT-5) | `/home/mudler/_git/vllm.cpp-release-binaries`; records/spec only, no build/GPU/download/service work | `row/ENG-RELEASE-BINARIES`, base `9cb799cb`, draft PR #129 | Accepted release-bundle spike plus its matrix/roadmap/coordination/state/NOW and public-doc lifecycle checkpoint; no CMake, workflow, source, test, or artifact implementation | `ACTIVE` | 2026-08-07 — user-reviewed revision complete: primary fat CUDA + adaptive CPU per host ABI, optional per-SM diagnostics; row stays `SPIKE`; awaiting fresh review | | `CLAIM-KIMI-LINEAR-W0` | `MODEL-TEXT-kimi-linear-kimi-linear-for-causal-lm` (stays SPIKE — dedicated W0 spike) | Claude Code (opus-4-8) | isolated worktree `.claude/worktrees/agent-a771cc029e6843be0`; CPU-only, records-only — NO build, NO GPU, NO download (two GPU jobs queued ahead; the W0 GPU golden capture is a SEPARATE later step) | branch `worktree-agent-a771cc029e6843be0`, base `origin/main` HEAD `10dd23ee` | The FULL dedicated W0 spike for `KimiLinearForCausalLM` (Kimi-Linear-48B-A3B) per the spike-first protocol, so W1 implementation can start immediately. Owns ONLY: NEW `.agents/specs/kimi-linear.md`, the `MODEL-TEXT-kimi-linear-*` matrix row (Spike link → the dedicated spec + note; STAYS `SPIKE` because it is actively claimed — protocol forbids a claimed row from being `READY`; checklist mark stays `📋`), this claim row + the narrative block below, the roadmap breadth note, `docs/STATUS.md`/`docs/BENCHMARKS.md`/`docs/FEATURES.md` one-liners, the `.agents/NOW.md` live-claim row + stamp, and one `.agents/state.md` entry. **NON-COLLISION:** records-only — touches NO model/kernel/registry/loader source, NO CMake, NO test. Co-owns the Kimi-Linear row with `CLAIM-MLA-DEEPSEEK` (MLA half) + `CLAIM-KDA-KERNEL` (KDA host refs), exactly as the K3 row is co-owned by `CLAIM-KIMI-K3-SCOPE`/`CLAIM-KIMI-K3-W2-W5`. | `DONE` | 2026-08-05 — **W0 DEDICATED SPIKE LANDED (records-only, NOT pushed).** Authoritative `config.json` fetched (27 layers = 20 KDA + 7 NoPE-MLA; 256e/top-8/1-shared sigmoid `noaux_tc` `routed_scaling=2.446` `first_k_dense_replace=1`; `num_nextn_predict_layers=0` ⇒ no MTP); reuse-vs-new map with our `file:line`; HW-fit (FITS one GB10, 91.5 GiB / 0.77× pool — real e2e SACRED gate reachable); W0-W7 breakdown + GPU golden-capture recipe. NET-NEW = KDA device kernel (host refs landed, `CLAIM-KDA-KERNEL`), NoPE-MLA branch, hybrid schedule/het-KV, loader name-map. Record checkers green (`check-model-checklist`, `check-agent-record`, `check-doc-checkpoint`, `check-now-current`). | | `CLAIM-KIMI-LINEAR-W1` | `MODEL-TEXT-kimi-linear-kimi-linear-for-causal-lm` (stays SPIKE — W1 registry/config/loader scaffolding; forward REFUSES-by-name) | Claude Code (opus-4-8) | isolated worktree `.claude/worktrees/agent-a11ae231a71e8694b`; CPU-only `build-cpu` (`-DVLLM_CPP_CUDA=OFF` Release); NO GPU, NO download — additive TU + one REGISTER line, forward is a `VT_CHECK(false)` stub | branch `worktree-agent-a11ae231a71e8694b`, base `origin/main` HEAD `053116df` | Kimi-Linear-48B-A3B W1 — registry + config + loader + KV-spec scaffolding so the W3-W6 forward can start. Owns ONLY: NEW `include/vllm/model_executor/models/kimi_linear.h`, NEW `src/vllm/model_executor/models/{kimi_linear_registry,kimi_linear_weights,kimi_linear}.cpp`, NEW `tests/vllm/models/test_kimi_linear_scaffold.cpp`, its two CMake registration lines (`CMakeLists.txt` source list + `tests/CMakeLists.txt`), the `KimiLinearForCausalLM` sorted-set + error-message + model-property inserts in `tests/vllm/models/test_model_registry.cpp`, the `MODEL-TEXT-kimi-linear-*` row cells + checklist Status (STAYS `SPIKE`/`📋` — the forward refuses, rollup UNCHANGED), this claim, `.agents/NOW.md`, `docs/STATUS.md`/`docs/BENCHMARKS.md`/`docs/FEATURES.md` one-liners, and one `.agents/state.md` entry. **NON-COLLISION:** additive TU + one REGISTER line ⇒ ZERO edit to any shared array; the forward is a `VT_CHECK(false)` refuse stub so NO production/device path changes; co-owns the Kimi-Linear row with `CLAIM-MLA-DEEPSEEK` + `CLAIM-KDA-KERNEL` + `CLAIM-KIMI-LINEAR-W0`; MUST NOT touch README, Metal/SACRED/apex/darwin, or any other model/kernel source. | `DONE` | 2026-08-05 — **W1 CPU SCAFFOLDING LANDED (foreground, NOT pushed).** Additive registry (`kimi_linear`/`KimiLinearForCausalLM`, `is_hybrid`, `supports_multimodal=false`) + `ParseKimiLinearParams` (20 KDA + 7 NoPE-MLA schedule, MLA 512/128/64/128 q_lora-null, 256e/top-8/1-shared sigmoid `noaux_tc` 2.446, `first_k_dense_replace=1`; asserts `mla_use_nope`/`q_lora==null`) + `EnumerateKimiLinearTensors` name-map VERIFIED vs the real HF safetensors index (MoE = `block_sparse_moe.*`, the correction of the DERIVED K3 `mlp.*`) + loader THROWS BY NAME on missing/mis-shaped tensors + het KV spec (MLA latent-576 + KDA mamba `12288×3`/`32×128×128`) + REFUSE-by-name forward. `test_kimi_linear_scaffold` **9/9·83** + `test_model_registry` **24/24** + `test_kimi_k3_scaffold` 6/6 unchanged; clean CPU Release build; `check-fusion-consistency`/`check-runner-routing-consistency` green (refuse stub skipped); record checkers rc=0. RESIDUAL (W3-W6): KDA device kernel + NoPE-MLA route + sigmoid-noaux MoE + het-KV born-on-runner forward + the e2e SACRED gate (spec §8 recipe). | | `CLAIM-KIMI-LINEAR-W2` | `MODEL-TEXT-kimi-linear-kimi-linear-for-causal-lm` (stays SPIKE — W2-W6 CPU REFERENCE forward; the DEVICE runner forward stays refuse-by-name) | Claude Code (opus-4-8) | isolated worktree `.claude/worktrees/agent-aec13af094cd4692f`; CPU-only `build` (`-DVLLM_CPP_CUDA=OFF` RelWithDebInfo); NO GPU, NO download — additive forward TU + a host-materialization pass in the existing loader + additive test | branch `laguna-s21-w7-speed-profile` (worktree), base `origin/main` HEAD `0988db48` | Kimi-Linear-48B-A3B W2-W6 — the CPU REFERENCE forward, per-op gated, replacing the refuse-by-name host `KimiLinearModel::Forward`. Owns ONLY: NEW `src/vllm/model_executor/models/kimi_linear_forward.cpp`, NEW `tests/vllm/models/test_kimi_linear_forward.cpp`, its two CMake lines (`CMakeLists.txt` source list + `tests/CMakeLists.txt`), ADDITIVE host-weight structs + per-op decls in `include/vllm/model_executor/models/kimi_linear.h`, the host-materialization pass in `src/vllm/model_executor/models/kimi_linear_weights.cpp`, the device-refuse doc/rename in `src/vllm/model_executor/models/kimi_linear.cpp`, the `MODEL-TEXT-kimi-linear-*` row cells + checklist Status (STAYS `SPIKE`/`📋` — the DEVICE forward refuses; rollup UNCHANGED), this claim, `.agents/NOW.md`, `docs/STATUS.md`/`docs/BENCHMARKS.md` one-liners, and one `.agents/state.md` entry. **NON-COLLISION:** additive TU + additive header structs; the host `Forward` becomes real but `ForwardDevice` (the DEFAULT gather_logits runner path) STAYS `VT_CHECK(false)` so the born-on-runner CI guard skips it (NO allowlist) and NO production/device path changes; co-owns the Kimi-Linear row with `CLAIM-MLA-DEEPSEEK` + `CLAIM-KDA-KERNEL` + `CLAIM-KIMI-LINEAR-W0/W1`; MUST NOT touch README, Metal/SACRED/apex/darwin, or any other model/kernel source. | `DONE` | 2026-08-05 — **W2-W6 CPU REFERENCE FORWARD LANDED (foreground, NOT pushed).** Real host `KimiLinearModel::Forward` composes the whole 27-layer hybrid from the landed primitives: KDA layer (`vllm::kimi_kda` refs + the gated-delta recurrence `fused_recurrent.py:122-149`), NoPE-MLA (materialized-MHA reference, scaling `qk_head**-0.5`, no RoPE), sigmoid `noaux_tc` MoE (bias-select/unbiased-weight, renormalize, `routed_scaling=2.446`) + shared expert, dense layer-0 SwiGLU; loader now MATERIALIZES the host float weights (bf16/f32→f32). `test_kimi_linear_forward` **6/6·246** (KDA==kimi_kda composition; NoPE-MLA==materialized-MHA ref; router+block==hand-computed; loader materializes; whole 2-layer forward finite/coherent; greedy-decodes 5 tokens, context advances) + `test_kimi_linear_scaffold` 9/9·83 + `test_kimi_kda` 14/14 UNCHANGED; clean CPU build; `check-fusion-consistency`/`check-runner-routing-consistency` green (DEVICE forward refuse-skipped, no allowlist). RESIDUAL: the DEVICE born-on-runner forward + the W0/W7 e2e SACRED token golden on GB10 (spec §8 recipe) + speed. | diff --git a/.agents/engine-matrix.md b/.agents/engine-matrix.md index 2f9809c1d..8bdd02e2d 100644 --- a/.agents/engine-matrix.md +++ b/.agents/engine-matrix.md @@ -207,7 +207,7 @@ claims it. | `SERVE-CLI-BENCH` | Serve and latency/throughput/serve benchmark modes | T0 | `vllm/entrypoints/cli/serve.py:44`; `vllm/entrypoints/cli/benchmark/main.py:29` | separate binaries + explicit scheduler-capacity flags `examples/server/main.cpp:63,96,116,170`; `examples/bench/main.cpp:40,109`; `examples/bench/bench_core.h:96,468` | server help contract `examples/CMakeLists.txt:34`; benchmark `tests/examples/test_bench.cpp:15,48` | `planned: specs/cli-serve-bench.md` | `PARTIAL` | - | | `SERVE-GATE-ONLINE` | Same-corpus online correctness, TTFT/TPOT/ITL, throughput and peak-memory gate vs vLLM v0.25.0 | T0 | `vllm/benchmarks/serve.py:1,581-615`; [v0.25 audit](sync/2026-07-12-702f481.md); `tests/benchmarks/test_serve_cli.py:1` | Schema-v5 harness plus [trace controller](../include/vt/cuda/cuda_profiler_control.h#L13), [production component driver](../scripts/dgx-gdn-packed-component.sh), and fail-closed [component finalizer](../tools/bench/gdn_packed_component.py) | **BINDING `9ecd9d0`: 114/124** (async default ON; mem 4/4, c1 20/20, c2 20/20, c16 19/20, c4 & c32 18/20, c8 15/20; `benchmark_binding` refers here, superseding `3f256ab` 55/124 and `246a23c` 49/124, both retained immutable). Two-grid totality with `f0fb727` (111/124) is 115/124 effective parity vs vLLM 0.25.0 (27B). Async CLOSED the c16/c32 ITL tails (ours now BEATS vLLM: c16 p99 1.055, c32 p90 1.034/p99 1.078) and leaves a stable c8 `p99_itl` ~0.86 residual, ROOT-CAUSED (2026-07-18, `CLAIM-C8-P99-TAIL-1`, [spec](specs/c8-p99-itl-tail-2026-07-18.md)) as IRREDUCIBLE-AS-MIRRORED: our deterministic synchronous forward keeps co-admitted c8 requests in byte-identical lockstep where vLLM's async-future jitter de-phases them; the c16/c32 INVERSION proves this is the trailing edge of the per-step determinism that wins c16/c32 + throughput, not a capability gap (scheduler + async placeholder byte-identical, `tests/vllm/v1/test_scheduler_wave.cpp:265`, [tail spec](specs/tail-stall-analysis-2026-07-16.md)). Full grid + per-binding forensics: roadmap_v1.md + parity ledger; no packed speed credit | [online serving gate](specs/cuda-online-serving-gate.md); [merged GDN projections](specs/gdn-merged-input-projections.md); [packed decode](specs/gdn-packed-decode.md) | `ANCHOR-BACKFILL` | CLAIM-SERVE-GATE-1 | | `SERVE-E2E-NIGHTLY` | Server conformance and real-model nightly suites for all release gates | T0 | `tests/entrypoints/openai/`; `tests/v1/e2e/`; `.buildkite/test-pipeline.yaml` | current unit/conformance tests only; no scheduled DGX suite | `tests/vllm/entrypoints/openai/test_conformance.cpp:1`; `tests/parity/test_qwen36_paged_engine.cpp:78`; `tests/parity/test_qwen27_paged_engine.cpp:110` | `planned: specs/server-e2e-nightly.md` | `INVENTORIED` | - | -| `ENG-RELEASE-BINARIES` | Downloadable, backend-specific `vllm-server` release bundles and fully-static feasibility boundary | T0 | vLLM release lanes `.buildkite/release-pipeline.yaml:1-18,34-170` @ `555967922`; release-image dependency boundary `docker/Dockerfile.cpu:262-290` | server target only `examples/CMakeLists.txt:54-64`; no install, archive, manifest, provenance, or release-publish implementation | help smoke only `examples/CMakeLists.txt:59-63`; issue `#117`; accepted matrix/package/gate contract in [release-binary-matrix.md](specs/release-binary-matrix.md) | [release-binary-matrix.md](specs/release-binary-matrix.md) | `SPIKE` | `CLAIM-ENG-RELEASE-BINARIES-SPIKE` | +| `ENG-RELEASE-BINARIES` | Downloadable host-ABI-specific `vllm-server` bundles: adaptive CPU and fat CUDA primary artifacts, optional per-SM diagnostics, and literal-static feasibility boundary | T0 | vLLM release lanes `.buildkite/release-pipeline.yaml:1-18,34-170` @ `555967922`; release-image dependency boundary `docker/Dockerfile.cpu:262-290` | server target only `examples/CMakeLists.txt:54-64`; CPU per-TU/runtime-dispatch baseline `CMakeLists.txt:870-890`, `src/vt/cpu/cpu_matmul_elem.cpp:553-612`, `src/vt/cpu/cpu_quant_dot_arm.cpp:39-77`; cross-family CUDA fat/per-source-gencode and multi-SM AOT gaps remain; no install/archive/publish implementation | help smoke only `examples/CMakeLists.txt:59-63`; issue `#117`; user-reviewed fat-CUDA/adaptive-CPU matrix and gates in [release-binary-matrix.md](specs/release-binary-matrix.md) | [release-binary-matrix.md](specs/release-binary-matrix.md) | `SPIKE` | `CLAIM-ENG-RELEASE-BINARIES-SPIKE` | | `SERVE-CLI-CHAT` | Interactive `chat` and `complete` commands against a running OpenAI-compatible server, plus preservation of the existing local-model completion invocation | T1 | registration `vllm/entrypoints/cli/main.py:17-37,73-98`; client/model resolution + stream shaping `vllm/entrypoints/cli/openai.py:30-100`; chat `:155-234`; complete `:237-312` at `5559679229` | current in-process completion only `examples/cli/main.cpp:1-207`; remote command implementation absent | C-ABI stream baseline `tests/capi/test_capi.cpp:567-711`; chat-template baseline `tests/capi/test_chat_prompt.cpp:37-89`; command/fake-server tests absent | [cli-chat-complete.md](specs/cli-chat-complete.md) | `ANCHOR-BACKFILL` | `CLAIM-SERVE-CLI-CHAT-SPIKE` | | `SERVE-POOLING-ENDPOINTS` | Embeddings, pooling, score, rerank, classify HTTP surface (`/v1/embeddings`, `/pooling`, `/score`, `/rerank`, `/classify`). **SPIKED 2026-07-28 (`CLAIM-POOLING`):** the whole pooling task class is scoped in [pooling-task-class.md](specs/pooling-task-class.md) — endpoints depend on the pooling RUNNER (`ENG-POOLING-RUNNER`, W3 LANDED 2026-07-29 — `PoolingRunner` returns pooled data via the model `Pooler`) and a concrete pooling model. Endpoint protocol + handler port is the W4 brick | T2 | `vllm/entrypoints/pooling/embed/api_router.py:28`; `vllm/entrypoints/pooling/scoring/api_router.py:37,71`; `vllm/entrypoints/pooling/classify/api_router.py:26` | - | - | [pooling-task-class.md](specs/pooling-task-class.md) | `SPIKE` | `CLAIM-POOLING` | | `ENG-POOLER-SEQ` | The non-generative POOLER OP — turn hidden states into a pooled embedding/logit row instead of a sampled token. **W1 LANDED + CPU-GATED 2026-07-28 (`CLAIM-POOLING`, NOT pushed):** the sequence pooling methods `CLSPool`/`LastPool`/`MeanPool` (+ `GetSeqPoolingMethod` factory) over a packed `[num_tokens, hidden]` CPU buffer keyed by a minimal `PoolingCursor` (CLS/MEAN reject partial prefill, LAST allows it, MeanPool upcasts to float32) and the activation heads `PoolerIdentity`/`PoolerNormalize` (L2 `F.normalize`)/`PoolerMultiLabelClassify` (sigmoid)/`PoolerClassify` (sigmoid if `num_labels<2` else `softmax`). Unit-gated vs DOUBLE-PRECISION references, RED-first. **W2 LANDED + CPU-GATED 2026-07-29 (`CLAIM-POOLING`, NOT pushed):** the pooler HEADS composite (`EmbeddingPoolerHead` = projector→matryoshka→normalize; `ClassifierPoolerHead` = classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` (method∩head task intersection) + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` groupby-task routing (`ForEmbedding`/`ForSeqCls` + a mixed embed+classify batch + ctor task-support validation), and the `PoolerConfig`/`PoolingParams`/`PoolingParamsUpdate` structs; `test_pooler_heads` 27/27 (240 asserts) vs double-precision refs, RED-first (disable matryoshka slice + logit_mean calibration → 8 cases / 50 asserts fail). RESIDUALS (named, spec §Work breakdown): the endpoints (W4), tokwise `AllPool`/`StepPool` (W5), a concrete pooling MODEL + real-oracle cosine gate (W3-model — see `ENG-POOLING-RUNNER`) | T2 | `vllm/model_executor/layers/pooler/seqwise/methods.py:35-121`; `vllm/model_executor/layers/pooler/activations.py:106-158`; `vllm/model_executor/layers/pooler/seqwise/heads.py:19-196`; `vllm/model_executor/layers/pooler/seqwise/poolers.py:41-138`; `vllm/model_executor/layers/pooler/special.py:23-140`; `vllm/model_executor/layers/pooler/common.py:12-30`; `vllm/pooling_params.py:35-70`; `vllm/config/pooler.py:16-90`; `vllm/v1/pool/metadata.py:13-71`; `tests/model_executor/layers/test_pooler_methods.py`, `tests/model_executor/layers/test_pooler_activations.py`, `tests/model_executor/layers/test_pooler_heads.py` | `include/vllm/model_executor/layers/pooler/{methods,activations,pooling_metadata,common,pooling_params,pooler_config,heads,poolers,dispatch_pooler}.h` + `src/vllm/model_executor/layers/pooler/{methods,activations,heads,poolers,dispatch_pooler}.cpp` — anchor `src/vllm/model_executor/layers/pooler/dispatch_pooler.cpp:13` | `tests/vllm/model_executor/layers/pooler/test_pooler.cpp` (CLS/LAST/MEAN + factory + activations, 50 asserts) + `test_pooler_heads.cpp` (Embedding/Classifier heads + SequencePooler + DispatchPooler, 240 asserts) — anchor `tests/vllm/model_executor/layers/pooler/test_pooler.cpp:81` | [pooling-task-class.md](specs/pooling-task-class.md) | `ACTIVE` | `CLAIM-POOLING` | diff --git a/.agents/roadmap_v1.md b/.agents/roadmap_v1.md index 10f779d73..49ccd4f26 100644 --- a/.agents/roadmap_v1.md +++ b/.agents/roadmap_v1.md @@ -80,7 +80,7 @@ models we already ship + benchmark. Full seam map + M0–M5 W-plan: | 14 | `ROAD-V1-D5` | LoRA, local KV/weight offload, expert streaming, wider model zoo | [engine matrix](engine-matrix.md), [model matrix](model-matrix.md) | corrected expert-streaming spike accepted (`ENG-EXPERT-STREAM` READY): bank-only safetensors→Marlin bank, fixed contiguous cache slots matching Marlin dense strides, logical→slot remap after explicit router D2H, chunked C` working everywhere `--dit ` did; gated CPU-only (72/72, 54497 post-rebase) on index/name mapping and on the REAL 535-tensor geometry read from a sparse 13-shard release. The DEVICE streamer landed 2026-08-07 (`row/H3-BF16-SHARDED-STREAM`, spec §8.14): one tensor at a time, zero host buffer for the bulk, bit-exact vs the non-streamed reference (73/73, 55203). **ENCODER + THE NUMBER 2026-08-07 (`row/H3-ENC-BF16-COND-DIFF`, spec §8.15)**: the 14-shard bf16 text encoder streams too and `--encoder-only` runs it alone; measured over 233 tokens, Q4_K_M-vs-bf16 conditioning is cos 0.99745 mean / 6.85% rel RMS excl. sink / 3.5 deg median rotation — as much as a one-word prompt edit, but DIFFUSE. Whether the RENDER changes is NOT established (75/75, 55609). This unblocks the bf16-vs-quantized quality A/B; no render or speed number is claimed. Spec §8.13. **W-FP4a LANDED (CPU) 2026-08-06 (`row/H3-FP4-SPEED`)**: the NVFP4 DiT projections now keep FP4 PACKED and route through the shared `dense_nvfp4::MatmulNvfp4W4A16D` (Marlin W4A16 — vLLM's own forced-a16 selection; SAME kernel as Laguna/dense-Qwen3 NVFP4; no new quant code); fp4-vs-bf16 WIRING gate GREEN (62/62·30039, W4A16 dispatcher runs all 11 quantized GEMMs). **GB10 leg LANDED 2026-08-06 (`row/H3-FP4-GPU-E2E`, PR #64):** Marlin W4A16 RAN on sm_121a (`dense_gemms==11` default / `marlin_gemms==11` VT_MARLIN_DENSE=0, `fallback_gemms==0`), fp4-vs-bf16 BYTE-EXACT; fp4 is a MEMORY win (~16 vs ~66 GB), ~0.79–0.83× the bf16 arm per diffusion forward (compute-bound large M; 3.47× faster at small decode-like M). Real-checkpoint fp4-resident t2va e2e RUNS end-to-end (real 18.75 GB NVFP4 DiT + VAEs + GGUF Qwen3-VL-32B encoder → valid mp4/wav; DiT s/step 5.45/20.0/209 s @512/768/REF-768×1344-209f) but frames are a non-scene patch-grid at 12/20/50 steps → OPEN render-coherence bug (device VAE decode / denoise), separate from the fp4 speed work. vLLM-Omni serves NO quantized H3 (BF16-only) -> HW/loader-forced-indirect (4×B300 209f render 86.964 s vs 1×GB10 209 s/forward). | | 15 | `ROAD-V1-D6` | **llama.cpp device breadth folded into scope (user-directed 2026-08-05):** the 11 ggml backends vLLM has no platform for — cann, musa, opencl, openvino, rpc, webgpu, zdnn, zendnn, hexagon, blas, virtgpu — inventoried as `BACKEND-GGML-*`. **SPIKES FIRST:** no implementation before each row's `.agents/specs/.md` clears the spike contract, per the standing directive. vLLM stays the mirror source; llama.cpp is the breadth reference. | [backend matrix](backend-matrix.md) | ☐ per-row spike required | `INVENTORIED` | first spike accepted | -| REL | `ROAD-V1-RELEASE` | Downloadable backend/host-specific static-core `vllm-server` bundles, with stable channels gated by matching runtime evidence and build-only targets labeled preview | [`ENG-RELEASE-BINARIES`](engine-matrix.md) | [release binary matrix](specs/release-binary-matrix.md) | `SPIKE` | Fresh review of PR #129, then claim W1: server install component and staged package target; no archive exists yet | +| REL | `ROAD-V1-RELEASE` | KISS downloads per OS+host ABI: one adaptive CPU binary and one fat CUDA binary covering every supported SM; per-SM CUDA artifacts are optional diagnostics; stable channels require matching runtime evidence and build-only paths stay preview | [`ENG-RELEASE-BINARIES`](engine-matrix.md) | [release binary matrix](specs/release-binary-matrix.md) | `SPIKE` | Fresh review of PR #129, then W1 cross-family per-source gencode; multi-SM AOT and CPU ISA audits precede bundle work; no archive exists | An area row cannot enter `READY` without a real spike under `specs/`, and cannot enter `DONE` without exact code and test/evidence anchors. Closed execution diff --git a/.agents/specs/release-binary-matrix.md b/.agents/specs/release-binary-matrix.md index b718b4651..0e086aa2c 100644 --- a/.agents/specs/release-binary-matrix.md +++ b/.agents/specs/release-binary-matrix.md @@ -24,10 +24,14 @@ not a copy from a build directory. It contains at least: - no model weights, tokenizer assets, Python, PyTorch, Triton runtime, compiler, or source/build directory. -The bundle name includes project version, backend, target, OS, host ABI, and -archive format. CPU architecture, CUDA SM, libc floor, macOS deployment target, -and optional provider are never implicit. A future server install component and -package target must stage this exact tree. The current CMake installs +The bundle name includes project version, backend, OS, host ABI, and archive +format. One binary never crosses an OS or host ABI. The primary CPU download is +one conservative-baseline, runtime-adaptive binary per OS+host ABI; the primary +CUDA download is one fat binary per OS+host ABI containing every supported SM. +Its manifest makes compiled CPU tiers or CUDA SMs explicit. Optional per-SM CUDA +archives are diagnostic/performance variants, not the primary downloads. A +future server install component and package target must stage this exact tree. +The current CMake installs `libvllm.{a,so}` and `vllm.h` but does not install the server (`CMakeLists.txt:1712-1783`; `examples/CMakeLists.txt:54-64`), so the install and package targets are implementation work, not present-tense capability. @@ -53,7 +57,20 @@ library install rules package `libvllm.a`, the shared C ABI library, and `include/vllm.h` (`CMakeLists.txt:1712-1783`). There is no server install rule, archive layout, release manifest, staged dependency audit, provenance, or publish workflow. Existing CUDA AOT trees are exactly the six directories named -below; ROCm's opt-in skeleton has never compiled on AMD hardware. +below. Cross-family CUDA fat builds currently fail because feature sources are +gencode'd for the whole list, and `cmake/TritonAOT.cmake:94-126` currently +rejects multi-arch AOT; both are prerequisites, not accepted release limits. + +The CPU baseline already demonstrates the intended shape. CMake applies ISA +flags surgically to selected translation units rather than globally +(`CMakeLists.txt:870-890` and the immediately following per-source blocks). +Element GEMM chooses portable/NEON or SSE2+F16C/AVX2/AVX-512 tables at runtime +and exposes a forced-tier test seam +(`src/vt/cpu/cpu_matmul_elem.cpp:553-612`). Arm quant i8mm is separately +compiled and selected through Linux HWCAP2 or Darwin sysctl +(`src/vt/cpu/cpu_quant_dot_arm.cpp:39-77`). This is a partial foundation, not +proof that every CPU kernel has every tier. ROCm's opt-in skeleton has never +compiled on AMD hardware. ## Evidence classes and publication policy @@ -71,7 +88,8 @@ independent booleans and evidence URLs for `build_verified`, `archive_smoke_verified`, `dependency_audit_verified`, `runtime_verified`, `correctness_verified`, and `performance_verified`. It also records the exact commit, clean-tree status, compiler/toolchain, CMake cache options, target -architecture, host ABI, dependency versions, and test commands. Stable +architecture or CPU baseline/tier set, host ABI, dependency versions, and test +commands. CUDA evidence is per SM and CPU evidence is per compiled tier. Stable publication fails closed unless every stable-required boolean is true; preview publication preserves false values rather than deriving or omitting them. @@ -88,8 +106,10 @@ hardware the project does not own. | Artifact tuple | Initial channel | Backend flags and evidence boundary | |---|---|---| -| `linux-x86_64-glibc-cpu` | stable after matching x86_64 runtime gate | CPU core; CUDA, Metal, MLX, Vulkan, and HIP explicitly off; record glibc and libstdc++ floors | -| `linux-aarch64-glibc-cpu` | stable after matching arm64 runtime gate | same CPU contract, independent arm64 runtime evidence; never inferred from x86_64 | +| `linux-x86_64-glibc-cpu` | stable after baseline and tiered runtime gates | one adaptive binary: SSE2/portable baseline without AVX2, plus only inventoried per-TU F16C/AVX2/AVX-512 and later VNNI/AMX tiers whose kernels and exact probes exist; record glibc/libstdc++ floors and compiled tiers | +| `linux-aarch64-glibc-cpu` | stable after baseline and tiered runtime gates | one adaptive binary: NEON/portable baseline plus only inventoried HWCAP/HWCAP2-gated DotProd/i8mm and future tiers whose kernels exist; independent arm64 evidence is never inferred from x86_64 | +| `linux-x86_64-glibc-cuda-fat` | preview until the fat prerequisite, archive gates, and per-SM evidence land | primary x86_64 CUDA download; explicit SM set `80,86,87,89,90a,100a,103a,110,120a,121a`; per-source gencode and per-SM runtime/AOT dispatch required | +| `linux-aarch64-glibc-cuda-fat` | preview until the fat prerequisite, archive gates, and per-SM evidence land | primary aarch64 CUDA download with the same ten SM device targets but a distinct host ELF ABI; cannot be the x86_64 archive | | `macos-arm64-metal` | stable after M-series runtime gate | native Metal, MLX off; record deployment target and required system frameworks | | `macos-arm64-metal-mlx` | preview until its exact bundled MLX tuple is runtime/correctness-gated | Metal plus opt-in MLX provider and redistribution/license audit; record MLX dylib and metallib versions | | `linux-x86_64-glibc-vulkan` | preview | Vulkan explicitly on; loader/device/driver remain external; only the Vulkan-supported model/quant surface is declared | @@ -102,40 +122,101 @@ explicitly from the canonical CMake surface: `VLLM_CPP_CUDA`, `VLLM_CPP_VULKAN`, `VLLM_CPP_HIP`, `VLLM_CPP_HIP_ARCHITECTURES`, `VLLM_CPP_SERVER`, `VLLM_CPP_BUILD_EXAMPLES`, `VLLM_CPP_BUILD_TESTS`, and `VLLM_CPP_TRITON` (`CMakeLists.txt:24-120`; `cmake/TritonAOT.cmake:52-87`). -The manifest records resolved feature-table output; it does not say “all CUDA -features” or “all models.” - -### CUDA artifacts: one SM and one host ABI each - -There is no generic CUDA archive and no cross-SM runtime claim. Every supported -release target is a separate artifact: - -| CUDA SM | Host ABI | AOT | Initial channel and honest boundary | -|---|---|---:|---| -| `80` | Linux x86_64, glibc | on | preview; derived/build-verified, no matching runtime board | -| `86` | Linux x86_64, glibc | on | preview; same-major derived/build evidence, no matching runtime board | -| `87` | Linux aarch64, glibc/Jetson | off | preview; portable synchronous path runtime-verified, default async path remains a known bug | -| `89` | Linux x86_64, glibc | on | preview; derived/build-verified, no matching runtime board | -| `90a` | Linux x86_64, glibc | on | preview; derived/build-verified, no matching runtime board | -| `100a` | Linux x86_64, glibc | on | preview; derived/build-verified, no matching runtime board | -| `103a` | Linux x86_64, glibc | off | preview; portable build-only target | -| `110` | Linux aarch64, glibc/Thor | off | preview until the archive itself repeats the existing portable runtime proof | -| `120a` | Linux x86_64, glibc | off | preview; build-supported, no matching runtime board | -| `121a` | Linux aarch64, glibc/GB10 | on | stable candidate; must rerun archive smoke, both gate-model correctness, and release performance on GB10 | - -The AOT split is exact: `80`, `86`, `89`, `90a`, `100a`, and `121a` consume -the complete vendored trees under `src/vt/cuda/triton_aot_vendored/`; -`87`, `103a`, `110`, and `120a` set `VLLM_CPP_TRITON=OFF`. No lane fabricates -or borrows another SM's cubin. AOT artifacts are single-architecture, so a -fat CUDA build with AOT is forbidden by both this matrix and -`cmake/TritonAOT.cmake:94-126`. This release program does not publish a -fat+AOT archive. - -The host ABI is part of the tuple because a CUDA cubin does not make its ELF -host executable portable. Preview labels remain even when SASS exists if the -exact archive has not run on matching silicon. The matrix makes no blanket -claim about fast-path coverage: each manifest must preserve the resolved -`VT_CUDA_FEATURE_TABLE` cells from `cmake/CudaArchFeatures.cmake:213-357`. +The manifest records resolved feature-table output and compiled CPU tiers; it +does not say “all acceleration,” “all CUDA features,” or “all models.” No +release preset uses global `-march=native`. + +### Primary CUDA artifacts: fat per OS and host ABI + +The KISS primary download is one fat CUDA binary for each OS+host ABI, not one +download per SM. The first two are separate Linux x86_64-glibc and Linux +aarch64-glibc archives. Both explicitly build the complete supported set +`80;86;87;89;90a;100a;103a;110;120a;121a`; their device code may match, but +their host ELF ABI cannot. A future OS/libc tuple gets another artifact rather +than widening either binary into an impossible universal host executable. + +Three current gaps must close before either primary CUDA bundle can build: + +1. **Per-source gencode.** The current cross-family fat build applies the whole + target list to incompatible fast-path translation units, so sm12x PTX is + rejected for other families. CMake must compile portable sources for every + supported target and narrow each feature source to the exact SM intersection + declared by `VT_CUDA_FEATURE_TABLE`. The fat-build gate inspects every TU's + SASS/cubin set; a green final link alone is insufficient. +2. **Multi-SM Triton AOT packaging.** `cmake/TritonAOT.cmake:94-126` currently + rejects multiple architectures because it selects one vendored tree. The + release prerequisite extends the builder/embedding contract to include every + available per-SM tree in one host binary/archive, namespace its launchers, + and select the exact cubin from the runtime device capability. It must never + load a same-family “close enough” cubin. +3. **Runtime dispatch.** Device capability selects only a compiled compatible + tactic and exact AOT tree. Missing optimized code takes the documented + portable C++/CUDA fallback; an unsupported device fails loudly. The manifest + lists compiled SASS, fast-path feature cells, AOT availability, and separate + build/runtime/correctness/performance evidence for every SM. + +The honest AOT/evidence inventory remains: + +| CUDA SM | AOT embedded in fat artifact | Current evidence and optional per-SM variant | +|---|---:|---| +| `80` | yes | preview; derived/build-verified, no matching runtime board | +| `86` | yes | preview; same-major derived/build evidence, no matching runtime board | +| `87` | no | preview; portable synchronous path runtime-verified, default async path remains a known bug | +| `89` | yes | preview; derived/build-verified, no matching runtime board | +| `90a` | yes | preview; derived/build-verified, no matching runtime board | +| `100a` | yes | preview; derived/build-verified, no matching runtime board | +| `103a` | no | preview; portable build-only target | +| `110` | no | preview until the archive repeats the existing portable runtime proof | +| `120a` | no | preview; build-supported, no matching runtime board | +| `121a` | yes | stable candidate after extracted-archive smoke, both gate-model correctness, and release performance rerun on GB10 | + +The six complete vendored AOT trees are exactly `80`, `86`, `89`, `90a`, +`100a`, and `121a`. For `87`, `103a`, `110`, and `120a`, the fat binary records +AOT unavailable and uses the portable CUDA fallback. No lane fabricates or +borrows another SM's cubin. Optional single-SM archives may still be published +for diagnosis, size comparison, or peak-performance experiments, using the +same evidence labels; they are not the primary downloads and cannot substitute +for either host-ABI fat-build gate. + +The overall fat artifact stays preview while any included path has only +build evidence. Per-SM manifest evidence remains independent, so the existing +sm_121a stable candidate and build-only targets are not flattened into a +blanket support claim. + +### Primary CPU artifacts: one adaptive binary per OS and host ABI + +CPU packaging follows the same KISS rule at the host-ABI level: one binary per +OS+host ABI, compiled to a conservative baseline and dispatching at runtime. +There is no artifact per ISA tier, no global `-march=native`, and no binary that +crosses Linux/macOS, glibc/musl, x86_64/aarch64, or deployment-target ABIs. + +For x86_64, the baseline must run without AVX2: portable/SSE2 code remains +callable, and higher instructions live only in per-function or per-TU tiers. +The current element-GEMM selector already exposes SSE2+F16C, AVX2, and AVX-512F +tiers (`src/vt/cpu/cpu_matmul_elem.cpp:553-612`), while CMake's per-source ISA +shape avoids changing the rest of the binary's baseline +(`CMakeLists.txt:870-890`). The release audit inventories the exact instruction +requirements of every tier. AVX/AVX2/AVX-512 require both CPU feature bits and +OS-enabled extended state; VNNI, AMX, BF16, or other tiers are added only where +a real kernel exists, with every required CPUID bit and OS state (including +XCR0/tile permission where applicable). The manifest never turns “CPU has +AVX-512” into “all kernels are AVX-512 accelerated.” + +For aarch64, the baseline is portable/NEON and optimized quant tiers are chosen +only after exact OS capability probes. Linux uses `getauxval(AT_HWCAP*)`; +Darwin uses the corresponding `sysctl` keys. DotProd and i8mm each require their +own compiled kernel and probe; the existing i8mm path demonstrates the required +compile/runtime split (`src/vt/cpu/cpu_quant_dot_arm.cpp:39-77`). SVE/SVE2, +SME, BF16, or later tiers remain absent from the manifest until kernels, exact +HWCAP/sysctl probes, and gates exist. + +Every adaptive CPU artifact records its conservative baseline, compiled tiers, +per-tier kernel families, required CPU bits, required OS state, detected tier, +and selected tier. Forced-tier mutation tests must execute each compiled path on +feature-rich hosts and force the baseline on the same binary; on feature-poor +hosts or emulation they must prove the baseline executes and forcing an +unsupported tier fails closed before an illegal instruction. Feature detection +alone is not execution evidence. ## Static and external-runtime boundary @@ -150,8 +231,10 @@ The one literal-static experiment is identifies a static executable, `ldd` reports no dynamic interpreter, the server help and loopback health smoke pass in a minimal container, and DNS/thread/file loading behavior is exercised. It stays experimental preview even when green; -the result decides whether a second arm64 musl lane is warranted. The archive -must not silently disable server functionality to obtain a static link. +the result decides whether a second arm64 musl lane is warranted. It follows the +same conservative x86_64 adaptive-tier contract but remains a distinct musl ABI, +not a replacement for the glibc artifact. The archive must not silently disable +server functionality to obtain a static link. Accelerator drivers are honest external boundaries. NVIDIA's kernel driver and CUDA driver ABI, the Vulkan loader/ICD and device driver, macOS Metal system @@ -188,14 +271,26 @@ Required gates: RPATH/RUNPATH, absolute developer path, or missing library. ELF RPATH must be absent or relative to the extracted bundle; Mach-O install names must use system paths, `@rpath`, or `@loader_path` as declared. -4. **Architecture audit:** `file`/ELF/Mach-O headers match the host ABI; CUDA - `cuobjdump` shows only the named SM; an AOT-on artifact's manifest and cubins - match that same SM; AOT-off artifacts contain no Triton AOT payload. -5. **Correctness:** CPU unit/conformance tests before packaging; after - extraction, representative endpoint and model checks. Stable `sm_121a` owes - both project gate models and the normal oracle comparison. Other stable - lanes owe the row's available backend/model correctness contract. -6. **Supply chain:** archive SHA256, SPDX JSON SBOM, source/dependency/license +4. **CUDA fat-architecture audit:** `file`/ELF headers match the host ABI; + `cuobjdump` proves all ten named SM targets are present in the primary fat + binary; per-TU inspection proves each fast-path source contains only its + compatible gencode set. All six available AOT trees are embedded and resolve + only on their exact SM; the other four capabilities select the portable + fallback. Removing any one SM/AOT mapping or routing one to a neighbour is a + red mutation. +5. **CPU adaptive-dispatch audit:** dependency/manifest data lists baseline, + compiled tiers, exact CPU bits and OS-state prerequisites per kernel family. + Forced-tier mutations cover every tier. The extracted binary executes on + feature-poor and feature-rich hosts or faithful emulation: baseline works + without AVX2 on x86_64, NEON/portable works without optional Arm extensions, + rich tiers execute when supported, and unsupported forced tiers refuse before + an illegal instruction. Build commands reject global `-march=native`. +6. **Correctness:** CPU unit/conformance tests before packaging; after + extraction, representative endpoint and model checks. Each CUDA SM and CPU + tier retains independent evidence. The sm_121a path owes both project gate + models and the normal oracle comparison before its stable-candidate flag can + turn true; another SM's result cannot satisfy it. +7. **Supply chain:** archive SHA256, SPDX JSON SBOM, source/dependency/license inventory, third-party notices, immutable build provenance, and a `VERSION` record containing tag, commit, clean-tree bit, compiler, backend, target, host ABI, and C ABI version. The checksum and provenance refer to the final @@ -210,7 +305,9 @@ applicable.” | Responsibility | Local destination | Contract | |---|---|---| | server install/package component | top-level and `examples/CMakeLists.txt` | stage the existing static-core server under its canonical output name without changing server behavior | -| release tuple/preset | focused release CMake presets or matrix data | explicit backend/provider/host ABI; no `AUTO`, wildcard SM, or blanket feature claim | +| release tuple/preset | focused release CMake presets or matrix data | explicit backend/provider/host ABI and exhaustive ten-SM fat list; no `AUTO`, implicit wildcard, `-march=native`, or blanket feature claim | +| CUDA fat compile/dispatch | CMake feature-source gencode plus CUDA/AOT runtime selector | narrow every fast-path TU to compatible SMs; embed all available AOT trees; dispatch exact SM with portable fallback where AOT is unavailable | +| CPU adaptive compile/dispatch | per-source ISA options and CPU selectors | conservative ABI baseline plus only real kernel tiers, guarded by exact CPU and OS-state probes | | manifest and supply-chain metadata | release scripts plus a versioned schema | independent evidence values, final-archive SHA256, SPDX SBOM, provenance, version and licenses | | staged archive validator | release checker tests/scripts | validate extracted bytes, dependencies, RPATH/install names, host architecture and AOT SM | | dry-run/tag automation | release workflow | build, verify, attest and publish as isolated least-privilege stages | @@ -219,7 +316,7 @@ applicable.” ## Tests to port The pinned upstream release pipeline is the executable structural spec; its -separate build/test/publish stages are mirrored by W4 rather than porting a +separate build/test/publish stages are mirrored by W8 rather than porting a Python runtime test. Existing local executable specs remain binding: - `test_server_help` (`examples/CMakeLists.txt:59-63`) runs from the extracted @@ -230,7 +327,13 @@ Python runtime test. Existing local executable specs remain binding: gate supplies runtime evidence; cross-builds record runtime false; and - new manifest/archive fixtures mutate every required evidence field, RPATH, dependency allowlist, architecture and AOT association to prove the checker - fails for the named reason. + fails for the named reason; +- CUDA mutations remove each SM in turn, mis-assign an AOT tree, or widen a + fast-path TU's gencode and must fail before publication; and +- CPU mutations force every compiled tier, remove each feature/OS-state probe, + and run the extracted binary on feature-poor and feature-rich hosts/emulation. + Unsupported force requests must fail closed; supported force requests must + execute the named tier, not merely report it. ## Least-privilege CI and release flow @@ -258,8 +361,12 @@ wildcards, and a failed lane cannot be replaced by an older workflow artifact. ## Dependencies - the existing server and library install boundaries; -- canonical CMake backend flags and CUDA feature-table resolution; -- complete per-SM Triton AOT trees where AOT is enabled; +- canonical CMake backend flags, CUDA feature-table resolution, and per-source + gencode narrowing for the full ten-SM list; +- multi-tree Triton AOT embedding/namespacing plus exact runtime selection of + the six complete per-SM trees; +- CPU per-source/per-function ISA compilation, exact OS capability APIs, and + feature-poor plus feature-rich hosts or emulation for both host architectures; - matching runtime hosts for any stable tuple and the normal GPU contention protocol for correctness/performance gates; - redistribution-compatible licenses for every bundled runtime file; @@ -275,18 +382,19 @@ the present checkpoint stays `SPIKE`. | Work | Deliverable | Exit gate | |---|---|---| -| W1 | canonical `vllm-server` output name, install component, and staging/package target for the existing static-core server | install into empty prefix; extracted help smoke; existing library install unchanged | -| W2 | versioned release-manifest generator and schema with independent evidence booleans | fixture tests distinguish absent, false, failed, and true evidence; mutation of each required field is red | -| W3 | staged archive validator: content allowlist, dependency/RPATH audit, architecture/AOT audit, SHA256, VERSION, licenses, SPDX SBOM | Linux fixture/archive tests red-first; no build-tree paths accepted | -| W4 | least-privilege dry-run/tag workflow skeleton, immutable artifact handoff, provenance and protected publish stages | permissions checker plus dry run proves no release is created and publish cannot consume unverified bytes | -| W5 | Linux glibc CPU x86_64 and arm64 bundles plus experimental x86_64 musl literal-static lane | both glibc tuples runtime-smoked on matching hosts before stable; musl static/loopback checks remain preview | -| W6 | macOS arm64 native-Metal bundle | extracted archive model smoke on M-series hardware; dependency/install-name audit; MLX absent | -| W7 | macOS arm64 MLX preview bundle | exact MLX dylib/metallib/license inventory and extracted runtime/correctness smoke; remains preview until all stable gates exist | -| W8 | Linux x86_64 Vulkan preview bundle | extracted archive runtime smoke on a Vulkan runner; loader/ICD stay external and declared | -| W9a | CUDA previews for `80`, `86`, `87`, `89` | four separate host/SM artifacts; exact AOT split; build/archive evidence never upgrades missing runtime booleans | -| W9b | CUDA previews for `90a`, `100a`, `103a`, `110` | same contract; `110` repeats matching-hardware archive smoke before any channel change | -| W9c | CUDA `120a` preview and `121a` stable candidate | separate artifacts; `121a` extracted archive reruns both gate models and release-performance gate on GB10 | -| W10 | release index/docs and retention policy generated from manifests | every link, checksum, channel, host ABI, driver boundary, and known limitation matches published bytes | +| W1 | cross-family CUDA fat-build prerequisite: per-source gencode narrowing over all ten supported SMs | clean x86_64-host fat build; per-TU `cuobjdump` mutation proves every compatible SM present and every incompatible SM absent | +| W2 | multi-SM Triton AOT embedding, namespacing, manifest and exact runtime dispatch | all six available trees coexist in one fat binary; exact-SM dispatch tests plus portable fallback for the four unavailable trees; wrong-tree mutation red | +| W3 | x86_64 CPU ISA-dispatch inventory and completion | SSE2/portable baseline runs without AVX2; current F16C/AVX2/AVX-512 tiers forced and executed; exact OS-state probes; VNNI/AMX listed only for real gated kernels; no `-march=native` | +| W4 | aarch64 CPU ISA-dispatch inventory and completion | NEON/portable baseline plus independently forced DotProd/i8mm where kernels exist; exact Linux HWCAP/Darwin sysctl gates; poor/rich host or emulation execution | +| W5 | versioned release-manifest generator and schema with independent per-SM and per-CPU-tier evidence | fixtures distinguish absent, false, failed and true; compiled tiers/SMs, required probes, dependencies and host ABI are mandatory | +| W6 | canonical `vllm-server` output name, install component, and staging/package target for the existing static-core server | install into empty prefix; extracted help smoke; existing library install unchanged | +| W7 | staged archive validator: allowlist, dependency/RPATH, fat-SM/AOT and adaptive-CPU audits, SHA256, VERSION, licenses and SPDX SBOM | Linux fixture/archive tests red-first; no build paths, missing SM, unsafe ISA tier or undeclared dependency accepted | +| W8 | least-privilege dry-run/tag workflow, immutable artifact handoff, provenance and protected publish stages | permissions checker plus dry run proves no release is created and publish cannot consume unverified bytes | +| W9 | primary adaptive CPU bundles: Linux glibc x86_64+aarch64 and experimental x86_64 musl literal-static | glibc binaries execute baseline and supported rich tiers on matching hosts/emulation before stable; musl remains preview | +| W10 | primary Linux CUDA fat bundles for x86_64 and aarch64 host ABIs | each extracted archive contains all ten SMs and six exact AOT trees; per-SM evidence remains independent; no host ABI is inferred from the other | +| W11 | macOS arm64 native-Metal/MLX and Linux Vulkan bundles | native Metal runtime-gated; MLX/Vulkan preview until exact archive gates; dependencies and install names audited | +| W12 | optional single-SM CUDA diagnostic/performance variants | generated from the same explicit matrix and evidence; never advertised as the primary KISS download or used to bypass W10 | +| W13 | release index/docs and retention policy generated from manifests | every link, checksum, channel, host ABI, compiled tier/SM, driver boundary and limitation matches published bytes | ROCm remains blocked outside these work units until its backend row first compiles on AMD hardware. A new lane is added by changing this matrix and its @@ -300,8 +408,13 @@ tests before workflow expansion, never by a wildcard build. backend family. A later toolchain or dependency change reruns the gates. - Cross-compilation can prove bytes and architecture, not execution. Preview is the honest publication channel for community hardware coverage. +- A fat CUDA link is not enough: per-source cubin inventory and exact runtime + dispatch are gates. Optional single-SM builds cannot substitute for them. - CUDA fast-path availability differs by SM. The feature-table output and AOT state are manifest data; archive names do not imply feature parity. +- CPU feature detection is not enough: each compiled tier must execute under a + forced-tier test, and every instruction requires its CPU plus OS-state gate. + One adaptive binary still cannot cross an OS or host ABI. - ffmpeg and accelerator drivers are external operational dependencies. Their absence must fail the relevant feature actionably, not corrupt a general server smoke. @@ -313,6 +426,8 @@ tests before workflow expansion, never by a wildcard build. The release program is feasible as backend-specific static-core bundles with a hybrid stable/preview channel. Literal-static scope is limited to the -experimental musl CPU lane. ROCm is blocked. The first implementation slice is -W1, followed by the manifest/validator substrate before any publishing workflow. +experimental musl CPU lane. ROCm is blocked. Primary downloads are adaptive CPU +or fat CUDA per OS+host ABI; per-SM CUDA archives are optional diagnostics. The +first implementation slice is W1's cross-family per-source-gencode prerequisite, +followed by multi-SM AOT dispatch and CPU ISA audits before any bundle workflow. `ENG-RELEASE-BINARIES` is `SPIKE`, not `READY`, `ACTIVE`, or `DONE`. diff --git a/.agents/state.md b/.agents/state.md index 1c3e73ca4..de34bcbc1 100644 --- a/.agents/state.md +++ b/.agents/state.md @@ -42253,3 +42253,29 @@ external ffmpeg/platform/driver boundaries, not a magically static GPU stack. First implementation after fresh review is W1: install and package the existing static-core server; manifest, validator, CI, platform, and CUDA bricks follow as separate claims. + +## 2026-08-07 — Release matrix revised: fat CUDA and adaptive CPU are the primary downloads + + +User review changed the KISS artifact unit without changing lifecycle: +`ENG-RELEASE-BINARIES` remains `SPIKE`, and no archive or implementation is +claimed. This explicitly supersedes the prior checkpoint's single-SM-primary +and fat+AOT-prohibited design. Primary downloads are now one binary per +OS+host ABI: adaptive CPU and fat CUDA. One ELF cannot cross x86_64/aarch64 or +libc/OS ABIs. + +CUDA primary artifacts cover all ten supported SMs +(`80,86,87,89,90a,100a,103a,110,120a,121a`) in each Linux x86_64/aarch64 host +binary. This makes two known gaps prerequisites: per-source gencode must make +cross-family fat builds compile, and Triton AOT must embed and runtime-select all +six available exact-SM trees (`80,86,89,90a,100a,121a`) while the other four +use portable CUDA fallbacks. Per-SM archives remain optional diagnostics, not +primary downloads. Stable/preview evidence stays per SM and is never inferred. + +CPU primary artifacts use a conservative baseline with runtime ISA dispatch, +never global `-march=native`: x86_64 portable/SSE2 plus only real F16C/AVX2/ +AVX-512 and later gated tiers; aarch64 portable/NEON plus independently probed +DotProd/i8mm and future real-kernel tiers. Exact CPU+OS-state probes, compiled +tier manifests, forced-tier mutation tests, and execution on feature-poor and +feature-rich hosts/emulation are release gates. W1 is now the CUDA per-source +gencode prerequisite; multi-SM AOT and CPU ISA audits precede bundle work. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index aff75e569..63bf6ea7d 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -4,7 +4,7 @@ | Reference | Workload | Headline | Tokens | |---|---|---|---| -| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: backend-specific static-core `vllm-server` archives; literal-static limited to an experimental musl CPU lane | **PENDING:** release contract accepted; no archive, staged smoke, runtime validation, or release-performance result exists yet | n/a | +| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** revised release contract accepted; no archive, staged smoke, runtime validation, or release-performance result exists yet | n/a | | **Developer agent entry point (implemented)** | `DOCS-AGENT-PROTOCOL-ENTRYPOINT`: public contribution guide + synchronized, mutation-gated pre-claim intake rule | Rebased documentation/protocol only; benchmark void | n/a | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **Record repair 2026-08-07** | `main` was red on `check-agent-record` + `check-env-doc`, blocking every PR. Dangling `kda-chunk-aot/` link and two undocumented env vars. No behaviour change | diff --git a/docs/STATUS.md b/docs/STATUS.md index 749c4ff29..21460339a 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -112,7 +112,7 @@ token-for-token correctness against the pinned oracle. | Tool-call parsing | 36 parser families / 40 accepted names, streaming | Every vLLM tool parser at the pin except the three Rust/Harmony-backed ones: pure-text parsers ported 1:1, the six engine-backed families reimplemented from their wire formats, all held to the upstream test suites. Selection via `--tool-call-parser` (server), `tool_parser` (C ABI), or template auto-detection; native-syntax forced tool_choice where expressible. Tables: docs/BENCHMARKS.md | | Reasoning parsing (`SAMPLE-REASONING`, ACTIVE, partial coverage) | 9 parsers, streaming | think_auto (auto-detect default: content unless markers appear), deepseek_r1, deepseek_v3 (passthrough) / holo2 (thinking→R1), mistral ([THINK]), minimax_m2 (+append_think), step3, olmo3 - reasoning split engine-side BEFORE tool parsing, streamed as `reasoning` deltas in the chat chunks. Coverage: 9 of upstream's ~28 registered names (remaining text families + engine-backed adapters tracked as W2/W3 in specs/reasoning-parsers.md); each ported parser doctest-gated vs its tests/reasoning case | | Unified streaming parser engine | Core, assembly, serving-SSE dispatch landed, gated; all 10 engine-backed families ported (family parity closed); JSON-schema tool-arg type coercion landed | The vLLM 0.26 declarative `parser/engine/` (shared state machine plus all 10 configs: qwen3, seed_oss, kimi_k2, minimax_m2, glm47_moe, deepseek_v4/v32, nemotron_v3, gemma4, inkling) and assembly layer, gated field-for-field vs vLLM 0.26. An engine-backed `--tool-call-parser` name drives the live chat SSE chunks, off by default. When a request's tools declare typed parameters, the assembled tool-call arguments are coerced to the declared JSON types (int/number/bool/string/array/null) 1:1 with vLLM `_fix_arg_types`, in both streaming and one-shot; no schema means the arguments pass through as strings unchanged. Details: .agents/specs/parser-assembly-c8.md | -| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): static-core backend/host bundles; runtime-gated stable vs build-only preview; no archive exists | +| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary host-ABI fat CUDA/adaptive CPU; per-SM diagnostics; no archive | | Pooling task class (embeddings / classify / score / rerank) | Spiked; pooler op + heads composite + pooling runner path landed (CPU), not yet servable end-to-end (no concrete model / endpoints) | The non-generative task class. W0 spike over the whole vLLM pooling surface (`.agents/specs/pooling-task-class.md`, `CLAIM-POOLING`). W1 landed the pooler OP (CLS/LAST/MEAN + Identity/Normalize/MultiLabelClassify/Classify activations, double-precision-gated). **W2 landed the pooler HEADS composite** — `EmbeddingPoolerHead` (projector→matryoshka→normalize), `ClassifierPoolerHead` (classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` task routing, and the `PoolerConfig`/`PoolingParams` structs (`test_pooler_heads` 27/27, 240 asserts, RED-first). **W3 landed the pooling RUNNER path** — `PoolingRunner` applies the model's `Pooler` to the last hidden state and returns pooled embeddings instead of sampled tokens, gated by a STRUCTURAL cosine-parity check vs a double-precision LAST+normalize reference (`test_pooling_runner` 5/5, 14 asserts, RED-first). NOT yet servable / honest residuals: a concrete pooling MODEL forward + the REAL-model oracle cosine gate (`vllm.LLM(task="embed").encode`) — no cosine-vs-oracle number is fabricated (W3-model); the `/v1/embeddings` + score/rerank/classify endpoints (W4); tokwise AllPool/StepPool (W5). See docs/BENCHMARKS.md | | Plugin system (out-of-core registration) | Spiked; first CPU brick landed, not yet wired into any production path | The extensibility-first discovery layer. W0 spike over vLLM's plugin surface (general / platform / io_processor / endpoint groups, the `register_model` an out-of-tree plugin calls, the invocation seams) is committed (`.agents/specs/plugin-system.md`, `ENG-PLUGIN-SYSTEM` ACTIVE, `CLAIM-PLUGIN-SYSTEM`). W1 landed `vllm::plugins::LoadGeneralPlugins()` + the out-of-core general-plugin registration seam (`RegisterGeneralPlugin` / `REGISTER_VLLM_GENERAL_PLUGIN`) over the existing `REGISTER_VLLM_MODEL`-style registries (the in-tree factory `MODEL-FACTORY-registry` is record-repaired `DONE` 2026-08-05: 28 self-registering TUs, dgx debt paid by the 2026-07-23 seven-gate run): a 1:1 mirror of `load_general_plugins` (load-once idempotence, the `VLLM_PLUGINS` allowlist, per-plugin failure isolation). Proven by an out-of-core toy-model plugin that registers a toy architecture through the public `RegisterModel` seam — unit-gated RED-first (`test_plugin_system` 1 case / 29 assertions: the toy arch resolves ONLY after LoadGeneralPlugins runs it, and not under `VLLM_PLUGINS=""`). Python entry points have no C++20 analogue, so discovery is the project's static-init/`dlopen` registration idiom (recorded porting-inventory §9). NOT yet wired: real shared-object `dlopen` + the C-ABI `vllm_plugin_register` entry (W2), the engine/CLI `--load-plugins` wiring that calls LoadGeneralPlugins from the construction paths (W3), the platform/quant plugin kinds (W4), and the io_processor/stat_logger/endpoint groups (W5) are named residuals. See docs/BENCHMARKS.md | | Offline Batch API (JSONL file runner) | Spiked; first CPU brick landed, not yet exposed as a CLI | The offline OpenAI Batch API: read a JSONL of OpenAI-format requests, run each through the engine, write a JSONL of responses. W0 spike over vLLM's `run_batch.py` (schema, endpoint dispatch, run loop, file I/O) is committed (`.agents/specs/batch-api.md`, `SERVE-BATCH-API` ACTIVE, `CLAIM-BATCH-API`). W1 landed `RunBatch` (`RunLine`/`RunLines`/`Run`) + `RunBatchFile` — a pure orchestrator over the existing `OpenAIServingChat::create_chat_completion` (NO reimplemented generation), 1:1 with vLLM's endpoint_registry url→handler map: `/v1/chat/completions` dispatch, the `BatchResponseData`/`BatchRequestOutput` schema (`vllm-` ids, custom_id echoed), the `run_request` AllResponse/ErrorResponse/stream branches, and the unsupported-endpoint/url error rows. Unit-gated RED-first (`test_openai_run_batch` 7 cases / 80 assertions over the synthetic serving engine: ordered rows + custom_id echo + per-line BatchRequestOutput schema round-trip, a malformed line isolated into an error row so the batch continues, dispatch + 404 error rows; dropping the custom_id echo fails 9 assertions). Recorded deviation: a malformed line is isolated (batch continues) where upstream aborts the job. NOT yet exposed: the `vllm run-batch` CLI + `BatchFrontendArgs` (W2), embeddings/score/rerank dispatch (W3, rides pooling endpoints), audio transcription/translation + media fetch (W4), and http(s)/data-URL file I/O + metrics + overlapped `AsyncLLM` submission (W5) are named residuals. See docs/BENCHMARKS.md | From 1b5b95963b8e0d0f1a47daba7feb31dc27f1e2c6 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 7 Aug 2026 22:59:46 +0000 Subject: [PATCH 04/11] test(release): gate accepted binary bundle contract Pin the user-approved fat-CUDA and adaptive-CPU artifact shapes, exact ten-SM set, W1-W13 dependency graph, and every canonical release anchor in a fail-closed checker wired into preflight and CI. Red-first evidence: test_check_release_binary_contract.py failed with 38 mutation subtest failures before the checker, contract block, and Deps column existed. The focused suite now passes 11/11 test methods, including every SM, work edge, and record-anchor deletion. The row remains SPIKE; archive and runtime evidence remain pending. FOLLOWING_AGENTS_PROTOCOL Assisted-by: Codex:gpt-5 [Codex] --- .agents/specs/release-binary-matrix.md | 62 ++++-- .github/workflows/ci.yml | 4 + docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- scripts/agent-preflight.sh | 2 + scripts/check-release-binary-contract.py | 199 ++++++++++++++++++ .../test_check_release_binary_contract.py | 199 ++++++++++++++++++ 7 files changed, 453 insertions(+), 17 deletions(-) create mode 100644 scripts/check-release-binary-contract.py create mode 100644 tests/scripts/test_check_release_binary_contract.py diff --git a/.agents/specs/release-binary-matrix.md b/.agents/specs/release-binary-matrix.md index 0e086aa2c..bfe84168e 100644 --- a/.agents/specs/release-binary-matrix.md +++ b/.agents/specs/release-binary-matrix.md @@ -9,6 +9,38 @@ request [#117](https://github.com/mudler/vllm.cpp/issues/117); claim `CLAIM-ENG-RELEASE-BINARIES-SPIKE` in draft PR [#129](https://github.com/mudler/vllm.cpp/pull/129). + +identity=ENG-RELEASE-BINARIES +lifecycle=SPIKE +primary_cuda_artifact=one-fat-binary-per-os-host-abi +primary_cuda_sms=80,86,87,89,90a,100a,103a,110,120a,121a +per_sm_cuda=optional-non-primary +primary_cpu_artifact=one-adaptive-binary-per-os-host-abi +x86_64_baseline=portable-sse2-without-avx2 +work_W12_policy=optional-non-blocking +archive_claims=pending +runtime_claims=pending +required_anchor_paths=.agents/engine-matrix.md,.agents/roadmap_v1.md,.agents/NOW.md,.agents/coordination.md,.agents/state.md,docs/STATUS.md,docs/BENCHMARKS.md +work_W1= +work_W2=W1 +work_W3= +work_W4= +work_W5= +work_W6= +work_W7=W1,W2,W3,W4,W5,W6 +work_W8=W5,W7 +work_W9=W3,W4,W5,W6,W7 +work_W10=W1,W2,W5,W6,W7 +work_W11=W5,W6,W7 +work_W12=W1,W2,W5,W6,W7 +work_W13=W5,W7,W8,W9,W10,W11 + + +The block above is consumed by `scripts/check-release-binary-contract.py`; its +values and the human-readable work table below must change together through a +new reviewed design decision. In particular W12 is optional and is deliberately +absent from W13's dependency set. + ## Scope and product contract The deliverable is a downloadable, backend-specific `vllm-server` bundle. The @@ -380,21 +412,21 @@ Each work unit is a separate claim with its own red-first checker change and fresh review. No unit advances this row beyond `ACTIVE` until its gates pass; the present checkpoint stays `SPIKE`. -| Work | Deliverable | Exit gate | -|---|---|---| -| W1 | cross-family CUDA fat-build prerequisite: per-source gencode narrowing over all ten supported SMs | clean x86_64-host fat build; per-TU `cuobjdump` mutation proves every compatible SM present and every incompatible SM absent | -| W2 | multi-SM Triton AOT embedding, namespacing, manifest and exact runtime dispatch | all six available trees coexist in one fat binary; exact-SM dispatch tests plus portable fallback for the four unavailable trees; wrong-tree mutation red | -| W3 | x86_64 CPU ISA-dispatch inventory and completion | SSE2/portable baseline runs without AVX2; current F16C/AVX2/AVX-512 tiers forced and executed; exact OS-state probes; VNNI/AMX listed only for real gated kernels; no `-march=native` | -| W4 | aarch64 CPU ISA-dispatch inventory and completion | NEON/portable baseline plus independently forced DotProd/i8mm where kernels exist; exact Linux HWCAP/Darwin sysctl gates; poor/rich host or emulation execution | -| W5 | versioned release-manifest generator and schema with independent per-SM and per-CPU-tier evidence | fixtures distinguish absent, false, failed and true; compiled tiers/SMs, required probes, dependencies and host ABI are mandatory | -| W6 | canonical `vllm-server` output name, install component, and staging/package target for the existing static-core server | install into empty prefix; extracted help smoke; existing library install unchanged | -| W7 | staged archive validator: allowlist, dependency/RPATH, fat-SM/AOT and adaptive-CPU audits, SHA256, VERSION, licenses and SPDX SBOM | Linux fixture/archive tests red-first; no build paths, missing SM, unsafe ISA tier or undeclared dependency accepted | -| W8 | least-privilege dry-run/tag workflow, immutable artifact handoff, provenance and protected publish stages | permissions checker plus dry run proves no release is created and publish cannot consume unverified bytes | -| W9 | primary adaptive CPU bundles: Linux glibc x86_64+aarch64 and experimental x86_64 musl literal-static | glibc binaries execute baseline and supported rich tiers on matching hosts/emulation before stable; musl remains preview | -| W10 | primary Linux CUDA fat bundles for x86_64 and aarch64 host ABIs | each extracted archive contains all ten SMs and six exact AOT trees; per-SM evidence remains independent; no host ABI is inferred from the other | -| W11 | macOS arm64 native-Metal/MLX and Linux Vulkan bundles | native Metal runtime-gated; MLX/Vulkan preview until exact archive gates; dependencies and install names audited | -| W12 | optional single-SM CUDA diagnostic/performance variants | generated from the same explicit matrix and evidence; never advertised as the primary KISS download or used to bypass W10 | -| W13 | release index/docs and retention policy generated from manifests | every link, checksum, channel, host ABI, compiled tier/SM, driver boundary and limitation matches published bytes | +| Work | Deps | Deliverable | Exit gate | +|---|---|---|---| +| W1 | — | cross-family CUDA fat-build prerequisite: per-source gencode narrowing over all ten supported SMs | clean x86_64-host fat build; per-TU `cuobjdump` mutation proves every compatible SM present and every incompatible SM absent | +| W2 | W1 | multi-SM Triton AOT embedding, namespacing, manifest and exact runtime dispatch | all six available trees coexist in one fat binary; exact-SM dispatch tests plus portable fallback for the four unavailable trees; wrong-tree mutation red | +| W3 | — | x86_64 CPU ISA-dispatch inventory and completion | SSE2/portable baseline runs without AVX2; current F16C/AVX2/AVX-512 tiers forced and executed; exact OS-state probes; VNNI/AMX listed only for real gated kernels; no `-march=native` | +| W4 | — | aarch64 CPU ISA-dispatch inventory and completion | NEON/portable baseline plus independently forced DotProd/i8mm where kernels exist; exact Linux HWCAP/Darwin sysctl gates; poor/rich host or emulation execution | +| W5 | — | versioned release-manifest generator and schema with independent per-SM and per-CPU-tier evidence | fixtures distinguish absent, false, failed and true; compiled tiers/SMs, required probes, dependencies and host ABI are mandatory | +| W6 | — | canonical `vllm-server` output name, install component, and staging/package target for the existing static-core server | install into empty prefix; extracted help smoke; existing library install unchanged | +| W7 | W1, W2, W3, W4, W5, W6 | staged archive validator: allowlist, dependency/RPATH, fat-SM/AOT and adaptive-CPU audits, SHA256, VERSION, licenses and SPDX SBOM | Linux fixture/archive tests red-first; no build paths, missing SM, unsafe ISA tier or undeclared dependency accepted | +| W8 | W5, W7 | least-privilege dry-run/tag workflow, immutable artifact handoff, provenance and protected publish stages | permissions checker plus dry run proves no release is created and publish cannot consume unverified bytes | +| W9 | W3, W4, W5, W6, W7 | primary adaptive CPU bundles: Linux glibc x86_64+aarch64 and experimental x86_64 musl literal-static | glibc binaries execute baseline and supported rich tiers on matching hosts/emulation before stable; musl remains preview | +| W10 | W1, W2, W5, W6, W7 | primary Linux CUDA fat bundles for x86_64 and aarch64 host ABIs | each extracted archive contains all ten SMs and six exact AOT trees; per-SM evidence remains independent; no host ABI is inferred from the other | +| W11 | W5, W6, W7 | macOS arm64 native-Metal/MLX and Linux Vulkan bundles | native Metal runtime-gated; MLX/Vulkan preview until exact archive gates; dependencies and install names audited | +| W12 | W1, W2, W5, W6, W7 | optional single-SM CUDA diagnostic/performance variants | generated from the same explicit matrix and evidence; never advertised as the primary KISS download or used to bypass W10 | +| W13 | W5, W7, W8, W9, W10, W11 | release index/docs and retention policy generated from manifests | every link, checksum, channel, host ABI, compiled tier/SM, driver boundary and limitation matches published bytes | ROCm remains blocked outside these work units until its backend row first compiles on AMD hardware. A new lane is added by changing this matrix and its diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 995864f59..cc8268cf0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,10 @@ jobs: python3 scripts/check-agent-record.py python3 tests/scripts/test_agent_record.py python3 tests/scripts/test_doc_checkpoint.py + - name: Accepted binary-release design and record anchors stay in sync + run: | + python3 scripts/check-release-binary-contract.py + python3 tests/scripts/test_check_release_binary_contract.py - name: README stays a human-readable user-facing document run: | python3 scripts/check-readme-structure.py diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 85541ae8e..80eb49cd6 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -4,7 +4,7 @@ | Reference | Workload | Headline | Tokens | |---|---|---|---| -| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** revised release contract accepted; no archive, staged smoke, runtime validation, or release-performance result exists yet | n/a | +| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** contract mutation-gated (10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13 graph, record anchors); no archive, staged smoke, runtime, or release-performance evidence | n/a | | **Developer agent entry point (implemented)** | `DOCS-AGENT-PROTOCOL-ENTRYPOINT`: public contribution guide + synchronized, mutation-gated pre-claim intake rule | Rebased documentation/protocol only; benchmark void | n/a | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **Record repair 2026-08-07** | `main` was red on `check-agent-record` + `check-env-doc`, blocking every PR. Dangling `kda-chunk-aot/` link and two undocumented env vars. No behaviour change | diff --git a/docs/STATUS.md b/docs/STATUS.md index 4fea67e5e..5a2e7713c 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -112,7 +112,7 @@ token-for-token correctness against the pinned oracle. | Tool-call parsing | 36 parser families / 40 accepted names, streaming | Every vLLM tool parser at the pin except the three Rust/Harmony-backed ones: pure-text parsers ported 1:1, the six engine-backed families reimplemented from their wire formats, all held to the upstream test suites. Selection via `--tool-call-parser` (server), `tool_parser` (C ABI), or template auto-detection; native-syntax forced tool_choice where expressible. Tables: docs/BENCHMARKS.md | | Reasoning parsing (`SAMPLE-REASONING`, ACTIVE, partial coverage) | 9 parsers, streaming | think_auto (auto-detect default: content unless markers appear), deepseek_r1, deepseek_v3 (passthrough) / holo2 (thinking→R1), mistral ([THINK]), minimax_m2 (+append_think), step3, olmo3 - reasoning split engine-side BEFORE tool parsing, streamed as `reasoning` deltas in the chat chunks. Coverage: 9 of upstream's ~28 registered names (remaining text families + engine-backed adapters tracked as W2/W3 in specs/reasoning-parsers.md); each ported parser doctest-gated vs its tests/reasoning case | | Unified streaming parser engine | Core, assembly, serving-SSE dispatch landed, gated; all 10 engine-backed families ported (family parity closed); JSON-schema tool-arg type coercion landed | The vLLM 0.26 declarative `parser/engine/` (shared state machine plus all 10 configs: qwen3, seed_oss, kimi_k2, minimax_m2, glm47_moe, deepseek_v4/v32, nemotron_v3, gemma4, inkling) and assembly layer, gated field-for-field vs vLLM 0.26. An engine-backed `--tool-call-parser` name drives the live chat SSE chunks, off by default. When a request's tools declare typed parameters, the assembled tool-call arguments are coerced to the declared JSON types (int/number/bool/string/array/null) 1:1 with vLLM `_fix_arg_types`, in both streaming and one-shot; no schema means the arguments pass through as strings unchanged. Details: .agents/specs/parser-assembly-c8.md | -| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary host-ABI fat CUDA/adaptive CPU; per-SM diagnostics; no archive | +| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary host-ABI fat CUDA/adaptive CPU and W1-W13 dependencies are mutation-gated; per-SM diagnostics remain optional; no archive or runtime claim | | Pooling task class (embeddings / classify / score / rerank) | Spiked; pooler op + heads composite + pooling runner path landed (CPU), not yet servable end-to-end (no concrete model / endpoints) | The non-generative task class. W0 spike over the whole vLLM pooling surface (`.agents/specs/pooling-task-class.md`, `CLAIM-POOLING`). W1 landed the pooler OP (CLS/LAST/MEAN + Identity/Normalize/MultiLabelClassify/Classify activations, double-precision-gated). **W2 landed the pooler HEADS composite** — `EmbeddingPoolerHead` (projector→matryoshka→normalize), `ClassifierPoolerHead` (classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` task routing, and the `PoolerConfig`/`PoolingParams` structs (`test_pooler_heads` 27/27, 240 asserts, RED-first). **W3 landed the pooling RUNNER path** — `PoolingRunner` applies the model's `Pooler` to the last hidden state and returns pooled embeddings instead of sampled tokens, gated by a STRUCTURAL cosine-parity check vs a double-precision LAST+normalize reference (`test_pooling_runner` 5/5, 14 asserts, RED-first). NOT yet servable / honest residuals: a concrete pooling MODEL forward + the REAL-model oracle cosine gate (`vllm.LLM(task="embed").encode`) — no cosine-vs-oracle number is fabricated (W3-model); the `/v1/embeddings` + score/rerank/classify endpoints (W4); tokwise AllPool/StepPool (W5). See docs/BENCHMARKS.md | | Plugin system (out-of-core registration) | Spiked; first CPU brick landed, not yet wired into any production path | The extensibility-first discovery layer. W0 spike over vLLM's plugin surface (general / platform / io_processor / endpoint groups, the `register_model` an out-of-tree plugin calls, the invocation seams) is committed (`.agents/specs/plugin-system.md`, `ENG-PLUGIN-SYSTEM` ACTIVE, `CLAIM-PLUGIN-SYSTEM`). W1 landed `vllm::plugins::LoadGeneralPlugins()` + the out-of-core general-plugin registration seam (`RegisterGeneralPlugin` / `REGISTER_VLLM_GENERAL_PLUGIN`) over the existing `REGISTER_VLLM_MODEL`-style registries (the in-tree factory `MODEL-FACTORY-registry` is record-repaired `DONE` 2026-08-05: 28 self-registering TUs, dgx debt paid by the 2026-07-23 seven-gate run): a 1:1 mirror of `load_general_plugins` (load-once idempotence, the `VLLM_PLUGINS` allowlist, per-plugin failure isolation). Proven by an out-of-core toy-model plugin that registers a toy architecture through the public `RegisterModel` seam — unit-gated RED-first (`test_plugin_system` 1 case / 29 assertions: the toy arch resolves ONLY after LoadGeneralPlugins runs it, and not under `VLLM_PLUGINS=""`). Python entry points have no C++20 analogue, so discovery is the project's static-init/`dlopen` registration idiom (recorded porting-inventory §9). NOT yet wired: real shared-object `dlopen` + the C-ABI `vllm_plugin_register` entry (W2), the engine/CLI `--load-plugins` wiring that calls LoadGeneralPlugins from the construction paths (W3), the platform/quant plugin kinds (W4), and the io_processor/stat_logger/endpoint groups (W5) are named residuals. See docs/BENCHMARKS.md | | Offline Batch API (JSONL file runner) | Spiked; first CPU brick landed, not yet exposed as a CLI | The offline OpenAI Batch API: read a JSONL of OpenAI-format requests, run each through the engine, write a JSONL of responses. W0 spike over vLLM's `run_batch.py` (schema, endpoint dispatch, run loop, file I/O) is committed (`.agents/specs/batch-api.md`, `SERVE-BATCH-API` ACTIVE, `CLAIM-BATCH-API`). W1 landed `RunBatch` (`RunLine`/`RunLines`/`Run`) + `RunBatchFile` — a pure orchestrator over the existing `OpenAIServingChat::create_chat_completion` (NO reimplemented generation), 1:1 with vLLM's endpoint_registry url→handler map: `/v1/chat/completions` dispatch, the `BatchResponseData`/`BatchRequestOutput` schema (`vllm-` ids, custom_id echoed), the `run_request` AllResponse/ErrorResponse/stream branches, and the unsupported-endpoint/url error rows. Unit-gated RED-first (`test_openai_run_batch` 7 cases / 80 assertions over the synthetic serving engine: ordered rows + custom_id echo + per-line BatchRequestOutput schema round-trip, a malformed line isolated into an error row so the batch continues, dispatch + 404 error rows; dropping the custom_id echo fails 9 assertions). Recorded deviation: a malformed line is isolated (batch continues) where upstream aborts the job. NOT yet exposed: the `vllm run-batch` CLI + `BatchFrontendArgs` (W2), embeddings/score/rerank dispatch (W3, rides pooling endpoints), audio transcription/translation + media fetch (W4), and http(s)/data-URL file I/O + metrics + overlapped `AsyncLLM` submission (W5) are named residuals. See docs/BENCHMARKS.md | diff --git a/scripts/agent-preflight.sh b/scripts/agent-preflight.sh index 561dc99c4..c9ad3ee65 100755 --- a/scripts/agent-preflight.sh +++ b/scripts/agent-preflight.sh @@ -56,6 +56,7 @@ done CHECKERS=( check-agent-record + check-release-binary-contract check-role-discipline claim-view check-readme-structure @@ -74,6 +75,7 @@ CHECKERS=( SUITES=( test_agent_record + test_check_release_binary_contract test_agent_role test_agent_onboard test_claim_view diff --git a/scripts/check-release-binary-contract.py b/scripts/check-release-binary-contract.py new file mode 100644 index 000000000..09521e3b7 --- /dev/null +++ b/scripts/check-release-binary-contract.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Fail closed when the accepted binary-release spike contract drifts.""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + + +BEGIN = "" +END = "" +SPEC_PATH = ".agents/specs/release-binary-matrix.md" + +IDENTITY = "ENG-RELEASE-BINARIES" +PRIMARY_CUDA_SMS = ( + "80", + "86", + "87", + "89", + "90a", + "100a", + "103a", + "110", + "120a", + "121a", +) +WORK_DEPS = { + "W1": (), + "W2": ("W1",), + "W3": (), + "W4": (), + "W5": (), + "W6": (), + "W7": ("W1", "W2", "W3", "W4", "W5", "W6"), + "W8": ("W5", "W7"), + "W9": ("W3", "W4", "W5", "W6", "W7"), + "W10": ("W1", "W2", "W5", "W6", "W7"), + "W11": ("W5", "W6", "W7"), + "W12": ("W1", "W2", "W5", "W6", "W7"), + "W13": ("W5", "W7", "W8", "W9", "W10", "W11"), +} + +ANCHORS = { + ".agents/engine-matrix.md": "| `ENG-RELEASE-BINARIES` |", + ".agents/roadmap_v1.md": "| REL | `ROAD-V1-RELEASE` |", + ".agents/NOW.md": "| Release | SPIKE | #129 |", + ".agents/coordination.md": ( + "| `CLAIM-ENG-RELEASE-BINARIES-SPIKE` | " + "`ENG-RELEASE-BINARIES` |" + ), + ".agents/state.md": ( + "## 2026-08-07 — Release matrix revised: fat CUDA and adaptive CPU " + "are the primary downloads" + ), + "docs/STATUS.md": ( + "[Release spike](../.agents/specs/release-binary-matrix.md)" + ), + "docs/BENCHMARKS.md": ( + "| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`:" + ), +} + +EXPECTED_FIELDS = { + "identity": IDENTITY, + "lifecycle": "SPIKE", + "primary_cuda_artifact": "one-fat-binary-per-os-host-abi", + "primary_cuda_sms": ",".join(PRIMARY_CUDA_SMS), + "per_sm_cuda": "optional-non-primary", + "primary_cpu_artifact": "one-adaptive-binary-per-os-host-abi", + "x86_64_baseline": "portable-sse2-without-avx2", + "work_W12_policy": "optional-non-blocking", + "archive_claims": "pending", + "runtime_claims": "pending", + "required_anchor_paths": ",".join(ANCHORS), + **{f"work_{work}": ",".join(deps) for work, deps in WORK_DEPS.items()}, +} + +WORK_ROW = re.compile(r"^\|\s*(W(?:[1-9]|1[0-3]))\s*\|\s*([^|]*)\|", re.M) + + +def parse_contract(text: str) -> tuple[dict[str, str], list[str]]: + if text.count(BEGIN) != 1 or text.count(END) != 1: + return {}, [ + f"{SPEC_PATH} must contain exactly one machine-readable release " + f"contract block ({BEGIN} ... {END})" + ] + start = text.find(BEGIN) + len(BEGIN) + end = text.find(END, start) + if end < start: + return {}, [f"{SPEC_PATH} has a malformed release contract block"] + + fields: dict[str, str] = {} + errors: list[str] = [] + for line in text[start:end].splitlines(): + stripped = line.strip() + if not stripped: + continue + if "=" not in stripped: + errors.append(f"release contract line is not key=value: {stripped!r}") + continue + key, value = stripped.split("=", 1) + if key in fields: + errors.append(f"release contract repeats field {key!r}") + fields[key] = value + return fields, errors + + +def _field_error(key: str, actual: str | None, expected: str) -> str: + names = { + "identity": "release spec identity", + "primary_cuda_artifact": "primary CUDA artifact", + "primary_cuda_sms": "primary CUDA SM set", + "per_sm_cuda": "per-SM CUDA policy", + "primary_cpu_artifact": "primary CPU artifact", + "x86_64_baseline": "x86_64 baseline", + "work_W12_policy": "W12 policy", + } + for work in WORK_DEPS: + names[f"work_{work}"] = f"{work} dependencies" + label = names.get(key, f"release contract field {key}") + return f"{label} is {actual!r}; expected {expected!r}" + + +def _normalize_deps(cell: str) -> tuple[str, ...]: + value = cell.replace("`", "").strip() + if value in {"", "—", "-", "[]"}: + return () + if value.startswith("[") and value.endswith("]"): + value = value[1:-1] + return tuple(part.strip() for part in value.split(",") if part.strip()) + + +def contract_errors(root: Path) -> list[str]: + spec = root / SPEC_PATH + if not spec.is_file(): + return [f"{SPEC_PATH} is missing"] + text = spec.read_text(encoding="utf-8") + fields, errors = parse_contract(text) + + missing = set(EXPECTED_FIELDS) - set(fields) + extra = set(fields) - set(EXPECTED_FIELDS) + if missing: + errors.append(f"release contract is missing fields: {sorted(missing)}") + if extra: + errors.append(f"release contract has unknown fields: {sorted(extra)}") + for key, expected in EXPECTED_FIELDS.items(): + if fields.get(key) != expected: + errors.append(_field_error(key, fields.get(key), expected)) + + if "Status: accepted spike for `ENG-RELEASE-BINARIES`." not in text: + errors.append( + "release spec identity/status line must name accepted spike " + "ENG-RELEASE-BINARIES" + ) + + header = "| Work | Deps | Deliverable | Exit gate |" + if header not in text: + errors.append("release work table is missing its explicit Deps column") + rows = {work: _normalize_deps(cell) for work, cell in WORK_ROW.findall(text)} + if set(rows) != set(WORK_DEPS): + errors.append( + f"release work table rows are {sorted(rows)}; expected W1-W13 exactly" + ) + for work, expected in WORK_DEPS.items(): + if rows.get(work) != expected: + errors.append( + f"{work} dependencies in work table are {rows.get(work)!r}; " + f"expected {expected!r}" + ) + + for relative, anchor in ANCHORS.items(): + path = root / relative + if not path.is_file() or anchor not in path.read_text(encoding="utf-8"): + errors.append(f"{relative} is missing required release anchor {anchor!r}") + return errors + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root", + type=Path, + default=Path(__file__).resolve().parents[1], + help="repository root (used by the mutation suite)", + ) + args = parser.parse_args(argv) + errors = contract_errors(args.root.resolve()) + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + print("Release binary contract: OK") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/scripts/test_check_release_binary_contract.py b/tests/scripts/test_check_release_binary_contract.py new file mode 100644 index 000000000..7459db2fa --- /dev/null +++ b/tests/scripts/test_check_release_binary_contract.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Mutation tests for the accepted binary-release spike contract.""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +CHECKER = ROOT / "scripts/check-release-binary-contract.py" + +CONTRACT_PATHS = ( + ".agents/specs/release-binary-matrix.md", + ".agents/engine-matrix.md", + ".agents/roadmap_v1.md", + ".agents/NOW.md", + ".agents/coordination.md", + ".agents/state.md", + "docs/STATUS.md", + "docs/BENCHMARKS.md", +) + +ANCHORS = { + ".agents/engine-matrix.md": "| `ENG-RELEASE-BINARIES` |", + ".agents/roadmap_v1.md": "| REL | `ROAD-V1-RELEASE` |", + ".agents/NOW.md": "| Release | SPIKE | #129 |", + ".agents/coordination.md": ( + "| `CLAIM-ENG-RELEASE-BINARIES-SPIKE` | " + "`ENG-RELEASE-BINARIES` |" + ), + ".agents/state.md": ( + "## 2026-08-07 — Release matrix revised: fat CUDA and adaptive CPU " + "are the primary downloads" + ), + "docs/STATUS.md": ( + "[Release spike](../.agents/specs/release-binary-matrix.md)" + ), + "docs/BENCHMARKS.md": ( + "| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`:" + ), +} + +EXPECTED_DEPS = { + "W1": "", + "W2": "W1", + "W3": "", + "W4": "", + "W5": "", + "W6": "", + "W7": "W1,W2,W3,W4,W5,W6", + "W8": "W5,W7", + "W9": "W3,W4,W5,W6,W7", + "W10": "W1,W2,W5,W6,W7", + "W11": "W5,W6,W7", + "W12": "W1,W2,W5,W6,W7", + "W13": "W5,W7,W8,W9,W10,W11", +} + + +def run_checker(root: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(CHECKER), "--root", str(root)], + capture_output=True, + text=True, + check=False, + ) + + +class RepoCopy: + def __enter__(self) -> Path: + self._tmp = tempfile.TemporaryDirectory() + root = Path(self._tmp.name) + for relative in CONTRACT_PATHS: + source = ROOT / relative + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + return root + + def __exit__(self, *unused: object) -> None: + self._tmp.cleanup() + + +def mutate(root: Path, relative: str, before: str, after: str = "") -> None: + path = root / relative + text = path.read_text(encoding="utf-8") + if before not in text: + raise AssertionError(f"mutation target missing in {relative}: {before!r}") + path.write_text(text.replace(before, after, 1), encoding="utf-8") + + +class LiveContract(unittest.TestCase): + def test_repository_contract_passes(self) -> None: + result = run_checker(ROOT) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +class AcceptedDesignMutations(unittest.TestCase): + SPEC = ".agents/specs/release-binary-matrix.md" + + def assert_mutation_fails(self, before: str, after: str, reason: str) -> None: + with RepoCopy() as root: + mutate(root, self.SPEC, before, after) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(reason, result.stdout + result.stderr) + + def test_spec_identity_is_fail_closed(self) -> None: + self.assert_mutation_fails( + "identity=ENG-RELEASE-BINARIES", + "identity=ENG-RELEASE-ARCHIVES", + "identity", + ) + + def test_each_primary_cuda_sm_is_required(self) -> None: + for sm in ("80", "86", "87", "89", "90a", "100a", "103a", "110", "120a", "121a"): + with self.subTest(sm=sm): + self.assert_mutation_fails( + "primary_cuda_sms=80,86,87,89,90a,100a,103a,110,120a,121a", + "primary_cuda_sms=" + ",".join( + value + for value in ("80", "86", "87", "89", "90a", "100a", "103a", "110", "120a", "121a") + if value != sm + ), + "primary CUDA SM set", + ) + + def test_primary_cuda_must_stay_one_fat_binary_per_host_abi(self) -> None: + self.assert_mutation_fails( + "primary_cuda_artifact=one-fat-binary-per-os-host-abi", + "primary_cuda_artifact=one-binary-per-sm", + "primary CUDA artifact", + ) + + def test_per_sm_cuda_must_not_become_primary(self) -> None: + self.assert_mutation_fails( + "per_sm_cuda=optional-non-primary", + "per_sm_cuda=primary", + "per-SM CUDA", + ) + + def test_primary_cpu_must_stay_one_adaptive_binary_per_host_abi(self) -> None: + self.assert_mutation_fails( + "primary_cpu_artifact=one-adaptive-binary-per-os-host-abi", + "primary_cpu_artifact=one-binary-per-isa", + "primary CPU artifact", + ) + + def test_x86_64_baseline_must_not_require_avx2(self) -> None: + self.assert_mutation_fails( + "x86_64_baseline=portable-sse2-without-avx2", + "x86_64_baseline=avx2-required", + "x86_64 baseline", + ) + + +class WorkGraphMutations(unittest.TestCase): + SPEC = ".agents/specs/release-binary-matrix.md" + + def test_work_table_has_explicit_deps_column(self) -> None: + with RepoCopy() as root: + mutate(root, self.SPEC, "| Work | Deps | Deliverable | Exit gate |", "| Work | Deliverable | Exit gate |") + result = run_checker(root) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Deps column", result.stdout + result.stderr) + + def test_each_work_dependency_edge_is_pinned(self) -> None: + for work, deps in EXPECTED_DEPS.items(): + with self.subTest(work=work), RepoCopy() as root: + mutate(root, self.SPEC, f"work_{work}={deps}", f"work_{work}=BROKEN") + result = run_checker(root) + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"{work} dependencies", result.stdout + result.stderr) + + def test_optional_w12_does_not_block_w13(self) -> None: + with RepoCopy() as root: + mutate(root, self.SPEC, "work_W13=W5,W7,W8,W9,W10,W11", "work_W13=W5,W7,W8,W9,W10,W11,W12") + result = run_checker(root) + self.assertNotEqual(result.returncode, 0) + self.assertIn("W13 dependencies", result.stdout + result.stderr) + + +class RecordAnchorMutations(unittest.TestCase): + def test_each_required_record_anchor_is_fail_closed(self) -> None: + for relative, anchor in ANCHORS.items(): + with self.subTest(path=relative), RepoCopy() as root: + mutate(root, relative, anchor) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(f"{relative} is missing required release anchor", result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() From 33e51d4be618e9f03840d33aef19dac260fc51fe Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 7 Aug 2026 23:14:32 +0000 Subject: [PATCH 05/11] test(release): harden accepted bundle contract Keep the public release state fail-closed at SPIKED/PENDING, bind W10 fat-CUDA and optional W12 diagnostics to the human work table, pin the adaptive-CPU/no-AVX2 prose, and require the exact 15-method mutation inventory by AST name and count. Reviewer mutations that previously stayed green now fail for their named reasons. The row remains SPIKE; no archive, runtime, correctness, or performance evidence is claimed. FOLLOWING_AGENTS_PROTOCOL Assisted-by: Codex:gpt-5 [Codex] --- docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- scripts/check-release-binary-contract.py | 166 +++++++++++++++++- .../test_check_release_binary_contract.py | 115 ++++++++++++ 4 files changed, 281 insertions(+), 4 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 80eb49cd6..71a5af62b 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -4,7 +4,7 @@ | Reference | Workload | Headline | Tokens | |---|---|---|---| -| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** contract mutation-gated (10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13 graph, record anchors); no archive, staged smoke, runtime, or release-performance evidence | n/a | +| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 15 tests. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | | **Developer agent entry point (implemented)** | `DOCS-AGENT-PROTOCOL-ENTRYPOINT`: public contribution guide + synchronized, mutation-gated pre-claim intake rule | Rebased documentation/protocol only; benchmark void | n/a | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **Record repair 2026-08-07** | `main` was red on `check-agent-record` + `check-env-doc`, blocking every PR. Dangling `kda-chunk-aot/` link and two undocumented env vars. No behaviour change | diff --git a/docs/STATUS.md b/docs/STATUS.md index 5a2e7713c..dd60c8e2b 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -112,7 +112,7 @@ token-for-token correctness against the pinned oracle. | Tool-call parsing | 36 parser families / 40 accepted names, streaming | Every vLLM tool parser at the pin except the three Rust/Harmony-backed ones: pure-text parsers ported 1:1, the six engine-backed families reimplemented from their wire formats, all held to the upstream test suites. Selection via `--tool-call-parser` (server), `tool_parser` (C ABI), or template auto-detection; native-syntax forced tool_choice where expressible. Tables: docs/BENCHMARKS.md | | Reasoning parsing (`SAMPLE-REASONING`, ACTIVE, partial coverage) | 9 parsers, streaming | think_auto (auto-detect default: content unless markers appear), deepseek_r1, deepseek_v3 (passthrough) / holo2 (thinking→R1), mistral ([THINK]), minimax_m2 (+append_think), step3, olmo3 - reasoning split engine-side BEFORE tool parsing, streamed as `reasoning` deltas in the chat chunks. Coverage: 9 of upstream's ~28 registered names (remaining text families + engine-backed adapters tracked as W2/W3 in specs/reasoning-parsers.md); each ported parser doctest-gated vs its tests/reasoning case | | Unified streaming parser engine | Core, assembly, serving-SSE dispatch landed, gated; all 10 engine-backed families ported (family parity closed); JSON-schema tool-arg type coercion landed | The vLLM 0.26 declarative `parser/engine/` (shared state machine plus all 10 configs: qwen3, seed_oss, kimi_k2, minimax_m2, glm47_moe, deepseek_v4/v32, nemotron_v3, gemma4, inkling) and assembly layer, gated field-for-field vs vLLM 0.26. An engine-backed `--tool-call-parser` name drives the live chat SSE chunks, off by default. When a request's tools declare typed parameters, the assembled tool-call arguments are coerced to the declared JSON types (int/number/bool/string/array/null) 1:1 with vLLM `_fix_arg_types`, in both streaming and one-shot; no schema means the arguments pass through as strings unchanged. Details: .agents/specs/parser-assembly-c8.md | -| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary host-ABI fat CUDA/adaptive CPU and W1-W13 dependencies are mutation-gated; per-SM diagnostics remain optional; no archive or runtime claim | +| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and 15-test inventory mutation-gated; per-SM diagnostics optional; no archive/runtime claim | | Pooling task class (embeddings / classify / score / rerank) | Spiked; pooler op + heads composite + pooling runner path landed (CPU), not yet servable end-to-end (no concrete model / endpoints) | The non-generative task class. W0 spike over the whole vLLM pooling surface (`.agents/specs/pooling-task-class.md`, `CLAIM-POOLING`). W1 landed the pooler OP (CLS/LAST/MEAN + Identity/Normalize/MultiLabelClassify/Classify activations, double-precision-gated). **W2 landed the pooler HEADS composite** — `EmbeddingPoolerHead` (projector→matryoshka→normalize), `ClassifierPoolerHead` (classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` task routing, and the `PoolerConfig`/`PoolingParams` structs (`test_pooler_heads` 27/27, 240 asserts, RED-first). **W3 landed the pooling RUNNER path** — `PoolingRunner` applies the model's `Pooler` to the last hidden state and returns pooled embeddings instead of sampled tokens, gated by a STRUCTURAL cosine-parity check vs a double-precision LAST+normalize reference (`test_pooling_runner` 5/5, 14 asserts, RED-first). NOT yet servable / honest residuals: a concrete pooling MODEL forward + the REAL-model oracle cosine gate (`vllm.LLM(task="embed").encode`) — no cosine-vs-oracle number is fabricated (W3-model); the `/v1/embeddings` + score/rerank/classify endpoints (W4); tokwise AllPool/StepPool (W5). See docs/BENCHMARKS.md | | Plugin system (out-of-core registration) | Spiked; first CPU brick landed, not yet wired into any production path | The extensibility-first discovery layer. W0 spike over vLLM's plugin surface (general / platform / io_processor / endpoint groups, the `register_model` an out-of-tree plugin calls, the invocation seams) is committed (`.agents/specs/plugin-system.md`, `ENG-PLUGIN-SYSTEM` ACTIVE, `CLAIM-PLUGIN-SYSTEM`). W1 landed `vllm::plugins::LoadGeneralPlugins()` + the out-of-core general-plugin registration seam (`RegisterGeneralPlugin` / `REGISTER_VLLM_GENERAL_PLUGIN`) over the existing `REGISTER_VLLM_MODEL`-style registries (the in-tree factory `MODEL-FACTORY-registry` is record-repaired `DONE` 2026-08-05: 28 self-registering TUs, dgx debt paid by the 2026-07-23 seven-gate run): a 1:1 mirror of `load_general_plugins` (load-once idempotence, the `VLLM_PLUGINS` allowlist, per-plugin failure isolation). Proven by an out-of-core toy-model plugin that registers a toy architecture through the public `RegisterModel` seam — unit-gated RED-first (`test_plugin_system` 1 case / 29 assertions: the toy arch resolves ONLY after LoadGeneralPlugins runs it, and not under `VLLM_PLUGINS=""`). Python entry points have no C++20 analogue, so discovery is the project's static-init/`dlopen` registration idiom (recorded porting-inventory §9). NOT yet wired: real shared-object `dlopen` + the C-ABI `vllm_plugin_register` entry (W2), the engine/CLI `--load-plugins` wiring that calls LoadGeneralPlugins from the construction paths (W3), the platform/quant plugin kinds (W4), and the io_processor/stat_logger/endpoint groups (W5) are named residuals. See docs/BENCHMARKS.md | | Offline Batch API (JSONL file runner) | Spiked; first CPU brick landed, not yet exposed as a CLI | The offline OpenAI Batch API: read a JSONL of OpenAI-format requests, run each through the engine, write a JSONL of responses. W0 spike over vLLM's `run_batch.py` (schema, endpoint dispatch, run loop, file I/O) is committed (`.agents/specs/batch-api.md`, `SERVE-BATCH-API` ACTIVE, `CLAIM-BATCH-API`). W1 landed `RunBatch` (`RunLine`/`RunLines`/`Run`) + `RunBatchFile` — a pure orchestrator over the existing `OpenAIServingChat::create_chat_completion` (NO reimplemented generation), 1:1 with vLLM's endpoint_registry url→handler map: `/v1/chat/completions` dispatch, the `BatchResponseData`/`BatchRequestOutput` schema (`vllm-` ids, custom_id echoed), the `run_request` AllResponse/ErrorResponse/stream branches, and the unsupported-endpoint/url error rows. Unit-gated RED-first (`test_openai_run_batch` 7 cases / 80 assertions over the synthetic serving engine: ordered rows + custom_id echo + per-line BatchRequestOutput schema round-trip, a malformed line isolated into an error row so the batch continues, dispatch + 404 error rows; dropping the custom_id echo fails 9 assertions). Recorded deviation: a malformed line is isolated (batch continues) where upstream aborts the job. NOT yet exposed: the `vllm run-batch` CLI + `BatchFrontendArgs` (W2), embeddings/score/rerank dispatch (W3, rides pooling endpoints), audio transcription/translation + media fetch (W4), and http(s)/data-URL file I/O + metrics + overlapped `AsyncLLM` submission (W5) are named residuals. See docs/BENCHMARKS.md | diff --git a/scripts/check-release-binary-contract.py b/scripts/check-release-binary-contract.py index 09521e3b7..5e0fcc91a 100644 --- a/scripts/check-release-binary-contract.py +++ b/scripts/check-release-binary-contract.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import ast import re import sys from pathlib import Path @@ -12,6 +13,7 @@ BEGIN = "" END = "" SPEC_PATH = ".agents/specs/release-binary-matrix.md" +TEST_PATH = "tests/scripts/test_check_release_binary_contract.py" IDENTITY = "ENG-RELEASE-BINARIES" PRIMARY_CUDA_SMS = ( @@ -62,6 +64,68 @@ ), } +BENCHMARKS_RELEASE_ROW = ( + "| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary " + "host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM " + "diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM " + "fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, " + "and 15 tests. No archive, staged smoke, runtime, correctness, or performance " + "evidence " + "| n/a |" +) + +STATUS_RELEASE_FRAGMENTS = ( + "Supported subset; bundles SPIKED, no artifacts", + "primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and " + "15-test inventory mutation-gated; per-SM diagnostics optional; no " + "archive/runtime claim", +) + +HUMAN_CONTRACT = { + "human primary CPU/CUDA contract": ( + "The primary CPU download is one conservative-baseline, runtime-adaptive " + "binary per OS+host ABI; the primary CUDA download is one fat binary per " + "OS+host ABI containing every supported SM." + ), + "human x86_64 no-AVX2 contract": ( + "For x86_64, the baseline must run without AVX2: portable/SSE2 code " + "remains callable, and higher instructions live only in per-function or " + "per-TU tiers." + ), +} + +WORK_CONTENT = { + "W10": ( + "primary Linux CUDA fat bundles for x86_64 and aarch64 host ABIs", + "each extracted archive contains all ten SMs and six exact AOT trees; " + "per-SM evidence remains independent; no host ABI is inferred from the " + "other", + ), + "W12": ( + "optional single-SM CUDA diagnostic/performance variants", + "generated from the same explicit matrix and evidence; never advertised " + "as the primary KISS download or used to bypass W10", + ), +} + +REQUIRED_TEST_METHODS = ( + "test_repository_contract_passes", + "test_spec_identity_is_fail_closed", + "test_each_primary_cuda_sm_is_required", + "test_primary_cuda_must_stay_one_fat_binary_per_host_abi", + "test_per_sm_cuda_must_not_become_primary", + "test_primary_cpu_must_stay_one_adaptive_binary_per_host_abi", + "test_x86_64_baseline_must_not_require_avx2", + "test_work_table_has_explicit_deps_column", + "test_each_work_dependency_edge_is_pinned", + "test_optional_w12_does_not_block_w13", + "test_each_required_record_anchor_is_fail_closed", + "test_public_release_rows_remain_pending", + "test_human_w12_is_optional_and_cannot_replace_w10", + "test_human_primary_artifact_contract_matches_machine_block", + "test_required_mutation_test_inventory_is_pinned", +) + EXPECTED_FIELDS = { "identity": IDENTITY, "lifecycle": "SPIKE", @@ -77,7 +141,10 @@ **{f"work_{work}": ",".join(deps) for work, deps in WORK_DEPS.items()}, } -WORK_ROW = re.compile(r"^\|\s*(W(?:[1-9]|1[0-3]))\s*\|\s*([^|]*)\|", re.M) +WORK_ROW = re.compile( + r"^\|\s*(W(?:[1-9]|1[0-3]))\s*\|\s*([^|]*)\|\s*([^|]*)\|\s*([^|]*)\|", + re.M, +) def parse_contract(text: str) -> tuple[dict[str, str], list[str]]: @@ -132,6 +199,56 @@ def _normalize_deps(cell: str) -> tuple[str, ...]: return tuple(part.strip() for part in value.split(",") if part.strip()) +def _normalize_prose(text: str) -> str: + return re.sub(r"\s+", " ", text).strip() + + +def _test_inventory_errors(root: Path) -> list[str]: + path = root / TEST_PATH + if not path.is_file(): + return [f"required mutation-test inventory file {TEST_PATH} is missing"] + try: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + except SyntaxError as error: + return [f"required mutation-test inventory is not valid Python: {error}"] + + methods = [ + node.name + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name.startswith("test_") + ] + expected = set(REQUIRED_TEST_METHODS) + actual = set(methods) + errors: list[str] = [] + if len(methods) != len(REQUIRED_TEST_METHODS) or actual != expected: + errors.append( + "required mutation-test inventory drifted: " + f"expected {len(REQUIRED_TEST_METHODS)} named methods " + f"{sorted(expected)}, found {len(methods)} {sorted(methods)}" + ) + + declarations = [ + node + for node in tree.body + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "REQUIRED_TEST_METHODS" + for target in node.targets + ) + ] + try: + declared = tuple(ast.literal_eval(declarations[0].value)) + except (IndexError, TypeError, ValueError, SyntaxError): + declared = () + if len(declarations) != 1 or declared != REQUIRED_TEST_METHODS: + errors.append( + "required mutation-test inventory declaration does not match the " + "checker's exact count and names" + ) + return errors + + def contract_errors(root: Path) -> list[str]: spec = root / SPEC_PATH if not spec.is_file(): @@ -158,7 +275,8 @@ def contract_errors(root: Path) -> list[str]: header = "| Work | Deps | Deliverable | Exit gate |" if header not in text: errors.append("release work table is missing its explicit Deps column") - rows = {work: _normalize_deps(cell) for work, cell in WORK_ROW.findall(text)} + parsed_rows = WORK_ROW.findall(text) + rows = {work: _normalize_deps(deps) for work, deps, _, _ in parsed_rows} if set(rows) != set(WORK_DEPS): errors.append( f"release work table rows are {sorted(rows)}; expected W1-W13 exactly" @@ -170,10 +288,54 @@ def contract_errors(root: Path) -> list[str]: f"expected {expected!r}" ) + work_content = { + work: (_normalize_prose(deliverable), _normalize_prose(exit_gate)) + for work, _, deliverable, exit_gate in parsed_rows + } + for work, (deliverable, exit_gate) in WORK_CONTENT.items(): + actual = work_content.get(work) + if actual is None or actual[0] != deliverable: + errors.append( + f"{work} deliverable is {None if actual is None else actual[0]!r}; " + f"expected {deliverable!r}" + ) + if actual is None or actual[1] != exit_gate: + errors.append( + f"{work} exit gate is {None if actual is None else actual[1]!r}; " + f"expected {exit_gate!r}" + ) + + normalized_spec = _normalize_prose(text) + for label, statement in HUMAN_CONTRACT.items(): + if statement not in normalized_spec: + errors.append( + f"{label} must match the accepted machine-readable release contract" + ) + for relative, anchor in ANCHORS.items(): path = root / relative if not path.is_file() or anchor not in path.read_text(encoding="utf-8"): errors.append(f"{relative} is missing required release anchor {anchor!r}") + benchmarks = root / "docs/BENCHMARKS.md" + if not benchmarks.is_file() or BENCHMARKS_RELEASE_ROW not in benchmarks.read_text( + encoding="utf-8" + ): + errors.append( + "docs/BENCHMARKS.md release row must stay PENDING with no archive, " + "runtime, correctness, or performance evidence" + ) + status = root / "docs/STATUS.md" + status_text = status.read_text(encoding="utf-8") if status.is_file() else "" + status_row = next( + (line for line in status_text.splitlines() if line.startswith("| OpenAI server |")), + "", + ) + if not all(fragment in status_row for fragment in STATUS_RELEASE_FRAGMENTS): + errors.append( + "docs/STATUS.md release row must stay SPIKED with no artifacts and no " + "runtime claim" + ) + errors.extend(_test_inventory_errors(root)) return errors diff --git a/tests/scripts/test_check_release_binary_contract.py b/tests/scripts/test_check_release_binary_contract.py index 7459db2fa..db4f0cb28 100644 --- a/tests/scripts/test_check_release_binary_contract.py +++ b/tests/scripts/test_check_release_binary_contract.py @@ -3,6 +3,7 @@ from __future__ import annotations +import ast import shutil import subprocess import sys @@ -23,6 +24,25 @@ ".agents/state.md", "docs/STATUS.md", "docs/BENCHMARKS.md", + "tests/scripts/test_check_release_binary_contract.py", +) + +REQUIRED_TEST_METHODS = ( + "test_repository_contract_passes", + "test_spec_identity_is_fail_closed", + "test_each_primary_cuda_sm_is_required", + "test_primary_cuda_must_stay_one_fat_binary_per_host_abi", + "test_per_sm_cuda_must_not_become_primary", + "test_primary_cpu_must_stay_one_adaptive_binary_per_host_abi", + "test_x86_64_baseline_must_not_require_avx2", + "test_work_table_has_explicit_deps_column", + "test_each_work_dependency_edge_is_pinned", + "test_optional_w12_does_not_block_w13", + "test_each_required_record_anchor_is_fail_closed", + "test_public_release_rows_remain_pending", + "test_human_w12_is_optional_and_cannot_replace_w10", + "test_human_primary_artifact_contract_matches_machine_block", + "test_required_mutation_test_inventory_is_pinned", ) ANCHORS = { @@ -94,6 +114,27 @@ def mutate(root: Path, relative: str, before: str, after: str = "") -> None: path.write_text(text.replace(before, after, 1), encoding="utf-8") +def delete_test_method(root: Path, method: str) -> None: + relative = "tests/scripts/test_check_release_binary_contract.py" + path = root / relative + text = path.read_text(encoding="utf-8") + lines = text.splitlines(keepends=True) + matches = [ + node + for node in ast.walk(ast.parse(text)) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == method + ] + if len(matches) != 1: + raise AssertionError(f"expected one method {method!r}, found {len(matches)}") + node = matches[0] + start = min( + [node.lineno, *(decorator.lineno for decorator in node.decorator_list)] + ) - 1 + del lines[start : node.end_lineno] + path.write_text("".join(lines), encoding="utf-8") + + class LiveContract(unittest.TestCase): def test_repository_contract_passes(self) -> None: result = run_checker(ROOT) @@ -195,5 +236,79 @@ def test_each_required_record_anchor_is_fail_closed(self) -> None: self.assertIn(f"{relative} is missing required release anchor", result.stdout + result.stderr) +class HumanContractMutations(unittest.TestCase): + SPEC = ".agents/specs/release-binary-matrix.md" + + def test_public_release_rows_remain_pending(self) -> None: + mutations = ( + ( + "docs/BENCHMARKS.md", + "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 15 tests. No archive, staged smoke, runtime, correctness, or performance evidence", + "**SHIPPED:** archive, runtime, correctness, and performance evidence complete", + "docs/BENCHMARKS.md release row", + ), + ( + "docs/STATUS.md", + "Supported subset; bundles SPIKED, no artifacts", + "Supported; bundles SHIPPED with runtime evidence", + "docs/STATUS.md release row", + ), + ) + for relative, before, after, reason in mutations: + with self.subTest(path=relative), RepoCopy() as root: + mutate(root, relative, before, after) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(reason, result.stdout + result.stderr) + + def test_human_w12_is_optional_and_cannot_replace_w10(self) -> None: + mutations = ( + ( + "optional single-SM CUDA diagnostic/performance variants", + "required primary single-SM CUDA release variants replacing W10", + "W12 deliverable", + ), + ( + "generated from the same explicit matrix and evidence; never advertised as the primary KISS download or used to bypass W10", + "the primary KISS download; W10 may be bypassed", + "W12 exit gate", + ), + ) + for before, after, reason in mutations: + with self.subTest(reason=reason), RepoCopy() as root: + mutate(root, self.SPEC, before, after) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(reason, result.stdout + result.stderr) + + def test_human_primary_artifact_contract_matches_machine_block(self) -> None: + mutations = ( + ( + "The primary CPU download is\none conservative-baseline, runtime-adaptive binary per OS+host ABI; the primary\nCUDA download is one fat binary per OS+host ABI containing every supported SM.", + "The primary CPU download is one binary per ISA; the primary CUDA download is one binary per SM.", + "human primary CPU/CUDA contract", + ), + ( + "For x86_64, the baseline must run without AVX2: portable/SSE2 code remains\ncallable, and higher instructions live only in per-function or per-TU tiers.", + "For x86_64, AVX2 is required by the baseline.", + "human x86_64 no-AVX2 contract", + ), + ) + for before, after, reason in mutations: + with self.subTest(reason=reason), RepoCopy() as root: + mutate(root, self.SPEC, before, after) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(reason, result.stdout + result.stderr) + + def test_required_mutation_test_inventory_is_pinned(self) -> None: + for method in REQUIRED_TEST_METHODS: + with self.subTest(method=method), RepoCopy() as root: + delete_test_method(root, method) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("required mutation-test inventory", result.stdout + result.stderr) + + if __name__ == "__main__": unittest.main() From ac68422b373a904bb1aaeaeee3c79c39db1d4831 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 7 Aug 2026 23:28:08 +0000 Subject: [PATCH 06/11] test(release): bind semantic mutation inventories Pin independently-authored literal mutation datasets for all ten CUDA SMs, the W1-W13 dependency graph, record anchors, public pending states, W10/W12 policy, and the primary fat-CUDA/adaptive-CPU no-AVX2 prose. AST checks now require each named test to consume its corresponding inventory, so retained-but-empty mutation loops fail closed. Red: removing 121a from the SM loop and changing W2 dependencies to empty each left the checker and all 15 tests green. Green: all ten reviewer mutations fail for their named reason, the checker passes, and 18/18 tests plus full and staged preflight pass. The row remains SPIKE/PENDING; no release artifact or runtime evidence is claimed. FOLLOWING_AGENTS_PROTOCOL Assisted-by: Codex:gpt-5 [Codex] --- docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- scripts/check-release-binary-contract.py | 198 +++++++++++++++-- .../test_check_release_binary_contract.py | 207 +++++++++++++----- 4 files changed, 339 insertions(+), 70 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 71a5af62b..7037b3884 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -4,7 +4,7 @@ | Reference | Workload | Headline | Tokens | |---|---|---|---| -| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 15 tests. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | +| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 18 tests. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | | **Developer agent entry point (implemented)** | `DOCS-AGENT-PROTOCOL-ENTRYPOINT`: public contribution guide + synchronized, mutation-gated pre-claim intake rule | Rebased documentation/protocol only; benchmark void | n/a | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **Record repair 2026-08-07** | `main` was red on `check-agent-record` + `check-env-doc`, blocking every PR. Dangling `kda-chunk-aot/` link and two undocumented env vars. No behaviour change | diff --git a/docs/STATUS.md b/docs/STATUS.md index dd60c8e2b..fd67bfaee 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -112,7 +112,7 @@ token-for-token correctness against the pinned oracle. | Tool-call parsing | 36 parser families / 40 accepted names, streaming | Every vLLM tool parser at the pin except the three Rust/Harmony-backed ones: pure-text parsers ported 1:1, the six engine-backed families reimplemented from their wire formats, all held to the upstream test suites. Selection via `--tool-call-parser` (server), `tool_parser` (C ABI), or template auto-detection; native-syntax forced tool_choice where expressible. Tables: docs/BENCHMARKS.md | | Reasoning parsing (`SAMPLE-REASONING`, ACTIVE, partial coverage) | 9 parsers, streaming | think_auto (auto-detect default: content unless markers appear), deepseek_r1, deepseek_v3 (passthrough) / holo2 (thinking→R1), mistral ([THINK]), minimax_m2 (+append_think), step3, olmo3 - reasoning split engine-side BEFORE tool parsing, streamed as `reasoning` deltas in the chat chunks. Coverage: 9 of upstream's ~28 registered names (remaining text families + engine-backed adapters tracked as W2/W3 in specs/reasoning-parsers.md); each ported parser doctest-gated vs its tests/reasoning case | | Unified streaming parser engine | Core, assembly, serving-SSE dispatch landed, gated; all 10 engine-backed families ported (family parity closed); JSON-schema tool-arg type coercion landed | The vLLM 0.26 declarative `parser/engine/` (shared state machine plus all 10 configs: qwen3, seed_oss, kimi_k2, minimax_m2, glm47_moe, deepseek_v4/v32, nemotron_v3, gemma4, inkling) and assembly layer, gated field-for-field vs vLLM 0.26. An engine-backed `--tool-call-parser` name drives the live chat SSE chunks, off by default. When a request's tools declare typed parameters, the assembled tool-call arguments are coerced to the declared JSON types (int/number/bool/string/array/null) 1:1 with vLLM `_fix_arg_types`, in both streaming and one-shot; no schema means the arguments pass through as strings unchanged. Details: .agents/specs/parser-assembly-c8.md | -| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and 15-test inventory mutation-gated; per-SM diagnostics optional; no archive/runtime claim | +| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and 18-test inventory mutation-gated; per-SM diagnostics optional; no archive/runtime claim | | Pooling task class (embeddings / classify / score / rerank) | Spiked; pooler op + heads composite + pooling runner path landed (CPU), not yet servable end-to-end (no concrete model / endpoints) | The non-generative task class. W0 spike over the whole vLLM pooling surface (`.agents/specs/pooling-task-class.md`, `CLAIM-POOLING`). W1 landed the pooler OP (CLS/LAST/MEAN + Identity/Normalize/MultiLabelClassify/Classify activations, double-precision-gated). **W2 landed the pooler HEADS composite** — `EmbeddingPoolerHead` (projector→matryoshka→normalize), `ClassifierPoolerHead` (classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` task routing, and the `PoolerConfig`/`PoolingParams` structs (`test_pooler_heads` 27/27, 240 asserts, RED-first). **W3 landed the pooling RUNNER path** — `PoolingRunner` applies the model's `Pooler` to the last hidden state and returns pooled embeddings instead of sampled tokens, gated by a STRUCTURAL cosine-parity check vs a double-precision LAST+normalize reference (`test_pooling_runner` 5/5, 14 asserts, RED-first). NOT yet servable / honest residuals: a concrete pooling MODEL forward + the REAL-model oracle cosine gate (`vllm.LLM(task="embed").encode`) — no cosine-vs-oracle number is fabricated (W3-model); the `/v1/embeddings` + score/rerank/classify endpoints (W4); tokwise AllPool/StepPool (W5). See docs/BENCHMARKS.md | | Plugin system (out-of-core registration) | Spiked; first CPU brick landed, not yet wired into any production path | The extensibility-first discovery layer. W0 spike over vLLM's plugin surface (general / platform / io_processor / endpoint groups, the `register_model` an out-of-tree plugin calls, the invocation seams) is committed (`.agents/specs/plugin-system.md`, `ENG-PLUGIN-SYSTEM` ACTIVE, `CLAIM-PLUGIN-SYSTEM`). W1 landed `vllm::plugins::LoadGeneralPlugins()` + the out-of-core general-plugin registration seam (`RegisterGeneralPlugin` / `REGISTER_VLLM_GENERAL_PLUGIN`) over the existing `REGISTER_VLLM_MODEL`-style registries (the in-tree factory `MODEL-FACTORY-registry` is record-repaired `DONE` 2026-08-05: 28 self-registering TUs, dgx debt paid by the 2026-07-23 seven-gate run): a 1:1 mirror of `load_general_plugins` (load-once idempotence, the `VLLM_PLUGINS` allowlist, per-plugin failure isolation). Proven by an out-of-core toy-model plugin that registers a toy architecture through the public `RegisterModel` seam — unit-gated RED-first (`test_plugin_system` 1 case / 29 assertions: the toy arch resolves ONLY after LoadGeneralPlugins runs it, and not under `VLLM_PLUGINS=""`). Python entry points have no C++20 analogue, so discovery is the project's static-init/`dlopen` registration idiom (recorded porting-inventory §9). NOT yet wired: real shared-object `dlopen` + the C-ABI `vllm_plugin_register` entry (W2), the engine/CLI `--load-plugins` wiring that calls LoadGeneralPlugins from the construction paths (W3), the platform/quant plugin kinds (W4), and the io_processor/stat_logger/endpoint groups (W5) are named residuals. See docs/BENCHMARKS.md | | Offline Batch API (JSONL file runner) | Spiked; first CPU brick landed, not yet exposed as a CLI | The offline OpenAI Batch API: read a JSONL of OpenAI-format requests, run each through the engine, write a JSONL of responses. W0 spike over vLLM's `run_batch.py` (schema, endpoint dispatch, run loop, file I/O) is committed (`.agents/specs/batch-api.md`, `SERVE-BATCH-API` ACTIVE, `CLAIM-BATCH-API`). W1 landed `RunBatch` (`RunLine`/`RunLines`/`Run`) + `RunBatchFile` — a pure orchestrator over the existing `OpenAIServingChat::create_chat_completion` (NO reimplemented generation), 1:1 with vLLM's endpoint_registry url→handler map: `/v1/chat/completions` dispatch, the `BatchResponseData`/`BatchRequestOutput` schema (`vllm-` ids, custom_id echoed), the `run_request` AllResponse/ErrorResponse/stream branches, and the unsupported-endpoint/url error rows. Unit-gated RED-first (`test_openai_run_batch` 7 cases / 80 assertions over the synthetic serving engine: ordered rows + custom_id echo + per-line BatchRequestOutput schema round-trip, a malformed line isolated into an error row so the batch continues, dispatch + 404 error rows; dropping the custom_id echo fails 9 assertions). Recorded deviation: a malformed line is isolated (batch continues) where upstream aborts the job. NOT yet exposed: the `vllm run-batch` CLI + `BatchFrontendArgs` (W2), embeddings/score/rerank dispatch (W3, rides pooling endpoints), audio transcription/translation + media fetch (W4), and http(s)/data-URL file I/O + metrics + overlapped `AsyncLLM` submission (W5) are named residuals. See docs/BENCHMARKS.md | diff --git a/scripts/check-release-binary-contract.py b/scripts/check-release-binary-contract.py index 5e0fcc91a..325289f94 100644 --- a/scripts/check-release-binary-contract.py +++ b/scripts/check-release-binary-contract.py @@ -69,7 +69,7 @@ "host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM " "diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM " "fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, " - "and 15 tests. No archive, staged smoke, runtime, correctness, or performance " + "and 18 tests. No archive, staged smoke, runtime, correctness, or performance " "evidence " "| n/a |" ) @@ -77,7 +77,7 @@ STATUS_RELEASE_FRAGMENTS = ( "Supported subset; bundles SPIKED, no artifacts", "primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and " - "15-test inventory mutation-gated; per-SM diagnostics optional; no " + "18-test inventory mutation-gated; per-SM diagnostics optional; no " "archive/runtime claim", ) @@ -108,6 +108,56 @@ ), } +PUBLIC_PENDING_MUTATIONS = ( + ( + "docs/BENCHMARKS.md", + "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " + "W1-W13/W10-W12 policy, public pending states, and 18 tests. No archive, " + "staged smoke, runtime, correctness, or performance evidence", + "**SHIPPED:** archive, runtime, correctness, and performance evidence " + "complete", + "docs/BENCHMARKS.md release row", + ), + ( + "docs/STATUS.md", + "Supported subset; bundles SPIKED, no artifacts", + "Supported; bundles SHIPPED with runtime evidence", + "docs/STATUS.md release row", + ), +) + +W10_W12_HUMAN_MUTATIONS = ( + ( + "optional single-SM CUDA diagnostic/performance variants", + "required primary single-SM CUDA release variants replacing W10", + "W12 deliverable", + ), + ( + "generated from the same explicit matrix and evidence; never advertised " + "as the primary KISS download or used to bypass W10", + "the primary KISS download; W10 may be bypassed", + "W12 exit gate", + ), +) + +PRIMARY_ARTIFACT_PROSE_MUTATIONS = ( + ( + "The primary CPU download is\none conservative-baseline, runtime-adaptive " + "binary per OS+host ABI; the primary\nCUDA download is one fat binary per " + "OS+host ABI containing every supported SM.", + "The primary CPU download is one binary per ISA; the primary CUDA " + "download is one binary per SM.", + "human primary CPU/CUDA contract", + ), + ( + "For x86_64, the baseline must run without AVX2: portable/SSE2 code " + "remains\ncallable, and higher instructions live only in per-function or " + "per-TU tiers.", + "For x86_64, AVX2 is required by the baseline.", + "human x86_64 no-AVX2 contract", + ), +) + REQUIRED_TEST_METHODS = ( "test_repository_contract_passes", "test_spec_identity_is_fail_closed", @@ -123,9 +173,54 @@ "test_public_release_rows_remain_pending", "test_human_w12_is_optional_and_cannot_replace_w10", "test_human_primary_artifact_contract_matches_machine_block", + "test_primary_cuda_mutation_inventory_literal_is_pinned", + "test_work_dependency_mutation_inventory_literal_is_pinned", + "test_each_semantic_inventory_consumer_is_pinned", "test_required_mutation_test_inventory_is_pinned", ) +TEST_LITERAL_INVENTORIES = { + "PRIMARY_CUDA_SMS": PRIMARY_CUDA_SMS, + "EXPECTED_DEPS": {work: ",".join(deps) for work, deps in WORK_DEPS.items()}, + "RECORD_ANCHORS": ANCHORS, + "PUBLIC_PENDING_MUTATIONS": PUBLIC_PENDING_MUTATIONS, + "W10_W12_HUMAN_MUTATIONS": W10_W12_HUMAN_MUTATIONS, + "PRIMARY_ARTIFACT_PROSE_MUTATIONS": PRIMARY_ARTIFACT_PROSE_MUTATIONS, +} + +TEST_INVENTORY_CONSUMERS = { + "PRIMARY_CUDA_SMS": ( + "test_each_primary_cuda_sm_is_required", + ("sm",), + False, + ), + "EXPECTED_DEPS": ( + "test_each_work_dependency_edge_is_pinned", + ("work", "deps"), + True, + ), + "RECORD_ANCHORS": ( + "test_each_required_record_anchor_is_fail_closed", + ("relative", "anchor"), + True, + ), + "PUBLIC_PENDING_MUTATIONS": ( + "test_public_release_rows_remain_pending", + ("relative", "before", "after", "reason"), + False, + ), + "W10_W12_HUMAN_MUTATIONS": ( + "test_human_w12_is_optional_and_cannot_replace_w10", + ("before", "after", "reason"), + False, + ), + "PRIMARY_ARTIFACT_PROSE_MUTATIONS": ( + "test_human_primary_artifact_contract_matches_machine_block", + ("before", "after", "reason"), + False, + ), +} + EXPECTED_FIELDS = { "identity": IDENTITY, "lifecycle": "SPIKE", @@ -203,6 +298,60 @@ def _normalize_prose(text: str) -> str: return re.sub(r"\s+", " ", text).strip() +def _top_level_literal(tree: ast.Module, name: str) -> tuple[object, int]: + declarations = [ + node + for node in tree.body + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == name + for target in node.targets + ) + ] + if len(declarations) != 1: + return None, len(declarations) + try: + return ast.literal_eval(declarations[0].value), 1 + except (TypeError, ValueError, SyntaxError): + return None, 1 + + +def _target_names(target: ast.expr) -> tuple[str, ...]: + if isinstance(target, ast.Name): + return (target.id,) + if isinstance(target, (ast.Tuple, ast.List)) and all( + isinstance(item, ast.Name) for item in target.elts + ): + return tuple(item.id for item in target.elts) + return () + + +def _iterates_inventory( + method: ast.FunctionDef | ast.AsyncFunctionDef, + inventory: str, + target_names: tuple[str, ...], + mapping_items: bool, +) -> bool: + for node in ast.walk(method): + if not isinstance(node, ast.For) or _target_names(node.target) != target_names: + continue + if not mapping_items and isinstance(node.iter, ast.Name): + if node.iter.id == inventory: + return True + if ( + mapping_items + and isinstance(node.iter, ast.Call) + and not node.iter.args + and not node.iter.keywords + and isinstance(node.iter.func, ast.Attribute) + and node.iter.func.attr == "items" + and isinstance(node.iter.func.value, ast.Name) + and node.iter.func.value.id == inventory + ): + return True + return False + + def _test_inventory_errors(root: Path) -> list[str]: path = root / TEST_PATH if not path.is_file(): @@ -212,12 +361,13 @@ def _test_inventory_errors(root: Path) -> list[str]: except SyntaxError as error: return [f"required mutation-test inventory is not valid Python: {error}"] - methods = [ - node.name + method_nodes = [ + node for node in ast.walk(tree) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test_") ] + methods = [node.name for node in method_nodes] expected = set(REQUIRED_TEST_METHODS) actual = set(methods) errors: list[str] = [] @@ -228,24 +378,36 @@ def _test_inventory_errors(root: Path) -> list[str]: f"{sorted(expected)}, found {len(methods)} {sorted(methods)}" ) - declarations = [ - node - for node in tree.body - if isinstance(node, ast.Assign) - and any( - isinstance(target, ast.Name) and target.id == "REQUIRED_TEST_METHODS" - for target in node.targets - ) - ] - try: - declared = tuple(ast.literal_eval(declarations[0].value)) - except (IndexError, TypeError, ValueError, SyntaxError): - declared = () - if len(declarations) != 1 or declared != REQUIRED_TEST_METHODS: + declared, declaration_count = _top_level_literal(tree, "REQUIRED_TEST_METHODS") + if declaration_count != 1 or declared != REQUIRED_TEST_METHODS: errors.append( "required mutation-test inventory declaration does not match the " "checker's exact count and names" ) + + for inventory, expected_value in TEST_LITERAL_INVENTORIES.items(): + declared_value, count = _top_level_literal(tree, inventory) + if count != 1 or declared_value != expected_value: + errors.append( + f"semantic mutation inventory {inventory} must be one explicit " + "top-level literal matching the checker's independent production " + f"contract; found {declared_value!r}" + ) + + methods_by_name = {node.name: node for node in method_nodes} + for inventory, ( + method_name, + target_names, + mapping_items, + ) in TEST_INVENTORY_CONSUMERS.items(): + method = methods_by_name.get(method_name) + if method is None or not _iterates_inventory( + method, inventory, target_names, mapping_items + ): + errors.append( + f"semantic mutation inventory {inventory} is not consumed by " + f"{method_name} through its required literal iteration" + ) return errors diff --git a/tests/scripts/test_check_release_binary_contract.py b/tests/scripts/test_check_release_binary_contract.py index db4f0cb28..dc999f779 100644 --- a/tests/scripts/test_check_release_binary_contract.py +++ b/tests/scripts/test_check_release_binary_contract.py @@ -42,10 +42,26 @@ "test_public_release_rows_remain_pending", "test_human_w12_is_optional_and_cannot_replace_w10", "test_human_primary_artifact_contract_matches_machine_block", + "test_primary_cuda_mutation_inventory_literal_is_pinned", + "test_work_dependency_mutation_inventory_literal_is_pinned", + "test_each_semantic_inventory_consumer_is_pinned", "test_required_mutation_test_inventory_is_pinned", ) -ANCHORS = { +PRIMARY_CUDA_SMS = ( + "80", + "86", + "87", + "89", + "90a", + "100a", + "103a", + "110", + "120a", + "121a", +) + +RECORD_ANCHORS = { ".agents/engine-matrix.md": "| `ENG-RELEASE-BINARIES` |", ".agents/roadmap_v1.md": "| REL | `ROAD-V1-RELEASE` |", ".agents/NOW.md": "| Release | SPIKE | #129 |", @@ -81,6 +97,56 @@ "W13": "W5,W7,W8,W9,W10,W11", } +PUBLIC_PENDING_MUTATIONS = ( + ( + "docs/BENCHMARKS.md", + "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " + "W1-W13/W10-W12 policy, public pending states, and 18 tests. No archive, " + "staged smoke, runtime, correctness, or performance evidence", + "**SHIPPED:** archive, runtime, correctness, and performance evidence " + "complete", + "docs/BENCHMARKS.md release row", + ), + ( + "docs/STATUS.md", + "Supported subset; bundles SPIKED, no artifacts", + "Supported; bundles SHIPPED with runtime evidence", + "docs/STATUS.md release row", + ), +) + +W10_W12_HUMAN_MUTATIONS = ( + ( + "optional single-SM CUDA diagnostic/performance variants", + "required primary single-SM CUDA release variants replacing W10", + "W12 deliverable", + ), + ( + "generated from the same explicit matrix and evidence; never advertised " + "as the primary KISS download or used to bypass W10", + "the primary KISS download; W10 may be bypassed", + "W12 exit gate", + ), +) + +PRIMARY_ARTIFACT_PROSE_MUTATIONS = ( + ( + "The primary CPU download is\none conservative-baseline, runtime-adaptive " + "binary per OS+host ABI; the primary\nCUDA download is one fat binary per " + "OS+host ABI containing every supported SM.", + "The primary CPU download is one binary per ISA; the primary CUDA " + "download is one binary per SM.", + "human primary CPU/CUDA contract", + ), + ( + "For x86_64, the baseline must run without AVX2: portable/SSE2 code " + "remains\ncallable, and higher instructions live only in per-function or " + "per-TU tiers.", + "For x86_64, AVX2 is required by the baseline.", + "human x86_64 no-AVX2 contract", + ), +) + def run_checker(root: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( @@ -159,14 +225,12 @@ def test_spec_identity_is_fail_closed(self) -> None: ) def test_each_primary_cuda_sm_is_required(self) -> None: - for sm in ("80", "86", "87", "89", "90a", "100a", "103a", "110", "120a", "121a"): + for sm in PRIMARY_CUDA_SMS: with self.subTest(sm=sm): self.assert_mutation_fails( "primary_cuda_sms=80,86,87,89,90a,100a,103a,110,120a,121a", "primary_cuda_sms=" + ",".join( - value - for value in ("80", "86", "87", "89", "90a", "100a", "103a", "110", "120a", "121a") - if value != sm + value for value in PRIMARY_CUDA_SMS if value != sm ), "primary CUDA SM set", ) @@ -228,7 +292,7 @@ def test_optional_w12_does_not_block_w13(self) -> None: class RecordAnchorMutations(unittest.TestCase): def test_each_required_record_anchor_is_fail_closed(self) -> None: - for relative, anchor in ANCHORS.items(): + for relative, anchor in RECORD_ANCHORS.items(): with self.subTest(path=relative), RepoCopy() as root: mutate(root, relative, anchor) result = run_checker(root) @@ -240,21 +304,7 @@ class HumanContractMutations(unittest.TestCase): SPEC = ".agents/specs/release-binary-matrix.md" def test_public_release_rows_remain_pending(self) -> None: - mutations = ( - ( - "docs/BENCHMARKS.md", - "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 15 tests. No archive, staged smoke, runtime, correctness, or performance evidence", - "**SHIPPED:** archive, runtime, correctness, and performance evidence complete", - "docs/BENCHMARKS.md release row", - ), - ( - "docs/STATUS.md", - "Supported subset; bundles SPIKED, no artifacts", - "Supported; bundles SHIPPED with runtime evidence", - "docs/STATUS.md release row", - ), - ) - for relative, before, after, reason in mutations: + for relative, before, after, reason in PUBLIC_PENDING_MUTATIONS: with self.subTest(path=relative), RepoCopy() as root: mutate(root, relative, before, after) result = run_checker(root) @@ -262,19 +312,7 @@ def test_public_release_rows_remain_pending(self) -> None: self.assertIn(reason, result.stdout + result.stderr) def test_human_w12_is_optional_and_cannot_replace_w10(self) -> None: - mutations = ( - ( - "optional single-SM CUDA diagnostic/performance variants", - "required primary single-SM CUDA release variants replacing W10", - "W12 deliverable", - ), - ( - "generated from the same explicit matrix and evidence; never advertised as the primary KISS download or used to bypass W10", - "the primary KISS download; W10 may be bypassed", - "W12 exit gate", - ), - ) - for before, after, reason in mutations: + for before, after, reason in W10_W12_HUMAN_MUTATIONS: with self.subTest(reason=reason), RepoCopy() as root: mutate(root, self.SPEC, before, after) result = run_checker(root) @@ -282,32 +320,101 @@ def test_human_w12_is_optional_and_cannot_replace_w10(self) -> None: self.assertIn(reason, result.stdout + result.stderr) def test_human_primary_artifact_contract_matches_machine_block(self) -> None: + for before, after, reason in PRIMARY_ARTIFACT_PROSE_MUTATIONS: + with self.subTest(reason=reason), RepoCopy() as root: + mutate(root, self.SPEC, before, after) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(reason, result.stdout + result.stderr) + + def test_primary_cuda_mutation_inventory_literal_is_pinned(self) -> None: + with RepoCopy() as root: + mutate( + root, + "tests/scripts/test_check_release_binary_contract.py", + ' "120a",\n "121a",\n)\n\nRECORD_ANCHORS', + ' "120a",\n)\n\nRECORD_ANCHORS', + ) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("PRIMARY_CUDA_SMS", result.stdout + result.stderr) + + def test_work_dependency_mutation_inventory_literal_is_pinned(self) -> None: + with RepoCopy() as root: + mutate( + root, + "tests/scripts/test_check_release_binary_contract.py", + ' "W2": "W1",', + ' "W2": "",', + ) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("EXPECTED_DEPS", result.stdout + result.stderr) + + def test_each_semantic_inventory_consumer_is_pinned(self) -> None: mutations = ( + ("for sm in PRIMARY_CUDA_SMS:", "for sm in ():", "PRIMARY_CUDA_SMS"), ( - "The primary CPU download is\none conservative-baseline, runtime-adaptive binary per OS+host ABI; the primary\nCUDA download is one fat binary per OS+host ABI containing every supported SM.", - "The primary CPU download is one binary per ISA; the primary CUDA download is one binary per SM.", - "human primary CPU/CUDA contract", + "for work, deps in EXPECTED_DEPS.items():", + "for work, deps in {}.items():", + "EXPECTED_DEPS", ), ( - "For x86_64, the baseline must run without AVX2: portable/SSE2 code remains\ncallable, and higher instructions live only in per-function or per-TU tiers.", - "For x86_64, AVX2 is required by the baseline.", - "human x86_64 no-AVX2 contract", + "for relative, anchor in RECORD_ANCHORS.items():", + "for relative, anchor in {}.items():", + "RECORD_ANCHORS", + ), + ( + "for relative, before, after, reason in PUBLIC_PENDING_MUTATIONS:", + "for relative, before, after, reason in ():", + "PUBLIC_PENDING_MUTATIONS", + ), + ( + "for before, after, reason in W10_W12_HUMAN_MUTATIONS:", + "for before, after, reason in ():", + "W10_W12_HUMAN_MUTATIONS", + ), + ( + "for before, after, reason in PRIMARY_ARTIFACT_PROSE_MUTATIONS:", + "for before, after, reason in ():", + "PRIMARY_ARTIFACT_PROSE_MUTATIONS", ), ) - for before, after, reason in mutations: - with self.subTest(reason=reason), RepoCopy() as root: - mutate(root, self.SPEC, before, after) + for before, after, inventory in mutations: + with self.subTest(inventory=inventory), RepoCopy() as root: + mutate( + root, + "tests/scripts/test_check_release_binary_contract.py", + before, + after, + ) result = run_checker(root) - self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertIn(reason, result.stdout + result.stderr) + self.assertNotEqual( + result.returncode, 0, result.stdout + result.stderr + ) + self.assertIn(inventory, result.stdout + result.stderr) def test_required_mutation_test_inventory_is_pinned(self) -> None: for method in REQUIRED_TEST_METHODS: - with self.subTest(method=method), RepoCopy() as root: - delete_test_method(root, method) - result = run_checker(root) - self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertIn("required mutation-test inventory", result.stdout + result.stderr) + for mutation in ("delete", "rename"): + with self.subTest(method=method, mutation=mutation), RepoCopy() as root: + if mutation == "delete": + delete_test_method(root, method) + else: + mutate( + root, + "tests/scripts/test_check_release_binary_contract.py", + f" def {method}(", + f" def renamed_{method}(", + ) + result = run_checker(root) + self.assertNotEqual( + result.returncode, 0, result.stdout + result.stderr + ) + self.assertIn( + "required mutation-test inventory", + result.stdout + result.stderr, + ) if __name__ == "__main__": From 129d9117787c6c6021b5fdf23262e4dcd2bee304 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 7 Aug 2026 23:40:56 +0000 Subject: [PATCH 07/11] test(release): bind spike lifecycle records Parse the release row lifecycle and no-artifact honesty contract on the engine, roadmap, coordination, and state surfaces. Pin the exact mutation dataset and its consumer so the four formerly escaping DONE mutations cannot be silently removed or bypassed. FOLLOWING_AGENTS_PROTOCOL Assisted-by: Codex:gpt-5 [Codex] --- docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- scripts/check-release-binary-contract.py | 190 +++++++++++++++++- .../test_check_release_binary_contract.py | 83 +++++++- 4 files changed, 271 insertions(+), 6 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 7037b3884..de0af9966 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -4,7 +4,7 @@ | Reference | Workload | Headline | Tokens | |---|---|---|---| -| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 18 tests. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | +| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 19 tests. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | | **Developer agent entry point (implemented)** | `DOCS-AGENT-PROTOCOL-ENTRYPOINT`: public contribution guide + synchronized, mutation-gated pre-claim intake rule | Rebased documentation/protocol only; benchmark void | n/a | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **Record repair 2026-08-07** | `main` was red on `check-agent-record` + `check-env-doc`, blocking every PR. Dangling `kda-chunk-aot/` link and two undocumented env vars. No behaviour change | diff --git a/docs/STATUS.md b/docs/STATUS.md index fd67bfaee..76bdc8ab8 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -112,7 +112,7 @@ token-for-token correctness against the pinned oracle. | Tool-call parsing | 36 parser families / 40 accepted names, streaming | Every vLLM tool parser at the pin except the three Rust/Harmony-backed ones: pure-text parsers ported 1:1, the six engine-backed families reimplemented from their wire formats, all held to the upstream test suites. Selection via `--tool-call-parser` (server), `tool_parser` (C ABI), or template auto-detection; native-syntax forced tool_choice where expressible. Tables: docs/BENCHMARKS.md | | Reasoning parsing (`SAMPLE-REASONING`, ACTIVE, partial coverage) | 9 parsers, streaming | think_auto (auto-detect default: content unless markers appear), deepseek_r1, deepseek_v3 (passthrough) / holo2 (thinking→R1), mistral ([THINK]), minimax_m2 (+append_think), step3, olmo3 - reasoning split engine-side BEFORE tool parsing, streamed as `reasoning` deltas in the chat chunks. Coverage: 9 of upstream's ~28 registered names (remaining text families + engine-backed adapters tracked as W2/W3 in specs/reasoning-parsers.md); each ported parser doctest-gated vs its tests/reasoning case | | Unified streaming parser engine | Core, assembly, serving-SSE dispatch landed, gated; all 10 engine-backed families ported (family parity closed); JSON-schema tool-arg type coercion landed | The vLLM 0.26 declarative `parser/engine/` (shared state machine plus all 10 configs: qwen3, seed_oss, kimi_k2, minimax_m2, glm47_moe, deepseek_v4/v32, nemotron_v3, gemma4, inkling) and assembly layer, gated field-for-field vs vLLM 0.26. An engine-backed `--tool-call-parser` name drives the live chat SSE chunks, off by default. When a request's tools declare typed parameters, the assembled tool-call arguments are coerced to the declared JSON types (int/number/bool/string/array/null) 1:1 with vLLM `_fix_arg_types`, in both streaming and one-shot; no schema means the arguments pass through as strings unchanged. Details: .agents/specs/parser-assembly-c8.md | -| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and 18-test inventory mutation-gated; per-SM diagnostics optional; no archive/runtime claim | +| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and 19-test inventory mutation-gated; per-SM diagnostics optional; no archive/runtime claim | | Pooling task class (embeddings / classify / score / rerank) | Spiked; pooler op + heads composite + pooling runner path landed (CPU), not yet servable end-to-end (no concrete model / endpoints) | The non-generative task class. W0 spike over the whole vLLM pooling surface (`.agents/specs/pooling-task-class.md`, `CLAIM-POOLING`). W1 landed the pooler OP (CLS/LAST/MEAN + Identity/Normalize/MultiLabelClassify/Classify activations, double-precision-gated). **W2 landed the pooler HEADS composite** — `EmbeddingPoolerHead` (projector→matryoshka→normalize), `ClassifierPoolerHead` (classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` task routing, and the `PoolerConfig`/`PoolingParams` structs (`test_pooler_heads` 27/27, 240 asserts, RED-first). **W3 landed the pooling RUNNER path** — `PoolingRunner` applies the model's `Pooler` to the last hidden state and returns pooled embeddings instead of sampled tokens, gated by a STRUCTURAL cosine-parity check vs a double-precision LAST+normalize reference (`test_pooling_runner` 5/5, 14 asserts, RED-first). NOT yet servable / honest residuals: a concrete pooling MODEL forward + the REAL-model oracle cosine gate (`vllm.LLM(task="embed").encode`) — no cosine-vs-oracle number is fabricated (W3-model); the `/v1/embeddings` + score/rerank/classify endpoints (W4); tokwise AllPool/StepPool (W5). See docs/BENCHMARKS.md | | Plugin system (out-of-core registration) | Spiked; first CPU brick landed, not yet wired into any production path | The extensibility-first discovery layer. W0 spike over vLLM's plugin surface (general / platform / io_processor / endpoint groups, the `register_model` an out-of-tree plugin calls, the invocation seams) is committed (`.agents/specs/plugin-system.md`, `ENG-PLUGIN-SYSTEM` ACTIVE, `CLAIM-PLUGIN-SYSTEM`). W1 landed `vllm::plugins::LoadGeneralPlugins()` + the out-of-core general-plugin registration seam (`RegisterGeneralPlugin` / `REGISTER_VLLM_GENERAL_PLUGIN`) over the existing `REGISTER_VLLM_MODEL`-style registries (the in-tree factory `MODEL-FACTORY-registry` is record-repaired `DONE` 2026-08-05: 28 self-registering TUs, dgx debt paid by the 2026-07-23 seven-gate run): a 1:1 mirror of `load_general_plugins` (load-once idempotence, the `VLLM_PLUGINS` allowlist, per-plugin failure isolation). Proven by an out-of-core toy-model plugin that registers a toy architecture through the public `RegisterModel` seam — unit-gated RED-first (`test_plugin_system` 1 case / 29 assertions: the toy arch resolves ONLY after LoadGeneralPlugins runs it, and not under `VLLM_PLUGINS=""`). Python entry points have no C++20 analogue, so discovery is the project's static-init/`dlopen` registration idiom (recorded porting-inventory §9). NOT yet wired: real shared-object `dlopen` + the C-ABI `vllm_plugin_register` entry (W2), the engine/CLI `--load-plugins` wiring that calls LoadGeneralPlugins from the construction paths (W3), the platform/quant plugin kinds (W4), and the io_processor/stat_logger/endpoint groups (W5) are named residuals. See docs/BENCHMARKS.md | | Offline Batch API (JSONL file runner) | Spiked; first CPU brick landed, not yet exposed as a CLI | The offline OpenAI Batch API: read a JSONL of OpenAI-format requests, run each through the engine, write a JSONL of responses. W0 spike over vLLM's `run_batch.py` (schema, endpoint dispatch, run loop, file I/O) is committed (`.agents/specs/batch-api.md`, `SERVE-BATCH-API` ACTIVE, `CLAIM-BATCH-API`). W1 landed `RunBatch` (`RunLine`/`RunLines`/`Run`) + `RunBatchFile` — a pure orchestrator over the existing `OpenAIServingChat::create_chat_completion` (NO reimplemented generation), 1:1 with vLLM's endpoint_registry url→handler map: `/v1/chat/completions` dispatch, the `BatchResponseData`/`BatchRequestOutput` schema (`vllm-` ids, custom_id echoed), the `run_request` AllResponse/ErrorResponse/stream branches, and the unsupported-endpoint/url error rows. Unit-gated RED-first (`test_openai_run_batch` 7 cases / 80 assertions over the synthetic serving engine: ordered rows + custom_id echo + per-line BatchRequestOutput schema round-trip, a malformed line isolated into an error row so the batch continues, dispatch + 404 error rows; dropping the custom_id echo fails 9 assertions). Recorded deviation: a malformed line is isolated (batch continues) where upstream aborts the job. NOT yet exposed: the `vllm run-batch` CLI + `BatchFrontendArgs` (W2), embeddings/score/rerank dispatch (W3, rides pooling endpoints), audio transcription/translation + media fetch (W4), and http(s)/data-URL file I/O + metrics + overlapped `AsyncLLM` submission (W5) are named residuals. See docs/BENCHMARKS.md | diff --git a/scripts/check-release-binary-contract.py b/scripts/check-release-binary-contract.py index 325289f94..d28f15113 100644 --- a/scripts/check-release-binary-contract.py +++ b/scripts/check-release-binary-contract.py @@ -64,12 +64,68 @@ ), } +LIFECYCLE_RECORD_MUTATIONS = ( + ( + ".agents/engine-matrix.md", + "`SPIKE` | `CLAIM-ENG-RELEASE-BINARIES-SPIKE` |", + "`DONE` | `CLAIM-ENG-RELEASE-BINARIES-SPIKE` |", + "engine-matrix release lifecycle", + ), + ( + ".agents/engine-matrix.md", + "gaps remain; no install/archive/publish implementation", + "gaps closed; install/archive/publish implementation complete", + "engine-matrix release lifecycle", + ), + ( + ".agents/roadmap_v1.md", + "`SPIKE` | Fresh review of PR #129", + "`DONE` | Fresh review of PR #129", + "roadmap release lifecycle", + ), + ( + ".agents/roadmap_v1.md", + "bundle work; no archive exists", + "bundle work complete; archive exists", + "roadmap release lifecycle", + ), + ( + ".agents/coordination.md", + "| `ACTIVE` | 2026-08-07 — user-reviewed revision complete: primary fat " + "CUDA + adaptive CPU per host ABI, optional per-SM diagnostics; row stays " + "`SPIKE`; awaiting fresh review |", + "| `DONE` | 2026-08-07 — user-reviewed revision complete: primary fat " + "CUDA + adaptive CPU per host ABI, optional per-SM diagnostics; row stays " + "`SPIKE`; awaiting fresh review |", + "coordination release lifecycle", + ), + ( + ".agents/coordination.md", + "no CMake, workflow, source, test, or artifact implementation", + "CMake, workflow, source, test, and artifact implementation complete", + "coordination release lifecycle", + ), + ( + ".agents/coordination.md", + "row stays `SPIKE`; awaiting fresh review", + "row is `DONE`; release shipped", + "coordination release lifecycle", + ), + ( + ".agents/state.md", + "`ENG-RELEASE-BINARIES` remains `SPIKE`, and no archive or implementation " + "is\nclaimed.", + "`ENG-RELEASE-BINARIES` is `DONE`, with archive and implementation.", + "state release lifecycle", + ), +) + BENCHMARKS_RELEASE_ROW = ( "| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary " "host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM " "diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM " "fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, " - "and 18 tests. No archive, staged smoke, runtime, correctness, or performance " + "and 19 tests. No archive, staged smoke, runtime, correctness, or performance " "evidence " "| n/a |" ) @@ -77,7 +133,7 @@ STATUS_RELEASE_FRAGMENTS = ( "Supported subset; bundles SPIKED, no artifacts", "primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and " - "18-test inventory mutation-gated; per-SM diagnostics optional; no " + "19-test inventory mutation-gated; per-SM diagnostics optional; no " "archive/runtime claim", ) @@ -112,7 +168,7 @@ ( "docs/BENCHMARKS.md", "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " - "W1-W13/W10-W12 policy, public pending states, and 18 tests. No archive, " + "W1-W13/W10-W12 policy, public pending states, and 19 tests. No archive, " "staged smoke, runtime, correctness, or performance evidence", "**SHIPPED:** archive, runtime, correctness, and performance evidence " "complete", @@ -170,6 +226,7 @@ "test_each_work_dependency_edge_is_pinned", "test_optional_w12_does_not_block_w13", "test_each_required_record_anchor_is_fail_closed", + "test_release_lifecycle_and_honesty_are_fail_closed", "test_public_release_rows_remain_pending", "test_human_w12_is_optional_and_cannot_replace_w10", "test_human_primary_artifact_contract_matches_machine_block", @@ -183,6 +240,7 @@ "PRIMARY_CUDA_SMS": PRIMARY_CUDA_SMS, "EXPECTED_DEPS": {work: ",".join(deps) for work, deps in WORK_DEPS.items()}, "RECORD_ANCHORS": ANCHORS, + "LIFECYCLE_RECORD_MUTATIONS": LIFECYCLE_RECORD_MUTATIONS, "PUBLIC_PENDING_MUTATIONS": PUBLIC_PENDING_MUTATIONS, "W10_W12_HUMAN_MUTATIONS": W10_W12_HUMAN_MUTATIONS, "PRIMARY_ARTIFACT_PROSE_MUTATIONS": PRIMARY_ARTIFACT_PROSE_MUTATIONS, @@ -204,6 +262,11 @@ ("relative", "anchor"), True, ), + "LIFECYCLE_RECORD_MUTATIONS": ( + "test_release_lifecycle_and_honesty_are_fail_closed", + ("relative", "before", "after", "reason"), + False, + ), "PUBLIC_PENDING_MUTATIONS": ( "test_public_release_rows_remain_pending", ("relative", "before", "after", "reason"), @@ -241,6 +304,15 @@ re.M, ) +STATE_RELEASE_HEADING = ( + "## 2026-08-07 — Release matrix revised: fat CUDA and adaptive CPU are the " + "primary downloads" +) +STATE_RELEASE_LIFECYCLE = ( + "`ENG-RELEASE-BINARIES` remains `SPIKE`, and no archive or implementation " + "is claimed." +) + def parse_contract(text: str) -> tuple[dict[str, str], list[str]]: if text.count(BEGIN) != 1 or text.count(END) != 1: @@ -298,6 +370,117 @@ def _normalize_prose(text: str) -> str: return re.sub(r"\s+", " ", text).strip() +def _table_record( + root: Path, + relative: str, + prefix: str, + cell_count: int, + label: str, + errors: list[str], +) -> tuple[str, ...] | None: + path = root / relative + if not path.is_file(): + errors.append(f"{label} record {relative} is missing") + return None + rows = [ + line + for line in path.read_text(encoding="utf-8").splitlines() + if line.startswith(prefix) + ] + if len(rows) != 1: + errors.append( + f"{label} record must have exactly one row starting {prefix!r}; " + f"found {len(rows)}" + ) + return None + cells = tuple(cell.strip() for cell in rows[0].split("|")[1:-1]) + if len(cells) != cell_count: + errors.append( + f"{label} record must have {cell_count} cells; found {len(cells)}" + ) + return None + return cells + + +def _release_lifecycle_errors(root: Path) -> list[str]: + errors: list[str] = [] + + engine = _table_record( + root, + ".agents/engine-matrix.md", + "| `ENG-RELEASE-BINARIES` |", + 9, + "engine-matrix release lifecycle", + errors, + ) + if engine is not None and ( + engine[7] != "`SPIKE`" + or "gaps remain; no install/archive/publish implementation" not in engine[4] + ): + errors.append( + "engine-matrix release lifecycle must stay SPIKE and state that the " + "gaps remain with no install/archive/publish implementation" + ) + + roadmap = _table_record( + root, + ".agents/roadmap_v1.md", + "| REL | `ROAD-V1-RELEASE` |", + 7, + "roadmap release lifecycle", + errors, + ) + if roadmap is not None and ( + roadmap[5] != "`SPIKE`" or "no archive exists" not in roadmap[6] + ): + errors.append( + "roadmap release lifecycle must stay SPIKE and state that no archive " + "exists" + ) + + coordination = _table_record( + root, + ".agents/coordination.md", + "| `CLAIM-ENG-RELEASE-BINARIES-SPIKE` |", + 8, + "coordination release lifecycle", + errors, + ) + if coordination is not None and ( + coordination[6] != "`ACTIVE`" + or "no CMake, workflow, source, test, or artifact implementation" + not in coordination[5] + or "row stays `SPIKE`; awaiting fresh review" not in coordination[7] + ): + errors.append( + "coordination release lifecycle must keep the claim ACTIVE while the " + "row stays SPIKE, awaiting review, with no implementation" + ) + + state_path = root / ".agents/state.md" + if not state_path.is_file(): + errors.append("state release lifecycle record .agents/state.md is missing") + else: + state_text = state_path.read_text(encoding="utf-8") + if state_text.count(STATE_RELEASE_HEADING) != 1: + errors.append( + "state release lifecycle must have exactly one revised-release " + "checkpoint" + ) + else: + start = state_text.index(STATE_RELEASE_HEADING) + len( + STATE_RELEASE_HEADING + ) + end = state_text.find("\n## ", start) + section = state_text[start:] if end < 0 else state_text[start:end] + if STATE_RELEASE_LIFECYCLE not in _normalize_prose(section): + errors.append( + "state release lifecycle must say ENG-RELEASE-BINARIES remains " + "SPIKE with no archive or implementation claimed" + ) + return errors + + def _top_level_literal(tree: ast.Module, name: str) -> tuple[object, int]: declarations = [ node @@ -478,6 +661,7 @@ def contract_errors(root: Path) -> list[str]: path = root / relative if not path.is_file() or anchor not in path.read_text(encoding="utf-8"): errors.append(f"{relative} is missing required release anchor {anchor!r}") + errors.extend(_release_lifecycle_errors(root)) benchmarks = root / "docs/BENCHMARKS.md" if not benchmarks.is_file() or BENCHMARKS_RELEASE_ROW not in benchmarks.read_text( encoding="utf-8" diff --git a/tests/scripts/test_check_release_binary_contract.py b/tests/scripts/test_check_release_binary_contract.py index dc999f779..ecb9e6502 100644 --- a/tests/scripts/test_check_release_binary_contract.py +++ b/tests/scripts/test_check_release_binary_contract.py @@ -39,6 +39,7 @@ "test_each_work_dependency_edge_is_pinned", "test_optional_w12_does_not_block_w13", "test_each_required_record_anchor_is_fail_closed", + "test_release_lifecycle_and_honesty_are_fail_closed", "test_public_release_rows_remain_pending", "test_human_w12_is_optional_and_cannot_replace_w10", "test_human_primary_artifact_contract_matches_machine_block", @@ -81,6 +82,62 @@ ), } +LIFECYCLE_RECORD_MUTATIONS = ( + ( + ".agents/engine-matrix.md", + "`SPIKE` | `CLAIM-ENG-RELEASE-BINARIES-SPIKE` |", + "`DONE` | `CLAIM-ENG-RELEASE-BINARIES-SPIKE` |", + "engine-matrix release lifecycle", + ), + ( + ".agents/engine-matrix.md", + "gaps remain; no install/archive/publish implementation", + "gaps closed; install/archive/publish implementation complete", + "engine-matrix release lifecycle", + ), + ( + ".agents/roadmap_v1.md", + "`SPIKE` | Fresh review of PR #129", + "`DONE` | Fresh review of PR #129", + "roadmap release lifecycle", + ), + ( + ".agents/roadmap_v1.md", + "bundle work; no archive exists", + "bundle work complete; archive exists", + "roadmap release lifecycle", + ), + ( + ".agents/coordination.md", + "| `ACTIVE` | 2026-08-07 — user-reviewed revision complete: primary fat " + "CUDA + adaptive CPU per host ABI, optional per-SM diagnostics; row stays " + "`SPIKE`; awaiting fresh review |", + "| `DONE` | 2026-08-07 — user-reviewed revision complete: primary fat " + "CUDA + adaptive CPU per host ABI, optional per-SM diagnostics; row stays " + "`SPIKE`; awaiting fresh review |", + "coordination release lifecycle", + ), + ( + ".agents/coordination.md", + "no CMake, workflow, source, test, or artifact implementation", + "CMake, workflow, source, test, and artifact implementation complete", + "coordination release lifecycle", + ), + ( + ".agents/coordination.md", + "row stays `SPIKE`; awaiting fresh review", + "row is `DONE`; release shipped", + "coordination release lifecycle", + ), + ( + ".agents/state.md", + "`ENG-RELEASE-BINARIES` remains `SPIKE`, and no archive or implementation " + "is\nclaimed.", + "`ENG-RELEASE-BINARIES` is `DONE`, with archive and implementation.", + "state release lifecycle", + ), +) + EXPECTED_DEPS = { "W1": "", "W2": "W1", @@ -101,7 +158,7 @@ ( "docs/BENCHMARKS.md", "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " - "W1-W13/W10-W12 policy, public pending states, and 18 tests. No archive, " + "W1-W13/W10-W12 policy, public pending states, and 19 tests. No archive, " "staged smoke, runtime, correctness, or performance evidence", "**SHIPPED:** archive, runtime, correctness, and performance evidence " "complete", @@ -299,6 +356,25 @@ def test_each_required_record_anchor_is_fail_closed(self) -> None: self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) self.assertIn(f"{relative} is missing required release anchor", result.stdout + result.stderr) + def test_release_lifecycle_and_honesty_are_fail_closed(self) -> None: + for relative, before, after, reason in LIFECYCLE_RECORD_MUTATIONS: + with self.subTest(path=relative, mutation=before), RepoCopy() as root: + mutate(root, relative, before, after) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(reason, result.stdout + result.stderr) + + with RepoCopy() as root: + mutate( + root, + "tests/scripts/test_check_release_binary_contract.py", + ' "engine-matrix release lifecycle",', + ' "renamed engine lifecycle",', + ) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("LIFECYCLE_RECORD_MUTATIONS", result.stdout + result.stderr) + class HumanContractMutations(unittest.TestCase): SPEC = ".agents/specs/release-binary-matrix.md" @@ -364,6 +440,11 @@ def test_each_semantic_inventory_consumer_is_pinned(self) -> None: "for relative, anchor in {}.items():", "RECORD_ANCHORS", ), + ( + "for relative, before, after, reason in LIFECYCLE_RECORD_MUTATIONS:", + "for relative, before, after, reason in ():", + "LIFECYCLE_RECORD_MUTATIONS", + ), ( "for relative, before, after, reason in PUBLIC_PENDING_MUTATIONS:", "for relative, before, after, reason in ():", From 7c0eb997c8199ed78f86627bc8c6e8258392ba62 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 8 Aug 2026 00:10:30 +0000 Subject: [PATCH 08/11] test(release): close guard-map shrinkage Keep the checker guard-map keysets independent from the mutable maps they validate. Exercise every guarded entry through delete and rename mutations, plus additions and an enforcement-bypass case, and pin the expanded 20-test public inventory. FOLLOWING_AGENTS_PROTOCOL Assisted-by: Codex:gpt-5 [Codex] --- docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- scripts/check-release-binary-contract.py | 81 ++++++++-- .../test_check_release_binary_contract.py | 151 +++++++++++++++++- 4 files changed, 221 insertions(+), 15 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index de0af9966..a1d6aad04 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -4,7 +4,7 @@ | Reference | Workload | Headline | Tokens | |---|---|---|---| -| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 19 tests. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | +| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 20 tests. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | | **Developer agent entry point (implemented)** | `DOCS-AGENT-PROTOCOL-ENTRYPOINT`: public contribution guide + synchronized, mutation-gated pre-claim intake rule | Rebased documentation/protocol only; benchmark void | n/a | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **Record repair 2026-08-07** | `main` was red on `check-agent-record` + `check-env-doc`, blocking every PR. Dangling `kda-chunk-aot/` link and two undocumented env vars. No behaviour change | diff --git a/docs/STATUS.md b/docs/STATUS.md index 76bdc8ab8..e84344e3b 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -112,7 +112,7 @@ token-for-token correctness against the pinned oracle. | Tool-call parsing | 36 parser families / 40 accepted names, streaming | Every vLLM tool parser at the pin except the three Rust/Harmony-backed ones: pure-text parsers ported 1:1, the six engine-backed families reimplemented from their wire formats, all held to the upstream test suites. Selection via `--tool-call-parser` (server), `tool_parser` (C ABI), or template auto-detection; native-syntax forced tool_choice where expressible. Tables: docs/BENCHMARKS.md | | Reasoning parsing (`SAMPLE-REASONING`, ACTIVE, partial coverage) | 9 parsers, streaming | think_auto (auto-detect default: content unless markers appear), deepseek_r1, deepseek_v3 (passthrough) / holo2 (thinking→R1), mistral ([THINK]), minimax_m2 (+append_think), step3, olmo3 - reasoning split engine-side BEFORE tool parsing, streamed as `reasoning` deltas in the chat chunks. Coverage: 9 of upstream's ~28 registered names (remaining text families + engine-backed adapters tracked as W2/W3 in specs/reasoning-parsers.md); each ported parser doctest-gated vs its tests/reasoning case | | Unified streaming parser engine | Core, assembly, serving-SSE dispatch landed, gated; all 10 engine-backed families ported (family parity closed); JSON-schema tool-arg type coercion landed | The vLLM 0.26 declarative `parser/engine/` (shared state machine plus all 10 configs: qwen3, seed_oss, kimi_k2, minimax_m2, glm47_moe, deepseek_v4/v32, nemotron_v3, gemma4, inkling) and assembly layer, gated field-for-field vs vLLM 0.26. An engine-backed `--tool-call-parser` name drives the live chat SSE chunks, off by default. When a request's tools declare typed parameters, the assembled tool-call arguments are coerced to the declared JSON types (int/number/bool/string/array/null) 1:1 with vLLM `_fix_arg_types`, in both streaming and one-shot; no schema means the arguments pass through as strings unchanged. Details: .agents/specs/parser-assembly-c8.md | -| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and 19-test inventory mutation-gated; per-SM diagnostics optional; no archive/runtime claim | +| OpenAI server | Supported subset; bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and 20-test inventory mutation-gated; per-SM diagnostics optional; no archive/runtime claim | | Pooling task class (embeddings / classify / score / rerank) | Spiked; pooler op + heads composite + pooling runner path landed (CPU), not yet servable end-to-end (no concrete model / endpoints) | The non-generative task class. W0 spike over the whole vLLM pooling surface (`.agents/specs/pooling-task-class.md`, `CLAIM-POOLING`). W1 landed the pooler OP (CLS/LAST/MEAN + Identity/Normalize/MultiLabelClassify/Classify activations, double-precision-gated). **W2 landed the pooler HEADS composite** — `EmbeddingPoolerHead` (projector→matryoshka→normalize), `ClassifierPoolerHead` (classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` task routing, and the `PoolerConfig`/`PoolingParams` structs (`test_pooler_heads` 27/27, 240 asserts, RED-first). **W3 landed the pooling RUNNER path** — `PoolingRunner` applies the model's `Pooler` to the last hidden state and returns pooled embeddings instead of sampled tokens, gated by a STRUCTURAL cosine-parity check vs a double-precision LAST+normalize reference (`test_pooling_runner` 5/5, 14 asserts, RED-first). NOT yet servable / honest residuals: a concrete pooling MODEL forward + the REAL-model oracle cosine gate (`vllm.LLM(task="embed").encode`) — no cosine-vs-oracle number is fabricated (W3-model); the `/v1/embeddings` + score/rerank/classify endpoints (W4); tokwise AllPool/StepPool (W5). See docs/BENCHMARKS.md | | Plugin system (out-of-core registration) | Spiked; first CPU brick landed, not yet wired into any production path | The extensibility-first discovery layer. W0 spike over vLLM's plugin surface (general / platform / io_processor / endpoint groups, the `register_model` an out-of-tree plugin calls, the invocation seams) is committed (`.agents/specs/plugin-system.md`, `ENG-PLUGIN-SYSTEM` ACTIVE, `CLAIM-PLUGIN-SYSTEM`). W1 landed `vllm::plugins::LoadGeneralPlugins()` + the out-of-core general-plugin registration seam (`RegisterGeneralPlugin` / `REGISTER_VLLM_GENERAL_PLUGIN`) over the existing `REGISTER_VLLM_MODEL`-style registries (the in-tree factory `MODEL-FACTORY-registry` is record-repaired `DONE` 2026-08-05: 28 self-registering TUs, dgx debt paid by the 2026-07-23 seven-gate run): a 1:1 mirror of `load_general_plugins` (load-once idempotence, the `VLLM_PLUGINS` allowlist, per-plugin failure isolation). Proven by an out-of-core toy-model plugin that registers a toy architecture through the public `RegisterModel` seam — unit-gated RED-first (`test_plugin_system` 1 case / 29 assertions: the toy arch resolves ONLY after LoadGeneralPlugins runs it, and not under `VLLM_PLUGINS=""`). Python entry points have no C++20 analogue, so discovery is the project's static-init/`dlopen` registration idiom (recorded porting-inventory §9). NOT yet wired: real shared-object `dlopen` + the C-ABI `vllm_plugin_register` entry (W2), the engine/CLI `--load-plugins` wiring that calls LoadGeneralPlugins from the construction paths (W3), the platform/quant plugin kinds (W4), and the io_processor/stat_logger/endpoint groups (W5) are named residuals. See docs/BENCHMARKS.md | | Offline Batch API (JSONL file runner) | Spiked; first CPU brick landed, not yet exposed as a CLI | The offline OpenAI Batch API: read a JSONL of OpenAI-format requests, run each through the engine, write a JSONL of responses. W0 spike over vLLM's `run_batch.py` (schema, endpoint dispatch, run loop, file I/O) is committed (`.agents/specs/batch-api.md`, `SERVE-BATCH-API` ACTIVE, `CLAIM-BATCH-API`). W1 landed `RunBatch` (`RunLine`/`RunLines`/`Run`) + `RunBatchFile` — a pure orchestrator over the existing `OpenAIServingChat::create_chat_completion` (NO reimplemented generation), 1:1 with vLLM's endpoint_registry url→handler map: `/v1/chat/completions` dispatch, the `BatchResponseData`/`BatchRequestOutput` schema (`vllm-` ids, custom_id echoed), the `run_request` AllResponse/ErrorResponse/stream branches, and the unsupported-endpoint/url error rows. Unit-gated RED-first (`test_openai_run_batch` 7 cases / 80 assertions over the synthetic serving engine: ordered rows + custom_id echo + per-line BatchRequestOutput schema round-trip, a malformed line isolated into an error row so the batch continues, dispatch + 404 error rows; dropping the custom_id echo fails 9 assertions). Recorded deviation: a malformed line is isolated (batch continues) where upstream aborts the job. NOT yet exposed: the `vllm run-batch` CLI + `BatchFrontendArgs` (W2), embeddings/score/rerank dispatch (W3, rides pooling endpoints), audio transcription/translation + media fetch (W4), and http(s)/data-URL file I/O + metrics + overlapped `AsyncLLM` submission (W5) are named residuals. See docs/BENCHMARKS.md | diff --git a/scripts/check-release-binary-contract.py b/scripts/check-release-binary-contract.py index d28f15113..9779d4b56 100644 --- a/scripts/check-release-binary-contract.py +++ b/scripts/check-release-binary-contract.py @@ -125,7 +125,7 @@ "host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM " "diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM " "fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, " - "and 19 tests. No archive, staged smoke, runtime, correctness, or performance " + "and 20 tests. No archive, staged smoke, runtime, correctness, or performance " "evidence " "| n/a |" ) @@ -133,7 +133,7 @@ STATUS_RELEASE_FRAGMENTS = ( "Supported subset; bundles SPIKED, no artifacts", "primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and " - "19-test inventory mutation-gated; per-SM diagnostics optional; no " + "20-test inventory mutation-gated; per-SM diagnostics optional; no " "archive/runtime claim", ) @@ -168,7 +168,7 @@ ( "docs/BENCHMARKS.md", "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " - "W1-W13/W10-W12 policy, public pending states, and 19 tests. No archive, " + "W1-W13/W10-W12 policy, public pending states, and 20 tests. No archive, " "staged smoke, runtime, correctness, or performance evidence", "**SHIPPED:** archive, runtime, correctness, and performance evidence " "complete", @@ -233,9 +233,37 @@ "test_primary_cuda_mutation_inventory_literal_is_pinned", "test_work_dependency_mutation_inventory_literal_is_pinned", "test_each_semantic_inventory_consumer_is_pinned", + "test_checker_guard_map_keysets_are_exact", "test_required_mutation_test_inventory_is_pinned", ) +EXPECTED_TEST_LITERAL_INVENTORY_KEYS = ( + "PRIMARY_CUDA_SMS", + "EXPECTED_DEPS", + "RECORD_ANCHORS", + "LIFECYCLE_RECORD_MUTATIONS", + "PUBLIC_PENDING_MUTATIONS", + "W10_W12_HUMAN_MUTATIONS", + "PRIMARY_ARTIFACT_PROSE_MUTATIONS", + "GUARD_MAP_KEYS", +) + +EXPECTED_TEST_INVENTORY_CONSUMER_KEYS = ( + "PRIMARY_CUDA_SMS", + "EXPECTED_DEPS", + "RECORD_ANCHORS", + "LIFECYCLE_RECORD_MUTATIONS", + "PUBLIC_PENDING_MUTATIONS", + "W10_W12_HUMAN_MUTATIONS", + "PRIMARY_ARTIFACT_PROSE_MUTATIONS", + "GUARD_MAP_KEYS", +) + +EXPECTED_GUARD_MAP_KEYS = { + "TEST_LITERAL_INVENTORIES": EXPECTED_TEST_LITERAL_INVENTORY_KEYS, + "TEST_INVENTORY_CONSUMERS": EXPECTED_TEST_INVENTORY_CONSUMER_KEYS, +} + TEST_LITERAL_INVENTORIES = { "PRIMARY_CUDA_SMS": PRIMARY_CUDA_SMS, "EXPECTED_DEPS": {work: ",".join(deps) for work, deps in WORK_DEPS.items()}, @@ -244,6 +272,7 @@ "PUBLIC_PENDING_MUTATIONS": PUBLIC_PENDING_MUTATIONS, "W10_W12_HUMAN_MUTATIONS": W10_W12_HUMAN_MUTATIONS, "PRIMARY_ARTIFACT_PROSE_MUTATIONS": PRIMARY_ARTIFACT_PROSE_MUTATIONS, + "GUARD_MAP_KEYS": EXPECTED_GUARD_MAP_KEYS, } TEST_INVENTORY_CONSUMERS = { @@ -282,6 +311,11 @@ ("before", "after", "reason"), False, ), + "GUARD_MAP_KEYS": ( + "test_checker_guard_map_keysets_are_exact", + ("guard_map", "keys"), + True, + ), } EXPECTED_FIELDS = { @@ -568,7 +602,33 @@ def _test_inventory_errors(root: Path) -> list[str]: "checker's exact count and names" ) - for inventory, expected_value in TEST_LITERAL_INVENTORIES.items(): + guard_maps = ( + ( + "TEST_LITERAL_INVENTORIES", + EXPECTED_TEST_LITERAL_INVENTORY_KEYS, + TEST_LITERAL_INVENTORIES, + ), + ( + "TEST_INVENTORY_CONSUMERS", + EXPECTED_TEST_INVENTORY_CONSUMER_KEYS, + TEST_INVENTORY_CONSUMERS, + ), + ) + for name, expected_keys, guard_map in guard_maps: + if tuple(guard_map) != expected_keys: + errors.append( + "semantic mutation guard map keyset drifted for " + f"{name}: expected {expected_keys!r}, found {tuple(guard_map)!r}" + ) + + for inventory in EXPECTED_TEST_LITERAL_INVENTORY_KEYS: + if inventory not in TEST_LITERAL_INVENTORIES: + errors.append( + "semantic mutation guard map TEST_LITERAL_INVENTORIES is missing " + f"expected entry {inventory}" + ) + continue + expected_value = TEST_LITERAL_INVENTORIES[inventory] declared_value, count = _top_level_literal(tree, inventory) if count != 1 or declared_value != expected_value: errors.append( @@ -578,11 +638,14 @@ def _test_inventory_errors(root: Path) -> list[str]: ) methods_by_name = {node.name: node for node in method_nodes} - for inventory, ( - method_name, - target_names, - mapping_items, - ) in TEST_INVENTORY_CONSUMERS.items(): + for inventory in EXPECTED_TEST_INVENTORY_CONSUMER_KEYS: + if inventory not in TEST_INVENTORY_CONSUMERS: + errors.append( + "semantic mutation guard map TEST_INVENTORY_CONSUMERS is missing " + f"expected entry {inventory}" + ) + continue + method_name, target_names, mapping_items = TEST_INVENTORY_CONSUMERS[inventory] method = methods_by_name.get(method_name) if method is None or not _iterates_inventory( method, inventory, target_names, mapping_items diff --git a/tests/scripts/test_check_release_binary_contract.py b/tests/scripts/test_check_release_binary_contract.py index ecb9e6502..b7548dc63 100644 --- a/tests/scripts/test_check_release_binary_contract.py +++ b/tests/scripts/test_check_release_binary_contract.py @@ -16,6 +16,7 @@ CHECKER = ROOT / "scripts/check-release-binary-contract.py" CONTRACT_PATHS = ( + "scripts/check-release-binary-contract.py", ".agents/specs/release-binary-matrix.md", ".agents/engine-matrix.md", ".agents/roadmap_v1.md", @@ -46,6 +47,7 @@ "test_primary_cuda_mutation_inventory_literal_is_pinned", "test_work_dependency_mutation_inventory_literal_is_pinned", "test_each_semantic_inventory_consumer_is_pinned", + "test_checker_guard_map_keysets_are_exact", "test_required_mutation_test_inventory_is_pinned", ) @@ -62,6 +64,29 @@ "121a", ) +GUARD_MAP_KEYS = { + "TEST_LITERAL_INVENTORIES": ( + "PRIMARY_CUDA_SMS", + "EXPECTED_DEPS", + "RECORD_ANCHORS", + "LIFECYCLE_RECORD_MUTATIONS", + "PUBLIC_PENDING_MUTATIONS", + "W10_W12_HUMAN_MUTATIONS", + "PRIMARY_ARTIFACT_PROSE_MUTATIONS", + "GUARD_MAP_KEYS", + ), + "TEST_INVENTORY_CONSUMERS": ( + "PRIMARY_CUDA_SMS", + "EXPECTED_DEPS", + "RECORD_ANCHORS", + "LIFECYCLE_RECORD_MUTATIONS", + "PUBLIC_PENDING_MUTATIONS", + "W10_W12_HUMAN_MUTATIONS", + "PRIMARY_ARTIFACT_PROSE_MUTATIONS", + "GUARD_MAP_KEYS", + ), +} + RECORD_ANCHORS = { ".agents/engine-matrix.md": "| `ENG-RELEASE-BINARIES` |", ".agents/roadmap_v1.md": "| REL | `ROAD-V1-RELEASE` |", @@ -158,7 +183,7 @@ ( "docs/BENCHMARKS.md", "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " - "W1-W13/W10-W12 policy, public pending states, and 19 tests. No archive, " + "W1-W13/W10-W12 policy, public pending states, and 20 tests. No archive, " "staged smoke, runtime, correctness, or performance evidence", "**SHIPPED:** archive, runtime, correctness, and performance evidence " "complete", @@ -207,7 +232,12 @@ def run_checker(root: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( - [sys.executable, str(CHECKER), "--root", str(root)], + [ + sys.executable, + str(root / "scripts/check-release-binary-contract.py"), + "--root", + str(root), + ], capture_output=True, text=True, check=False, @@ -258,6 +288,74 @@ def delete_test_method(root: Path, method: str) -> None: path.write_text("".join(lines), encoding="utf-8") +def mutate_checker_guard_map( + root: Path, guard_map: str, mutation: str, key: str +) -> None: + relative = "scripts/check-release-binary-contract.py" + path = root / relative + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + assignments = [ + node + for node in tree.body + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == guard_map + for target in node.targets + ) + ] + if len(assignments) != 1 or not isinstance(assignments[0].value, ast.Dict): + raise AssertionError(f"expected one literal guard map {guard_map!r}") + mapping = assignments[0].value + keys = [ast.literal_eval(node) for node in mapping.keys] + + if mutation == "add": + if key in keys: + raise AssertionError(f"guard key already exists in {guard_map}: {key}") + mapping.keys.append(ast.Constant(value=key)) + mapping.values.append(ast.Constant(value=None)) + else: + if keys.count(key) != 1: + raise AssertionError( + f"expected one guard key {key!r} in {guard_map}, found {keys.count(key)}" + ) + index = keys.index(key) + if mutation == "delete": + del mapping.keys[index] + del mapping.values[index] + elif mutation == "rename": + mapping.keys[index] = ast.Constant(value=f"RENAMED_{key}") + else: + raise AssertionError(f"unknown guard-map mutation {mutation!r}") + + path.write_text(ast.unparse(ast.fix_missing_locations(tree)) + "\n", encoding="utf-8") + + +def bypass_checker_keyset_enforcement(root: Path) -> None: + relative = "scripts/check-release-binary-contract.py" + path = root / relative + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + matches = [] + for node in ast.walk(tree): + if not isinstance(node, ast.If) or not isinstance(node.test, ast.Compare): + continue + left = node.test.left + if ( + isinstance(left, ast.Call) + and isinstance(left.func, ast.Name) + and left.func.id == "tuple" + and len(left.args) == 1 + and isinstance(left.args[0], ast.Name) + and left.args[0].id == "guard_map" + ): + matches.append(node) + if len(matches) != 1: + raise AssertionError( + f"expected one guard-map keyset enforcement, found {len(matches)}" + ) + matches[0].test = ast.Constant(value=False) + path.write_text(ast.unparse(ast.fix_missing_locations(tree)) + "\n", encoding="utf-8") + + class LiveContract(unittest.TestCase): def test_repository_contract_passes(self) -> None: result = run_checker(ROOT) @@ -408,8 +506,8 @@ def test_primary_cuda_mutation_inventory_literal_is_pinned(self) -> None: mutate( root, "tests/scripts/test_check_release_binary_contract.py", - ' "120a",\n "121a",\n)\n\nRECORD_ANCHORS', - ' "120a",\n)\n\nRECORD_ANCHORS', + ' "120a",\n "121a",\n)\n\nGUARD_MAP_KEYS', + ' "120a",\n)\n\nGUARD_MAP_KEYS', ) result = run_checker(root) self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) @@ -475,6 +573,51 @@ def test_each_semantic_inventory_consumer_is_pinned(self) -> None: ) self.assertIn(inventory, result.stdout + result.stderr) + def test_checker_guard_map_keysets_are_exact(self) -> None: + for guard_map, keys in GUARD_MAP_KEYS.items(): + for key in keys: + for mutation in ("delete", "rename"): + with ( + self.subTest( + guard_map=guard_map, + key=key, + mutation=mutation, + ), + RepoCopy() as root, + ): + mutate_checker_guard_map(root, guard_map, mutation, key) + result = run_checker(root) + self.assertNotEqual( + result.returncode, 0, result.stdout + result.stderr + ) + self.assertIn( + "semantic mutation guard map keyset", + result.stdout + result.stderr, + ) + + with self.subTest(guard_map=guard_map, mutation="add"), RepoCopy() as root: + mutate_checker_guard_map(root, guard_map, "add", "EXTRA_GUARD") + result = run_checker(root) + self.assertNotEqual( + result.returncode, 0, result.stdout + result.stderr + ) + self.assertIn( + "semantic mutation guard map keyset", + result.stdout + result.stderr, + ) + + with ( + self.subTest(guard_map=guard_map, mutation="bypass"), + RepoCopy() as root, + ): + mutate_checker_guard_map(root, guard_map, "delete", keys[0]) + bypass_checker_keyset_enforcement(root) + result = run_checker(root) + self.assertNotEqual( + result.returncode, 0, result.stdout + result.stderr + ) + self.assertIn(keys[0], result.stdout + result.stderr) + def test_required_mutation_test_inventory_is_pinned(self) -> None: for method in REQUIRED_TEST_METHODS: for mutation in ("delete", "rename"): From 8cecabead7087bddd9fd05863d28f5d75d92de2c Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 8 Aug 2026 01:17:37 +0000 Subject: [PATCH 09/11] test(release): kill remaining contract bypasses Pin every inventory consumer body, independently reject unknown machine fields, and mirror the complete human W1-W13 dependency table. Expand the mutation contract to 25 named methods and retain honest pending release claims. Row: ENG-RELEASE-BINARIES FOLLOWING_AGENTS_PROTOCOL Assisted-by: Codex:gpt-5 [Codex] --- docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- scripts/check-release-binary-contract.py | 122 ++++++++++- .../test_check_release_binary_contract.py | 199 +++++++++++++++++- 4 files changed, 311 insertions(+), 14 deletions(-) diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 318806df1..69913058b 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -4,7 +4,7 @@ | Reference | Workload | Headline | Tokens | |---|---|---|---| -| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 22 tests. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | +| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 25 tests. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | | **Developer agent entry point (implemented)** | `DOCS-AGENT-PROTOCOL-ENTRYPOINT`: public contribution guide + synchronized, mutation-gated pre-claim intake rule | Rebased documentation/protocol only; benchmark void | n/a | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **Record repair 2026-08-07** | `main` was red on `check-agent-record` + `check-env-doc`, blocking every PR. Dangling `kda-chunk-aot/` link and two undocumented env vars. No behaviour change | diff --git a/docs/STATUS.md b/docs/STATUS.md index 6b7d6c9c7..03c479230 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -112,7 +112,7 @@ token-for-token correctness against the pinned oracle. | Tool-call parsing | 36 parser families / 40 accepted names, streaming | Every vLLM tool parser at the pin except the three Rust/Harmony-backed ones: pure-text parsers ported 1:1, the six engine-backed families reimplemented from their wire formats, all held to the upstream test suites. Selection via `--tool-call-parser` (server), `tool_parser` (C ABI), or template auto-detection; native-syntax forced tool_choice where expressible. Tables: docs/BENCHMARKS.md | | Reasoning parsing (`SAMPLE-REASONING`, ACTIVE, partial coverage) | 9 parsers, streaming | think_auto (auto-detect default: content unless markers appear), deepseek_r1, deepseek_v3 (passthrough) / holo2 (thinking→R1), mistral ([THINK]), minimax_m2 (+append_think), step3, olmo3 - reasoning split engine-side BEFORE tool parsing, streamed as `reasoning` deltas in the chat chunks. Coverage: 9 of upstream's ~28 registered names (remaining text families + engine-backed adapters tracked as W2/W3 in specs/reasoning-parsers.md); each ported parser doctest-gated vs its tests/reasoning case | | Unified streaming parser engine | Core, assembly, serving-SSE dispatch landed, gated; all 10 engine-backed families ported (family parity closed); JSON-schema tool-arg type coercion landed | The vLLM 0.26 declarative `parser/engine/` (shared state machine plus all 10 configs: qwen3, seed_oss, kimi_k2, minimax_m2, glm47_moe, deepseek_v4/v32, nemotron_v3, gemma4, inkling) and assembly layer, gated field-for-field vs vLLM 0.26. An engine-backed `--tool-call-parser` name drives the live chat SSE chunks, off by default. When a request's tools declare typed parameters, the assembled tool-call arguments are coerced to the declared JSON types (int/number/bool/string/array/null) 1:1 with vLLM `_fix_arg_types`, in both streaming and one-shot; no schema means the arguments pass through as strings unchanged. Details: .agents/specs/parser-assembly-c8.md | -| OpenAI server | Supported (subset); bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and 22-test inventory mutation-gated; per-SM diagnostics optional; no archive/runtime claim | +| OpenAI server | Supported (subset); bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and 25-test inventory mutation-gated; per-SM diagnostics optional; no archive/runtime claim | | Pooling task class (embeddings / classify / score / rerank) | Spiked; pooler op + heads composite + pooling runner path landed (CPU), not yet servable end-to-end (no concrete model / endpoints) | The non-generative task class. W0 spike over the whole vLLM pooling surface (`.agents/specs/pooling-task-class.md`, `CLAIM-POOLING`). W1 landed the pooler OP (CLS/LAST/MEAN + Identity/Normalize/MultiLabelClassify/Classify activations, double-precision-gated). **W2 landed the pooler HEADS composite** — `EmbeddingPoolerHead` (projector→matryoshka→normalize), `ClassifierPoolerHead` (classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` task routing, and the `PoolerConfig`/`PoolingParams` structs (`test_pooler_heads` 27/27, 240 asserts, RED-first). **W3 landed the pooling RUNNER path** — `PoolingRunner` applies the model's `Pooler` to the last hidden state and returns pooled embeddings instead of sampled tokens, gated by a STRUCTURAL cosine-parity check vs a double-precision LAST+normalize reference (`test_pooling_runner` 5/5, 14 asserts, RED-first). NOT yet servable / honest residuals: a concrete pooling MODEL forward + the REAL-model oracle cosine gate (`vllm.LLM(task="embed").encode`) — no cosine-vs-oracle number is fabricated (W3-model); the `/v1/embeddings` + score/rerank/classify endpoints (W4); tokwise AllPool/StepPool (W5). See docs/BENCHMARKS.md | | Plugin system (out-of-core registration) | Spiked; first CPU brick landed, not yet wired into any production path | The extensibility-first discovery layer. W0 spike over vLLM's plugin surface (general / platform / io_processor / endpoint groups, the `register_model` an out-of-tree plugin calls, the invocation seams) is committed (`.agents/specs/plugin-system.md`, `ENG-PLUGIN-SYSTEM` ACTIVE, `CLAIM-PLUGIN-SYSTEM`). W1 landed `vllm::plugins::LoadGeneralPlugins()` + the out-of-core general-plugin registration seam (`RegisterGeneralPlugin` / `REGISTER_VLLM_GENERAL_PLUGIN`) over the existing `REGISTER_VLLM_MODEL`-style registries (the in-tree factory `MODEL-FACTORY-registry` is record-repaired `DONE` 2026-08-05: 28 self-registering TUs, dgx debt paid by the 2026-07-23 seven-gate run): a 1:1 mirror of `load_general_plugins` (load-once idempotence, the `VLLM_PLUGINS` allowlist, per-plugin failure isolation). Proven by an out-of-core toy-model plugin that registers a toy architecture through the public `RegisterModel` seam — unit-gated RED-first (`test_plugin_system` 1 case / 29 assertions: the toy arch resolves ONLY after LoadGeneralPlugins runs it, and not under `VLLM_PLUGINS=""`). Python entry points have no C++20 analogue, so discovery is the project's static-init/`dlopen` registration idiom (recorded porting-inventory §9). NOT yet wired: real shared-object `dlopen` + the C-ABI `vllm_plugin_register` entry (W2), the engine/CLI `--load-plugins` wiring that calls LoadGeneralPlugins from the construction paths (W3), the platform/quant plugin kinds (W4), and the io_processor/stat_logger/endpoint groups (W5) are named residuals. See docs/BENCHMARKS.md | | Offline Batch API (JSONL file runner) | Spiked; first CPU brick landed, not yet exposed as a CLI | The offline OpenAI Batch API: read a JSONL of OpenAI-format requests, run each through the engine, write a JSONL of responses. W0 spike over vLLM's `run_batch.py` (schema, endpoint dispatch, run loop, file I/O) is committed (`.agents/specs/batch-api.md`, `SERVE-BATCH-API` ACTIVE, `CLAIM-BATCH-API`). W1 landed `RunBatch` (`RunLine`/`RunLines`/`Run`) + `RunBatchFile` — a pure orchestrator over the existing `OpenAIServingChat::create_chat_completion` (NO reimplemented generation), 1:1 with vLLM's endpoint_registry url→handler map: `/v1/chat/completions` dispatch, the `BatchResponseData`/`BatchRequestOutput` schema (`vllm-` ids, custom_id echoed), the `run_request` AllResponse/ErrorResponse/stream branches, and the unsupported-endpoint/url error rows. Unit-gated RED-first (`test_openai_run_batch` 7 cases / 80 assertions over the synthetic serving engine: ordered rows + custom_id echo + per-line BatchRequestOutput schema round-trip, a malformed line isolated into an error row so the batch continues, dispatch + 404 error rows; dropping the custom_id echo fails 9 assertions). Recorded deviation: a malformed line is isolated (batch continues) where upstream aborts the job. NOT yet exposed: the `vllm run-batch` CLI + `BatchFrontendArgs` (W2), embeddings/score/rerank dispatch (W3, rides pooling endpoints), audio transcription/translation + media fetch (W4), and http(s)/data-URL file I/O + metrics + overlapped `AsyncLLM` submission (W5) are named residuals. See docs/BENCHMARKS.md | diff --git a/scripts/check-release-binary-contract.py b/scripts/check-release-binary-contract.py index df8f481e5..837addb91 100644 --- a/scripts/check-release-binary-contract.py +++ b/scripts/check-release-binary-contract.py @@ -5,6 +5,7 @@ import argparse import ast +import hashlib import re import sys from collections import Counter @@ -126,7 +127,7 @@ "host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM " "diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM " "fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, " - "and 22 tests. No archive, staged smoke, runtime, correctness, or performance " + "and 25 tests. No archive, staged smoke, runtime, correctness, or performance " "evidence " "| n/a |" ) @@ -134,7 +135,7 @@ STATUS_RELEASE_FRAGMENTS = ( "Supported (subset); bundles SPIKED, no artifacts", "primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and " - "22-test inventory mutation-gated; per-SM diagnostics optional; no " + "25-test inventory mutation-gated; per-SM diagnostics optional; no " "archive/runtime claim", ) @@ -169,7 +170,7 @@ ( "docs/BENCHMARKS.md", "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " - "W1-W13/W10-W12 policy, public pending states, and 22 tests. No archive, " + "W1-W13/W10-W12 policy, public pending states, and 25 tests. No archive, " "staged smoke, runtime, correctness, or performance evidence", "**SHIPPED:** archive, runtime, correctness, and performance evidence " "complete", @@ -233,9 +234,12 @@ "test_public_release_rows_remain_pending", "test_human_w12_is_optional_and_cannot_replace_w10", "test_human_primary_artifact_contract_matches_machine_block", + "test_unknown_machine_fields_are_fail_closed", + "test_each_human_work_dependency_is_pinned", "test_primary_cuda_mutation_inventory_literal_is_pinned", "test_work_dependency_mutation_inventory_literal_is_pinned", "test_each_semantic_inventory_consumer_is_pinned", + "test_each_semantic_inventory_consumer_body_is_pinned", "test_checker_guard_map_keysets_are_exact", "test_required_mutation_test_inventory_is_pinned", ) @@ -250,6 +254,10 @@ "PUBLIC_PENDING_MUTATIONS", "W10_W12_HUMAN_MUTATIONS", "PRIMARY_ARTIFACT_PROSE_MUTATIONS", + "INVENTORY_CONSUMER_METHODS", + "CONSUMER_FLOW_MUTATIONS", + "UNKNOWN_MACHINE_FIELD_MUTATIONS", + "HUMAN_WORK_DEPS", "GUARD_MAP_KEYS", ) @@ -263,6 +271,10 @@ "PUBLIC_PENDING_MUTATIONS", "W10_W12_HUMAN_MUTATIONS", "PRIMARY_ARTIFACT_PROSE_MUTATIONS", + "INVENTORY_CONSUMER_METHODS", + "CONSUMER_FLOW_MUTATIONS", + "UNKNOWN_MACHINE_FIELD_MUTATIONS", + "HUMAN_WORK_DEPS", "GUARD_MAP_KEYS", ) @@ -291,6 +303,29 @@ "PUBLIC_PENDING_MUTATIONS": PUBLIC_PENDING_MUTATIONS, "W10_W12_HUMAN_MUTATIONS": W10_W12_HUMAN_MUTATIONS, "PRIMARY_ARTIFACT_PROSE_MUTATIONS": PRIMARY_ARTIFACT_PROSE_MUTATIONS, + "INVENTORY_CONSUMER_METHODS": { + "PRIMARY_CUDA_SMS": "test_each_primary_cuda_sm_is_required", + "EXACT_MACHINE_FIELDS": "test_each_exact_machine_field_is_fail_closed", + "EXPECTED_DEPS": "test_each_work_dependency_edge_is_pinned", + "HUMAN_WORK_IDS": "test_each_human_work_row_id_occurs_exactly_once", + "RECORD_ANCHORS": "test_each_required_record_anchor_is_fail_closed", + "LIFECYCLE_RECORD_MUTATIONS": ( + "test_release_lifecycle_and_honesty_are_fail_closed" + ), + "PUBLIC_PENDING_MUTATIONS": "test_public_release_rows_remain_pending", + "W10_W12_HUMAN_MUTATIONS": ( + "test_human_w12_is_optional_and_cannot_replace_w10" + ), + "PRIMARY_ARTIFACT_PROSE_MUTATIONS": ( + "test_human_primary_artifact_contract_matches_machine_block" + ), + "GUARD_MAP_KEYS": "test_checker_guard_map_keysets_are_exact", + }, + "CONSUMER_FLOW_MUTATIONS": ("continue", "break", "wrap_false"), + "UNKNOWN_MACHINE_FIELD_MUTATIONS": (("unexpected_field", "x"),), + "HUMAN_WORK_DEPS": { + work: ",".join(deps) for work, deps in WORK_DEPS.items() + }, "GUARD_MAP_KEYS": EXPECTED_GUARD_MAP_KEYS, } @@ -340,6 +375,26 @@ ("before", "after", "reason"), False, ), + "INVENTORY_CONSUMER_METHODS": ( + "test_each_semantic_inventory_consumer_body_is_pinned", + ("inventory", "method"), + True, + ), + "CONSUMER_FLOW_MUTATIONS": ( + "test_each_semantic_inventory_consumer_body_is_pinned", + ("mutation",), + False, + ), + "UNKNOWN_MACHINE_FIELD_MUTATIONS": ( + "test_unknown_machine_fields_are_fail_closed", + ("field", "value"), + False, + ), + "HUMAN_WORK_DEPS": ( + "test_each_human_work_dependency_is_pinned", + ("work", "expected"), + True, + ), "GUARD_MAP_KEYS": ( "test_checker_guard_map_keysets_are_exact", ("guard_map", "keys"), @@ -347,6 +402,23 @@ ), } +TEST_INVENTORY_BODY_DIGESTS = { + "PRIMARY_CUDA_SMS": "5dc05132f5f24f0b2add406cb5682031b05d1940d2189728543b53a375d15125", + "EXACT_MACHINE_FIELDS": "8fa2ec5fca092a1092052a1426c37b66b8b8be49290e0e6e414e555dcbdad8dc", + "EXPECTED_DEPS": "4f8d345df7b467312869355f15153e04b77a0fd7005ea60ad86db40cc55ae6a2", + "HUMAN_WORK_IDS": "363c9494815086a53c3112792afe3c1651256d06a519e6fbeed43b78e865d922", + "RECORD_ANCHORS": "0e15f59b43bd4e70535055ccbcb41c500f89006fe9df4c2e58b0cfd0a00205f1", + "LIFECYCLE_RECORD_MUTATIONS": "79de0584503bb8c1cc1463169d8741455714c1d252136a89294e532878ec996a", + "PUBLIC_PENDING_MUTATIONS": "507e58eb51f04ab03db0cea484513da5cb0a7fdd2d6f6f332004e09106682fd3", + "W10_W12_HUMAN_MUTATIONS": "1fcb8914c91dc01052f7f26a45cbbaebe01c211bc0e692f70df3d62b13a963bf", + "PRIMARY_ARTIFACT_PROSE_MUTATIONS": "1fcb8914c91dc01052f7f26a45cbbaebe01c211bc0e692f70df3d62b13a963bf", + "GUARD_MAP_KEYS": "06691dc7239166ac458c9999d545938f7c8afb5021cb347d0fa6bdd0dac2a082", + "INVENTORY_CONSUMER_METHODS": "5754b33de9ca699665d4f612f8089371dbc7d9ce583422ffeb48a33499575eab", + "CONSUMER_FLOW_MUTATIONS": "a2d05b5bea24a09c6c5313f4e65d259a9b7e984f6450aa74309f790b3e81de8a", + "UNKNOWN_MACHINE_FIELD_MUTATIONS": "1d9242dabe625e43b909709ed0a40915d55fd6610c06bab3e696dc803745e0b8", + "HUMAN_WORK_DEPS": "800c69c64c995030fff12ccfbe2bb0002193c17cbe28aeca52c584cf15c2b675", +} + EXACT_MACHINE_FIELDS = { "lifecycle": "SPIKE", "work_W12_policy": "optional-non-blocking", @@ -580,18 +652,19 @@ def _target_names(target: ast.expr) -> tuple[str, ...]: return () -def _iterates_inventory( +def _inventory_loop( method: ast.FunctionDef | ast.AsyncFunctionDef, inventory: str, target_names: tuple[str, ...], mapping_items: bool, -) -> bool: +) -> ast.For | None: + matches: list[ast.For] = [] for node in ast.walk(method): if not isinstance(node, ast.For) or _target_names(node.target) != target_names: continue if not mapping_items and isinstance(node.iter, ast.Name): if node.iter.id == inventory: - return True + matches.append(node) if ( mapping_items and isinstance(node.iter, ast.Call) @@ -602,8 +675,14 @@ def _iterates_inventory( and isinstance(node.iter.func.value, ast.Name) and node.iter.func.value.id == inventory ): - return True - return False + matches.append(node) + return matches[0] if len(matches) == 1 else None + + +def _consumer_body_digest(loop: ast.For) -> str: + body = ast.Module(body=loop.body, type_ignores=[]) + serialized = ast.dump(body, include_attributes=False).encode("utf-8") + return hashlib.sha256(serialized).hexdigest() def _test_inventory_errors(root: Path) -> list[str]: @@ -684,13 +763,24 @@ def _test_inventory_errors(root: Path) -> list[str]: continue method_name, target_names, mapping_items = TEST_INVENTORY_CONSUMERS[inventory] method = methods_by_name.get(method_name) - if method is None or not _iterates_inventory( - method, inventory, target_names, mapping_items - ): + loop = ( + None + if method is None + else _inventory_loop(method, inventory, target_names, mapping_items) + ) + if loop is None: errors.append( f"semantic mutation inventory {inventory} is not consumed by " f"{method_name} through its required literal iteration" ) + continue + actual_digest = _consumer_body_digest(loop) + expected_digest = TEST_INVENTORY_BODY_DIGESTS[inventory] + if actual_digest != expected_digest: + errors.append( + f"semantic mutation inventory {inventory} consumer body drifted; " + f"expected {expected_digest}, found {actual_digest}" + ) return errors @@ -707,6 +797,11 @@ def contract_errors(root: Path) -> list[str]: errors.append(f"release contract is missing fields: {sorted(missing)}") if extra: errors.append(f"release contract has unknown fields: {sorted(extra)}") + if set(fields) != set(EXPECTED_FIELDS): + errors.append( + "release contract schema has missing or unknown fields: " + f"expected {sorted(EXPECTED_FIELDS)}, found {sorted(fields)}" + ) for key, expected in EXPECTED_FIELDS.items(): if fields.get(key) != expected: errors.append(_field_error(key, fields.get(key), expected)) @@ -739,6 +834,11 @@ def contract_errors(root: Path) -> list[str]: f"{work} dependencies in work table are {rows.get(work)!r}; " f"expected {expected!r}" ) + if rows != WORK_DEPS: + errors.append( + "human work-table dependency mirror drifted: " + f"expected {WORK_DEPS!r}, found {rows!r}" + ) work_content = { work: (_normalize_prose(deliverable), _normalize_prose(exit_gate)) diff --git a/tests/scripts/test_check_release_binary_contract.py b/tests/scripts/test_check_release_binary_contract.py index 95151eec8..036a647f1 100644 --- a/tests/scripts/test_check_release_binary_contract.py +++ b/tests/scripts/test_check_release_binary_contract.py @@ -46,9 +46,12 @@ "test_public_release_rows_remain_pending", "test_human_w12_is_optional_and_cannot_replace_w10", "test_human_primary_artifact_contract_matches_machine_block", + "test_unknown_machine_fields_are_fail_closed", + "test_each_human_work_dependency_is_pinned", "test_primary_cuda_mutation_inventory_literal_is_pinned", "test_work_dependency_mutation_inventory_literal_is_pinned", "test_each_semantic_inventory_consumer_is_pinned", + "test_each_semantic_inventory_consumer_body_is_pinned", "test_checker_guard_map_keysets_are_exact", "test_required_mutation_test_inventory_is_pinned", ) @@ -77,6 +80,10 @@ "PUBLIC_PENDING_MUTATIONS", "W10_W12_HUMAN_MUTATIONS", "PRIMARY_ARTIFACT_PROSE_MUTATIONS", + "INVENTORY_CONSUMER_METHODS", + "CONSUMER_FLOW_MUTATIONS", + "UNKNOWN_MACHINE_FIELD_MUTATIONS", + "HUMAN_WORK_DEPS", "GUARD_MAP_KEYS", ), "TEST_INVENTORY_CONSUMERS": ( @@ -89,6 +96,10 @@ "PUBLIC_PENDING_MUTATIONS", "W10_W12_HUMAN_MUTATIONS", "PRIMARY_ARTIFACT_PROSE_MUTATIONS", + "INVENTORY_CONSUMER_METHODS", + "CONSUMER_FLOW_MUTATIONS", + "UNKNOWN_MACHINE_FIELD_MUTATIONS", + "HUMAN_WORK_DEPS", "GUARD_MAP_KEYS", ), } @@ -205,7 +216,7 @@ ( "docs/BENCHMARKS.md", "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " - "W1-W13/W10-W12 policy, public pending states, and 22 tests. No archive, " + "W1-W13/W10-W12 policy, public pending states, and 25 tests. No archive, " "staged smoke, runtime, correctness, or performance evidence", "**SHIPPED:** archive, runtime, correctness, and performance evidence " "complete", @@ -263,6 +274,41 @@ ), } +INVENTORY_CONSUMER_METHODS = { + "PRIMARY_CUDA_SMS": "test_each_primary_cuda_sm_is_required", + "EXACT_MACHINE_FIELDS": "test_each_exact_machine_field_is_fail_closed", + "EXPECTED_DEPS": "test_each_work_dependency_edge_is_pinned", + "HUMAN_WORK_IDS": "test_each_human_work_row_id_occurs_exactly_once", + "RECORD_ANCHORS": "test_each_required_record_anchor_is_fail_closed", + "LIFECYCLE_RECORD_MUTATIONS": "test_release_lifecycle_and_honesty_are_fail_closed", + "PUBLIC_PENDING_MUTATIONS": "test_public_release_rows_remain_pending", + "W10_W12_HUMAN_MUTATIONS": "test_human_w12_is_optional_and_cannot_replace_w10", + "PRIMARY_ARTIFACT_PROSE_MUTATIONS": ( + "test_human_primary_artifact_contract_matches_machine_block" + ), + "GUARD_MAP_KEYS": "test_checker_guard_map_keysets_are_exact", +} + +CONSUMER_FLOW_MUTATIONS = ("continue", "break", "wrap_false") + +UNKNOWN_MACHINE_FIELD_MUTATIONS = (("unexpected_field", "x"),) + +HUMAN_WORK_DEPS = { + "W1": "", + "W2": "W1", + "W3": "", + "W4": "", + "W5": "", + "W6": "", + "W7": "W1,W2,W3,W4,W5,W6", + "W8": "W5,W7", + "W9": "W3,W4,W5,W6,W7", + "W10": "W1,W2,W5,W6,W7", + "W11": "W5,W6,W7", + "W12": "W1,W2,W5,W6,W7", + "W13": "W5,W7,W8,W9,W10,W11", +} + def run_checker(root: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( @@ -390,6 +436,107 @@ def bypass_checker_keyset_enforcement(root: Path) -> None: path.write_text(ast.unparse(ast.fix_missing_locations(tree)) + "\n", encoding="utf-8") +def mutate_inventory_consumer_flow( + root: Path, inventory: str, mutation: str +) -> None: + relative = "tests/scripts/test_check_release_binary_contract.py" + path = root / relative + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + method_name = INVENTORY_CONSUMER_METHODS[inventory] + method = next( + node + for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef) and node.name == method_name + ) + loops = [] + for node in ast.walk(method): + if not isinstance(node, ast.For): + continue + source = node.iter + if isinstance(source, ast.Name) and source.id == inventory: + loops.append(node) + elif ( + isinstance(source, ast.Call) + and isinstance(source.func, ast.Attribute) + and isinstance(source.func.value, ast.Name) + and source.func.value.id == inventory + and source.func.attr == "items" + ): + loops.append(node) + if len(loops) != 1: + raise AssertionError( + f"expected one {inventory} consumer loop in {method_name}, found {len(loops)}" + ) + loop = loops[0] + if mutation == "continue": + loop.body.insert(0, ast.Continue()) + elif mutation == "break": + loop.body.insert(0, ast.Break()) + elif mutation == "wrap_false": + loop.body = [ast.If(test=ast.Constant(False), body=loop.body, orelse=[])] + else: + raise AssertionError(f"unknown consumer-flow mutation {mutation!r}") + path.write_text(ast.unparse(ast.fix_missing_locations(tree)) + "\n", encoding="utf-8") + + +def bypass_unknown_field_enforcement(root: Path) -> None: + relative = "scripts/check-release-binary-contract.py" + path = root / relative + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + matches = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.If) + and isinstance(node.test, ast.Name) + and node.test.id == "extra" + ] + if len(matches) != 1: + raise AssertionError( + f"expected one unknown-field equality, found {len(matches)}" + ) + matches[0].test = ast.Constant(False) + path.write_text(ast.unparse(ast.fix_missing_locations(tree)) + "\n", encoding="utf-8") + + +def mutate_human_work_dependency(root: Path, work: str, replacement: str) -> None: + relative = ".agents/specs/release-binary-matrix.md" + path = root / relative + lines = path.read_text(encoding="utf-8").splitlines(keepends=True) + matches = [index for index, line in enumerate(lines) if line.startswith(f"| {work} |")] + if len(matches) != 1: + raise AssertionError(f"expected one human work row {work}, found {len(matches)}") + index = matches[0] + cells = lines[index].split("|") + cells[2] = f" {replacement} " + lines[index] = "|".join(cells) + path.write_text("".join(lines), encoding="utf-8") + + +def bypass_human_work_dependency_enforcement(root: Path) -> None: + relative = "scripts/check-release-binary-contract.py" + path = root / relative + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + matches = [] + for node in ast.walk(tree): + if not isinstance(node, ast.If) or not isinstance(node.test, ast.Compare): + continue + left = node.test.left + if ( + isinstance(left, ast.Call) + and isinstance(left.func, ast.Attribute) + and isinstance(left.func.value, ast.Name) + and left.func.value.id == "rows" + and left.func.attr == "get" + ): + matches.append(node) + if len(matches) != 1: + raise AssertionError( + f"expected one human dependency equality, found {len(matches)}" + ) + matches[0].test = ast.Constant(False) + path.write_text(ast.unparse(ast.fix_missing_locations(tree)) + "\n", encoding="utf-8") + + class LiveContract(unittest.TestCase): def test_repository_contract_passes(self) -> None: result = run_checker(ROOT) @@ -467,6 +614,22 @@ def test_each_exact_machine_field_is_fail_closed(self) -> None: ) self.assertIn("expected", result.stdout + result.stderr) + def test_unknown_machine_fields_are_fail_closed(self) -> None: + for field, value in UNKNOWN_MACHINE_FIELD_MUTATIONS: + with self.subTest(field=field), RepoCopy() as root: + mutate( + root, + self.SPEC, + "lifecycle=SPIKE", + f"lifecycle=SPIKE\n{field}={value}", + ) + bypass_unknown_field_enforcement(root) + result = run_checker(root) + self.assertNotEqual( + result.returncode, 0, result.stdout + result.stderr + ) + self.assertIn("unknown fields", result.stdout + result.stderr) + class WorkGraphMutations(unittest.TestCase): SPEC = ".agents/specs/release-binary-matrix.md" @@ -486,6 +649,20 @@ def test_each_work_dependency_edge_is_pinned(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn(f"{work} dependencies", result.stdout + result.stderr) + def test_each_human_work_dependency_is_pinned(self) -> None: + for work, expected in HUMAN_WORK_DEPS.items(): + with self.subTest(work=work, expected=expected), RepoCopy() as root: + mutate_human_work_dependency(root, work, "W99") + bypass_human_work_dependency_enforcement(root) + result = run_checker(root) + self.assertNotEqual( + result.returncode, 0, result.stdout + result.stderr + ) + self.assertIn( + "human work-table dependency mirror", + result.stdout + result.stderr, + ) + def test_each_human_work_row_id_occurs_exactly_once(self) -> None: for work in HUMAN_WORK_IDS: with self.subTest(work=work, mutation="duplicate"), RepoCopy() as root: @@ -654,6 +831,26 @@ def test_each_semantic_inventory_consumer_is_pinned(self) -> None: ) self.assertIn(inventory, result.stdout + result.stderr) + def test_each_semantic_inventory_consumer_body_is_pinned(self) -> None: + for inventory, method in INVENTORY_CONSUMER_METHODS.items(): + for mutation in CONSUMER_FLOW_MUTATIONS: + with ( + self.subTest( + inventory=inventory, + method=method, + mutation=mutation, + ), + RepoCopy() as root, + ): + mutate_inventory_consumer_flow(root, inventory, mutation) + result = run_checker(root) + self.assertNotEqual( + result.returncode, 0, result.stdout + result.stderr + ) + output = result.stdout + result.stderr + self.assertIn(inventory, output) + self.assertIn("consumer body", output) + def test_checker_guard_map_keysets_are_exact(self) -> None: for guard_map, keys in GUARD_MAP_KEYS.items(): for key in keys: From 8ab5fb78ce378c6f5a1527aadf0c449e3b47cde0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 8 Aug 2026 01:22:03 +0000 Subject: [PATCH 10/11] docs(release): preserve latest-main keyed text Restore origin/main's full NOW guidance and OpenAI endpoint inventory, then reapply only the release SPIKE clauses. Keep NOW and STATUS within their existing size ratchets through concise release wording and one semantics-preserving NOW compaction. Reconciled-main: b44ad337f2524ddce38ace492599a0e595ad0388 Row: ENG-RELEASE-BINARIES FOLLOWING_AGENTS_PROTOCOL Assisted-by: Codex:gpt-5 [Codex] --- .agents/NOW.md | 4 ++-- docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- scripts/check-release-binary-contract.py | 10 ++++------ tests/scripts/test_check_release_binary_contract.py | 2 +- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.agents/NOW.md b/.agents/NOW.md index b805c4e19..9db8860fb 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -2,7 +2,7 @@ -Read this FIRST. A SNAPSHOT: what is live, +Read this FIRST, every session. A SNAPSHOT, rewritten in place: what is live, the gate being chased, what to do next. Never a log — evidence lives in the append-only [state.md](state.md), [parity-ledger.md](parity-ledger.md) and the benchmark record. Budget: 100 lines. @@ -42,7 +42,7 @@ both gate models, reproduced 2–3x on an idle box. See [gates.md](gates.md) and [benchmark-protocol.md](benchmark-protocol.md). Parity pin: vLLM `555967922` (0.26.0.dev0). -Method rules hardened (AGENTS.md): the STRUCTURAL lens (same kernel, different +AGENTS.md: STRUCTURAL lens (same kernel, different throughput ⇒ audit the context; per-shape MEASUREMENT arbitrates). ## Next actions diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 69913058b..37d5ad741 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -4,7 +4,7 @@ | Reference | Workload | Headline | Tokens | |---|---|---|---| -| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, and 25 tests. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | +| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states; 25 tests GREEN. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | | **Developer agent entry point (implemented)** | `DOCS-AGENT-PROTOCOL-ENTRYPOINT`: public contribution guide + synchronized, mutation-gated pre-claim intake rule | Rebased documentation/protocol only; benchmark void | n/a | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **Record repair 2026-08-07** | `main` was red on `check-agent-record` + `check-env-doc`, blocking every PR. Dangling `kda-chunk-aot/` link and two undocumented env vars. No behaviour change | diff --git a/docs/STATUS.md b/docs/STATUS.md index 03c479230..de09ec11a 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -112,7 +112,7 @@ token-for-token correctness against the pinned oracle. | Tool-call parsing | 36 parser families / 40 accepted names, streaming | Every vLLM tool parser at the pin except the three Rust/Harmony-backed ones: pure-text parsers ported 1:1, the six engine-backed families reimplemented from their wire formats, all held to the upstream test suites. Selection via `--tool-call-parser` (server), `tool_parser` (C ABI), or template auto-detection; native-syntax forced tool_choice where expressible. Tables: docs/BENCHMARKS.md | | Reasoning parsing (`SAMPLE-REASONING`, ACTIVE, partial coverage) | 9 parsers, streaming | think_auto (auto-detect default: content unless markers appear), deepseek_r1, deepseek_v3 (passthrough) / holo2 (thinking→R1), mistral ([THINK]), minimax_m2 (+append_think), step3, olmo3 - reasoning split engine-side BEFORE tool parsing, streamed as `reasoning` deltas in the chat chunks. Coverage: 9 of upstream's ~28 registered names (remaining text families + engine-backed adapters tracked as W2/W3 in specs/reasoning-parsers.md); each ported parser doctest-gated vs its tests/reasoning case | | Unified streaming parser engine | Core, assembly, serving-SSE dispatch landed, gated; all 10 engine-backed families ported (family parity closed); JSON-schema tool-arg type coercion landed | The vLLM 0.26 declarative `parser/engine/` (shared state machine plus all 10 configs: qwen3, seed_oss, kimi_k2, minimax_m2, glm47_moe, deepseek_v4/v32, nemotron_v3, gemma4, inkling) and assembly layer, gated field-for-field vs vLLM 0.26. An engine-backed `--tool-call-parser` name drives the live chat SSE chunks, off by default. When a request's tools declare typed parameters, the assembled tool-call arguments are coerced to the declared JSON types (int/number/bool/string/array/null) 1:1 with vLLM `_fix_arg_types`, in both streaming and one-shot; no schema means the arguments pass through as strings unchanged. Details: .agents/specs/parser-assembly-c8.md | -| OpenAI server | Supported (subset); bundles SPIKED, no artifacts | Completions/chat (including SSE), models, health/version/ping, metrics, tokenize/detokenize, tokenizer/server info, prefix reset and abort; videos in Sora shape plus content GET, with image/video/audio references. Tokenizer info and abort are flag-gated; metrics and prefix reset lack live async backing. Endpoint/flag source: docs/USAGE.md. Depth-2 async heap fix landed. [Release spike](../.agents/specs/release-binary-matrix.md): primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and 25-test inventory mutation-gated; per-SM diagnostics optional; no archive/runtime claim | +| OpenAI server | Supported (subset); bundles SPIKED, no artifacts | `/v1/completions`, `/v1/chat/completions`, streaming SSE, `/v1/models`, `/health`, `/version`, `/ping`, `/metrics`, `/tokenize` (raw-`prompt` and chat-`messages`), `/detokenize`, `/tokenizer_info`, `/server_info`, `/reset_prefix_cache`, `/abort_requests`; `/v1/videos` in OpenAI's Sora shape + `GET /v1/videos/{id}/content`, conditioned on an `input_reference` image or the `metadata` video/audio references. `/tokenizer_info` and `/abort_requests` are flag-gated; `/metrics` and `/reset_prefix_cache` have handlers but no live backing on the async path. Endpoint list and flags: docs/USAGE.md. Depth-2 async serving no longer corrupts the host heap under `ignore_eos`. [Release spike](../.agents/specs/release-binary-matrix.md): pending. | | Pooling task class (embeddings / classify / score / rerank) | Spiked; pooler op + heads composite + pooling runner path landed (CPU), not yet servable end-to-end (no concrete model / endpoints) | The non-generative task class. W0 spike over the whole vLLM pooling surface (`.agents/specs/pooling-task-class.md`, `CLAIM-POOLING`). W1 landed the pooler OP (CLS/LAST/MEAN + Identity/Normalize/MultiLabelClassify/Classify activations, double-precision-gated). **W2 landed the pooler HEADS composite** — `EmbeddingPoolerHead` (projector→matryoshka→normalize), `ClassifierPoolerHead` (classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` task routing, and the `PoolerConfig`/`PoolingParams` structs (`test_pooler_heads` 27/27, 240 asserts, RED-first). **W3 landed the pooling RUNNER path** — `PoolingRunner` applies the model's `Pooler` to the last hidden state and returns pooled embeddings instead of sampled tokens, gated by a STRUCTURAL cosine-parity check vs a double-precision LAST+normalize reference (`test_pooling_runner` 5/5, 14 asserts, RED-first). NOT yet servable / honest residuals: a concrete pooling MODEL forward + the REAL-model oracle cosine gate (`vllm.LLM(task="embed").encode`) — no cosine-vs-oracle number is fabricated (W3-model); the `/v1/embeddings` + score/rerank/classify endpoints (W4); tokwise AllPool/StepPool (W5). See docs/BENCHMARKS.md | | Plugin system (out-of-core registration) | Spiked; first CPU brick landed, not yet wired into any production path | The extensibility-first discovery layer. W0 spike over vLLM's plugin surface (general / platform / io_processor / endpoint groups, the `register_model` an out-of-tree plugin calls, the invocation seams) is committed (`.agents/specs/plugin-system.md`, `ENG-PLUGIN-SYSTEM` ACTIVE, `CLAIM-PLUGIN-SYSTEM`). W1 landed `vllm::plugins::LoadGeneralPlugins()` + the out-of-core general-plugin registration seam (`RegisterGeneralPlugin` / `REGISTER_VLLM_GENERAL_PLUGIN`) over the existing `REGISTER_VLLM_MODEL`-style registries (the in-tree factory `MODEL-FACTORY-registry` is record-repaired `DONE` 2026-08-05: 28 self-registering TUs, dgx debt paid by the 2026-07-23 seven-gate run): a 1:1 mirror of `load_general_plugins` (load-once idempotence, the `VLLM_PLUGINS` allowlist, per-plugin failure isolation). Proven by an out-of-core toy-model plugin that registers a toy architecture through the public `RegisterModel` seam — unit-gated RED-first (`test_plugin_system` 1 case / 29 assertions: the toy arch resolves ONLY after LoadGeneralPlugins runs it, and not under `VLLM_PLUGINS=""`). Python entry points have no C++20 analogue, so discovery is the project's static-init/`dlopen` registration idiom (recorded porting-inventory §9). NOT yet wired: real shared-object `dlopen` + the C-ABI `vllm_plugin_register` entry (W2), the engine/CLI `--load-plugins` wiring that calls LoadGeneralPlugins from the construction paths (W3), the platform/quant plugin kinds (W4), and the io_processor/stat_logger/endpoint groups (W5) are named residuals. See docs/BENCHMARKS.md | | Offline Batch API (JSONL file runner) | Spiked; first CPU brick landed, not yet exposed as a CLI | The offline OpenAI Batch API: read a JSONL of OpenAI-format requests, run each through the engine, write a JSONL of responses. W0 spike over vLLM's `run_batch.py` (schema, endpoint dispatch, run loop, file I/O) is committed (`.agents/specs/batch-api.md`, `SERVE-BATCH-API` ACTIVE, `CLAIM-BATCH-API`). W1 landed `RunBatch` (`RunLine`/`RunLines`/`Run`) + `RunBatchFile` — a pure orchestrator over the existing `OpenAIServingChat::create_chat_completion` (NO reimplemented generation), 1:1 with vLLM's endpoint_registry url→handler map: `/v1/chat/completions` dispatch, the `BatchResponseData`/`BatchRequestOutput` schema (`vllm-` ids, custom_id echoed), the `run_request` AllResponse/ErrorResponse/stream branches, and the unsupported-endpoint/url error rows. Unit-gated RED-first (`test_openai_run_batch` 7 cases / 80 assertions over the synthetic serving engine: ordered rows + custom_id echo + per-line BatchRequestOutput schema round-trip, a malformed line isolated into an error row so the batch continues, dispatch + 404 error rows; dropping the custom_id echo fails 9 assertions). Recorded deviation: a malformed line is isolated (batch continues) where upstream aborts the job. NOT yet exposed: the `vllm run-batch` CLI + `BatchFrontendArgs` (W2), embeddings/score/rerank dispatch (W3, rides pooling endpoints), audio transcription/translation + media fetch (W4), and http(s)/data-URL file I/O + metrics + overlapped `AsyncLLM` submission (W5) are named residuals. See docs/BENCHMARKS.md | diff --git a/scripts/check-release-binary-contract.py b/scripts/check-release-binary-contract.py index 837addb91..5127c959c 100644 --- a/scripts/check-release-binary-contract.py +++ b/scripts/check-release-binary-contract.py @@ -126,17 +126,15 @@ "| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary " "host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM " "diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM " - "fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states, " - "and 25 tests. No archive, staged smoke, runtime, correctness, or performance " + "fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states; " + "25 tests GREEN. No archive, staged smoke, runtime, correctness, or performance " "evidence " "| n/a |" ) STATUS_RELEASE_FRAGMENTS = ( "Supported (subset); bundles SPIKED, no artifacts", - "primary fat CUDA/adaptive CPU, W1-W13/W10-W12 policy, pending claims, and " - "25-test inventory mutation-gated; per-SM diagnostics optional; no " - "archive/runtime claim", + "[Release spike](../.agents/specs/release-binary-matrix.md): pending.", ) HUMAN_CONTRACT = { @@ -170,7 +168,7 @@ ( "docs/BENCHMARKS.md", "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " - "W1-W13/W10-W12 policy, public pending states, and 25 tests. No archive, " + "W1-W13/W10-W12 policy, public pending states; 25 tests GREEN. No archive, " "staged smoke, runtime, correctness, or performance evidence", "**SHIPPED:** archive, runtime, correctness, and performance evidence " "complete", diff --git a/tests/scripts/test_check_release_binary_contract.py b/tests/scripts/test_check_release_binary_contract.py index 036a647f1..d1aebf317 100644 --- a/tests/scripts/test_check_release_binary_contract.py +++ b/tests/scripts/test_check_release_binary_contract.py @@ -216,7 +216,7 @@ ( "docs/BENCHMARKS.md", "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " - "W1-W13/W10-W12 policy, public pending states, and 25 tests. No archive, " + "W1-W13/W10-W12 policy, public pending states; 25 tests GREEN. No archive, " "staged smoke, runtime, correctness, or performance evidence", "**SHIPPED:** archive, runtime, correctness, and performance evidence " "complete", From 897b1d0890fadb2131b9ed46f544345be773dcea Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 8 Aug 2026 05:34:03 +0000 Subject: [PATCH 11/11] fix(release): bind PR #129 contract execution Pin ENG-RELEASE-BINARIES backend policy, active CI/preflight wiring, and the compact pending public status without replacing the OpenAI endpoint record. Row: ENG-RELEASE-BINARIES PR: #129 FOLLOWING_AGENTS_PROTOCOL Assisted-by: Codex:gpt-5 [Codex] --- .agents/NOW.md | 2 +- .agents/specs/release-binary-matrix.md | 14 + .agents/state.md | 19 + docs/BENCHMARKS.md | 2 +- docs/STATUS.md | 2 +- scripts/check-release-binary-contract.py | 488 +++++++++++++++++- .../test_check_release_binary_contract.py | 202 +++++++- 7 files changed, 713 insertions(+), 16 deletions(-) diff --git a/.agents/NOW.md b/.agents/NOW.md index 3729ead7b..e2bafa840 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -28,7 +28,7 @@ Working head: `row/backend-rocm-w0` (#41). Prior: benchmark checkpoint | Supported-models list | **LANDED**: FEATURES arch table CI-bound (33 archs) | — | | `/v1/videos` OpenAI shape | **MERGED** (#71): Sora `model`/`size`/`seconds` + `GET /{id}/content` | `row/SERVE-VIDEOS-REFS` PR open: reference conditioning | | `BACKEND-ROCM` W0 | Skeleton in; **HIP never compiled** (no AMD HW) | #41 contributors build it; a compile error IS the deliverable | -| Release | SPIKE | #129 | +| Release | SPIKE; 30/30 | #129 | | Surface coverage (`ARCH-ONE-SURFACE`) | **ROW 8 LANDED; #139 repair CPU-GREEN**: ABI v14 stable; registry-resolved named platform; DSR 39→32; execution guard 52/52 | Fresh re-review #139; CUDA A/B residual | In-flight (default-OFF, not pushed): `laguna-fp4proj-prod`, laguna diff --git a/.agents/specs/release-binary-matrix.md b/.agents/specs/release-binary-matrix.md index bfe84168e..a0ec7caaa 100644 --- a/.agents/specs/release-binary-matrix.md +++ b/.agents/specs/release-binary-matrix.md @@ -20,6 +20,13 @@ x86_64_baseline=portable-sse2-without-avx2 work_W12_policy=optional-non-blocking archive_claims=pending runtime_claims=pending +metal_channel=stable-after-runtime-gate +mlx_channel=preview +vulkan_channel=preview +musl_channel=experimental-preview +musl_scope=cpu-only-no-gpu +rocm_channel=blocked +gpu_driver_boundary=external-host-never-bundled required_anchor_paths=.agents/engine-matrix.md,.agents/roadmap_v1.md,.agents/NOW.md,.agents/coordination.md,.agents/state.md,docs/STATUS.md,docs/BENCHMARKS.md work_W1= work_W2=W1 @@ -41,6 +48,13 @@ values and the human-readable work table below must change together through a new reviewed design decision. In particular W12 is optional and is deliberately absent from W13's dependency set. +The same checker owns its execution path: it requires itself and its 30-test +mutation suite as direct commands in an unconditional CI step, and executes an +instrumented copy of preflight to prove both named arrays reach their real +loops. The suite independently mutates the Metal, MLX, Vulkan, musl, ROCm and +external-driver policies above; quoted, conditional, inert or deleted wiring is +not accepted as execution. + ## Scope and product contract The deliverable is a downloadable, backend-specific `vllm-server` bundle. The diff --git a/.agents/state.md b/.agents/state.md index 0c76c2e98..93f52a64c 100644 --- a/.agents/state.md +++ b/.agents/state.md @@ -42669,3 +42669,22 @@ the shipped-tree check and M43, while deleting the production path guard still fails M42 itself. Baseline checker plus suite are green at 52/52. This remains CPU-only governance hardening; selector behavior, ABI, DSR 32, performance, model state, CUDA A/B and release state are unchanged. + +## 2026-08-08 — PR #129 release contract binds backend policy and guard execution + + +Final mutation review found that the accepted release checker did not bind five +backend channel decisions or its own CI/preflight execution, and that its +`docs/STATUS.md` edit replaced unrelated OpenAI/Sora endpoint prose. The repair +adds exact machine and human guards for Metal, MLX, Vulkan, experimental +CPU-only musl, blocked ROCm, and the external host GPU-driver boundary. + +The release checker now accepts only direct argv in an unconditional Actions +job/step and executes preflight under Python/Git shims to prove the checker and +suite each run once through the named loops. Nineteen new semantic/wiring +mutations are red, and their inventories, consumer bodies and 30 named tests are +production-pinned. The full release suite is 30/30 and the shared executable +registration guard remains 52/52. The OpenAI row is byte-identical to main after +removing only `; #129: SPIKE∅`; the compact clause consumes the existing +279150-character ratchet exactly. `ENG-RELEASE-BINARIES` remains `SPIKE`: there +is no archive, runtime, correctness or performance evidence. diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 2421aee88..c08de4de2 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -4,7 +4,7 @@ | Reference | Workload | Headline | Tokens | |---|---|---|---| -| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states; 25 tests GREEN. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | +| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`: primary host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states; 30 tests GREEN. No archive, staged smoke, runtime, correctness, or performance evidence | n/a | | **Developer agent entry point (implemented)** | `DOCS-AGENT-PROTOCOL-ENTRYPOINT`: public contribution guide + synchronized, mutation-gated pre-claim intake rule | Rebased documentation/protocol only; benchmark void | n/a | | **ARCH audit: ABI is text-only** | 4 capabilities (H3 video, Laguna, Kimi-Linear, DeepSeek-V4) reachable only from `examples/`, none registry-backed. No gate asks whether a CONSUMER can reach a capability. Documentation only | | **Record repair 2026-08-07** | `main` was red on `check-agent-record` + `check-env-doc`, blocking every PR. Dangling `kda-chunk-aot/` link and two undocumented env vars. No behaviour change | diff --git a/docs/STATUS.md b/docs/STATUS.md index 0af20cad0..e6b692110 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -112,7 +112,7 @@ token-for-token correctness against the pinned oracle. | Tool-call parsing | 36 parser families / 40 accepted names, streaming | Every vLLM tool parser at the pin except the three Rust/Harmony-backed ones: pure-text parsers ported 1:1, the six engine-backed families reimplemented from their wire formats, all held to the upstream test suites. Selection via `--tool-call-parser` (server), `tool_parser` (C ABI), or template auto-detection; native-syntax forced tool_choice where expressible. Tables: docs/BENCHMARKS.md | | Reasoning parsing (`SAMPLE-REASONING`, ACTIVE, partial coverage) | 9 parsers, streaming | think_auto (auto-detect default: content unless markers appear), deepseek_r1, deepseek_v3 (passthrough) / holo2 (thinking→R1), mistral ([THINK]), minimax_m2 (+append_think), step3, olmo3 - reasoning split engine-side BEFORE tool parsing, streamed as `reasoning` deltas in the chat chunks. Coverage: 9 of upstream's ~28 registered names (remaining text families + engine-backed adapters tracked as W2/W3 in specs/reasoning-parsers.md); each ported parser doctest-gated vs its tests/reasoning case | | Unified streaming parser engine | Core, assembly, serving-SSE dispatch landed, gated; all 10 engine-backed families ported (family parity closed); JSON-schema tool-arg type coercion landed | The vLLM 0.26 declarative `parser/engine/` (shared state machine plus all 10 configs: qwen3, seed_oss, kimi_k2, minimax_m2, glm47_moe, deepseek_v4/v32, nemotron_v3, gemma4, inkling) and assembly layer, gated field-for-field vs vLLM 0.26. An engine-backed `--tool-call-parser` name drives the live chat SSE chunks, off by default. When a request's tools declare typed parameters, the assembled tool-call arguments are coerced to the declared JSON types (int/number/bool/string/array/null) 1:1 with vLLM `_fix_arg_types`, in both streaming and one-shot; no schema means the arguments pass through as strings unchanged. Details: .agents/specs/parser-assembly-c8.md | -| OpenAI server | Supported (subset); bundles SPIKED, no artifacts | `/v1/completions`, `/v1/chat/completions`, streaming SSE, `/v1/models`, `/health`, `/version`, `/ping`, `/metrics`, `/tokenize` (raw-`prompt` and chat-`messages`), `/detokenize`, `/tokenizer_info`, `/server_info`, `/reset_prefix_cache`, `/abort_requests`; `/v1/videos` + `GET /v1/videos/{id}/content` with `input_reference` image or `metadata` video/audio refs. `/tokenizer_info`, `/abort_requests`: flag-gated; `/metrics`, `/reset_prefix_cache`: handlers without live async backing. [Usage](USAGE.md). Depth-2 async serving is heap-safe under `ignore_eos`. [Release spike](../.agents/specs/release-binary-matrix.md): pending. | +| OpenAI server | Supported (subset); #129: SPIKE∅ | `/v1/completions`, `/v1/chat/completions`, streaming SSE, `/v1/models`, `/health`, `/version`, `/ping`, `/metrics`, `/tokenize` (raw-`prompt` and chat-`messages`), `/detokenize`, `/tokenizer_info`, `/server_info`, `/reset_prefix_cache`, `/abort_requests`; `/v1/videos` in OpenAI's Sora shape + `GET /v1/videos/{id}/content`, conditioned on an `input_reference` image or the `metadata` video/audio references. `/tokenizer_info` and `/abort_requests` are flag-gated; `/metrics` and `/reset_prefix_cache` have handlers but no live backing on the async path. Endpoint list and flags: docs/USAGE.md. Depth-2 async serving no longer corrupts the host heap under `ignore_eos` | | Pooling task class (embeddings / classify / score / rerank) | Spiked; pooler op + heads composite + pooling runner path landed (CPU), not yet servable end-to-end (no concrete model / endpoints) | The non-generative task class. W0 spike over the whole vLLM pooling surface (`.agents/specs/pooling-task-class.md`, `CLAIM-POOLING`). W1 landed the pooler OP (CLS/LAST/MEAN + Identity/Normalize/MultiLabelClassify/Classify activations, double-precision-gated). **W2 landed the pooler HEADS composite** — `EmbeddingPoolerHead` (projector→matryoshka→normalize), `ClassifierPoolerHead` (classifier→`(logit-mean)/sigma`→activation), the `SequencePooler` + `PoolerForEmbed`/`PoolerForClassify` factories, the `DispatchPooler` task routing, and the `PoolerConfig`/`PoolingParams` structs (`test_pooler_heads` 27/27, 240 asserts, RED-first). **W3 landed the pooling RUNNER path** — `PoolingRunner` applies the model's `Pooler` to the last hidden state and returns pooled embeddings instead of sampled tokens, gated by a STRUCTURAL cosine-parity check vs a double-precision LAST+normalize reference (`test_pooling_runner` 5/5, 14 asserts, RED-first). NOT yet servable / honest residuals: a concrete pooling MODEL forward + the REAL-model oracle cosine gate (`vllm.LLM(task="embed").encode`) — no cosine-vs-oracle number is fabricated (W3-model); the `/v1/embeddings` + score/rerank/classify endpoints (W4); tokwise AllPool/StepPool (W5). See docs/BENCHMARKS.md | | Plugin system (out-of-core registration) | Spiked; first CPU brick landed, not yet wired into any production path | The extensibility-first discovery layer. W0 spike over vLLM's plugin surface (general / platform / io_processor / endpoint groups, the `register_model` an out-of-tree plugin calls, the invocation seams) is committed (`.agents/specs/plugin-system.md`, `ENG-PLUGIN-SYSTEM` ACTIVE, `CLAIM-PLUGIN-SYSTEM`). W1 landed `vllm::plugins::LoadGeneralPlugins()` + the out-of-core general-plugin registration seam (`RegisterGeneralPlugin` / `REGISTER_VLLM_GENERAL_PLUGIN`) over the existing `REGISTER_VLLM_MODEL`-style registries (the in-tree factory `MODEL-FACTORY-registry` is record-repaired `DONE` 2026-08-05: 28 self-registering TUs, dgx debt paid by the 2026-07-23 seven-gate run): a 1:1 mirror of `load_general_plugins` (load-once idempotence, the `VLLM_PLUGINS` allowlist, per-plugin failure isolation). Proven by an out-of-core toy-model plugin that registers a toy architecture through the public `RegisterModel` seam — unit-gated RED-first (`test_plugin_system` 1 case / 29 assertions: the toy arch resolves ONLY after LoadGeneralPlugins runs it, and not under `VLLM_PLUGINS=""`). Python entry points have no C++20 analogue, so discovery is the project's static-init/`dlopen` registration idiom (recorded porting-inventory §9). NOT yet wired: real shared-object `dlopen` + the C-ABI `vllm_plugin_register` entry (W2), the engine/CLI `--load-plugins` wiring that calls LoadGeneralPlugins from the construction paths (W3), the platform/quant plugin kinds (W4), and the io_processor/stat_logger/endpoint groups (W5) are named residuals. See docs/BENCHMARKS.md | | Offline Batch API (JSONL file runner) | Spiked; first CPU brick landed, not yet exposed as a CLI | The offline OpenAI Batch API: read a JSONL of OpenAI-format requests, run each through the engine, write a JSONL of responses. W0 spike over vLLM's `run_batch.py` (schema, endpoint dispatch, run loop, file I/O) is committed (`.agents/specs/batch-api.md`, `SERVE-BATCH-API` ACTIVE, `CLAIM-BATCH-API`). W1 landed `RunBatch` (`RunLine`/`RunLines`/`Run`) + `RunBatchFile` — a pure orchestrator over the existing `OpenAIServingChat::create_chat_completion` (NO reimplemented generation), 1:1 with vLLM's endpoint_registry url→handler map: `/v1/chat/completions` dispatch, the `BatchResponseData`/`BatchRequestOutput` schema (`vllm-` ids, custom_id echoed), the `run_request` AllResponse/ErrorResponse/stream branches, and the unsupported-endpoint/url error rows. Unit-gated RED-first (`test_openai_run_batch` 7 cases / 80 assertions over the synthetic serving engine: ordered rows + custom_id echo + per-line BatchRequestOutput schema round-trip, a malformed line isolated into an error row so the batch continues, dispatch + 404 error rows; dropping the custom_id echo fails 9 assertions). Recorded deviation: a malformed line is isolated (batch continues) where upstream aborts the job. NOT yet exposed: the `vllm run-batch` CLI + `BatchFrontendArgs` (W2), embeddings/score/rerank dispatch (W3, rides pooling endpoints), audio transcription/translation + media fetch (W4), and http(s)/data-URL file I/O + metrics + overlapped `AsyncLLM` submission (W5) are named residuals. See docs/BENCHMARKS.md | diff --git a/scripts/check-release-binary-contract.py b/scripts/check-release-binary-contract.py index 5127c959c..ce424ca17 100644 --- a/scripts/check-release-binary-contract.py +++ b/scripts/check-release-binary-contract.py @@ -6,8 +6,12 @@ import argparse import ast import hashlib +import os import re +import shlex +import subprocess import sys +import tempfile from collections import Counter from pathlib import Path @@ -16,6 +20,8 @@ END = "" SPEC_PATH = ".agents/specs/release-binary-matrix.md" TEST_PATH = "tests/scripts/test_check_release_binary_contract.py" +PREFLIGHT_PATH = "scripts/agent-preflight.sh" +CI_PATH = ".github/workflows/ci.yml" IDENTITY = "ENG-RELEASE-BINARIES" PRIMARY_CUDA_SMS = ( @@ -49,7 +55,7 @@ ANCHORS = { ".agents/engine-matrix.md": "| `ENG-RELEASE-BINARIES` |", ".agents/roadmap_v1.md": "| REL | `ROAD-V1-RELEASE` |", - ".agents/NOW.md": "| Release | SPIKE | #129 |", + ".agents/NOW.md": "| Release | SPIKE; 30/30 | #129 |", ".agents/coordination.md": ( "| `CLAIM-ENG-RELEASE-BINARIES-SPIKE` | " "`ENG-RELEASE-BINARIES` |" @@ -59,7 +65,7 @@ "are the primary downloads" ), "docs/STATUS.md": ( - "[Release spike](../.agents/specs/release-binary-matrix.md)" + "#129: SPIKE∅" ), "docs/BENCHMARKS.md": ( "| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`:" @@ -127,14 +133,134 @@ "host-ABI fat-CUDA + adaptive-CPU static-core bundles; optional per-SM " "diagnostics; experimental literal-static musl CPU | **PENDING:** pins 10-SM " "fat CUDA, adaptive no-AVX2 CPU, W1-W13/W10-W12 policy, public pending states; " - "25 tests GREEN. No archive, staged smoke, runtime, correctness, or performance " + "30 tests GREEN. No archive, staged smoke, runtime, correctness, or performance " "evidence " "| n/a |" ) STATUS_RELEASE_FRAGMENTS = ( - "Supported (subset); bundles SPIKED, no artifacts", - "[Release spike](../.agents/specs/release-binary-matrix.md): pending.", + "Supported (subset); #129: SPIKE∅", +) + +BACKEND_POLICY_PROSE = { + "Metal release channel": ( + "| `macos-arm64-metal` | stable after M-series runtime gate |" + ), + "MLX release channel": ( + "| `macos-arm64-metal-mlx` | preview until its exact bundled MLX tuple " + "is runtime/correctness-gated |" + ), + "Vulkan release channel": "| `linux-x86_64-glibc-vulkan` | preview |", + "musl experimental CPU-only policy": ( + "| `linux-x86_64-musl-cpu-static` | experimental preview | " + "literal-static feasibility lane; CPU only; see the static boundary below |" + ), + "ROCm release channel": "| ROCm/HIP | blocked |", + "external host GPU-driver boundary": "it never claims to bundle a GPU driver.", + "musl CPU-only/no-GPU boundary": ( + "The one literal-static experiment is " + "`linux-x86_64-musl-cpu-static`. It is CPU-only" + ), +} + +BACKEND_POLICY_PROSE_MUTATIONS = ( + ( + "| `macos-arm64-metal` | stable after M-series runtime gate |", + "| `macos-arm64-metal` | preview |", + "Metal release channel", + ), + ( + "| `macos-arm64-metal-mlx` | preview until its exact bundled MLX tuple " + "is runtime/correctness-gated |", + "| `macos-arm64-metal-mlx` | stable |", + "MLX release channel", + ), + ( + "| `linux-x86_64-glibc-vulkan` | preview |", + "| `linux-x86_64-glibc-vulkan` | stable |", + "Vulkan release channel", + ), + ( + "| `linux-x86_64-musl-cpu-static` | experimental preview | literal-static " + "feasibility lane; CPU only; see the static boundary below |", + "| `linux-x86_64-musl-cpu-static` | stable | literal-static feasibility " + "lane with CUDA |", + "musl experimental CPU-only policy", + ), + ( + "| ROCm/HIP | blocked |", + "| ROCm/HIP | preview |", + "ROCm release channel", + ), + ( + "it never claims to bundle a GPU driver.", + "it bundles the GPU driver.", + "external host GPU-driver boundary", + ), + ( + "The one literal-static experiment is\n" + "`linux-x86_64-musl-cpu-static`. It is CPU-only", + "The one literal-static experiment is\n" + "`linux-x86_64-musl-cpu-static`. It includes GPU runtimes", + "musl CPU-only/no-GPU boundary", + ), +) + +PREFLIGHT_WIRING_MUTATIONS = ( + (" check-release-binary-contract\n", "", "preflight CHECKERS"), + (" test_check_release_binary_contract\n", "", "preflight SUITES"), + ( + 'for checker in "${CHECKERS[@]}"; do', + 'for checker in "${CHECKERS[@]}"; do\n continue', + "execute release CHECKERS", + ), + ( + 'for suite in "${SUITES[@]}"; do', + 'for suite in "${SUITES[@]}"; do\n continue', + "execute release SUITES", + ), + ("CHECKERS=(\n", "INERT_CHECKERS=(\n", "preflight CHECKERS"), + ("SUITES=(\n", "INERT_SUITES=(\n", "preflight SUITES"), +) + +CI_WIRING_MUTATIONS = ( + ( + " python3 scripts/check-release-binary-contract.py\n", + "", + "CI checker", + ), + ( + " python3 tests/scripts/test_check_release_binary_contract.py\n", + "", + "CI step", + ), + ( + " python3 scripts/check-release-binary-contract.py\n" + " python3 tests/scripts/test_check_release_binary_contract.py\n", + " if false; then\n" + " python3 scripts/check-release-binary-contract.py\n" + " python3 tests/scripts/test_check_release_binary_contract.py\n" + " fi\n", + "direct active commands", + ), + ( + " python3 scripts/check-release-binary-contract.py\n", + ' echo "python3 scripts/check-release-binary-contract.py"\n', + "direct active commands", + ), + ( + " - name: Accepted binary-release design and record anchors stay in sync\n" + " run: |\n", + " - name: Accepted binary-release design and record anchors stay in sync\n" + " if: ${{ false }}\n" + " run: |\n", + "direct active commands", + ), + ( + " agent-record:\n", + " agent-record:\n if: ${{ false }}\n", + "direct active commands", + ), ) HUMAN_CONTRACT = { @@ -168,7 +294,7 @@ ( "docs/BENCHMARKS.md", "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " - "W1-W13/W10-W12 policy, public pending states; 25 tests GREEN. No archive, " + "W1-W13/W10-W12 policy, public pending states; 30 tests GREEN. No archive, " "staged smoke, runtime, correctness, or performance evidence", "**SHIPPED:** archive, runtime, correctness, and performance evidence " "complete", @@ -176,8 +302,8 @@ ), ( "docs/STATUS.md", - "Supported (subset); bundles SPIKED, no artifacts", - "Supported; bundles SHIPPED with runtime evidence", + "Supported (subset); #129: SPIKE∅", + "Supported; #129: SHIPPED", "docs/STATUS.md release row", ), ) @@ -240,6 +366,11 @@ "test_each_semantic_inventory_consumer_body_is_pinned", "test_checker_guard_map_keysets_are_exact", "test_required_mutation_test_inventory_is_pinned", + "test_backend_policy_machine_fields_are_required", + "test_backend_policy_prose_is_fail_closed", + "test_preflight_and_ci_wiring_is_an_executable_contract", + "test_preflight_wiring_mutations_fail", + "test_ci_wiring_mutations_fail", ) EXPECTED_TEST_LITERAL_INVENTORY_KEYS = ( @@ -257,6 +388,9 @@ "UNKNOWN_MACHINE_FIELD_MUTATIONS", "HUMAN_WORK_DEPS", "GUARD_MAP_KEYS", + "BACKEND_POLICY_PROSE_MUTATIONS", + "PREFLIGHT_WIRING_MUTATIONS", + "CI_WIRING_MUTATIONS", ) EXPECTED_TEST_INVENTORY_CONSUMER_KEYS = ( @@ -274,6 +408,9 @@ "UNKNOWN_MACHINE_FIELD_MUTATIONS", "HUMAN_WORK_DEPS", "GUARD_MAP_KEYS", + "BACKEND_POLICY_PROSE_MUTATIONS", + "PREFLIGHT_WIRING_MUTATIONS", + "CI_WIRING_MUTATIONS", ) EXPECTED_GUARD_MAP_KEYS = { @@ -288,6 +425,13 @@ "work_W12_policy": "optional-non-blocking", "archive_claims": "pending", "runtime_claims": "pending", + "metal_channel": "stable-after-runtime-gate", + "mlx_channel": "preview", + "vulkan_channel": "preview", + "musl_channel": "experimental-preview", + "musl_scope": "cpu-only-no-gpu", + "rocm_channel": "blocked", + "gpu_driver_boundary": "external-host-never-bundled", "required_anchor_paths": ( ".agents/engine-matrix.md,.agents/roadmap_v1.md,.agents/NOW.md," ".agents/coordination.md,.agents/state.md,docs/STATUS.md," @@ -318,6 +462,9 @@ "test_human_primary_artifact_contract_matches_machine_block" ), "GUARD_MAP_KEYS": "test_checker_guard_map_keysets_are_exact", + "BACKEND_POLICY_PROSE_MUTATIONS": "test_backend_policy_prose_is_fail_closed", + "PREFLIGHT_WIRING_MUTATIONS": "test_preflight_wiring_mutations_fail", + "CI_WIRING_MUTATIONS": "test_ci_wiring_mutations_fail", }, "CONSUMER_FLOW_MUTATIONS": ("continue", "break", "wrap_false"), "UNKNOWN_MACHINE_FIELD_MUTATIONS": (("unexpected_field", "x"),), @@ -325,6 +472,9 @@ work: ",".join(deps) for work, deps in WORK_DEPS.items() }, "GUARD_MAP_KEYS": EXPECTED_GUARD_MAP_KEYS, + "BACKEND_POLICY_PROSE_MUTATIONS": BACKEND_POLICY_PROSE_MUTATIONS, + "PREFLIGHT_WIRING_MUTATIONS": PREFLIGHT_WIRING_MUTATIONS, + "CI_WIRING_MUTATIONS": CI_WIRING_MUTATIONS, } TEST_INVENTORY_CONSUMERS = { @@ -398,6 +548,21 @@ ("guard_map", "keys"), True, ), + "BACKEND_POLICY_PROSE_MUTATIONS": ( + "test_backend_policy_prose_is_fail_closed", + ("before", "after", "reason"), + False, + ), + "PREFLIGHT_WIRING_MUTATIONS": ( + "test_preflight_wiring_mutations_fail", + ("before", "after", "reason"), + False, + ), + "CI_WIRING_MUTATIONS": ( + "test_ci_wiring_mutations_fail", + ("before", "after", "reason"), + False, + ), } TEST_INVENTORY_BODY_DIGESTS = { @@ -415,6 +580,9 @@ "CONSUMER_FLOW_MUTATIONS": "a2d05b5bea24a09c6c5313f4e65d259a9b7e984f6450aa74309f790b3e81de8a", "UNKNOWN_MACHINE_FIELD_MUTATIONS": "1d9242dabe625e43b909709ed0a40915d55fd6610c06bab3e696dc803745e0b8", "HUMAN_WORK_DEPS": "800c69c64c995030fff12ccfbe2bb0002193c17cbe28aeca52c584cf15c2b675", + "BACKEND_POLICY_PROSE_MUTATIONS": "df80167a6da67c499ba1fdebda15e8c5250b46eccebb329973d7e63f7c6d0763", + "PREFLIGHT_WIRING_MUTATIONS": "635f1f49d2e04eb662e61d97f8e4128569d98b8af327fd33d3093451f3f24f7e", + "CI_WIRING_MUTATIONS": "ce2cb4b0c5c71431861c9af57b15468205ee335387120b48b150d7244a008885", } EXACT_MACHINE_FIELDS = { @@ -422,6 +590,13 @@ "work_W12_policy": "optional-non-blocking", "archive_claims": "pending", "runtime_claims": "pending", + "metal_channel": "stable-after-runtime-gate", + "mlx_channel": "preview", + "vulkan_channel": "preview", + "musl_channel": "experimental-preview", + "musl_scope": "cpu-only-no-gpu", + "rocm_channel": "blocked", + "gpu_driver_boundary": "external-host-never-bundled", "required_anchor_paths": ( ".agents/engine-matrix.md,.agents/roadmap_v1.md,.agents/NOW.md," ".agents/coordination.md,.agents/state.md,docs/STATUS.md," @@ -511,6 +686,291 @@ def _normalize_prose(text: str) -> str: return re.sub(r"\s+", " ", text).strip() +def backend_policy_errors(spec_text: str) -> list[str]: + """Return drift from the accepted backend release-channel boundaries.""" + + normalized = _normalize_prose(spec_text) + return [ + f"{reason} must remain {statement!r}" + for reason, statement in BACKEND_POLICY_PROSE.items() + if _normalize_prose(statement) not in normalized + ] + + +def _without_line_comments(text: str) -> str: + """Remove shell comments while preserving quoted hash characters.""" + + cleaned: list[str] = [] + for line in text.splitlines(): + quoted = False + escaped = False + kept: list[str] = [] + for char in line: + if escaped: + kept.append(char) + escaped = False + continue + if char == "\\" and quoted: + kept.append(char) + escaped = True + continue + if char == '"': + quoted = not quoted + kept.append(char) + continue + if char == "#" and not quoted: + break + kept.append(char) + cleaned.append("".join(kept)) + return "\n".join(cleaned) + + +def _indent(line: str) -> int: + return len(line) - len(line.lstrip()) + + +def _literal_block(lines: list[str], header_index: int) -> list[str]: + parent_indent = _indent(lines[header_index]) + raw: list[str] = [] + for candidate in lines[header_index + 1 :]: + if not candidate.strip(): + raw.append("") + continue + if _indent(candidate) <= parent_indent: + break + raw.append(candidate) + nonblank = [line for line in raw if line.strip()] + if not nonblank: + return [] + content_indent = min(_indent(line) for line in nonblank) + return [line[content_indent:] if line.strip() else "" for line in raw] + + +def _yaml_mapping( + line: str, indent: int, sequence: bool = False +) -> tuple[str, str] | None: + if _indent(line) != indent: + return None + content = line[indent:] + if sequence: + if not content.startswith("-"): + return None + content = content[1:].lstrip() + if not content: + return None + match = re.match( + r"^(?P'[^']*'|\"[^\"]*\"|[^:#]+?)\s*:\s*(?P.*)$", + content, + ) + if match is None: + return None + key = match.group("key").strip() + if len(key) >= 2 and key[0] == key[-1] and key[0] in {"'", '"'}: + key = key[1:-1] + return key.strip(), match.group("value").strip() + + +def _unconditional_ci_run_blocks(text: str) -> list[list[str]]: + """Return direct run blocks owned by unconditional Actions jobs and steps.""" + + lines = text.splitlines() + blocks: list[list[str]] = [] + jobs_index = next((i for i, line in enumerate(lines) if line == "jobs:"), None) + if jobs_index is None: + return blocks + job_starts = [ + i + for i in range(jobs_index + 1, len(lines)) + if (mapping := _yaml_mapping(lines[i], 2)) is not None + and mapping[1] == "" + ] + for job_pos, job_start in enumerate(job_starts): + job_end = ( + job_starts[job_pos + 1] + if job_pos + 1 < len(job_starts) + else len(lines) + ) + job_lines = lines[job_start + 1 : job_end] + job_fields = { + mapping[0] + for line in job_lines + if (mapping := _yaml_mapping(line, 4)) is not None + } + if "if" in job_fields: + continue + steps_offset = next( + ( + i + for i, line in enumerate(job_lines) + if _yaml_mapping(line, 4) == ("steps", "") + ), + None, + ) + if steps_offset is None: + continue + steps_start = job_start + 1 + steps_offset + 1 + step_starts = [ + i + for i in range(steps_start, job_end) + if _indent(lines[i]) == 6 and lines[i][6:].startswith("-") + ] + for step_pos, step_start in enumerate(step_starts): + step_end = ( + step_starts[step_pos + 1] + if step_pos + 1 < len(step_starts) + else job_end + ) + step_fields: dict[str, tuple[str, int]] = {} + first = _yaml_mapping(lines[step_start], 6, sequence=True) + if first is not None: + step_fields[first[0]] = (first[1], step_start) + for index in range(step_start + 1, step_end): + mapping = _yaml_mapping(lines[index], 8) + if mapping is not None: + step_fields[mapping[0]] = (mapping[1], index) + if {"if", "continue-on-error", "shell"} & step_fields.keys(): + continue + run = step_fields.get("run") + if run is not None and re.fullmatch(r"\|[-+]?", run[0]): + blocks.append(_literal_block(lines, run[1])) + return blocks + + +def _direct_commands(block: list[str]) -> list[list[str]] | None: + commands: list[list[str]] = [] + for line in block: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if line != line.lstrip(): + return None + try: + argv = shlex.split(stripped, comments=True, posix=True) + except ValueError: + return None + if not argv or any(token in {";", "&&", "||", "|", "&"} for token in argv): + return None + commands.append(argv) + return commands + + +def _active_ci_commands(ci_text: str) -> set[tuple[str, ...]]: + commands: set[tuple[str, ...]] = set() + for block in _unconditional_ci_run_blocks(ci_text): + parsed = _direct_commands(block) + if parsed is not None: + commands.update(tuple(command) for command in parsed) + return commands + + +def _ci_has_active_release_step(ci_text: str) -> bool: + expected = [ + ["python3", "scripts/check-release-binary-contract.py"], + ["python3", "tests/scripts/test_check_release_binary_contract.py"], + ] + return any( + _direct_commands(block) == expected + for block in _unconditional_ci_run_blocks(ci_text) + ) + + +def _bash_array_values(text: str, name: str) -> list[str] | None: + lines = text.splitlines() + starts = [i for i, line in enumerate(lines) if line.strip() == f"{name}=("] + if len(starts) != 1: + return None + values: list[str] = [] + for line in lines[starts[0] + 1 :]: + if line.strip() == ")": + return values + try: + values.extend(shlex.split(line, comments=True, posix=True)) + except ValueError: + return None + return None + + +def _trace_preflight_commands(text: str) -> tuple[int, list[tuple[str, ...]]]: + """Execute preflight with shims and return every Python argv it owns.""" + + with tempfile.TemporaryDirectory(prefix="vllm-release-preflight-trace-") as temporary: + root = Path(temporary) + script = root / PREFLIGHT_PATH + script.parent.mkdir(parents=True) + script.write_text(text, encoding="utf-8") + script.chmod(0o700) + (root / ".agents").mkdir() + (root / ".agents/NOW.md").write_text("trace-only\n", encoding="utf-8") + shim_dir = root / "shim" + shim_dir.mkdir() + trace = root / "python.trace" + python = shim_dir / "python3" + python.write_text( + "#!/bin/sh\n" + "printf '%s\\0' \"$@\" >> \"$VLLM_RELEASE_TRACE\"\n" + "printf '\\0' >> \"$VLLM_RELEASE_TRACE\"\n", + encoding="utf-8", + ) + python.chmod(0o700) + git = shim_dir / "git" + git.write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") + git.chmod(0o700) + environment = os.environ.copy() + environment["PATH"] = f"{shim_dir}{os.pathsep}{environment.get('PATH', '')}" + environment["VLLM_RELEASE_TRACE"] = str(trace) + result = subprocess.run( + ["bash", str(script), "--quiet", "--no-require-role"], + cwd=root, + env=environment, + text=True, + capture_output=True, + check=False, + ) + raw = trace.read_bytes() if trace.exists() else b"" + invocations = [] + for record in raw.split(b"\0\0"): + if record: + invocations.append( + tuple(token.decode("utf-8") for token in record.split(b"\0") if token) + ) + return result.returncode, invocations + + +def wiring_errors(preflight_text: str, ci_text: str) -> list[str]: + """Require this checker and suite to execute through preflight and CI.""" + + errors: list[str] = [] + uncommented = _without_line_comments(preflight_text) + checkers = _bash_array_values(uncommented, "CHECKERS") + suites = _bash_array_values(uncommented, "SUITES") + if checkers is None or "check-release-binary-contract" not in checkers: + errors.append("release checker is missing from preflight CHECKERS") + if suites is None or "test_check_release_binary_contract" not in suites: + errors.append("release mutation suite is missing from preflight SUITES") + returncode, invocations = _trace_preflight_commands(preflight_text) + checker_argv = ("scripts/check-release-binary-contract.py",) + suite_argv = ("tests/scripts/test_check_release_binary_contract.py",) + if invocations.count(checker_argv) != 1: + errors.append("preflight does not execute release CHECKERS through its checker loop") + if invocations.count(suite_argv) != 1: + errors.append("preflight does not execute release SUITES through its suite loop") + if returncode != 0: + errors.append(f"instrumented preflight execution failed with rc={returncode}") + active = _active_ci_commands(ci_text) + if ("python3", "scripts/check-release-binary-contract.py") not in active: + errors.append("release checker is missing from the explicit CI checker step") + if ( + "python3", + "tests/scripts/test_check_release_binary_contract.py", + ) not in active: + errors.append("release mutation suite is missing from the explicit CI step") + if not _ci_has_active_release_step(ci_text): + errors.append( + "CI release step must contain checker and suite as direct active commands" + ) + return errors + + def _table_record( root: Path, relative: str, @@ -861,6 +1321,7 @@ def contract_errors(root: Path) -> list[str]: errors.append( f"{label} must match the accepted machine-readable release contract" ) + errors.extend(backend_policy_errors(text)) for relative, anchor in ANCHORS.items(): path = root / relative @@ -886,6 +1347,17 @@ def contract_errors(root: Path) -> list[str]: "docs/STATUS.md release row must stay SPIKED with no artifacts and no " "runtime claim" ) + preflight = root / PREFLIGHT_PATH + ci = root / CI_PATH + if not preflight.is_file() or not ci.is_file(): + errors.append("release checker wiring inputs are missing") + else: + errors.extend( + wiring_errors( + preflight.read_text(encoding="utf-8"), + ci.read_text(encoding="utf-8"), + ) + ) errors.extend(_test_inventory_errors(root)) return errors diff --git a/tests/scripts/test_check_release_binary_contract.py b/tests/scripts/test_check_release_binary_contract.py index d1aebf317..8336d82ef 100644 --- a/tests/scripts/test_check_release_binary_contract.py +++ b/tests/scripts/test_check_release_binary_contract.py @@ -4,6 +4,7 @@ from __future__ import annotations import ast +import importlib.util import shutil import subprocess import sys @@ -14,6 +15,12 @@ ROOT = Path(__file__).resolve().parents[2] CHECKER = ROOT / "scripts/check-release-binary-contract.py" +CHECKER_SPEC = importlib.util.spec_from_file_location( + "check_release_binary_contract", CHECKER +) +assert CHECKER_SPEC is not None and CHECKER_SPEC.loader is not None +checker = importlib.util.module_from_spec(CHECKER_SPEC) +CHECKER_SPEC.loader.exec_module(checker) CONTRACT_PATHS = ( "scripts/check-release-binary-contract.py", @@ -25,6 +32,8 @@ ".agents/state.md", "docs/STATUS.md", "docs/BENCHMARKS.md", + ".github/workflows/ci.yml", + "scripts/agent-preflight.sh", "tests/scripts/test_check_release_binary_contract.py", ) @@ -54,6 +63,11 @@ "test_each_semantic_inventory_consumer_body_is_pinned", "test_checker_guard_map_keysets_are_exact", "test_required_mutation_test_inventory_is_pinned", + "test_backend_policy_machine_fields_are_required", + "test_backend_policy_prose_is_fail_closed", + "test_preflight_and_ci_wiring_is_an_executable_contract", + "test_preflight_wiring_mutations_fail", + "test_ci_wiring_mutations_fail", ) PRIMARY_CUDA_SMS = ( @@ -85,6 +99,9 @@ "UNKNOWN_MACHINE_FIELD_MUTATIONS", "HUMAN_WORK_DEPS", "GUARD_MAP_KEYS", + "BACKEND_POLICY_PROSE_MUTATIONS", + "PREFLIGHT_WIRING_MUTATIONS", + "CI_WIRING_MUTATIONS", ), "TEST_INVENTORY_CONSUMERS": ( "PRIMARY_CUDA_SMS", @@ -101,13 +118,16 @@ "UNKNOWN_MACHINE_FIELD_MUTATIONS", "HUMAN_WORK_DEPS", "GUARD_MAP_KEYS", + "BACKEND_POLICY_PROSE_MUTATIONS", + "PREFLIGHT_WIRING_MUTATIONS", + "CI_WIRING_MUTATIONS", ), } RECORD_ANCHORS = { ".agents/engine-matrix.md": "| `ENG-RELEASE-BINARIES` |", ".agents/roadmap_v1.md": "| REL | `ROAD-V1-RELEASE` |", - ".agents/NOW.md": "| Release | SPIKE | #129 |", + ".agents/NOW.md": "| Release | SPIKE; 30/30 | #129 |", ".agents/coordination.md": ( "| `CLAIM-ENG-RELEASE-BINARIES-SPIKE` | " "`ENG-RELEASE-BINARIES` |" @@ -117,7 +137,7 @@ "are the primary downloads" ), "docs/STATUS.md": ( - "[Release spike](../.agents/specs/release-binary-matrix.md)" + "#129: SPIKE∅" ), "docs/BENCHMARKS.md": ( "| **Binary release matrix (spiked)** | `ENG-RELEASE-BINARIES`:" @@ -216,7 +236,7 @@ ( "docs/BENCHMARKS.md", "**PENDING:** pins 10-SM fat CUDA, adaptive no-AVX2 CPU, " - "W1-W13/W10-W12 policy, public pending states; 25 tests GREEN. No archive, " + "W1-W13/W10-W12 policy, public pending states; 30 tests GREEN. No archive, " "staged smoke, runtime, correctness, or performance evidence", "**SHIPPED:** archive, runtime, correctness, and performance evidence " "complete", @@ -224,8 +244,8 @@ ), ( "docs/STATUS.md", - "Supported (subset); bundles SPIKED, no artifacts", - "Supported; bundles SHIPPED with runtime evidence", + "Supported (subset); #129: SPIKE∅", + "Supported; #129: SHIPPED", "docs/STATUS.md release row", ), ) @@ -267,6 +287,13 @@ "work_W12_policy": "optional-non-blocking", "archive_claims": "pending", "runtime_claims": "pending", + "metal_channel": "stable-after-runtime-gate", + "mlx_channel": "preview", + "vulkan_channel": "preview", + "musl_channel": "experimental-preview", + "musl_scope": "cpu-only-no-gpu", + "rocm_channel": "blocked", + "gpu_driver_boundary": "external-host-never-bundled", "required_anchor_paths": ( ".agents/engine-matrix.md,.agents/roadmap_v1.md,.agents/NOW.md," ".agents/coordination.md,.agents/state.md,docs/STATUS.md," @@ -287,6 +314,9 @@ "test_human_primary_artifact_contract_matches_machine_block" ), "GUARD_MAP_KEYS": "test_checker_guard_map_keysets_are_exact", + "BACKEND_POLICY_PROSE_MUTATIONS": "test_backend_policy_prose_is_fail_closed", + "PREFLIGHT_WIRING_MUTATIONS": "test_preflight_wiring_mutations_fail", + "CI_WIRING_MUTATIONS": "test_ci_wiring_mutations_fail", } CONSUMER_FLOW_MUTATIONS = ("continue", "break", "wrap_false") @@ -309,6 +339,132 @@ "W13": "W5,W7,W8,W9,W10,W11", } +BACKEND_POLICY_FIELDS = { + "metal_channel": "stable-after-runtime-gate", + "mlx_channel": "preview", + "vulkan_channel": "preview", + "musl_channel": "experimental-preview", + "musl_scope": "cpu-only-no-gpu", + "rocm_channel": "blocked", + "gpu_driver_boundary": "external-host-never-bundled", +} + +BACKEND_POLICY_PROSE_MUTATIONS = ( + ( + "| `macos-arm64-metal` | stable after M-series runtime gate |", + "| `macos-arm64-metal` | preview |", + "Metal release channel", + ), + ( + "| `macos-arm64-metal-mlx` | preview until its exact bundled MLX tuple " + "is runtime/correctness-gated |", + "| `macos-arm64-metal-mlx` | stable |", + "MLX release channel", + ), + ( + "| `linux-x86_64-glibc-vulkan` | preview |", + "| `linux-x86_64-glibc-vulkan` | stable |", + "Vulkan release channel", + ), + ( + "| `linux-x86_64-musl-cpu-static` | experimental preview | literal-static " + "feasibility lane; CPU only; see the static boundary below |", + "| `linux-x86_64-musl-cpu-static` | stable | literal-static feasibility " + "lane with CUDA |", + "musl experimental CPU-only policy", + ), + ( + "| ROCm/HIP | blocked |", + "| ROCm/HIP | preview |", + "ROCm release channel", + ), + ( + "it never claims to bundle a GPU driver.", + "it bundles the GPU driver.", + "external host GPU-driver boundary", + ), + ( + "The one literal-static experiment is\n" + "`linux-x86_64-musl-cpu-static`. It is CPU-only", + "The one literal-static experiment is\n" + "`linux-x86_64-musl-cpu-static`. It includes GPU runtimes", + "musl CPU-only/no-GPU boundary", + ), +) + +PREFLIGHT_WIRING_MUTATIONS = ( + ( + " check-release-binary-contract\n", + "", + "preflight CHECKERS", + ), + ( + " test_check_release_binary_contract\n", + "", + "preflight SUITES", + ), + ( + 'for checker in "${CHECKERS[@]}"; do', + 'for checker in "${CHECKERS[@]}"; do\n continue', + "execute release CHECKERS", + ), + ( + 'for suite in "${SUITES[@]}"; do', + 'for suite in "${SUITES[@]}"; do\n continue', + "execute release SUITES", + ), + ( + "CHECKERS=(\n", + "INERT_CHECKERS=(\n", + "preflight CHECKERS", + ), + ( + "SUITES=(\n", + "INERT_SUITES=(\n", + "preflight SUITES", + ), +) + +CI_WIRING_MUTATIONS = ( + ( + " python3 scripts/check-release-binary-contract.py\n", + "", + "CI checker", + ), + ( + " python3 tests/scripts/test_check_release_binary_contract.py\n", + "", + "CI step", + ), + ( + " python3 scripts/check-release-binary-contract.py\n" + " python3 tests/scripts/test_check_release_binary_contract.py\n", + " if false; then\n" + " python3 scripts/check-release-binary-contract.py\n" + " python3 tests/scripts/test_check_release_binary_contract.py\n" + " fi\n", + "direct active commands", + ), + ( + " python3 scripts/check-release-binary-contract.py\n", + ' echo "python3 scripts/check-release-binary-contract.py"\n', + "direct active commands", + ), + ( + " - name: Accepted binary-release design and record anchors stay in sync\n" + " run: |\n", + " - name: Accepted binary-release design and record anchors stay in sync\n" + " if: ${{ false }}\n" + " run: |\n", + "direct active commands", + ), + ( + " agent-record:\n", + " agent-record:\n if: ${{ false }}\n", + "direct active commands", + ), +) + def run_checker(root: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( @@ -543,6 +699,42 @@ def test_repository_contract_passes(self) -> None: self.assertEqual(result.returncode, 0, result.stdout + result.stderr) +class MissingReviewGuardTests(unittest.TestCase): + def test_backend_policy_machine_fields_are_required(self) -> None: + for field, value in BACKEND_POLICY_FIELDS.items(): + with self.subTest(field=field): + self.assertEqual(checker.EXPECTED_FIELDS[field], value) + + def test_preflight_and_ci_wiring_is_an_executable_contract(self) -> None: + preflight = (ROOT / "scripts/agent-preflight.sh").read_text(encoding="utf-8") + ci = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") + self.assertEqual(checker.wiring_errors(preflight, ci), []) + + def test_backend_policy_prose_is_fail_closed(self) -> None: + for before, after, reason in BACKEND_POLICY_PROSE_MUTATIONS: + with self.subTest(reason=reason), RepoCopy() as root: + mutate(root, ".agents/specs/release-binary-matrix.md", before, after) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(reason, result.stdout + result.stderr) + + def test_preflight_wiring_mutations_fail(self) -> None: + for before, after, reason in PREFLIGHT_WIRING_MUTATIONS: + with self.subTest(reason=reason, mutation=before), RepoCopy() as root: + mutate(root, "scripts/agent-preflight.sh", before, after) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(reason, result.stdout + result.stderr) + + def test_ci_wiring_mutations_fail(self) -> None: + for before, after, reason in CI_WIRING_MUTATIONS: + with self.subTest(reason=reason, mutation=before), RepoCopy() as root: + mutate(root, ".github/workflows/ci.yml", before, after) + result = run_checker(root) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(reason, result.stdout + result.stderr) + + class AcceptedDesignMutations(unittest.TestCase): SPEC = ".agents/specs/release-binary-matrix.md"