From 3d501c6fc7bc831c53e2b9bbb62bc40427dc5d63 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 7 Aug 2026 22:00:14 +0000 Subject: [PATCH] =?UTF-8?q?feat(abi):=20MiniMax-H3=20video+audio=20generat?= =?UTF-8?q?ion=20through=20the=20ONE=20surface=20=E2=80=94=20vllm=5Fvideo?= =?UTF-8?q?=5F*,=20ABI=20v12,=20/v1/videos=20via=20the=20library=20seam,?= =?UTF-8?q?=20thin-client=20examples?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ONE-SURFACE fold ROW 2 (row/H3-VIDEO-ABI, task #283; the #121 precedent's shape: three-arm fold gate, guard-ratchet shrink). The library absorbs everything the examples privately owned. GOLDENS FIRST: minimax_h3_video_fold_fixture.h writes a deterministic tiny checkpoint set (857KB F32 ComfyUI-GGUF DiT at the test_minimax_h3.cpp:3786 reduced geometry, on-disk reduced ViT3D/BigVGAN VAEs + config.json, prompt embeds); the PRE-fold minimax-h3-gen binary AT THE BRANCH BASE (fc636c76) rendered it (--partition fl2va --steps 3 --frames 5 --height/width 32, CPU, keep-quant; run-to-run deterministic, diff -r clean) and its 8 frames + WAV + both minimax-h3-mux --print-only argv lines are the committed goldens in fixtures/minimax_h3_video_fold/. W1 seam: vllm::multimodal::MiniMaxH3VideoEngine absorbs the 1293-line example driver AND the server's 354-line /v1/videos twin — 4 DiT loader arms (GGUF keep-quant / dequant-bf16 host+streamed / bf16-shard stream / NVFP4 bf16+fp4-resident), VAE decoder + lazy encoder halves, the H3-Encoder tower staged once (GGUF or bf16 shards), the #77 partition guard, fl2va/ref2va conditioning, the byte-exact splitmix64 Box-Muller noise streams (VT_H3_GAUSSIAN_NOISE honored; seeded requests derive the audio stream via splitmix64), artifact writing and mux-argv assembly. The library SPAWNS NOTHING (mkdir -p became std::filesystem; ffmpeg stays caller-side — the ratified 2026-08-03 boundary). MiniMaxH3VideoGenParamsFromRequest is the ONE /v1/videos mapping so HTTP and FFI cannot drift. W2 ABI: the ratified vllm_video_* slice (one-surface-abi.md:92-108) — vllm_video_engine_load/free, vllm_video_model_params/_default, vllm_video_params/_default, vllm_video_generate, vllm_video_result/_free — plus the engine-free vllm_video_mux_argv(+_params_default,+_free) composer the mux thin client needs. VLLM_ABI_VERSION 11 -> 12; test_capi floor pin >= 12 (the == checks move with the macro). As-shipped deltas vs the proposal argued in the spec (+prompt_embeds_path/+partition/+fp4_resident/+output_dir; -task/-duration; ONE ref_image). Zero values preserve behaviour. Refusals both directions: vllm_video_engine_load on a text dir names vllm_engine_load; vllm_engine_load on the H3 dir fails byte-for-byte as captured at v11. W3 server: the VideoState block is DELETED; /v1/videos = seam Load + a runner lambda (FromRequest -> Generate -> fork/execvp(mux_argv)). Direct MiniMaxH3 refs 54 -> 7, all seam type names. DISCLOSED numeric deltas on the server arm (no goldens existed; the drift WAS the defect): legacy single-stream uniform noise -> the shared ratified recipe; host-f32 GGUF default -> keep-quant; new --video-dequant-bf16 flag. W4 examples: minimax_h3_gen (1293 -> 216 lines) + minimax_h3_mux are vllm.h+vllm::shared thin clients, BOTH byte-identical to the pre-fold binaries on the fixture (frames+WAV cmp clean; mux argv diff clean; the pre-fold diagnostic modes died with the private pipeline — named residual). Ratchet: both example-abi-allowlist rows removed, MAX_INTERNAL_REACHING 11 -> 9 with the equality pin + spec claims moved; the abi-capability video row closes; FEATURES video row -> reachable naming the four symbols. Gates: three-arm fold gate test_minimax_h3_video_fold 4 cases / 131 assertions GREEN (seam == replicated pre-fold pipeline == committed goldens, byte-identical; partition-guard + input refusals; FromRequest field-complete); test_capi 40/373 incl. the v12 golden e2e THROUGH the C marshalling; vllm_capi_c_check strict-C11 green; test_minimax_h3 75/75 unchanged; test_openai_api_server 45/566 incl. the socket-level routes-do-not-exist-without-a-runner 404 pin; test_video_api 14/14; check-surface-coverage green with the SHRINK enforced + its 46-test mutation suite. GB10 real-video re-verification through the v12 ABI is a NAMED RESIDUAL (box on the Kimi campaign); records updated (STATUS/BENCHMARKS/FEATURES/NOW/state/model-matrix + both specs). FOLLOWING_AGENTS_PROTOCOL Assisted-by: Claude Code:claude-fable-5 [ClaudeCode] --- .agents/NOW.md | 2 +- .agents/model-matrix.md | 2 +- .agents/specs/one-surface-abi.md | 60 +- .agents/specs/surface-coverage-2026-08-07.md | 18 +- .agents/state.md | 65 + CMakeLists.txt | 6 + docs/BENCHMARKS.md | 2 +- docs/FEATURES.md | 6 +- docs/STATUS.md | 2 +- examples/CMakeLists.txt | 19 +- examples/minimax_h3_gen/main.cpp | 1394 ++--------------- examples/minimax_h3_mux/main.cpp | 159 +- examples/server/main.cpp | 524 +------ include/vllm.h | 129 +- include/vllm/multimodal/minimax_h3_video.h | 144 ++ scripts/abi-capability-allowlist.txt | 1 - scripts/check-surface-coverage.py | 12 +- scripts/example-abi-allowlist.txt | 2 - src/capi/vllm_c.cpp | 219 +++ src/vllm/multimodal/minimax_h3_video.cpp | 730 +++++++++ tests/CMakeLists.txt | 9 +- tests/capi/test_capi.cpp | 267 +++- tests/scripts/test_check_surface_coverage.py | 16 +- .../fixtures/minimax_h3_video_fold/audio.wav | Bin 0 -> 108 bytes .../minimax_h3_video_fold/frame_000000.ppm | 4 + .../minimax_h3_video_fold/frame_000001.ppm | 4 + .../minimax_h3_video_fold/frame_000002.ppm | 4 + .../minimax_h3_video_fold/frame_000003.ppm | 4 + .../minimax_h3_video_fold/frame_000004.ppm | 4 + .../minimax_h3_video_fold/frame_000005.ppm | 4 + .../minimax_h3_video_fold/frame_000006.ppm | 4 + .../minimax_h3_video_fold/frame_000007.ppm | 4 + .../minimax_h3_video_fold/golden_mux_argv.txt | 1 + .../golden_mux_argv_silent.txt | 1 + .../models/minimax_h3_video_fold_fixture.h | 342 ++++ .../models/test_minimax_h3_video_fold.cpp | 406 +++++ 36 files changed, 2756 insertions(+), 1814 deletions(-) create mode 100644 include/vllm/multimodal/minimax_h3_video.h create mode 100644 src/vllm/multimodal/minimax_h3_video.cpp create mode 100644 tests/vllm/models/fixtures/minimax_h3_video_fold/audio.wav create mode 100644 tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000000.ppm create mode 100644 tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000001.ppm create mode 100644 tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000002.ppm create mode 100644 tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000003.ppm create mode 100644 tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000004.ppm create mode 100644 tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000005.ppm create mode 100644 tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000006.ppm create mode 100644 tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000007.ppm create mode 100644 tests/vllm/models/fixtures/minimax_h3_video_fold/golden_mux_argv.txt create mode 100644 tests/vllm/models/fixtures/minimax_h3_video_fold/golden_mux_argv_silent.txt create mode 100644 tests/vllm/models/minimax_h3_video_fold_fixture.h create mode 100644 tests/vllm/models/test_minimax_h3_video_fold.cpp diff --git a/.agents/NOW.md b/.agents/NOW.md index e237ef1f9..c9edbb8a1 100644 --- a/.agents/NOW.md +++ b/.agents/NOW.md @@ -28,7 +28,7 @@ Working head: `row/backend-rocm-w0` (#41). Prior: benchmark checkpoint | Supported-models list | **LANDED**: FEATURES arch table CI-bound (33 archs) | — | | `/v1/videos` OpenAI shape | **MERGED** (#71): Sora `model`/`size`/`seconds` + `GET /{id}/content` | `row/SERVE-VIDEOS-REFS` PR open: reference conditioning | | `BACKEND-ROCM` W0 | Skeleton in; **HIP never compiled** (no AMD HW) | #41 contributors build it; a compile error IS the deliverable | -| Surface coverage (`ARCH-ONE-SURFACE`) | **ROW 1 LANDED (#121)**: Parakeet ASR on the surface (`vllm_transcribe` v11, `/v1/audio/transcriptions`, ratchet 12→11) | Next fold row | +| Surface coverage (`ARCH-ONE-SURFACE`) | **ROW 2 LANDED (#123)**: H3 video on the surface (`vllm_video_*` v12, `/v1/videos` via the seam, both examples thin clients, ratchet 11→9); ROW 1 (#121) before it | GB10 re-verify residual; next fold row | In-flight (default-OFF, not pushed): `laguna-fp4proj-prod`, laguna bf16/legacy/pipeline-gemv, `ds4-hc-expand-fuse`. diff --git a/.agents/model-matrix.md b/.agents/model-matrix.md index 4e7a56944..3a87c2ff7 100644 --- a/.agents/model-matrix.md +++ b/.agents/model-matrix.md @@ -83,7 +83,7 @@ Engaged architectures (the 47 non-`INVENTORIED` rows): | ✅ | `Glm4MoeLiteForCausalLM` | GLM-4.7-Flash (31.2B MLA + GLM MoE) | SACRED gate 8/8 vs vLLM 0.25.0 (STRICT token-exact 1/8 + near-tie-band 7/8, 69/128 tokens strictly exact, max teacher-forced gap 0.0 nats, 0 forward-divergent; vLLM K=5 self-deterministic → STRICT bar); FIRST e2e coverage of the q_lora query branch AND the noaux_tc sigmoid router (closes the MLA campaign's two gaps, C2); speed pending | `MODEL-TEXT-glm4-moe-lite-glm4-moe-lite-for-causal-lm` | | 🚧 | `KimiLinearForCausalLM` | Kimi-Linear-48B-A3B | **PAGED-INCREMENTAL DECODE LANDS the 5× speed win (2026-08-07, §19, `row/KIMI-PAGED-INCREMENTAL` #113):** the §18 real lever (e) BUILT — `KimiDecodeCache` + `ForwardPrefillIncremental`/`ForwardDecodeStepIncremental` (`kimi_linear_device.cpp`): prefill-once (KDA recurrent+conv state carried via `vt::KdaGatedDeltaRule` state in/out + `vt::CausalConv1dFwd` tap-carry; NoPE-MLA latent-KV cached) + recurrent decode-step, MIRRORING vLLM `kimi_gdn_linear_attn._forward` (prefill=`chunk_kda_with_fused_gate` output_final_state / decode=`fused_recurrent_kda` initial_state, `vllm-src` `a4e3cb4`; divergences: host state vs paged slot cache, materialized-MHA MLA vs paged-FA2 — named residuals). CPU byte-exact state-carry gate `test_kimi_linear_forward` **15/15·875** (NEW case l: carried decode == fresh full-recompute byte-identical + greedy-identical). Full 48.9B GB10 (single-load/config, flock, drop_caches, min-avail 18-21 GiB, no reboot, §12 golden md5 `bfa5bdbf`): recompute 122/128 @ 4.23 tok/s (reproduces #111); incremental+recurrence 120/128 @ 16.63; **incremental+chunk-prefill 122/128 @ 18.87/19.03 tok/s (2 runs) — token-IDENTICAL to recompute (Gate A PASS, p7 `got` byte-exact) = 4.5× over recompute, 0.90× of vLLM ~21 (the 5× decode gap 0.20×→0.90×)**. Gate B STRICT NOT reached (122/128): chunk-prefill in the RIGHT vehicle reproduces recompute EXACTLY, does NOT close p7 — REFUTES the #111 "p7 in the right vehicle → STRICT" hypothesis; p7 intrinsic (§13/§14 f32-vs-bf16 near-tie at a comma). Decode decomposition (nsys, ours, 99 steps, same-tool): **~90% is the SAME cuBLAS `internal::gemvx::kernel` vLLM calls (batch-1 GEMV-parity)**, KdaScanKernel 2.3%, MoE glue 2.3%, CastBf16 3%; chunk kernels 20 inst = prefill only (prefill=chunk/decode=recurrent IN VIVO). Killing O(n²) ALONE reaches parity-class; no lever load-bearing beyond it — residual = ~15% host-orchestration idle + 3% CastBf16 (a bf16 residual stream = the ONE lever ALSO closing the p7-STRICT near-tie) + paged-FA2 MLA decode. vLLM-live-nsys@0.82 NOT run (box-safety: 95-98 GiB reservation + nsys below the 15 GiB LIFE-CRITICAL floor). `--incremental` opt-in; `VT_KIMI_DEVICE_KDA`/`_CHUNK` STAY OFF (122/128 ≠ STRICT). Row STAYS 🚧. **chunk_kda PREFILL PHASE-2 MEASURED — op CORRECT, chunk-EVERY-STEP REGRESSES 122→102 (2026-08-07, §18, `row/KIMI-CHUNK-KDA-P2` #111):** the `chunk_kda` prefill family regenerated + vendored for ALL 6 arches (reproducible — only new `kda_*`+MANIFEST; GDN cubins byte-identical; drift GREEN) + wired through the new op `vt::KdaChunkPrefill` (the 6-cubin `_chunk_kda_fwd_with_cumulative_g`; `cuda_gdn.cu.o` -Werror clean; RED-first unit `test_ops_kda_chunk_prefill` **2/2·4** on GB10 — chunk-vs-recurrence mean_abs **4.68e-5**, wrong-gate **72×**; GDN untouched 66/66·4242). Full 48.9B GB10 gate (flock, min-avail 21 GiB, no reboot): control device-KDA reproduces **122/128, 4.24 tok/s** EXACTLY; **+chunk-prefill (`VT_KIMI_DEVICE_KDA_CHUNK`) REGRESSES to 102/128, 4.08 tok/s** (p3 16→3, p6 16→11). Root cause: the island's O(n²) recompute applies chunk EVERY decode step over the growing sequence — NOT vLLM's prefill=chunk/decode=recurrent split — so it coin-flips near-ties the recurrence-every-step (control) doesn't (the recurrence matches vLLM's DECODE; chunk only matches its PREFILL). vLLM speed arm (§12 recipe, util 0.82, triton MoE, eager, single-seq; min-avail 15 GiB, no reboot): **~21 tok/s median** 16-token aggregate (25.3 cold-discarded; TTFT not isolable in 0.25.0) vs ours **4.24** (recurrence) / **4.08** (chunk) STEADY decode → **ours/vLLM ≈ 0.20** (vLLM ~5× faster on decode — the O(n²)-recompute vs paged-incremental distance, = the coupled STRICT+speed lever). `VT_KIMI_DEVICE_KDA_CHUNK` STAYS OFF (a regression isn't a flip); device-KDA (122, OFF) still best. The op + regen are the validated prefill half of the named real lever (e) paged-incremental decode (chunk-prefill ONCE + recurrent-decode over PERSISTENT state — kills the O(n²); the STRICT + speed lever, coupled). Row STAYS 🚧. **DEVICE-KDA GB10 122/128 + 4.24 tok/s (§15, #104); device NoPE-MLA lever MEASURED-NEGATIVE (2026-08-07, §16, `row/KIMI-STRICT-CLOSE` #107):** the per-channel-decay device recurrence `vt::KdaGatedDeltaRule` moves 106→**122/128** (p0-p6 16/16; sole p7 pos-6 comma near-tie) AND **1.35→4.24 tok/s (3.1×)** — vLLM's ACTUAL f32-on-bf16 arithmetic, beats §14's host-precision 120. The §15 residual (d) was attempted in device-COMPUTE form: `VT_KIMI_DEVICE_MLA` routes the 7 NoPE-MLA layers' softmax core through `vt::Attention` (pad-V: value zero-padded qk_nope+qk_rope=192 vs v=128, `out[:,:,:v]` byte-exact). CPU RED-first gate GREEN (`test_kimi_linear_forward` **14/14·825**, pad-V==f64 ref rtol 3e-3; perturbation fails 108). Full 48.9B GB10 gate (single-load, flock, min-avail 21 GiB, no reboot): control device-KDA reproduces **122/128, 4.24 tok/s** EXACTLY; **+device-MLA REGRESSES to 109/128 AND 3.89 tok/s** — `vt::Attention`'s f32 online-softmax is the right math but a DIFFERENT reduction order than vLLM's FA2, so it coin-flips near-ties (breaks p3 16→3 into §14's `163586×` repeat) and the per-(t,h) build slows the O(n²) recompute. `VT_KIMI_DEVICE_MLA` STAYS OFF, kept as a documented-MEASURED-NEGATIVE A/B knob (§14 `ISLAND_F32ACC` precedent). MLA dims VERIFIED from the real config (nah=32, qk_nope=128, qk_rope=64, v=128, kv_lora=512, q_lora=None; 7 full-attn/20 KDA). Both device knobs default OFF (122 ≠ STRICT, K=3-deterministic golden). STRICT residual, sharpened: needs vLLM's ACTUAL kernels — (c) chunk_kda prefill family (Triton-AOT regen for sm_121a) + (d) paged FA2 `mla::ForwardMlaAttentionBlock` (NOT the vt::Attention approximation) + (e) paged-incremental decode (needs a decode/paged-attn op, query_len≠key_len; kills the O(n²)) — each a substantial multi-kernel brick (§16). Row STAYS 🚧. **FULL-MODEL GB10 e2e RUNS — NEAR-TIE 106/128 (2026-08-06, `row/MODEL-KIMI-LINEAR-BF16`):** the bf16-resident path CLEARS the f32-loader block — the full 48.9B model now runs e2e on one GB10. dgx CUDA build (`-Werror` clean, 14 GDN AOT symbols nm-linked, `test_kimi_linear_forward` 13/13·656 in the CUDA binary); `kimi-linear-gen --gpu` greedy-decodes the §12 8-prompt battery x16 vs `greedy_ids.npy`. MEMORY: load 117.6s, host RSS PEAK **1.7 GiB** (stage-then-ReleaseHost), device peak 98.5 GiB, min-avail **21.6 GiB** (above the 15 GiB floor, matches the ~25 GiB pool-math headroom), NO OOM/reboot. TOKEN gate **NEAR-TIE 106/128 (82.8%)** — prompts 0,1,3,4,5,6 are 16/16 token-exact, p2/p7 diverge at punctuation/word near-ties; 96 consecutive exact tokens across 6 prompts prove the WIRING (a wiring bug can't). Root cause (honest): the f32 residual stream + host-f64 islands are MORE precise than vLLM's bf16 device kernels, so they flip the argmax where vLLM's deterministic bf16 top-1 has a small margin. STRICT path = the named W7-speed residuals (device GDN/MLA islands -> bf16 stream matching vLLM's rounding). 1.59 tok/s (recompute+island rate). `VT_KIMI_DEVICE_COMPUTE` STAYS OFF (parity-enablers: near-tie != token-exact). Row STAYS 🚧. **bf16-RESIDENT loader/forward IMPLEMENTED + CPU-gated (2026-08-06, `row/MODEL-KIMI-LINEAR-BF16`):** the §13 design is coded — `LoadKimiLinearResidentBf16Weights`/`StageKimiResidentBf16`/`BuildKimiResidentFromHost` (`kimi_linear_weights.cpp`; `LoadBf16Direct` -> `OwnedTensor`, per-tensor stage-to-`d_dev` + `ReleaseHost`, tiny vectors host f32), `KimiLinearResidentWeights` (`kimi_linear.h`), bf16 device forward `DeviceForwardBodyBf16` + `Gemm Bf16` cast-act at ~20 GEMM sites with the two host-fallback islands EXTRACTED+shared (`kimi_linear_device.cpp`), `ForwardDevice` resident-path dispatch (`kimi_linear.cpp`), and the `kimi-linear-gen` e2e harness. CPU **13/13·656** (12/12·614 f32 path UNTOUCHED + NEW tiny-config bf16-vs-f32 gate). PENDING: dgx CUDA build + full-model GB10 e2e vs the STRICT golden. Row STAYS 🚧. **bf16-RESIDENT brick POOL-MATH+DESIGN (2026-08-06, `row/MODEL-KIMI-LINEAR-BF16`):** pool math CLOSES (91.5 GiB bf16 device-resident + ~2.4 GiB act/norms/ctx ≈ 94 GiB, ~25 GiB headroom); design grounded §13 (Laguna `GemmBf16` cast-act + `OwnedTensor::d_dev`, `LoadBf16Direct`, f32 `MaterializeHost` kept for the unit gate). Impl (loader/forward rewrite + gate + e2e) pending. Row STAYS 🚧. **§8 GOLDEN CAPTURED — STRICT (2026-08-06, `row/MODEL-KIMI-LINEAR-E2E`):** the §8 SACRED oracle golden is captured on GB10 (0.25.0-stage, util 0.82, moe=triton, min 15 GiB avail, NO reboot), **8/8 prompts DETERMINISTIC over K=3 → STRICT gate**, committed at `tests/parity/goldens/kimi_linear_greedy/`. Full our-engine e2e BLOCKED on OUR f32 loader (materializes ~183 GiB > 119 pool), the bf16-residency residual; row STAYS 🚧. **W7 GPU-VERIFY (2026-08-06, branch `row/MODEL-KIMI-LINEAR-GPU`):** the device compute runs **12/12·614 GREEN on GB10 sm_121a CUDA build**, BOTH arms (`VT_KIMI_DEVICE_COMPUTE=1` + host-ref); prod stack (CUTLASS-NVFP4 GEMM + FA2 ENABLED + Triton-AOT GDN, 14 cubins nm-verified); f32 device==W2 ref, no divergence, no DeepSeek-class trap. Oracle gateability re-confirmed (0.25.0-stage registers `KimiLinearForCausalLM`). e2e §8 SACRED golden STILL disk-blocked (91.5 GiB checkpoint absent, dgx root 100% full, 34G free). Row STAYS 🚧. **W7 DBuf-resident device COMPUTE landed, CPU-gated** (`CLAIM-KIMI-LINEAR-W7`): the real device compute (`ForwardDeviceCompute`, `kimi_linear_device.cpp`) composes the whole 27-layer KDA/NoPE-MLA + 256-expert-MoE hybrid over pooled f32 `DBuf`s through the SHARED `vt::` ops (embed/`FusedChain` add+RMSNorm/`MatmulBT` projections/`CausalConv1dFwd` convs/`L2Norm`/`RmsNormGated`/`MoeRouterTopK` sigmoid-`noaux_tc`/`MoeSiluMul`/`MoeCombine`/lm_head), returning DEVICE-RESIDENT logits; 2 documented HOST-FALLBACK islands (the KDA per-k-channel gated-delta recurrence + its exp/softplus decay gate — `vt::GdnDecode` carries only a per-HEAD scalar decay; the NoPE-MLA softmax core — the paged `mla::ForwardMlaAttentionBlock` device path is born-on-runner) are the W7-speed residuals. CPU-gated vs the W2 host reference (the CPU backend runs the SAME `vt::` dispatch): `test_kimi_linear_forward` **12/12·614** (per-op KDA/NoPE-MLA/MoE/dense device==ref within f32-accumulation tolerance; the whole `ForwardDeviceCompute` == ref logits + greedy-token-identical + device-resident). Runner opt-in via `VT_KIMI_DEVICE_COMPUTE=1` (default OFF keeps the CPU-verified W6 host-ref compose). GPU numerics (bf16 activations, GDN Triton-AOT cubins, paged het-KV, grouped-MoE slabs) + the e2e SACRED golden stay a NAMED pending (box down) — row STAYS 🚧. ON TOP OF **W6 DEVICE forward SEAM** (`CLAIM-KIMI-LINEAR-W6`): the born-on-the-runner `ForwardDevice` (the DEFAULT `gather_logits` runner path) no longer refuses — it composes the `[rows,vocab]` logits via the CPU reference and hands them back DEVICE-RESIDENT (a pooled `DBuf`, wrapped like deepseek_v2 `WrapDeviceLogits`; `on_device()==true` on CPU+CUDA) so the on-GPU sampler consumes them with NO host download. Kimi-Linear now ROUTES device-resident (`check-runner-routing-consistency` reclassifies it, refuse-skipped stubs 2→1, NO allowlist; `check-fusion-consistency` green); `test_kimi_linear_forward` **7/7·300** (adds the `ForwardDevice`==host-ref device-resident gate). The DBuf-resident device COMPUTE (KDA via the GDN family, NoPE-MLA via `mla::ForwardMlaAttentionBlock`, DeepSeek-V2 grouped-MoE over the paged het-KV; full plan in `kimi_linear.cpp`) is the GPU-verify-pending W7 residual. ON TOP OF **W2-W6 CPU REFERENCE forward** (`CLAIM-KIMI-LINEAR-W2`): the real host `KimiLinearModel::Forward` composes the whole 27-layer hybrid from the landed primitives (KDA layer via `vllm::kimi_kda` refs + the gated-delta recurrence; NoPE-MLA materialized-MHA ref; sigmoid `noaux_tc` MoE + shared expert; dense SwiGLU); loader now materializes host float weights; `test_kimi_linear_forward` 6/6·246 (per-op gates + finite whole forward + greedy decode). ON TOP OF **W1 scaffolding** (registry + `ParseKimiLinearParams` 20 KDA + 7 NoPE-MLA + index-verified name-map + het-KV spec). e2e-gateable (FITS one GB10, 0.77× pool). RESIDUAL = the DEVICE born-on-runner forward (KDA kernel/absorbed-MLA/grouped-MoE slabs) + the W0/W7 e2e SACRED golden. Row → `ACTIVE` (device SEAM wired; the DBuf device compute + e2e SACRED golden pending) | `MODEL-TEXT-kimi-linear-kimi-linear-for-causal-lm` | | 📋 | `KimiK3ForConditionalGeneration` | Kimi K3 (2.8T MoE + MoonViT-V2, DERIVE-AND-SHIP) | **W2/W5 CPU scaffolding landed** (registry stub + nested text/vision/quant config descent + text-backbone structural name-map + REFUSE-by-name forward + MXFP4-refuse loader; clean CPU build, scaffold gate 6/6). text backbone IS `KimiLinearForCausalLM` (KDA+MLA+MoE hybrid, HEAVY reuse); **does NOT fit GB10 (~1.56 TB MXFP4, ~12×)** and NOT in the pinned oracle ⇒ no on-box golden — DERIVED, proxy-gated on Kimi-Linear-48B; forward + MXFP4 + KDA delta + MoonViT-V2 not implemented (NOT-YET-BUILDABLE) | `MODEL-MM-kimi-k3-kimi-k3-for-conditional-generation` | -| 🚧 | `MiniMaxH3DiTModel` | MiniMax-H3 (33.1B omni-modal video+audio DiT, DERIVE-AND-SHIP) | **W1/W2 landed**: packed layout (fl2va + ref2va, fp64 position grid BIT-EXACT), latent packing, euler-ancestral eta0 scheduler, and the full DiT forward all parity-gated against the UPSTREAM vLLM-Omni modules executed at reduced dimensions (**max abs diff 1.6e-7**, 10/10 cases / 2539 assertions). NOT autoregressive (no KV cache, no sampler, no logits) and **e2e HW-BLOCKED** (~354 GB checkpoint, ~133 GB/rank on 4x B300 vs 119 GiB unified); bf16 production stream + request planning + the ComfyUI-GGUF arm also landed (535 REAL tensors resolve onto our contract, geometry from shapes alone). **HW verdict CORRECTED: quantized arms FIT (~41 GB in 119 GiB)**, so e2e + speed are reachable; encoder/VAEs/audio VAE DONE (4.2e-9 vs the checkpoint's remote code); NVFP4 layout GATED as identical to ours (speed path is loader wiring); BOTH VAE DECODERS done (audio 4.2e-9, video ViT3D 8.9e-8); video tiling + 3D-CNN encoder (conditioning only) pending; encoder TEXT tower done (1.2e-7); **serving `/v1/videos` DONE and the DEVICE-RESIDENT forward (W2b, f32) LANDED + GPU-VERIFIED on Thor sm_110 at video 1.49e-7 / audio 8.94e-8**; bf16 stream + fusion folds + the FP4 path (needs sm_121a) + a real-checkpoint run pending. **2026-08-05: the AUDIO-VAE ENCODER is ported** (DAC analysis stack + `pre_block` AttnProjection + `mean_proj`, gated stage by stage vs the checkpoint's own remote code at 2.98e-8 / 1.64e-7 / 1.86e-8) with its own checkpoint loader gated on the real 1087-tensor manifest — so **ref2va AUDIO and VIDEO+AUDIO references are now WIRED** (audio rows move by 0.51 / 0.71; a different waveform still moves them by 7.1e-4). Both VAEs are now complete in both directions. **bf16 13-SHARD RELEASE INDEXES 2026-08-07 (`row/H3-BF16-SHARDED-DIT`)**: `MiniMaxH3ShardedCheckpoint` resolves the ORIGINAL 66.3 GB release through its own `model.safetensors.index.json` (a tensor named in the index but missing from its shard throws BY NAME), `EnumerateMiniMaxH3ShardedTensors` feeds the shared shapes-only geometry parser, and `LoadMiniMaxH3DitFromShards` is the host-f32 reference loader. Gated CPU-only at 72/72/54497 (post-rebase): every tensor resolves to the shard the index named AND to the bytes written there, the derived geometry equals the single-file path field for field, and a SPARSE 13-shard release with the REAL 535 tensors at REAL shapes (66.3 GB declared, 144 KB on disk) derives the SHIPPED geometry (50/5376/56/128/14336/24/32/1x2x2/5120). **STREAMS 2026-08-07 (`row/H3-BF16-SHARDED-STREAM`)**: `StreamMiniMaxH3ShardedToDeviceBf16` uploads it one tensor at a time — a BF16 tensor bound for a bf16 device slot goes straight from the mmap with ZERO host buffer, so peak host is bounded by ONE tensor (observed `host_peak=8192`, `direct=37 converted=9`); bit-exact vs the non-streamed `StageMiniMaxH3DitWeights` reference over all 46 views with identical logits, rope.inv_freq host-resident, 73/73/55203. Spec §8.14. **bf16 TEXT ENCODER + THE CONDITIONING NUMBER 2026-08-07 (`row/H3-ENC-BF16-COND-DIFF`)**: the 14-shard 63 GB bf16 Qwen3-VL-32B encoder streams to device too (`StreamMiniMaxH3EncoderShardsToDevice`, q/k/v and gate/up fused ON DEVICE), `--encoder-only` runs the tower alone (peak ~96 -> ~49 GiB by not loading the DiT first), and the widening is gated BIT-IDENTICAL vs an f32-staged tower so the A/B cannot be confounded. MEASURED on Thor over 233 tokens: Q4_K_M vs bf16 conditioning is cos 0.99745 mean / 0.909 min, rel RMS 6.85% excluding the attention sink, median rotation 3.5 deg — same energy as a ONE-WORD prompt edit but DIFFUSE (232/233 tokens rotate vs 172/233). Whether the RENDER changes is NOT established. 75/75/55609. Spec §8.15. This UNBLOCKS the quantization-quality question; no bf16-vs-quant render or speed number is claimed. Spec §8.13. **W-FP4a LANDED (CPU) 2026-08-06 (`row/H3-FP4-SPEED`)**: the device DiT forward now routes the NVFP4 projections through the shared Marlin W4A16 dispatcher (fp4 kept packed; no new quant code), fp4-vs-bf16 wiring gate GREEN (62/62·30039). **W-FP4a GB10 leg LANDED 2026-08-06 (`row/H3-FP4-GPU-E2E`, PR #64):** on sm_121a the Marlin W4A16 path RAN for all 11 projections (`dense_gemms==11` default — VT_MARLIN_DENSE is default-ON → vLLM's own DENSE Marlin GEMM, not the grouped route; `marlin_gemms==11` under VT_MARLIN_DENSE=0; `fallback_gemms==0`), fp4-vs-bf16 BYTE-EXACT (max\|diff\|=0), and the fp4 arm is a MEMORY win not a diffusion-forward speed win (per-forward bf16/fp4 3.47× @seq64 → 0.79–0.83× @seq4224–7040; ~16 vs ~66 GB device). Real-checkpoint fp4-resident t2va e2e RUNS (real 18.75 GB NVFP4 DiT + VAEs + GGUF Qwen3-VL-32B encoder → valid mp4/wav; DiT s/step 5.45/20.0/209 s @512/768/REF-209f) but frames are a non-scene patch-grid at 12/20/50 steps → OPEN render bug (device VAE/denoise). vLLM-Omni has no quantized H3 arm (BF16-only) so any comparison is HW/loader-forced-indirect — spec §8 | `MODEL-DIFFUSION-minimax-h3-mini-max-h3-dit` | +| 🚧 | `MiniMaxH3DiTModel` | MiniMax-H3 (33.1B omni-modal video+audio DiT, DERIVE-AND-SHIP) | **W1/W2 landed**: packed layout (fl2va + ref2va, fp64 position grid BIT-EXACT), latent packing, euler-ancestral eta0 scheduler, and the full DiT forward all parity-gated against the UPSTREAM vLLM-Omni modules executed at reduced dimensions (**max abs diff 1.6e-7**, 10/10 cases / 2539 assertions). NOT autoregressive (no KV cache, no sampler, no logits) and **e2e HW-BLOCKED** (~354 GB checkpoint, ~133 GB/rank on 4x B300 vs 119 GiB unified); bf16 production stream + request planning + the ComfyUI-GGUF arm also landed (535 REAL tensors resolve onto our contract, geometry from shapes alone). **HW verdict CORRECTED: quantized arms FIT (~41 GB in 119 GiB)**, so e2e + speed are reachable; encoder/VAEs/audio VAE DONE (4.2e-9 vs the checkpoint's remote code); NVFP4 layout GATED as identical to ours (speed path is loader wiring); BOTH VAE DECODERS done (audio 4.2e-9, video ViT3D 8.9e-8); video tiling + 3D-CNN encoder (conditioning only) pending; encoder TEXT tower done (1.2e-7); **serving `/v1/videos` DONE and the DEVICE-RESIDENT forward (W2b, f32) LANDED + GPU-VERIFIED on Thor sm_110 at video 1.49e-7 / audio 8.94e-8**; bf16 stream + fusion folds + the FP4 path (needs sm_121a) + a real-checkpoint run pending. **2026-08-05: the AUDIO-VAE ENCODER is ported** (DAC analysis stack + `pre_block` AttnProjection + `mean_proj`, gated stage by stage vs the checkpoint's own remote code at 2.98e-8 / 1.64e-7 / 1.86e-8) with its own checkpoint loader gated on the real 1087-tensor manifest — so **ref2va AUDIO and VIDEO+AUDIO references are now WIRED** (audio rows move by 0.51 / 0.71; a different waveform still moves them by 7.1e-4). Both VAEs are now complete in both directions. **bf16 13-SHARD RELEASE INDEXES 2026-08-07 (`row/H3-BF16-SHARDED-DIT`)**: `MiniMaxH3ShardedCheckpoint` resolves the ORIGINAL 66.3 GB release through its own `model.safetensors.index.json` (a tensor named in the index but missing from its shard throws BY NAME), `EnumerateMiniMaxH3ShardedTensors` feeds the shared shapes-only geometry parser, and `LoadMiniMaxH3DitFromShards` is the host-f32 reference loader. Gated CPU-only at 72/72/54497 (post-rebase): every tensor resolves to the shard the index named AND to the bytes written there, the derived geometry equals the single-file path field for field, and a SPARSE 13-shard release with the REAL 535 tensors at REAL shapes (66.3 GB declared, 144 KB on disk) derives the SHIPPED geometry (50/5376/56/128/14336/24/32/1x2x2/5120). **STREAMS 2026-08-07 (`row/H3-BF16-SHARDED-STREAM`)**: `StreamMiniMaxH3ShardedToDeviceBf16` uploads it one tensor at a time — a BF16 tensor bound for a bf16 device slot goes straight from the mmap with ZERO host buffer, so peak host is bounded by ONE tensor (observed `host_peak=8192`, `direct=37 converted=9`); bit-exact vs the non-streamed `StageMiniMaxH3DitWeights` reference over all 46 views with identical logits, rope.inv_freq host-resident, 73/73/55203. Spec §8.14. **bf16 TEXT ENCODER + THE CONDITIONING NUMBER 2026-08-07 (`row/H3-ENC-BF16-COND-DIFF`)**: the 14-shard 63 GB bf16 Qwen3-VL-32B encoder streams to device too (`StreamMiniMaxH3EncoderShardsToDevice`, q/k/v and gate/up fused ON DEVICE), `--encoder-only` runs the tower alone (peak ~96 -> ~49 GiB by not loading the DiT first), and the widening is gated BIT-IDENTICAL vs an f32-staged tower so the A/B cannot be confounded. MEASURED on Thor over 233 tokens: Q4_K_M vs bf16 conditioning is cos 0.99745 mean / 0.909 min, rel RMS 6.85% excluding the attention sink, median rotation 3.5 deg — same energy as a ONE-WORD prompt edit but DIFFUSE (232/233 tokens rotate vs 172/233). Whether the RENDER changes is NOT established. 75/75/55609. Spec §8.15. This UNBLOCKS the quantization-quality question; no bf16-vs-quant render or speed number is claimed. Spec §8.13. **W-FP4a LANDED (CPU) 2026-08-06 (`row/H3-FP4-SPEED`)**: the device DiT forward now routes the NVFP4 projections through the shared Marlin W4A16 dispatcher (fp4 kept packed; no new quant code), fp4-vs-bf16 wiring gate GREEN (62/62·30039). **W-FP4a GB10 leg LANDED 2026-08-06 (`row/H3-FP4-GPU-E2E`, PR #64):** on sm_121a the Marlin W4A16 path RAN for all 11 projections (`dense_gemms==11` default — VT_MARLIN_DENSE is default-ON → vLLM's own DENSE Marlin GEMM, not the grouped route; `marlin_gemms==11` under VT_MARLIN_DENSE=0; `fallback_gemms==0`), fp4-vs-bf16 BYTE-EXACT (max\|diff\|=0), and the fp4 arm is a MEMORY win not a diffusion-forward speed win (per-forward bf16/fp4 3.47× @seq64 → 0.79–0.83× @seq4224–7040; ~16 vs ~66 GB device). Real-checkpoint fp4-resident t2va e2e RUNS (real 18.75 GB NVFP4 DiT + VAEs + GGUF Qwen3-VL-32B encoder → valid mp4/wav; DiT s/step 5.45/20.0/209 s @512/768/REF-209f) but frames are a non-scene patch-grid at 12/20/50 steps → OPEN render bug (device VAE/denoise). vLLM-Omni has no quantized H3 arm (BF16-only) so any comparison is HW/loader-forced-indirect — spec §8. **ONE-SURFACE ROW 2 LANDED 2026-08-08 (`row/H3-VIDEO-ABI`)**: the whole assembly pipeline is library-owned (`vllm::multimodal::MiniMaxH3VideoEngine`, `minimax_h3_video.cpp`) behind the ABI v12 `vllm_video_*` entry points; `/v1/videos` routes through the SAME seam; `minimax_h3_gen`+`minimax_h3_mux` are thin `vllm.h` clients, frames+WAV byte-identical to the pre-fold binary on the committed fold fixture (`test_minimax_h3_video_fold` 3-arm gate + the v12 `test_capi` section); GB10 real-video re-verify via the v12 ABI = named residual | `MODEL-DIFFUSION-minimax-h3-mini-max-h3-dit` | | ✅ | `LagunaForCausalLM` | Poolside Laguna-S-2.1 (118B/8B MoE) | **LONG-CTX DECODE LEVERS LANDED + MEASURED (2026-08-03, `CLAIM-LAGUNA-LONGCTX-LEVERS`): window-bounded SWA reads (`VT_LAGUNA_SWA_WINDOW`, default-ON, BYTE-EXACT) bound the four `DecodeAttnGqa*` kernels' read to the ~512 sliding window (vLLM `laguna.py:412`) — GB10 A/B token-IDENTICAL `=1` vs `=0` at 520-token context (truncation active), MEASURED −0.30 ms/step at ~2k (~0 at ≤512, grows linearly). bf16 paged KV (`VT_LAGUNA_KV_BF16`, default-OFF opt-in) a distributional near-tie left UNRATIFIED. See BENCHMARKS `CLAIM-LAGUNA-LONGCTX-LEVERS`.** — **NVFP4 W4A4 ARM RAN on GB10 (N4, 2026-08-01, `CLAIM-LAGUNA-NVFP4-N4`): the additive safetensors NVFP4 arm (N1a/N1b/N2/N3 — `Nvfp4Weight` expert fields + `LoadLagunaForCausalLMWeights` + `LqGemmNvfp4Fp4` per-expert TRUE-W4A4 + `LagunaFfnBlock` `fp4` branch + `laguna_gen` dir-autodetect; CPU-gated `test_laguna_nvfp4_loader` 3/3·61, GGUF path byte-identical) generates COHERENTLY on the real 67 GiB `poolside/Laguna-S-2.1-NVFP4`. vs the vLLM MARLIN golden (vLLM's exact prompt ids injected): FIRST 2 TOKENS MATCH exactly, then near-tie divergence (our TRUE-W4A4 fp4-activations vs the MARLIN golden's W4A16 bf16-activations — different precision, EXPECTED; shares golden vocab). SPEED (N5, trace-driven, 2026-08-01): 0.16 → ~4.5 tok/s (~28× THIS SESSION), now ~4× from vLLM 18.8. **Lever #2** (nsys found the bf16 tower running host `MatmulNK` on the CUDA queue): route it to the GPU (`LqGemm` bf16 → `CastBf16` + `MatmulBT`, weight stays bf16) → 6.34 → 0.39 s/tok (16×). **Lever #1** (nsys found the emulation expert GEMM at 92%, GPU 87% busy): the engine's native sm120a fp4 tensor-core MMA (`MatmulNvfp4Fp4Native`) reads the SAME linear scales — it was gated OFF behind `VT_NVFP4_FP4_NATIVE`; default it ON in the driver → 0.39 → ~0.20-0.24 s/tok (~2×). Both coherent + near-tie (byte-identical ids to emulation; first token matches golden). Two GB10 memory fixes landed to run (shard-release + context-before-load). OPEN #234 (remaining ~4×): grouped W4A4 MoE (top_k×3 launches → 3), `ResidentNvfp4`, decode CUDA-graph + on-GPU sampling (the host-orchestration tail). Spec `.agents/specs/laguna-nvfp4-arm-2026-07-31.md` §N4/§N5. The GGUF-Q4_K track (below) is the separate keep-quant vehicle.** Prior **FASTER DECODE (W9, 2026-07-31, `CLAIM-LAGUNA-W9-GROUPED`): the 30 un-grouped per-expert keep-quant GEMV launches/step (top_k × {gate,up,down} `LqGemmRowSlice`) fold onto the SHARED `vt::MatmulBTQuantGrouped` op — per token, Pk experts' gate/up/down each collapse to ONE grouped launch over the already-stacked `[E*N,H]` tower (no loader change). Same-binary A/B on real UD-Q4_K_XL (GB10, `--gpu`, drop_caches cold, 24 tok): grouped (`VT_LAGUNA_GROUPED_MOE=1`, default) == per-expert (`=0`) BYTE-IDENTICAL (md5 `754728c6`, both == W6 golden) + decode 0.18 → 0.13 s/tok (1.38×). Routes through the shared vt op (fold policy). Cumulative with W8: decode 0.66 → 0.13 s/tok (5.1×; 1.5 → 7.7 tok/s; 18× → 3.6× vs llama.cpp 27.8). Next lever: device-resident decode (#1). See spec §W9.** Prior **FASTER DECODE (W8, 2026-07-31, `CLAIM-LAGUNA-W8-EMBED`): `LagunaEmbed` no longer converts the whole 1.23 GB embed table to f32 every token (it gathered T rows out of the whole [Vsz,H] table via `ReadF32` — ~311M host element-converts/token, the DOMINANT decode cost the W7 profile under-filed as "#5"); now gathers only the T needed rows directly (BIT-IDENTICAL — same per-element conversion, same rows). GATED on the real 3-shard UD-Q4_K_XL GGUF (GB10, `--gpu`, W6 cached, drop_caches cold, 24 tok): TOKEN-IDENTICAL to the W5/W6 golden (`22345 83 350 785 …`, coherent " Paris.") + decode 0.66 → 0.17 s/tok = 3.9× (1.5 → 5.9 tok/s; 18× → 4.7× vs llama.cpp 27.8). See `.agents/specs/laguna-s21-w7-speed-2026-07-31.md` §W8. Next: grouped-expert GEMM (=A3) then device-resident decode.** Prior **DECODE-SPEED ATTRIBUTED (W7 profile-only, 2026-07-31, `CLAIM-LAGUNA-W7-SPEED`): `nsys` of the W6 decode (real UD-Q4_K_XL GGUF, GB10) attributes the 0.66 s/tok (~1.5 tok/s vs llama.cpp 27.8 on identical bytes, ~15-18x) to HOST-ORCHESTRATION, not kernel compute — GPU active only 32.7% of the step, 67.3% host/idle; 22,115 `cudaStreamSynchronize` (~2,764/step, zero GPU overlap) from the ~1,795 per-GEMM `DrainQueue` in `LagunaForwardGgufCached` + scalar host glue; 39.4% of GPU time is `QuantizeQ8K` activation-quant (per-GEMM), weight GEMVs un-grouped at ~22% of the 240 GB/s peak (llama.cpp ~76%); no H2D/D2H (unified memory). Ranked levers (all in-tree from ds4): device-resident decode 1.5->~5-7 tok/s, grouped-expert GEMM (`MatmulBTQuantGrouped`) +1.5-2x + dedupes the activation-quant, decode CUDA-graph, tuned MMVQ; + free host cleanups (`LagunaEmbed` copies the whole 1.23 GB embed table/token, per-token RoPE-cache rebuild). Honest reachable ~13-20 tok/s, 27.8 a stretch. NO code changed. See `.agents/specs/laguna-s21-w7-speed-2026-07-31.md`. Prior RUNNABLE + FAST DECODE (W6, 2026-07-31): a per-layer K/V cache + single-token incremental decode replaces W5's O(n²) STATELESS recompute — TOKEN-IDENTICAL (byte-equal ids, md5 `754728c6…` match, == the W5 golden) and 5.05× faster per token: decode 3.33 → 0.66 s/tok on the real UD-Q4_K_XL GGUF (GB10, `--gpu`, keep-quant), same " Paris.…" text. `LagunaKvCache` (mirrors `DeepseekV4KvCache`, MLA-latent → GQA multi-head K/V; caches post-QK-RMSNorm/post-RoPE K + raw V at f32 — bit-exact since RoPE/QK-norm are position-only and attention is causal), MIXED attention per-layer: 12 GLOBAL layers grow unbounded + 36 SLIDING-WINDOW-512 layers EVICT rows beyond the 512 window (gemma2/3 `is_sliding`); `LagunaForwardGgufCached` + shared `LagunaAttention`/`LagunaFfnBlock` helpers used by BOTH forwards (identical float ops; recompute ids unchanged after refactor), `examples/laguna_gen --stateless` A/B flag. No cache bug (bit-exact first run). Next speed = grouped-expert GEMM + device-resident decode (both in-tree from ds4). See `.agents/specs/laguna-s21-w6-2026-07-31.md`. Prior RUNNABLE (W5, 2026-07-31): our engine greedy-generates COHERENT text on the REAL 3-shard UD-Q4_K_XL GGUF (GB10 keep-quant) — "The capital of France is" → " Paris. …", first token "Paris." matches the llama.cpp-Poolside reference. Multi-shard GGUF reader + keep-quant tower (`LoadLagunaFromGgufShards`) + `LagunaForwardGguf` (ds4 keep-quant Gemm/GemmRowSlice) + `examples/laguna_gen`; load 20.6s, peak 71 GiB, 3.27 s/tok stateless recompute (speed=W6).** Prior W3: **W3 REAL forward + 3 new ops landed** (`laguna_ops.cpp`: per-head softplus attn out-gate + ungrouped sigmoid-noaux router + dual per-layer RoPE cos/sin builders; `LagunaModel::Forward` now a REAL runnable host-reference composition — variable-Q-head GQA + dual RoPE + sliding-window mask + softplus gate + dense L0 / ungrouped-MoE L1..47 + untied lm_head — replacing the W1/W2 `VT_CHECK(false)` stub; CPU `-Werror` full-library build clean; `test_laguna_scaffold` **8/8·166** incl. softplus math, router selection+tie-break RED-first, dual-RoPE cos/sin bit-match, variable-Q-head shapes, forward composition on synthetic weights; `test_model_registry` 24/24). W1 oracle DECISION: vLLM native `laguna.py` in pin ⇒ config constructs; dual-oracle = vLLM-NVFP4/-FP8 (fits GB10, BF16 235 GiB does NOT) + llama.cpp-Q4_K token-exact. DEFERRED to W4 (needs 73 GB checkpoint): GGUF keep-quant tower materialization + device/paged production forward + strict dual-oracle greedy gate. ~85-90% reuse (ds4-MoE + gemma-sliding + olmo3-dual-rope + landed Q4_K keep-quant); NEW = the 3 landed host ops + name-map + variable-Q-head device runner. **W4 (2026-07-31, `CLAIM-LAGUNA-W4`, in progress):** the UD-Q4_K_XL GGUF (73.4 GiB, 3 shards) FETCHED to dgx + its metadata/tensor-map READ AUTHORITATIVELY (814 tensors, arch `laguna`, `expert_gating_func=2` sigmoid, `leading_dense_block_count=1`, `expert_weights_scale=2.5`). Three CPU-verified FIDELITY corrections the W1-W3 scaffold got wrong, each grounded in the real GGUF + llama.cpp: (1) **per-head QK-RMSNorm** (`attn_q_norm`/`attn_k_norm` F32[128]) added to params+forward — the scope MISSED it (surfaces only in the tensor map); (2) **dual-RoPE mscale** now uses llama.cpp's `yarn_attn_factor·(1+0.1·ln(factor))` off the GGUF-authoritative `factor=32`/`yarn_attn_factor=1.0` (256K-ctx build, NOT HF's factor-128/1.4852 1M-ctx scalar) — resolves the numerics-delicate residual; (3) **separate** `ffn_gate_exps`/`ffn_up_exps` (Q4_K) + `ffn_down_exps` (Q5_K) + Q8_0 shared/attn (the scaffold assumed merged gate_up). GGUF keep-quant tower materialization (`Mw`/`Sew` mirror of ds4) + keep-quant `ForwardGguf` (vt::MatmulBT/GemmRowSlice) + the real-model greedy run vs the llama.cpp-laguna same-quant oracle remain the W5 close (73 GB single-GB10, host-orchestrated) | `MODEL-TEXT-laguna-laguna-for-causal-lm` | | 🚫 | `DeepseekV3ForCausalLM` / `DeepseekV32ForCausalLM` | DeepSeek-V3 / V3.2 | HW-blocked (671B, ~642 GiB fp8 vs 119 GiB unified memory); V3.2 additionally DEP-blocked (DSA indexer) | `MODEL-TEXT-deepseek-v2-deepseek-v3-for-causal-lm` | | 🚫 | `GlmMoeDsaForCausalLM` | GLM-5 (DSA) | HW-blocked (1404 GiB bf16) and DEP-blocked (GLM-5.x is DeepSeek-V3.2 verbatim) | `MODEL-TEXT-deepseek-v2-glm-moe-dsa-for-causal-lm` | diff --git a/.agents/specs/one-surface-abi.md b/.agents/specs/one-surface-abi.md index 4344c7111..ad4a273b4 100644 --- a/.agents/specs/one-surface-abi.md +++ b/.agents/specs/one-surface-abi.md @@ -1,6 +1,6 @@ # ONE SURFACE — every capability ships through the C ABI -Row: `ARCH-ONE-SURFACE`. Status: **AUDIT DONE; remediation IN PROGRESS — ROW 1 (Parakeet ASR / audio transcription) LANDED 2026-08-07: ABI v11 `vllm_transcribe`, live `/v1/audio/transcriptions`, registry refuse-by-task, example folded, ratchet 12 -> 11.** +Row: `ARCH-ONE-SURFACE`. Status: **AUDIT DONE; remediation IN PROGRESS — ROW 1 (Parakeet ASR / audio transcription) LANDED 2026-08-07: ABI v11 `vllm_transcribe`, live `/v1/audio/transcriptions`, registry refuse-by-task, example folded, ratchet 12 -> 11. ROW 2 (MiniMax-H3 video+audio generation) LANDED 2026-08-08 (`row/H3-VIDEO-ABI`, task #283): ABI v12 `vllm_video_engine_load`/`vllm_video_generate`/`vllm_video_result_free` + `vllm_video_mux_argv` over the `MiniMaxH3VideoEngine` library seam, `/v1/videos` routed through the SAME seam, both H3 examples rewritten as `vllm.h` clients byte-identical to the pre-fold binary, ratchet 11 -> 9. GB10 real-video re-verification is a NAMED RESIDUAL (box on the Kimi campaign).** ## The defect @@ -28,7 +28,7 @@ stay OPEN. The complete, code-grounded matrix (all 30 archs + every off-registry | capability | registered? | why still off-surface | only real path via | example size | |---|---|---|---|---| -| MiniMax-H3 video+audio gen | NO | no arch, no video C-ABI; served via example-injected `VideoRunner` | `examples/minimax_h3_gen`, `examples/server` `/v1/videos` | 1293 lines | +| MiniMax-H3 video+audio gen | NO (diffusion lane) | **CLOSED (ROW 2)** — `vllm_video_*` on ABI v12; `/v1/videos` drives the library seam | library seam `MiniMaxH3VideoEngine`; examples are thin clients | 216 lines (was 1293) | | Laguna | YES | keep-quant/NVFP4 decode example-only; registry forward `VT_CHECK`s non-bf16; GGUF dispatch unreachable | `examples/laguna_gen` | 415 lines | | Kimi-Linear | YES | registry leg is a stateless recompute reference; §18/§19 incremental entry points are private | `examples/kimi_linear_gen` | 318 lines | | DeepSeek-V4 | YES | KV spec is a "never exercised" stub; registry forward discards attn_meta/kv | `examples/deepseek_v4_gen` | 240 lines | @@ -71,11 +71,39 @@ behaviour. 4. `examples/server` routes through the SAME entry point, so HTTP and FFI cannot drift. -## Proposed ABI shape for video (first slice) +## Proposed ABI shape for video (first slice) — SHIPPED (ROW 2, ABI v12) Mirrors the existing engine idiom: an opaque handle, a params struct with a `_default()`, an explicit free, `vllm_last_error` for diagnosis. +**As-shipped deltas from the proposal below (each argued during ROW 2, +2026-08-08; `include/vllm.h` is the binding text):** +- `vllm_video_model_params` GAINS `prompt_embeds_path` (without an encoder + there is NO conditioning path — both pre-fold consumers had this arm), + `partition` (the #77 guard refuses every full render without a declared + partition, so omitting it would make the ABI unable to render at all) and + `fp4_resident` (the gated NVFP4 Marlin arm); `encoder_max_layers` stays a + C++-seam knob. +- `vllm_video_params` GAINS `output_dir` (the result needs a destination) and + DROPS `duration_seconds`/`task` (derivable: `num_frames` expresses duration, + and the task is resolved from the references exactly as upstream + `_resolve_task` does; both remain on the C++ seam for the server). +- `vllm_video_result.mux_argv` is `char**`, NULL-terminated (execvp-ready), + freed via `vllm_video_result_free`. +- ADDED `vllm_video_mux_params(_default)` + `vllm_video_mux_argv(_free)`: the + engine-free composer `minimax-h3-mux` needs to be a `vllm.h` client (W4); + the encoding contract stays the library's, the caller execs. +- NAMED RESIDUALS of the first slice: ONE `ref_image` (multi-image ref2va is + C++-seam-reachable only), and the pre-fold example's diagnostic modes + (`--denoise-only`, `--dump-params`, `--encoder-only`/`--save-embeds`, + `--decode-latent`, `--roundtrip`, `--prompt-image`, `--cond-image`, + `--dry-run`) were deleted with the private pipeline — the capabilities they + probed are gated by `test_minimax_h3`/`test_minimax_h3_video_fold`, and the + GB10 speed recipe moves to the seam/ABI (re-verification residual below). +- The pre-fold CPU host-f32 GGUF arm (the example default with NEITHER + `--keep-quant` nor `--dequant-bf16`) is not on the ABI: `dequant_bf16=0` is + keep-quant (the gated arm the fold goldens were captured on). + ```c typedef struct vllm_video_engine vllm_video_engine; @@ -140,11 +168,21 @@ embedder produce an MP4 without reinventing the command: the library composes it ## NOT claimed -The VIDEO ABI shape above is a proposal, not an implementation, and it has not -been reviewed against an embedder other than LocalAI. The ONE landed slice is -audio transcription (ROW 1, 2026-08-07): `vllm_transcribe` + params/result -structs on ABI v11, the `ParakeetTranscriber` library seam, task-conditional -`/v1/audio/transcriptions`, registry refuse-by-task, and the example as a thin -`vllm.h` client — gated byte-identical to the pre-fold transcripts. Video, -Laguna/DeepSeek/Kimi fast decode, embeddings and multimodal input remain open -rows of this program. +The video ABI has not been reviewed against an embedder other than LocalAI. +TWO slices are landed: audio transcription (ROW 1, 2026-08-07 — ABI v11 +`vllm_transcribe`, the `ParakeetTranscriber` seam, task-conditional +`/v1/audio/transcriptions`, example as a thin client, byte-identical +transcripts) and video+audio generation (ROW 2, 2026-08-08 — ABI v12 +`vllm_video_*`, the `MiniMaxH3VideoEngine` seam, `/v1/videos` through it, both +H3 examples as thin clients, frames+WAV byte-identical to the pre-fold binary +on the committed fold fixture; three-arm gate `test_minimax_h3_video_fold` + +the v12 `test_capi` section). ROW 2 residuals, named: (1) GB10 real-video +re-verification (real checkpoints through the v12 ABI + the folded server; +the box is running the Kimi campaign — CPU fold gates are the landed +evidence); (2) DISCLOSED server-path numeric deltas (no goldens existed): the +pre-fold `/v1/videos` runner drew single-stream legacy-uniform noise and +defaulted to the host-f32 GGUF arm — it now shares the ratified recipe +(dual-stream splitmix64 Gaussian, keep-quant default), which kills the exact +HTTP-vs-CLI drift this row exists to prevent; (3) multi-image ref2va on the +ABI. Laguna/DeepSeek/Kimi fast decode, embeddings and multimodal input remain +open rows of this program. diff --git a/.agents/specs/surface-coverage-2026-08-07.md b/.agents/specs/surface-coverage-2026-08-07.md index cde95df33..aac30e84d 100644 --- a/.agents/specs/surface-coverage-2026-08-07.md +++ b/.agents/specs/surface-coverage-2026-08-07.md @@ -24,7 +24,7 @@ The four surfaces, and the public boundary the guard draws: | Rank | Gap | Registered | Servable | C-ABI | Where the capability actually lives | |---|---|---|---|---|---| -| 1 | **MiniMax-H3 video+audio generation** | NO (off-registry diffusion) | `/v1/videos` ONLY via an example-injected `VideoRunner` | NO | `examples/minimax_h3_gen` + `examples/minimax_h3_mux` (whole pipeline) | +| 1 | **MiniMax-H3 video+audio generation** | **CLOSED (ROW 2)**: still off-registry (a diffusion lane, not a text arch) but library-owned end to end | **`/v1/videos` through the library seam** (`MiniMaxH3VideoEngine` + `MiniMaxH3VideoGenParamsFromRequest`; the server keeps only flag plumbing + the ffmpeg exec) | **`vllm_video_*` (ABI v12)** | library seam `vllm::multimodal::MiniMaxH3VideoEngine`; both examples are clean ABI clients | | 2 | **Laguna fast decode** | yes, but forward is a stub | no (stub `VT_CHECK`s non-bf16) | no | `examples/laguna_gen` (keep-quant GGUF + NVFP4 W4A4) | | 3 | **DeepSeek-V4 fast decode** | yes, but forward is a W3 stub | no (stub) | no | `examples/deepseek_v4_gen` (keep-quant GGUF) | | 4 | **Audio transcription** | **CLOSED (ROW 1)**: Parakeet CTC/RNNT/TDT registered (transcription-only; Whisper/Voxtral still off-registry) | **live `/v1/audio/transcriptions`** (task-conditional; the run_batch line stays a residual) | **`vllm_transcribe` (ABI v11)** | library seam `ParakeetTranscriber`; example is a clean ABI client | @@ -60,7 +60,7 @@ All three drivers run a PRIVATE host-argmax greedy loop, not the on-GPU sampler. | Lane | Registered | Code (file:line) | Server | C-ABI | Driver | |---|---|---|---|---|---| -| MiniMax-H3 video+audio GEN | NO | `minimax_h3*.cpp` (~22 TUs), vt op `kMiniMaxH3` | `/v1/videos` via `set_video_runner` (example-injected only; `api_server.h:167`, `api_server.cpp:818`; library never sets it) | NO | `examples/minimax_h3_gen`, `examples/minimax_h3_mux` | +| MiniMax-H3 video+audio GEN | NO (diffusion lane; **ROW 2** made it library-owned without a registry entry) | `minimax_h3*.cpp` (~22 TUs) + the **`minimax_h3_video.cpp` seam** (ROW 2), vt op `kMiniMaxH3` | `/v1/videos` via `set_video_runner`, the runner now a thin exec wrapper over the LIBRARY seam (`MiniMaxH3VideoGenParamsFromRequest` -> `Generate` -> exec argv) | **`vllm_video_engine_load` / `vllm_video_generate` / `vllm_video_result_free` / `vllm_video_mux_argv` (ABI v12)** | `examples/minimax_h3_gen`, `examples/minimax_h3_mux` = thin `vllm.h` clients | | Parakeet/FastConformer ASR | **YES (ROW 1)**: ParakeetForCTC/RNNT/TDT, `parakeet_registry.cpp` (SupportsTranscription-only; text paths refuse by task) | `parakeet_transcription.cpp` seam composes encoder/transducer/audio-processor; the example's private `ReadWav16BitMono`/`LoadVocab`/`DecodeIds` are DELETED (`vllm::Tokenizer` now decodes Metaspace split=true) | **`/v1/audio/transcriptions`** (task-conditional) | **`vllm_transcribe` (ABI v11)** | `examples/parakeet_transcribe` = thin `vllm.h` client | | Voxtral audio->text | NO (`VoxtralForConditionalGeneration` unregistered) | `voxtral.cpp` (`vllm::multimodal`) | NO (`/v1/audio/transcriptions` = `run_batch.cpp:188` residual) | NO | tests-only reachability | | Whisper audio encoder | NO | `whisper_audio.cpp:174` | NO | NO | tests-only callers | @@ -77,11 +77,13 @@ Public boundary = `#include "vllm.h"` only. 13 example units; `examples/cli` (vl links `vllm::shared`, `#include "vllm.h"` only, `cli/main.cpp:16`) is the sole clean ABI client. **ROW 1 UPDATE (2026-08-07): `parakeet_transcribe` is the SECOND clean ABI client** — the Parakeet fold rewrote it against `vllm.h` + `vllm::shared` only, and the -ratchet fell 12 -> 11. The remaining 11 reach `include/vllm/**` / `vt/**` and are -transition-tracked in `scripts/example-abi-allowlist.txt`: +ratchet fell 12 -> 11. **ROW 2 UPDATE (2026-08-08): `minimax_h3_gen` and +`minimax_h3_mux` are the THIRD and FOURTH clean ABI clients** (the video fold, ABI +v12 `vllm_video_*`), and the ratchet fell 11 -> 9. The remaining 9 reach +`include/vllm/**` / `vt/**` and are transition-tracked in +`scripts/example-abi-allowlist.txt`: -- Capability drivers: `deepseek_v4_gen`, `laguna_gen`, `kimi_linear_gen`, - `minimax_h3_gen`, `minimax_h3_mux`, `server`. +- Capability drivers: `deepseek_v4_gen`, `laguna_gen`, `kimi_linear_gen`, `server`. - Dev/diagnostic (internal-by-nature, folded for consistency): `bench` (via `bench_core.h`), `tokenize`, `dump_container`, `dequant_nvfp4`, `quant_gemm_bench`. - Out of the gated `examples/` tree: `benchmarks/vulkan_gemm_ab.cpp` (Vulkan A/B harness). @@ -89,7 +91,7 @@ transition-tracked in `scripts/example-abi-allowlist.txt`: **Policy (developer-directed 2026-08-07): no permanent exemptions.** Every allowlist entry — drivers AND dev/diagnostic tools — is a transition-tracker pointing at a fold row; the guard fails on any internal include not tracked, and a shrink-only ratchet -(`MAX_INTERNAL_REACHING`, 11 since ROW 1) means the count can only fall as folds land, never grow to +(`MAX_INTERNAL_REACHING`, 9 since ROW 2; 11 since ROW 1) means the count can only fall as folds land, never grow to admit a new violation. The public header set is DERIVED from the CMake install rules (exactly `include/vllm.h` today), not hardcoded. The guard catches BOTH breach vectors: a `#include "vllm/..."|"vt/..."|"src/..."` AND a CMake `-I` grant into the internal tree @@ -128,7 +130,7 @@ lanes are leaves of `ARCH-ONE-SURFACE` (do not open parallel rows). | # | Fold | Grow ABI (new `vllm.h` surface) | Then rewrite / delete | Effort | Depends on | |---|---|---|---|---|---| -| 1 | Video+audio gen | `vllm_video_generate` + job/status/content entry points (mirror `/v1/videos`; carry the `VideoRunner` internally, not example-injected) | rewrite `minimax_h3_gen`/`server` as clients; delete driver glue | L | H3 loaders; ffmpeg-mux boundary (ratified in `examples/`) | +| 1 | Video+audio gen | **DONE (ROW 2, 2026-08-08)**: `vllm_video_engine_load`/`vllm_video_generate`/`vllm_video_result_free` + `vllm_video_mux_argv` (ABI v12); `/v1/videos` routes through the SAME `MiniMaxH3VideoEngine` seam (job/status/content stay `VideoJobStore`-served; the runner is now a thin exec wrapper the example injects, because the SPAWN stays in examples/ — the ratified ffmpeg boundary) | **DONE**: `minimax_h3_gen` + `minimax_h3_mux` rewritten as `vllm.h` clients (frames+WAV byte-identical to the pre-fold binary on the fold fixture); server driver glue deleted (54 -> 7 H3 refs, all seam type names) | L | H3 loaders; ffmpeg-mux boundary (ratified in `examples/`) | | 2 | Laguna fast decode | make the registered `LagunaForCausalLM` forward route the keep-quant/NVFP4 device path (retire the stub); load keep-quant GGUF/NVFP4 dirs through `vllm_engine_load` | rewrite `laguna_gen`; delete `LagunaForwardGguf*` | M | keep-quant load in the engine loader | | 3 | DeepSeek-V4 fast decode | same as (2) for `DeepseekV4ForCausalLM`; real MLA paged KV (retire the W3 stub) | rewrite `deepseek_v4_gen`; delete `DeepseekV4ForwardGguf*` | M | MLA paged-KV topology | | 4 | Audio transcription | **DONE (ROW 1, 2026-08-07)**: `vllm_transcribe` (ABI v11) + live `/v1/audio/transcriptions`; ParakeetForCTC/RNNT/TDT registered (SupportsTranscription mirror, refuse-by-task) | **DONE**: `parakeet_transcribe` rewritten as a `vllm.h` client (byte-identical transcript goldens); route live, task-conditional | M | encoder→text seam (LANDED: `ParakeetTranscriber`) | diff --git a/.agents/state.md b/.agents/state.md index a50cfeec9..f0309b308 100644 --- a/.agents/state.md +++ b/.agents/state.md @@ -42148,3 +42148,68 @@ synthetic pair + the P4/P6 pretrained evidence. (4) A real Parakeet tokenizer.json may carry normalizer fields `FromHfJson` refuses loudly — untestable CPU-side without the checkpoint, fails loud not wrong. (5) Whisper/ Voxtral remain off-registry (fold #9/#10 of the audit). + +## 2026-08-08 — ARCH-ONE-SURFACE ROW 2: MiniMax-H3 video+audio generation folded onto the ONE surface (PR #123) + + +**What landed (`row/H3-VIDEO-ABI`, task #283; fold order: grow ABI -> rewrite +examples -> delete parallel impl).** +- FOLD GATE FIRST: `tests/vllm/models/minimax_h3_video_fold_fixture.h` writes a + deterministic tiny checkpoint set (857KB F32 ComfyUI-GGUF DiT at the :3786 + reduced geometry + on-disk reduced ViT3D/BigVGAN VAEs + prompt embeds); the + PRE-fold `minimax-h3-gen` binary at the branch base (fc636c76) rendered it + (--partition fl2va --steps 3 --frames 5 --height/width 32, CPU, keep-quant) + and its 8 frames + WAV + both `minimax-h3-mux --print-only` argv lines are + COMMITTED goldens (`fixtures/minimax_h3_video_fold/`). Determinism proven + (two runs, diff clean). +- W1 seam: `vllm::multimodal::MiniMaxH3VideoEngine` (`minimax_h3_video.{h,cpp}`) + absorbs the example's 1293-line assembly driver AND the server's 354-line + /v1/videos twin: 4 DiT loader arms (GGUF keep-quant / dequant-bf16 + host+streamed / bf16-shard stream / NVFP4 bf16+fp4-resident stream), VAE + decoder + lazy encoder halves, the H3-Encoder tower staged once (GGUF or + bf16 shards), the #77 partition guard, fl2va/ref2va conditioning, the + byte-exact splitmix64 Box-Muller noise streams (VT_H3_GAUSSIAN_NOISE + honored; seeded requests derive audio via splitmix64), artifact writing + + mux argv. Library SPAWNS NOTHING (mkdir -p became std::filesystem; ffmpeg + stays caller-side). `MiniMaxH3VideoGenParamsFromRequest` = the ONE + /v1/videos mapping. +- W2 ABI: `vllm_video_engine(_load/_free)`, `vllm_video_model_params(_default)`, + `vllm_video_params(_default)`, `vllm_video_generate`, `vllm_video_result(_free)` + + `vllm_video_mux_argv((_params_default)/_free)`; VLLM_ABI_VERSION 11 -> 12, + test_capi floor >= 12. As-shipped deltas vs the ratified proposal argued in + the spec (+prompt_embeds_path/+partition/+fp4_resident/+output_dir/+mux + composer; -task/-duration; ONE ref_image). Refuse-both-directions pinned: + video-load on a text dir names vllm_engine_load; vllm_engine_load on the H3 + dir fails byte-for-byte as captured at v11. +- W3 server: the VideoState block is DELETED; /v1/videos = seam Load + a + runner lambda (FromRequest -> Generate -> fork/execvp(mux_argv)). Direct + MiniMaxH3 refs 54 -> 7 (all seam type names). DISCLOSED numeric deltas on + the server arm (no goldens existed; the drift WAS the defect): legacy + single-stream uniform noise -> the ratified shared recipe; host-f32 GGUF + default -> keep-quant; new `--video-dequant-bf16`. +- W4 examples: `minimax_h3_gen` (1293 -> 216 lines) + `minimax_h3_mux` are + `vllm.h`+`vllm::shared` thin clients; BOTH byte-identical to the pre-fold + binaries on the fixture (frames+WAV cmp clean; mux argv diff clean). + Ratchet: both allowlist rows removed, `MAX_INTERNAL_REACHING` 11 -> 9 + + equality pin + spec claims moved; abi-capability video row closed; FEATURES + video row -> reachable naming the four symbols. + +**Gates.** 3-arm fold gate `test_minimax_h3_video_fold` 4/131 GREEN (seam == +replicated pre-fold pipeline == committed goldens, byte-identical; guard +refusals; FromRequest field-complete). test_capi 40/373 incl. the v12 golden +e2e THROUGH the C marshalling; vllm_capi_c_check strict-C11 green. +test_minimax_h3 75/75 unchanged. test_openai_api_server 45/566 incl. the +socket-level "routes do not exist without a runner" 404 pin; test_video_api +14/14. check-surface-coverage green with the SHRINK enforced + its 46-test +mutation suite green. + +**Residuals (honest).** (1) GB10 real-video re-verification through the v12 +ABI + folded server (real checkpoints; box on the Kimi campaign — CPU fold +gates are the landed evidence). (2) The pre-fold example's diagnostic modes +(--denoise-only/--dump-params/--encoder-only/--save-embeds/--decode-latent/ +--roundtrip/--prompt-image/--cond-image/--dry-run, multi --ref-image) were +deleted with the private pipeline; capabilities remain library/test-reachable; +the GB10 speed recipe must move to the seam. (3) Server-arm numeric deltas +disclosed above. (4) The CPU host-f32 GGUF arm is off the ABI (keep-quant is +the gated arm). (5) /v1/videos job/status/content stay VideoJobStore-served +(unchanged); no async-job C-ABI shape yet. diff --git a/CMakeLists.txt b/CMakeLists.txt index b5168a919..a7906a079 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -673,6 +673,12 @@ add_library(vllm STATIC # Parakeet head archs so config.json RESOLVES (SupportsTranscription mirror). src/vllm/multimodal/parakeet_transcription.cpp src/vllm/model_executor/models/parakeet_registry.cpp + # The ONE video-generation seam every consumer drives (C ABI vllm_video_*, + # the server's /v1/videos, the minimax-h3-gen example) — ARCH-ONE-SURFACE + # ROW 2: absorbs the assembly pipeline examples/minimax_h3_gen and the + # server's private /v1/videos twin. Builds artifacts + the ffmpeg argv and + # SPAWNS NOTHING (the ratified process boundary). + src/vllm/multimodal/minimax_h3_video.cpp src/vllm/v1/core/encoder_cache_manager.cpp src/vllm/tokenizer/unicode_data.cpp src/vllm/tokenizer/pretokenizer.cpp diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index b59ca697c..b3f13f8a0 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -317,7 +317,7 @@ built on it rather than keeping the flattering one. | DeepSeek-V4-Flash vs vLLM | Infeasible on one Spark | 2x GB10 with TP2 over the NCCL seam | | DFlash speculative decode | **CLOSED 2026-07-27 (D14)**: warp-scoped draft attention (242.9 → 77.9 ms), c1 our-on 29.32 vs vLLM-on 29.24 tok/s, non-overlapping 3-rep bands, 1.003x | none, closed | | Multimodal image, audio, video | Correctness gated, speed unmeasured | Per-modality speed grids | -| `/v1/videos` OpenAI (Sora) shape | **No number owed**: a CPU serving-surface change (request aliases, the MP4 content route, and reference conditioning wiring), unit-gated only, no kernel or generation path touched | Video generation speed stays the MiniMax-H3 FP4 row below | +| `/v1/videos` OpenAI (Sora) shape + ONE-SURFACE fold ROW 2 | **No number owed**: CPU serving-surface changes only; the video fold (ABI v12, seam-routed `/v1/videos`, thin clients) is byte-identical plumbing, no speed claim; server defaults now the ratified recipe (disclosed) | Speed stays the MiniMax-H3 FP4 row below; GB10 re-verify via the v12 ABI = named residual | | Qwen3-dense decode CUDA-graph | Token-exact pass, ~4.3% e2e directional | Steady-state per-step tok/s | | Kimi-Linear-48B-A3B (KDA+MLA+MoE) | **Paged-incremental (§19) GB10: 18.9 tok/s @ 122/128 = coherent best.** bf16 stream REFUTED (§20/#118: 122→4/128 KDA repeat-loop, no speed win); STRICT unreachable (§14-§20 levers closed), p7 near-tie | vLLM ~21 (16-tok aggregate floor); ours **0.90× vLLM**. Last 0.10× + STRICT need vLLM's real kernels via the SERVER fold (scoped) | | vLLM 0.26 re-benchmark | Pending | Re-run the binding grids on the advanced pin | diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 48964fb76..cd2d2e643 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -162,7 +162,7 @@ model architecture is wired. | Image | ✅ correctness-gated | ✅ | ✅ | ◐ | | Video | ✅ correctness-gated | ✅ | ✅ | ☐ | | Audio | ✅ correctness-gated | ✅ | ◐ | ◐ | -| Video+audio GENERATION (MiniMax-H3 DiT, vLLM-Omni lane) | ◐ t2va+fl2va COHERENT on GB10; ref2va NVFP4 grid = the community ckpt's own quant fidelity, NO loader bug (§8.12); DiT loads GGUF/NVFP4/bf16-13-shard, encoder loads GGUF or bf16-14-shard | ✅ (vllm-omni, BF16-only, no quantized H3 arm) | ☐ | ☐ | +| Video+audio GENERATION (MiniMax-H3 DiT, vLLM-Omni lane) | ◐ t2va+fl2va COHERENT on GB10; ref2va NVFP4 grid = that ckpt's own quant fidelity (§8.12); GGUF/NVFP4/bf16-shard loaders; **embedder-reachable since ABI v12** (`vllm_video_*`, `/v1/videos` = same seam) | ✅ (vllm-omni, BF16-only, no quantized H3 arm) | ☐ | ☐ | | Multimodal over the OpenAI server | ☐ | ✅ | ✅ | ◐ | Image, video and audio are correct through the CLI and library. Serving them @@ -232,7 +232,7 @@ Build with `-DVLLM_CPP_VULKAN=ON`; off by default. | Flat C ABI for embedding in other languages | ✅ versioned | ☐ | ☐ | ✅ | #### C-ABI capability coverage -- Which capabilities an embedder drives through the flat C ABI (`include/vllm.h`, the only installed header), gated by `scripts/check-surface-coverage.py`: a `reachable` row names an entry point that exists; an `embedder-unreachable` row is tracked in `scripts/abi-capability-allowlist.txt` against its fold row (`ARCH-ONE-SURFACE`). The ABI is text-generation-complete; the four `embedder-unreachable` rows are the open capability gaps. +- Which capabilities an embedder drives through the flat C ABI (`include/vllm.h`, the only installed header), gated by `scripts/check-surface-coverage.py`: a `reachable` row names an entry point that exists; an `embedder-unreachable` row is tracked in `scripts/abi-capability-allowlist.txt` against its fold row (`ARCH-ONE-SURFACE`). The ABI is text-generation-complete; the two `embedder-unreachable` rows are the open capability gaps. | Capability | C-ABI surface | Embedder-reachable | |---|---|---| @@ -245,7 +245,7 @@ Build with `-DVLLM_CPP_VULKAN=ON`; off by default. | Custom logits processor | `vllm_logits_processor` | reachable | | Embeddings / pooling | none | embedder-unreachable | | Audio transcription (Parakeet ASR) | `vllm_transcribe`, `vllm_transcription_params_default`, `vllm_transcription_free` | reachable | -| Video+audio generation (MiniMax-H3) | none | embedder-unreachable | +| Video+audio generation (MiniMax-H3) | `vllm_video_engine_load`, `vllm_video_generate`, `vllm_video_result_free`, `vllm_video_mux_argv` | reachable | | Multimodal input (image/audio/video) | none | embedder-unreachable | ## Parallelism and scale-out diff --git a/docs/STATUS.md b/docs/STATUS.md index 82bdc35a0..102421ddd 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -86,7 +86,7 @@ token-for-token correctness against the pinned oracle. | OLMo-3 dense (dual rope, interleaved sliding window) | Implemented, oracle-blocked | Loads + runs in our engine (dual rope: plain sliding + YaRN full-attn, per-layer sliding window); no SACRED gate: vLLM 0.25.0 oracle cannot run OLMo-3-1025-7B (`KeyError: 'rope_theta'`; transformers 5.13.1 nests `rope_parameters` per layer-type, no flat `rope_theta`; run-verified W0 2026-07-26) | | Laguna-S-2.1 MoE (`LagunaForCausalLM`, 118B/8B) | **BINDING 2026-08-04: 87% of vLLM (37.55 vs 43.10, SAME-TOOL nsys both engines); the whole +3.1 ms/step is the bf16 M=1 GEMV bucket (2/3 o_proj, ~196-204 vs 139 us/call, identical `gemvx` kernel); attention/MoE/glue tied or ours-ahead. Invocation match (bf16-out `cublasGemmEx`) A/B'd = WASH, ruled out; ROOT CAUSE FOUND 2026-08-04 (`VT_LAGUNA_RESIDENT_BF16W`): the bf16 projections read UNIFIED/ATS host memory, not `cudaMalloc`'d device memory — staging them device-resident (byte-exact ids) gives 38.8→44.6 tok/s (o_proj 194→131, lm_head 2410→1620 us/call), parity+ vs vLLM 43.1; **default-ON** (flip smoke-verified: canonical byte-exact ids, 44.6 clean-median). Earlier ceiling/diffuse verdicts below were cross-tool artifacts.** **REAL vLLM BAR ESTABLISHED (2026-07-31, `CLAIM-LAGUNA-VLLM-NVFP4`): FIRST-EVER vLLM Laguna run** — prior numbers (incl. the correctness oracle) were all llama.cpp, never vLLM. vLLM on official `poolside/Laguna-S-2.1-NVFP4` (single GB10, greedy, eager, MARLIN backend forced via `VLLM_TEST_FORCE_FP8_MARLIN=1` because the auto-default `FLASHINFER_CUTLASS` needs an absent `nvcc`): **~18.8 tok/s** (64-tok steady) — a LOWER bound. Our GGUF-Q4_K engine = 7.7 tok/s (vLLM ~2.4×); llama.cpp GGUF = 27.8 (still fastest at batch-1). llama.cpp is now a labeled SECONDARY "beat best-in-class GGUF" note; vLLM-NVFP4 is the headline bar. TRUE apples-to-apple still owes OUR NVFP4 Laguna forward arm (same tensor-core path as 27B/35B) — bring-up W-plan SPEC'D in `.agents/specs/laguna-nvfp4-arm-2026-07-31.md` (~85% reuse of the 35B NVFP4 W4A4 MoE infra + a name-map; bf16 attn/dense + fp4 experts; N1-N5 bricks, DGX-gated). **N1-scaffold LANDED (2026-07-31):** additive `LagunaMoeWeights.experts_{gate,up,down}_fp4` + `shared_{gate,up,down}_fp4` (`Nvfp4Weight`, mirror qwen3_5), dead until the N1 loader; CPU build clean + `test_laguna_scaffold` 8/8·167 unchanged. **N1b loader IMPLEMENTED (2026-07-31, build-verified):** `LoadLagunaForCausalLMWeights` (`laguna_weights.cpp`) replaces the `VT_CHECK(false)` stub — resolver + per-layer `LoadBf16Direct` (attn/dense/norms/embed/lm_head/router/shared-expert) + F32 `e_score_correction_bias` + `LnLoadCtNvfp4Raw` W4A4 experts. Name-map + dtypes VERIFIED against the real `poolside/Laguna-S-2.1-NVFP4` index (router `mlp.gate` BF16, bias F32, experts W4A4, shared-expert BF16). **N1b RUN-VERIFIED (2026-07-31):** loader round-trips a synthetic NVFP4 checkpoint byte-identically (`test_laguna_nvfp4_loader` 2/2·29; full detail in the benchmark record). **N2 FORWARD-BRANCH LANDED + CPU-GATED (2026-07-31):** `LqGemmNvfp4Fp4` (per-expert TRUE-W4A4: `ScaledFp4Quant(input_global_scale_inv)`→`MatmulNvfp4Fp4(alpha)`, unified-memory pattern like `LqGemm`) + `LagunaFfnBlock` branches on `fp4=!experts_gate_fp4.empty()` (routed experts fp4; keep-quant grouped fast-path gated off `!fp4`; bf16 attn/dense/router/shared-expert/lm_head unchanged) + both `LagunaForwardGguf{,Cached}` guards relaxed to `has_gguf_weights||has_nvfp4_weights`. **CORRECTION:** routed experts are W4A4 ⇒ per-expert `MatmulNvfp4Fp4`, NOT the grouped W4A16 `MoeGroupedGemmNvfp4` (grouped W4A4 deferred to N5 speed). `test_laguna_nvfp4_loader` 3/3·61 (added a forward run-gate: fp4 MoE branch runs through the real `LagunaForwardGguf` → finite+deterministic logits + routed-experts-consumed); `test_laguna_scaffold` 8/8 unchanged (GGUF byte-identical). **N3 DRIVER LANDED + CPU-SMOKE-VERIFIED (2026-07-31):** `examples/laguna_gen` auto-detects a safetensors DIRECTORY (→ NVFP4: `LoadHfConfig(config.json)` + `LoadLagunaForCausalLMWeights` + `LagunaForwardGguf{,Cached}`) vs a `.gguf` FILE (→ keep-quant), sharing the greedy loop; `--token-ids` bypass the tokenizer for the id-vs-golden gate. Verified on a synthetic NVFP4 dir with a REAL config.json (exercises the `LoadHfConfig`→`ParseLagunaParams` seam the loader test bypassed) → `has_nvfp4=1`, KV-cache decode runs finite. **N4 RAN on GB10 (2026-08-01) — the arm works end-to-end; correctness coherent+near-tie, speed 120× off.** git-archived `84fab587` → clean CUDA build (`121a`) → `laguna-gen --gpu` on the real 67 GiB `ckpt` with vLLM's exact prompt ids injected (`2,785,9626,377,15360,395`, captured via the HF tokenizer). Two GB10 memory fixes landed to run: release the mmap'd shards after the loader's memcpy-copy (114→67 GiB RSS), and create the CUDA context BEFORE the load (the 67 GiB reclaimable page cache otherwise starves `cudaStreamCreate`). **Correctness:** ours `22345 83 350 71070 395 340 9626 372 1703 …` vs golden `22345 83 290 350 674 330 5541 966 340 9626 377 15360 …` — **first 2 tokens match vLLM exactly**, then near-tie divergence; coherent ("France is" = 9626/377/15360; shares golden vocab). EXPECTED: our TRUE-W4A4 (fp4 activations) vs the MARLIN golden's W4A16 (bf16 activations) — different precision, not a bug. **Speed: 6.34 s/tok (0.16 tok/s), prefill 17.3s — ~120× slower than vLLM 18.8.** ROOT CAUSE (source-confirmed): `LqGemmNvfp4Fp4` uses the generic `vt::MatmulNvfp4Fp4` = the hand-written EMULATION CUDA kernel, NOT the cutlass sm120a fp4 tensor-core path the 27B/35B W4A4 use (`MatmulNvfp4Fp4DirectD`); + per-expert loop + per-GEMM host sync + no device residency. **nsys (2026-08-01) trace-confirmed + refined:** only 2 GPU kernels — `MatmulNvfp4Fp4Naive` = 99.3% of GPU time + fp4-quant 0.7%; GPU busy only ~18% of wall. NO bf16 GEMM on the GPU ⇒ `LqGemm`'s bf16 branch runs the host `MatmulNK` reference on the CUDA queue (attention/dense/router/shared/lm_head are CPU-bound, ~4.8 s/tok) — a second lever the source scan missed. **N5 LEVER #2 LANDED (2026-08-01) — 16× decode.** Routed the bf16 tower (attention/dense/router/shared/lm_head) off the host `MatmulNK` onto the GPU (`LqGemm` bf16 branch: `vt::CastBf16` the small activation + `vt::MatmulBT` bf16×bf16→f32, weight stays bf16 — no per-token `ReadF32` of `lm_head [100352,H]`): **decode 6.34 → 0.39 s/tok (16.3×; 0.16 → 2.56 tok/s), prefill 17.3 → 2.24s**; coherence preserved (near-tie). CPU path unchanged (run-gate byte-identical). **N5 LEVER #1 LANDED (2026-08-01) — native fp4 tensor-core, another ~2×.** The engine's native sm120a fp4 tensor-core MMA (`MatmulNvfp4Fp4Native`, `mma.sync kind::mxf4nvf4`) reads the same linear scale layout `LqGemmNvfp4Fp4` produces — it was gated OFF behind `VT_NVFP4_FP4_NATIVE`; the Laguna driver now defaults it ON (scoped; 27B/35B untouched). **decode 0.39 → ~0.20-0.24 s/tok (~2×; ~4.2-5.0 tok/s)**; coherent (byte-identical ids to the emulation path — numerically equivalent), first token matches the golden. **Cumulative N5: 0.16 → ~4.5 tok/s (~28×), now ~4× from vLLM 18.8.** **Device-resident MoE block LANDED + MEASURED (2026-08-01, `LagunaMoeResidentFp4`, `VT_LAGUNA_RESIDENT_MOE` default-ON):** the whole token's routed experts as ONE async device chain (fp4-quant→GEMM gate/up, `MoeSiluMul`, →down stacked, ONE `MoeCombine`), draining once vs ~Pk×3 syncs. **Speed EAGER-NEUTRAL (0.20 s/tok)** — empirically confirms the ds4 precedent (per-op syncs overlap GPU compute; wall is GPU-serial-bound; the graph is the payoff). **CORRECTNESS WIN: golden-token match 2 → 13** (the device `MoeSiluMul`/`MoeCombine` mirror vLLM's fused MoE faithfully). Lands default-ON (better correctness, no speed cost, graph prerequisite). **CORRECTED CEILING (from the measured state): a perfect decode graph caps at ~5.9 tok/s** (GPU already ~87% busy at 0.20 s/tok), still 3.3× short of vLLM 18.8 — the graph is necessary but NOT sufficient; the remaining 3.3× is KERNEL EFFICIENCY (native fp4 MMA ~302µs/M=1 expert GEMM vs vLLM's tuned cutlass sm120a fp4 + fused norm/quant/silu). Parity = TWO campaigns: (A) device-resident+graph → ~5.9; (B) cutlass DirectD experts + fused ops + M=1-tuned GEMV → the rest. **CAMPAIGN-B FIRST BRICK LANDED (2026-08-01): coalesced M=1 fp4 GEMV** (`MatmulNvfp4Fp4Gemv`, one warp/column, coalesced weight-row reads, `VT_NVFP4_FP4_GEMV` default-ON) — same-binary A/B: **decode 0.20 → 0.15 s/tok (1.33×; → ~6.7 tok/s), prefill 1.14 → 0.86s**, coherent+near-tie. **Cumulative this session: 0.16 → ~6.7 tok/s (~42×), now ~2.8× from vLLM 18.8.** (ILP variant `kCpw=4` measured SLOWER — 0.21 s/tok, occupancy loss > activation-reuse gain — reverted to `kCpw=1`; kernel kept templated as a re-measurable knob.) **ncu of the GEMV (sudo): sm__throughput 35-71%, DRAM n/a — COMPUTE/LATENCY-bound, not BW-bound.** Corrects the earlier "~6× BW → ~16-17 tok/s" estimate: the next GEMV lever is HARDWARE fp4 dequant (`cvt.e2m1x2`), not more bandwidth. Parity (18.8) is a multi-brick campaign (decode graph + fused norm/quant + hardware-dequant GEMV), not one more kernel. **B0 hw-fp8 SCALE-decode: MEASURED NEGATIVE, reverted (2026-08-01, `ab7a1c1e`).** Replacing the GEMV's per-byte software fp8-e4m3 group-scale decode (`F8E4M3ToF32Dev`/`ldexpf`) with hardware `cvt.rn.f16.e4m3` (`__nv_fp8_e4m3`→float) is bit-exact (ids byte-identical on the real ckpt) but paging-immune ncu shows it NEUTRAL-to-slightly-WORSE (grid768 41.2 vs 41.9µs tie; mean 53.6 vs 49.4µs) — GPU `ldexpf` is a cheap exponent-bit add, not a libcall. NOTE this is the fp8 SCALE decode, NOT the fp4-e2m1 WEIGHT dequant (the `kE2M1` `__constant__` LUT); the LUT→arithmetic/`cvt.e2m1x2` weight-dequant is a SEPARATE still-open lever (spec brick B1). Also: end-to-end wall-clock is unusable for kernel A/B here (67 GiB unified reload swings TPOT 0.16↔1.08 s/tok run-to-run) — kernel-duration ncu is the only honest anchor. **★ B2 SCOPED + DE-RISKED (2026-08-01, zero-DGX) — the real 18.8 lever:** vLLM's 18.8 bar is MARLIN W4A16 (`VLLM_TEST_FORCE_FP8_MARLIN=1`), which is LOW-M-optimized (decode-correct, unlike a tensor-core W4A4 GEMM that wastes M=1 tile rows). The engine already ships the EXACT kernel `vt::MoeGroupedGemmNvfp4Marlin` (1:1 lift of vLLM `moe_wna16_marlin_gemm`) + shared `MarlinRepackExpertWeight`, and qwen3_5 (27B/35B) already routes its NVFP4 experts through it (default-ON `VT_NVFP4_MARLIN`, 16/16-vs-oracle, +22% gate/+80% decode) via `BuildMoeMarlinResident`. So B2 = mirror that for `LagunaMoeWeights.experts_*_fp4` (a `BuildLagunaMoeMarlinResident` reusing the shared repack + route `LagunaFfnBlock`'s fp4 branch to the Marlin grouped GEMM, GEMV kept as the `=0` escape hatch) — pure reuse, no new kernel, matches vLLM's exact W4A16 numerics. **B2 IMPLEMENTED (2026-08-01, `3c49ef37`) — COMPILES CLEAN on GB10 sm_121a, runtime bug pending.** `LagunaMoeResidentMarlin` + `BuildLagunaMoeMarlinResident` (laguna.cpp, `#ifdef VT_MARLIN_NVFP4`) reconstruct the MoE Marlin path over the SHARED `dense_nvfp4::Dev`/`DBuf`/`ResidentNvfp4` + shared `vt::cuda` Marlin repack/align ops + `vt::MoeGroupedGemmNvfp4Marlin`; SACRED 27B/35B path BYTE-UNTOUCHED; gated `VT_LAGUNA_MARLIN_MOE=1` **default-OFF** (zero regression to the default GEMV path). Compiles clean on the full CUDA build. RUN: loads OK (48 layers, 256 experts) but the FIRST FORWARD device-faults silently on the Marlin path — a layout/param bug (suspects: `MoeCombine` bf16-in/f32-out dtype, the down-GEMM reusing the gate/up align, or the fp4-original free omitted → mem ~doubles). NEXT: `compute-sanitizer` localize → fix → near-tie vs the vLLM-Marlin golden + kernel-duration ncu → flip default-ON. Default path unaffected. **UPDATE (`22d6e146`): added the qwen3_5-style fp4-original free after repack** (device transients + host bytes; peak was ~3× the expert tower → past the 119 GiB pool → null-alloc → silent fault the likely cause); compiles clean. The runtime gate stayed INCONCLUSIVE this session (contended/orphaned processes on the shared box, no captured ids) — rerun on a clean uncontended session, compute-sanitizer if it still faults. **★★ B2 VALIDATED on GB10 (2026-08-01, with the mem-free fix): RUN_EXIT=0, coherent, first 13 generated tokens MATCH the vLLM-Marlin golden EXACTLY** (`22345 83 290 350 674 330 5541 966 340 9626 377 15360 81` — the best Laguna-NVFP4 correctness yet, W4A16 matching vLLM's config). **Steady-state decode 0.10 s/tok = ~10 tok/s** (steps 10-17 all 0.10; the TPOT-0.56 average is warmup-polluted — the DevicePool warms over ~9 decode steps then reuses). vs the GEMV path's 6.7 tok/s = **~1.5× faster; the gap to vLLM 18.8 closes from ~3× to ~1.9×.** Memory flat (7.9 GiB host RSS — the fp4-original free worked; it also fixed the first-forward fault). Still `VT_LAGUNA_MARLIN_MOE=1` default-OFF. TO DONE: move the lazy Marlin-resident build (216s first-forward, 48L×256E repack) to model-LOAD time → clean warm A/B + ncu → flip default-ON → matrix/roadmap. Remaining ~1.9×: vLLM graphs its decode (ours still eager) — decode CUDA-graph is the next lever. **REPRODUCED 3× (reproduction gate MET): GB10 runs deterministic — first 18-20 tokens byte-identical, steady-state 0.10 s/tok confirmed each — so the ~10 tok/s + golden-match is gated, not a single sample.** **#234 item (1) — load-time resident-build LANDED (`LagunaBuildMarlinResidents`, called from the example after load; mirrors vLLM process_weights_after_loading): builds all 48L×256E Marlin residents at LOAD so the repack is not a first-token TTFT spike. Fixed an anon-namespace linkage bug (public fn was defined with internal linkage → moved outside the anon namespace); BUILD CLEAN + links on GB10 sm_121a, default-OFF. Runtime prewarm-fires-at-load timing UNVERIFIED this session (repeated ssh-drops ate the run capture); the forward's lazy build is the validated fallback so it cannot regress. Owed: one clean run to confirm the build moved to load + then flip default-ON.** **★★ DONE (2026-08-01): Marlin is now the UNCONDITIONAL DEFAULT (`LagunaMarlinMoeEnabled` default-ON; `=0` is a code-level A/B opt-out no user needs) — "it just works" with NO env. Confirmed in a no-env GB10 run captured via tmux: `MARLIN residents built at load in 238.4s`, prefill 14.78s (build moved OUT of first-forward), golden-matching ids, steady-state 0.10 s/tok = ~10 tok/s (4th reproduction), RSS ~5-8 GiB. So a default Laguna-NVFP4 load on GB10 gets vLLM's own W4A16 Marlin decode (~10 tok/s, ~1.9× from vLLM 18.8) with zero flags. The 238s load-time repack is a one-time cost (mirrors vLLM process_weights_after_loading); optimizing its 48×256 per-expert sync count is a follow-up. Residual to 18.8 = decode CUDA-graph (deferred; user refocusing on DeepSeek next).** Post-lever-1 nsys: the remaining ~4× is HOST-SYNC-bound — 22,115 `cudaStreamSynchronize` (78.6% of API time, ~2,760/token, the per-GEMM `DrainQueue`), GPU kernels fast. Remaining levers: grouped W4A4 MoE (design input: `vt::MoeGroupedGemmNvfp4` is W4A16, so true-W4A4 grouped needs a new fp4×fp4 op or the `use_a16` mode + expert-stacking — needs a spike), device-resident decode (RECOMMENDED — the current forward is host-style so every GEMM drains; keep activations on-device, drain once/step; reuse qwen3_5's `Dev`/`Nvfp4Dev`/`ResidentNvfp4`/device-SwiGLU machinery; kills the 22k syncs; converges with the pending GGUF #228 and lifts both quant paths), decode CUDA-graph. Binding number needs a clean 2-3× re-run. See `docs/BENCHMARKS.md` + the spec N5 plan. See `docs/BENCHMARKS.md` `CLAIM-LAGUNA-VLLM-NVFP4`. Prior W7 nsys attribution: host-orchestration-bound, levers ranked (spec `laguna-s21-w7-speed-2026-07-31.md`, ledger `CLAIM-LAGUNA-W7-SPEED`). Prior RUNNABLE + FAST DECODE (W6, 2026-07-31): a per-layer K/V cache + single-token incremental decode replaces W5's O(n²) STATELESS full-recompute — TOKEN-IDENTICAL (byte-equal ids, md5 match, == the W5 golden) and 5.05× faster per token: decode 3.33 → 0.66 s/tok on the real 3-shard UD-Q4_K_XL GGUF (GB10, `--gpu`, keep-quant), same "The capital of France is" → " Paris.\n\nThe user is seeking a detailed explanation of the concept of \"cultural capital\"…". `LagunaKvCache` (mirrors `DeepseekV4KvCache`, MLA-latent → GQA multi-head K/V) caches post-QK-RMSNorm/post-RoPE K + raw V at f32 (bit-exact by construction: RoPE/QK-norm are position-only and attention is causal). MIXED attention handled per-layer: 12 GLOBAL layers grow the cache unbounded (full causal); 36 SLIDING-WINDOW-512 layers EVICT the oldest rows beyond the 512 window (gemma2/3 `is_sliding`), capping their K/V. `LagunaForwardGgufCached` + shared `LagunaAttention`/`LagunaFfnBlock` helpers used by BOTH forwards (identical float ops — the recompute path's ids are unchanged after the refactor); `examples/laguna_gen --stateless` forces the W5 recompute for the A/B gate. No cache bug: bit-exact on the first run. Next speed: grouped-expert GEMM + device-resident decode (both in-tree from ds4). See `.agents/specs/laguna-s21-w6-2026-07-31.md`. Prior RUNNABLE (W5, 2026-07-31): our engine greedy-generates COHERENT text on the REAL 3-shard UD-Q4_K_XL GGUF (GB10, keep-quant). `laguna-gen` "The capital of France is" → " Paris.\n\nThe user is seeking a detailed explanation of the concept of \"cultural capital\" as developed by French soci…" — the FIRST token is "Paris.", matching the llama.cpp-Poolside reference on the identical bytes. Multi-shard GGUF reader (LagunaGgufCtx routes each of 814 tensors to its shard; shard-1 = header only) + keep-quant tower (attn/dense/shared/experts/lm_head stay Q8_0/Q4_K/Q5_K COMPRESSED, consumed via `vt::MatmulBT`; norms/router/bias/embed → f32) + `LagunaForwardGguf` (the f32 composition with the ~9 GEMM sites swapped to keep-quant Gemm/GemmRowSlice, ds4 precedent) + `examples/laguna_gen`. Real GGUF metadata verified: dual-RoPE freq_base 500000/10000, dims 64/128, YaRN factor 32, sigmoid ungrouped-noaux router (scale 2.5), per-layer Q-head [48 global/72 sliding], per-head softplus out-gate, QK-RMSNorm. Load 20.6s, peak 71 GiB (fits 119 pool). Prior W4 IN PROGRESS (2026-07-31): 73.4 GiB UD-Q4_K_XL GGUF FETCHED + read authoritatively (814 tensors); 3 CPU-verified fidelity corrections grounded in the real GGUF + llama.cpp — per-head QK-RMSNorm (`attn_q/k_norm`, the scope MISSED it), GGUF-authoritative dual-RoPE mscale (llama.cpp `yarn_attn_factor·(1+0.1·ln(factor))`, factor 32 not HF 128), separate `ffn_gate/up_exps`. Keep-quant tower materialization + `ForwardGguf` + the real-model greedy run vs llama.cpp-laguna same-quant oracle = W5 close. Prior: W3 REAL host-reference forward + 3 new ops (`laguna_ops.cpp`, CPU `-Werror` clean, `test_laguna_scaffold` unit-gated)** | Poolside Laguna: 48 layers (12 global + 36 sliding-window-512), 256 routed top-10 + 1 shared expert, per-head **softplus attention output gate**, sigmoid `noaux_tc` router, dual per-layer RoPE (YaRN full-attn / plain sliding), GQA 8 KV / 128 head-dim, 1M ctx. **W3 (2026-07-31):** the 3 genuinely-NEW small host ops landed in `laguna_ops.cpp` — per-head softplus attn out-gate (`LagunaSoftplusHeadGate`), ungrouped sigmoid-noaux router (`LagunaUngroupedRouterTopK`, ds3 noaux_tc MINUS the group step + tie-break razor), dual per-layer RoPE cos/sin builders (`BuildLaguna{FullYarn,Sliding}CosSin`, reusing the pinned YaRN inv_freq over the partial-64 dims); `LagunaModel::Forward` is now a REAL runnable host-reference composition (variable-Q-head GQA + dual RoPE + sliding-window mask + softplus gate + dense L0 / ungrouped-MoE L1..47 + untied lm_head) replacing the `VT_CHECK(false)` stub; `test_laguna_scaffold` **8/8·166** (softplus math, router selection+tie-break RED-first, dual-RoPE bit-match, variable-Q-head shapes, forward composition on synthetic weights), `test_model_registry` 24/24. **W2 (2026-07-30):** registered, `ParseLagunaParams`, GGUF `blk.N.*` name-map + UD-Q4_K_XL quant-mix (Q4_K/Q5_K/Q6_K/Q8_0 ALL already decoded → ZERO new kernel). **W1 oracle DECISION:** vLLM NATIVE `laguna.py` (in pin → config constructs); dual-oracle = vLLM-NVFP4/-FP8 (fits GB10 119 GiB; BF16 235 GiB does NOT) + llama.cpp-Q4_K token-exact. ~85–90% reuse (ds4-MoE + Gemma-sliding + OLMo-3-dual-rope + Q4_K keep-quant, ALREADY landed). DEFERRED (W4): GGUF keep-quant tower materialization + device/paged production forward (loaders still LOUDLY throw) + strict dual-oracle greedy gate on a fetched checkpoint + `poolside_v1` parser. See `.agents/specs/laguna-s21-w3-2026-07-31.md` (+ W1/W2 `laguna-s21-w1w2-2026-07-30.md`, W0 `laguna-s21-scope-2026-07-30.md`). **Decode attention-glue fusion LANDED (2026-08-02, `CLAIM-LAGUNA-GLUE-FUSED`, default-ON `VT_LAGUNA_GLUE_FUSED`, `=0` A/B):** BYTE-EXACT L1 (softplus out-gate → `DecodeAttnCombineKernel` store) + L4 (residual-Add+RMSNorm pairs → the shared `vt::FusedChain(kFusedAddRmsNormStd)` seam) on the resident decode-graph — same-binary A/B ids byte-identical (159/159 @160), paging-immune nsys steady decode **−4.2% GPU-busy (28.90→27.69 ms/step), −120 graph nodes/step (−10%)**, wall drop_caches-tied (no regression). C shared-into-MoeCombine SKIPPED (Laguna's bf16 `MoeCombine` → not byte-exact); L2 qk-norm+RoPE preamble DEFERRED (needs a device-position kernel variant). See BENCHMARKS.md `CLAIM-LAGUNA-GLUE-FUSED`. **On-device greedy sample LANDED (2026-08-02, `CLAIM-LAGUNA-ONDEV-SAMPLE`, default-ON `VT_LAGUNA_ONDEV_SAMPLE`, `=0` A/B):** the resident decode graph used to Synchronize, return the whole `[100352]` logits, and argmax on the HOST between replays (+ host embed-gather of the next token) — the off-framework "born-on-host" seam the decode-framework-routing audit flagged. Now BOTH run ON-DEVICE inside the captured graph: `vt::GreedyArgmax` (lowest-index tie = the exact host winner) → 1-elem device token buffer, + a new capture-safe `embed_gather` kernel gathers the next input embedding from it (the stock `vt::Embedding` is NOT capture-safe: per-call event-sync + D2H ring). BYTE-EXACT (160-id stream identical `=0`/`=1` on `~/laguna-xs-nvfp4`) + faster: paired drop_caches decode wall **+0.28% median** (8/8 reps ≥0; removes ~150 us/step host argmax) at GPU-busy parity (nsys 2-length 27.44→27.42 ms/step). Aligns Laguna decode with vLLM on-device sampling. **Lever 2 (lm_head GEMV DRAM eff) MEASURED, NOT landed:** `[M=1,100352,2048]` bf16 = **170 GB/s (2.41 ms)** = ~91% of the cuBLAS M=1×large-N reference (~187 GB/s / 2.2 ms) — at the M=1 practical floor (the 273 GB/s ceiling is streaming-only, unreachable for a once-read GEMV); ≤0.7%-of-step headroom needs a reduction reorder (near-tie re-gate) ⇒ not chased, per prior "lm_head optimal". See BENCHMARKS.md `CLAIM-LAGUNA-ONDEV-SAMPLE`. **MoE add_rms_norm fold LANDED (2026-08-02, `CLAIM-LAGUNA-MOE-ADDNORM`, default-ON `VT_LAGUNA_MOE_ADDNORM_FUSED`, `=0` A/B):** the glue-fused MoE tail ran its residual update as TWO graph nodes — `vt::Add(hidden,routed)` [`AddKernel`] + `FusedChain(kFusedAddRmsNormStd)` [shared-add+RMSNorm, `RmsNormRowKernel`] — now ONE `fused_add2_rmsnorm` device node/MoE-layer (`hidden=(hidden+routed)+shared; hn=rms_norm(hidden)*w`). BYTE-EXACT (IEEE add commutes + the identical 256-thread shared-tree norm reduction; 160-id stream byte-identical `=0`/`=1` on `~/laguna-xs-nvfp4`) + faster: **−39 `AddKernel` graph nodes/step** (2.63ms→0 over 69 steps), paging-immune nsys 2-length **~−46 us/tok GPU (27339→27293)**, nsys wall **+0.4% (34.00→34.14 tok/s @70-tok)**. Small (byte-exact node-count trim on the graph-captured, GPU-bound decode; the dominant ~72% cost is the bf16 projection GEMVs — see the Lever-B negative in BENCHMARKS.md). See BENCHMARKS.md `CLAIM-LAGUNA-MOE-ADDNORM`. **Shared expert kept fp4 LANDED (2026-08-03, `CLAIM-LAGUNA-SHARED-FP4`, default-ON `VT_LAGUNA_SHARED_FP4`, `=0` A/B):** the XS-NVFP4 shared expert was DEQUANTIZED to bf16 at load (`LnLoadSharedExpertBf16`) → the M=1 decode GEMV read 4× the DRAM bytes of vLLM (which keeps it fp4). Now kept fp4-resident and routed through the SAME Marlin W4A16 single-expert (num_experts=1) grouped GEMM the routed experts win on (`dense_nvfp4::GateUpFusedMarlinD`+`MatmulNvfp4MarlinD`); the decode GEMV drops to router-ONLY (`moe.router`), shared gate/up/down go fp4. ADDITIVE new `laguna_shared_fp4.cpp` re-reads the on-disk fp4 from the gen driver before shard release (does NOT touch SACRED `laguna_weights.cpp`); bf16 shared KEPT for the T>1 prefill. NEAR-TIE (fp4≠bf16): coherent, first-20 ids == documented golden, byte-identical to bf16 for ~85 tokens then diverges; **DISTRIBUTIONAL GATE PASS 40/40** (ours' first-40 ids ∈ vLLM's 8-run greedy candidate set; vLLM XS-greedy is bf16-non-det, 8 unique of 8). FASTER: paging-immune nsys 2-length **GPU 27.24→26.53 ms/step (−2.6%)**, wall drop_caches **35.8→36.3 tok/s (+1.4%, fp4 wins all 3 reps)**; shared-expert kernel bucket ~1.68→~0.90 ms/step (halved); vs vLLM ~43 tok/s 83.3%→84.4%; RSS 22.2→22.1 GiB (freed the decode-only fused router-shared projection). Modest by design — XS's shared expert is small (`shared_expert_intermediate_size==moe_intermediate_size==512`). Default-ON per parity (matches vLLM's fp4 shared). See BENCHMARKS.md `CLAIM-LAGUNA-SHARED-FP4`. **qk-norm+RoPE preamble fusion LANDED (2026-08-03, `CLAIM-LAGUNA-PREAMBLE-FUSED`, default-ON `VT_LAGUNA_PREAMBLE_FUSED`, `=0` A/B):** closes the `CLAIM-LAGUNA-GLUE-FUSED` L2 deferral — the decode graph ran the per-layer attention preamble as FOUR under-occupied M=1 nodes (`rms_norm_seq(q)`+`rms_norm_seq(k)`+`rope_from_cache_g(q)`+`rope_from_cache_g(k)`); now ONE capture-safe `fused_qk_norm_rope_g` node/layer (`FusedQkNormRopeGKernel`, one block/head, reads the decode position from DEVICE `*pos_buf`, handles the per-layer dual-RoPE 64/128 + `Hq` 48/64). BYTE-EXACT BY CONSTRUCTION: it replicates the composed path's f32 MEMORY round-trip (Phase A 256-thread Σx² == `RmsNormSeqKernel`; Phase B the same `(x*inv)*w` store; `__syncthreads`; Phase C the `RopeFromCacheGKernel` rope read back) — an earlier register-only recompute was numerically-equivalent but diverged at a token-110 near-tie via compiler fma-contraction; the memory boundary forces bit-identity. 160-id stream byte-identical `=0`/`=1` on `~/laguna-xs-nvfp4` (determinism verified `=0`×3/`=1`×3 each run-to-run identical). FASTER: preamble norm+rope kernels **160→40 launches/tok, 326→154 us/tok (−0.17 ms/step)**; all decode-scaling kernels 26.53→26.37 ms/step; wall drop_caches **36.42→36.64 tok/s (+0.6%, fused wins all 3 paired reps)**; vs vLLM ~43 84.7%→85.2%. Modest (preamble ~1.2% of the 26.5 ms/step decode; the dominant cost stays the bf16 projection GEMVs at cuBLAS parity) — a byte-exact graph-node/launch trim (the glue-fusion residual mechanism). Default-ON per parity. See BENCHMARKS.md `CLAIM-LAGUNA-PREAMBLE-FUSED`. **W7 two-front pass LANDED (2026-08-03, `CLAIM-LAGUNA-W7-DECODE`):** FRONT 1 — the example driver logged `[gen] step N …(RSS)` EVERY decode step, and the RSS arg calls `CurResidentGiB()` (a `/proc/self/status` read) + an unbuffered stderr write in the GPU-idle gap between replays; guarded behind `VT_LAGUNA_STEP_LOG` (default OFF) + added a `decode_wall` line (TRUE end-to-end throughput incl. per-step gaps) next to the gap-free `decode_hp`. Since the fprintf sat OUTSIDE the `s0→s1` timer, `decode_hp` was ALREADY honest; with the log off `decode_wall == decode_hp` (within 0.001 tok/s, every LOG_OFF rep) and the recovered host tax is only ~0.1% (drop_caches noise floor). CONCLUSION: the ~86% gap to vLLM 43 is genuine device compute, NOT a harness artifact. FRONT 2 — `VT_LAGUNA_MOE_ONECAST` (default ON): a MoE layer cast the same `hn[1,H]` f32→bf16 THREE times (router GEMV + routed Marlin + shared Marlin); now cast ONCE into a persistent buffer and reuse (`CastHnBf16`/`GemmBf16Pre` + optional pre-cast param on both `…Into` helpers). BYTE-EXACT (deterministic truncation; `=1` vs `=0` byte-identical 300-tok ids); `CastBf16` **200→122 nodes/step (−78 = 2×39 MoE layers)**, GPU-busy parity within nsys noise, decode_hp +0.29%. Combined (onecast on + log off) **36.97 tok/s = 86.0% of vLLM-NVFP4 43** (from 36.64/85.2%). See BENCHMARKS.md `CLAIM-LAGUNA-W7-DECODE`. **Tail-fold follow-up LANDED (2026-08-03, `CLAIM-LAGUNA-TAIL-FUSED`, default-ON `VT_LAGUNA_TAIL_FUSED`, `=0` A/B):** a fresh node-ranking of the baseline decode graph found the routed-MoE `CastF32` as the one clean byte-exact fold left; it folds into the trailing `fused_add2_rmsnorm` via a new bf16-x1 sibling kernel (`AddAdd2RmsNormStdBf16Kernel` — `MoeCombine` writes bf16 straight to a persistent buffer, widened in-kernel by `__bfloat162float`). BYTE-EXACT (`=1` vs `=0` byte-identical 160-tok ids), `CastF32` **78→39 nodes/step**, total graph nodes **919→880**, GPU-busy parity; decode_hp a WASH (median +0.14% / mean −0.04%, at the drop_caches noise floor). Lands on the deterministic node-count basis (like onecast/preamble/addnorm), NOT a wall win; combined headline UNCHANGED **36.97 tok/s = 86.0%**. The ranking confirms the byte-exact decode-tail fold tier is now essentially EXHAUSTED (residual tail = already-folded norms + attention compute + cuBLAS-adjacent router/topk + ported-Marlin `MoeAlign`/`SiluAndMul`/`MoeCombine`); the gap to vLLM 43 is genuine device compute at the practical ceiling. See BENCHMARKS.md `CLAIM-LAGUNA-TAIL-FUSED`. **KERNEL-EFFICIENCY tier (2026-08-03, `VT_LAGUNA_FAST_NORM` default ON + f32 ext of `VT_RMSNORM_DECODE_FAST`):** the fold tier was exhausted but the residual-stream norm KERNELS were still under-occupied — `ncu` on the shipped `<<<1,256>>>` `AddAdd2RmsNormStdBf16`/`RmsNormRow` decode norms: `launch__waves_per_multiprocessor≈0.00`, `sm__throughput≈0.06%` (one 256-thread block on 1 SM of ~100+, latency-bound). Porting the PROVEN bit-identical `RmsNormRowFastKernel` structure (1024-thread float4 memory passes; 256-strided-partial + tree reduction reproduced byte-for-byte) to the f32 kernels cut each **286→~155 µs/tok (1.85×)**, **byte-exact** (160-tok ids identical `=1`vs`=0`; the f32 fix vs the bf16 sibling: store `v` not `v²` and square in the reduction so nvcc emits shipped's `acc += v*v` **fma** — a pre-squared f32 `v²` is not exact and flipped an XS near-tie at tok 108). **−0.81% decode-step GPU time** (paging-immune 70-vs-20 2-length diff, 26192→25980 µs/step); wall-clock ON/OFF overlap (noise floor). Residual: the byte-exact 256-strided reduction can't reach vLLM's per-kernel norm floor (~2.4× vLLM) without breaking byte-exactness → that remainder is byte-exactness-BLOCKED. See BENCHMARKS.md `CLAIM-LAGUNA-FAST-NORM`. **Router top-k warp-shuffle LANDED (2026-08-03, `CLAIM-LAGUNA-TOPK-SHFL`, default-ON `VT_LAGUNA_TOPK_SHFL`, `=0` A/B): BYTE-EXACT** — an nsys 2-length rank of the remaining small kernels (past the at-parity `gemvx` projection GEMVs ~69% of step + Marlin MoE) put the router `SigmoidTopKKernel` top (415 µs/step); `ncu` showed it `<<<1,256>>>` at `waves≈0.000`/`sm≈0.2%` — pure latency (8 serially-dependent rounds × a ~10-sync `sh[256]` argmax tree). New `SigmoidTopKShflKernel` reduces each round by warp-shuffle argmax (2 syncs/round; argmax over the total order is associative ⇒ SAME winner) → **`SigmoidTopK` 414.6→248.8 µs/step (1.67×)**, decode-step GPU **−0.57%** (26.018→25.869 ms/step), 37.39→37.49 tok/s decode_hp (**87.2% of vLLM-NVFP4 43**); 160-id stream byte-identical `=1`vs`=0`. **NOT landed — norm warp-shuffle (`VT_LAGUNA_NORM_SHFL`):** a near-tie register-accumulate+shuffle reduce for the Laguna `AddAdd2RmsNormStd{,Bf16}Fast` norms PASSED the distributional gate (coherent, in-set 38/40 = baseline, one near-tie fork at pos 37) and was −19.3% per-kernel (`AddAdd2RmsNormStdBf16` 150.3→121.3 µs/step) BUT washed at whole-step (0.6% of step; +0.02% within noise) — a near-tie fork isn't justified by a below-noise gain, so it was dropped. The small-kernel norm tail is at its occupancy floor; the decode step is dominated by the at-parity projection GEMVs. See BENCHMARKS.md `CLAIM-LAGUNA-TOPK-SHFL`. **Shared-expert 2-stream overlap LANDED (2026-08-03, `CLAIM-LAGUNA-SHARED-AUX`, default-ON `VT_LAGUNA_SHARED_AUX`, `=0` A/B):** mirror of vLLM's `MULTI_STREAM_OVERLAPPED` — in `LagunaGraph::RunChain` the fp4-shared arm's shared expert is EARLY-forked onto a second CUDA stream from the post-attn hidden `hn` BEFORE the router GEMV (aux reads `hn` f32 + does its own byte-identical cast; scratch from `AuxPool`), overlapping router+`sigmoid_topk`+routed grouped GEMM, joined before the combine — the SAME machinery the 35B ships default-ON (ENG-MOE-SHARED-AUX, runs inside the captured graph). This is the EARLY fork the prior fused-`router_shared_gu` attempt (`89e0d074`, −0.35% wash) could not reach. Capture-safe (aux stream+2 events in the ctor; gstate-0 warm-run builds residents + warms `AuxPool`). **BYTE-EXACT** (`=1`vs`=0` byte-identical 63-tok ids). REAL concurrency: nsys `--cuda-graph-trace=node` 20↔70 sum-vs-union → OVERLAP **2.34 ms/step** (SUM/UNION 1.092) vs `=0`'s 0.0004 ms; net GPU-busy wall **26.213→25.467 ms/step (−2.9%, 38.15→39.27 tok/s)**, wall @200 37.08→37.93 (+2.3%). Net9.** t2va+fl2va COHERENT; bf16 shards STREAM | ref2va = NVFP4 ckpt fidelity §8.12; encoder A/B §8.15; GB10 re-verify via the v12 ABI = named residual; embedder-unreachable caps now 2 | | Command-R / Cohere dense (`CohereForCausalLM`) | Implemented, gate-blocked | ZERO-new-kernel port grounded in vLLM `commandr.py`: weight-only Cohere LayerNorm + GPT-J full-width RoPE + PARALLEL residual + `logit_scale` + tied embeddings, all reuse; compiles, links, self-registers. No SACRED gate yet (real checkpoints HF-gated, ungated ones tiny-random, GPU box disk-full); oracle run-verified at W0. See docs/BENCHMARKS.md | | Phi-1 / Phi-2 dense (`PhiForCausalLM`, parallel residual) | Correctness-complete, speed-pending | Token-exact 16/16 (microsoft/phi-2): 9/16 strict + 7/16 bf16 near-ties (max gap 0.25 nats), 0 forward-divergent; the OLDER Microsoft Phi arch, DISTINCT from Phi-3/Phi-4; ZERO new compute kernel (GPT-J parallel residual, LayerNorm-with-bias, biased qkv/dense, partial NeoX rope 32/80, non-gated NewGELU MLP reusing `vt::GeluTanh`, untied biased lm_head); F16 dtype-aware loader | | MiniCPM dense (`MiniCPMForCausalLM`, three scalars) | Correctness-complete, speed-pending | Token-exact 16/16 (openbmb/MiniCPM-2B-sft-bf16): 10/16 strict + 6/16 bf16 near-ties (max gap 0.0 nats), 0 forward-divergent; first OpenBMB MiniCPM model; ZERO new compute kernel (the Llama/Granite dense forward plus three scalars: scale_emb, scale_depth/sqrt(layers) residual, dim_model_base logit scaling), tied lm_head; `.bin`-only weights converted to safetensors via trusted torch | diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index f6d7e14dc..5de59e4ac 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -64,17 +64,22 @@ if(VLLM_CPP_SERVER) endif() # minimax-h3-mux: muxes MiniMax-H3 PPM frames + WAV into an MP4 by invoking -# ffmpeg. This is the RATIFIED home of that process spawn (developer decision -# 2026-08-03): the library builds the artifacts and the argv, examples/ runs it. +# ffmpeg — a THIN client of the public C ABI (include/vllm.h + vllm::shared +# ONLY, like vllm-cli), per the ONE SURFACE directive. This is the RATIFIED +# home of that process spawn (developer decision 2026-08-03): the library +# composes the argv (vllm_video_mux_argv), examples/ runs it. add_executable(minimax-h3-mux minimax_h3_mux/main.cpp) -target_link_libraries(minimax-h3-mux PRIVATE vllm::vllm) +target_link_libraries(minimax-h3-mux PRIVATE vllm::shared) vllm_cpp_set_warnings(minimax-h3-mux) -# minimax-h3-gen: the ASSEMBLY driver — opens the real checkpoints, runs the whole -# t2va path and writes an MP4. Lives here because it invokes ffmpeg (the ratified -# boundary) and because it is a driver, not library surface. +# minimax-h3-gen: MiniMax-H3 video generation end to end — a THIN client of +# the public C ABI (ARCH-ONE-SURFACE ROW 2). The assembly pipeline it used to +# own privately lives in the library behind vllm_video_engine_load + +# vllm_video_generate; the fold gate holds its frames + WAV byte-identical to +# the pre-fold binary. Lives here because it invokes ffmpeg (the ratified +# boundary). add_executable(minimax-h3-gen minimax_h3_gen/main.cpp) -target_link_libraries(minimax-h3-gen PRIVATE vllm::vllm) +target_link_libraries(minimax-h3-gen PRIVATE vllm::shared) vllm_cpp_set_warnings(minimax-h3-gen) # vulkan-gemm-ab: the VK-C tactic A/B — cooperative matrix vs the portable scalar diff --git a/examples/minimax_h3_gen/main.cpp b/examples/minimax_h3_gen/main.cpp index 3c9565214..f0703f870 100644 --- a/examples/minimax_h3_gen/main.cpp +++ b/examples/minimax_h3_gen/main.cpp @@ -1,1293 +1,203 @@ -// minimax-h3-gen: the ASSEMBLY driver — open the real checkpoints, run the whole -// t2va path, and write an MP4. +// minimax-h3-gen — MiniMax-H3 video+audio generation end to end, as a THIN +// CLIENT of the public C ABI (include/vllm.h) and NOTHING else, per the ONE +// SURFACE directive (ARCH-ONE-SURFACE ROW 2). // -// Everything below this line was gated component by component (packed layout, DiT -// forward, both VAE decoders, the encoder towers, the loaders). This is the piece -// that puts them together over REAL files, which is the only way the remaining -// integration questions — config plumbing, latent statistics, shard assembly — -// actually get answered. -// -// It lives in examples/ for the same reason the muxer does: this is where the -// ffmpeg invocation is allowed (developer-ratified 2026-08-03). src/vllm/ builds -// artifacts and argv and spawns nothing. +// The pre-fold version of this example owned the whole assembly pipeline +// privately (1293 lines: loader-arm dispatch, encoder conditioning, reference +// encoding, the noise streams, artifact writing, the ffmpeg mux). All of that +// now lives in the library behind vllm_video_engine_load + vllm_video_generate +// — the SAME entry points any embedder gets — and this file keeps exactly what +// an example may own: argv parsing, printing, and the ONE process spawn (the +// ffmpeg invocation, ratified into examples/ 2026-08-03; the library only +// composes the argv). The rendered frames + WAV are byte-identical to the +// pre-fold binary (gated by tests/vllm/models/test_minimax_h3_video_fold.cpp +// and the committed goldens in tests/vllm/models/fixtures/minimax_h3_video_fold). // // Usage: // minimax-h3-gen --dit -// # a DIRECTORY holding the original bf16 release's shards plus -// # model.safetensors.index.json is accepted wherever a single -// # DiT file is; every existing --dit form is unchanged. -// --video-vae --video-vae-config -// --audio-vae --audio-vae-config -// --prompt-embeds (rows of text_dim, little-endian f32) -// --out -// [--partition fl2va|ref2va] (REQUIRED for a full render: the -// served checkpoint partition — community GGUF/NVFP4 strip it and -// the FL2VA/Ref2VA DiTs are indistinguishable. t2va/fl2va need -// fl2va, ref2va needs ref2va; recipe:50-51,289) -// [--keep-quant] [--steps N] [--frames N] [--height N] [--width N] -// [--workdir DIR] [--ffmpeg PATH] [--dry-run] -// -// minimax-h3-gen --encoder-only -// --encoder -// # a DIRECTORY of the original bf16 encoder release's shards -// # plus model.safetensors.index.json is accepted wherever the -// # Q4_K_M GGUF is. -// --prompt --tokenizer -// --save-embeds [--encoder-max-layers N] +// --video-vae [--video-vae-config ] +// --audio-vae [--audio-vae-config ] +// --partition fl2va|ref2va (REQUIRED for a full render: the +// served checkpoint partition — community GGUF/NVFP4 files +// strip it and the FL2VA/Ref2VA DiTs are indistinguishable) +// (--prompt-embeds | --encoder +// --prompt [--tokenizer ]) +// --workdir DIR [--out ] [--ffmpeg PATH] +// [--steps N] [--frames N] [--height N] [--width N] [--seed N] +// [--device cpu|cuda] [--dequant-bf16 | --keep-quant] [--fp4-resident] +// [--first-frame f.ppm] [--last-frame f.ppm] [--noise-aug A] +// [--ref-image f.ppm] [--ref-video DIR] [--ref-audio f.wav] // -// PROMPT EMBEDDINGS are taken as a file rather than computed here, deliberately: -// the encoder tower needs a tokenizer + a 32B forward, which is its own driver. -// This keeps the assembly question ("do the checkpoints compose into a video?") -// separable from the encoding question. `--encoder-only` is the other side of -// that seam: run the tower alone, write the conditioning, exit — no DiT, no VAEs. -// On a 122 GiB UNIFIED pool that is what makes encoding the same prompt with two -// different encoders affordable, which is how "what does quantizing the encoder -// cost?" gets a number instead of an opinion. +// The pre-fold binary's DIAGNOSTIC modes (--dry-run, --denoise-only, +// --dump-params, --encoder-only/--save-embeds, --decode-latent, --roundtrip, +// --prompt-image, --cond-image, multiple --ref-image) were part of the private +// pipeline and are gone with it; the capabilities they probed are gated by +// test_minimax_h3 / test_minimax_h3_video_fold, and multi-image ref2va remains +// reachable through the C++ seam (a named residual of the ABI's first slice). #include #include -#include -#include -#include #include -#include #include #include #include -#include -#include -#include -#include -#include -#include #include #include -#include - -#include "vllm/model_executor/model_loader/gguf_reader.h" -#include "vllm/model_executor/model_loader/safetensors_reader.h" -#include "vllm/model_executor/models/minimax_h3.h" -#include "vllm/model_executor/models/qwen3_vl_text.h" // Qwen3VLGetRopeIndex + MmImageSpan -#include "vllm/multimodal/qwen3vl_processor.h" -#include "vllm/tokenizer/tokenizer.h" -#include "vt/backend.h" -#include "vt/dtype.h" +#include "vllm.h" namespace { +// The ONE process spawn, and it is in examples/ by decision: the library +// composed `args`; this runs it. int RunFfmpeg(const std::vector& args) { std::vector argv; argv.reserve(args.size() + 1); for (const std::string& a : args) argv.push_back(const_cast(a.c_str())); argv.push_back(nullptr); const pid_t pid = fork(); - if (pid < 0) throw std::runtime_error("fork failed"); + if (pid < 0) { + std::fprintf(stderr, "error: fork failed\n"); + return -1; + } if (pid == 0) { execvp(argv[0], argv.data()); _exit(127); } int status = 0; - if (waitpid(pid, &status, 0) < 0) throw std::runtime_error("waitpid failed"); + if (waitpid(pid, &status, 0) < 0) { + std::fprintf(stderr, "error: waitpid failed\n"); + return -1; + } if (WIFSIGNALED(status)) { - throw std::runtime_error("ffmpeg died on signal " + std::to_string(WTERMSIG(status))); + std::fprintf(stderr, "error: ffmpeg died on signal %d\n", WTERMSIG(status)); + return -1; } return WIFEXITED(status) ? WEXITSTATUS(status) : -1; } -nlohmann::json ReadJson(const std::string& path) { - std::ifstream in(path); - if (!in) throw std::runtime_error("cannot open " + path); - nlohmann::json j; - in >> j; - return j; -} - -// A binary PPM (P6) reader, so a reference frame can be handed in with no image -// dependency -- the same format this example already WRITES for its own output, -// so a frame from one run can condition the next. Returns [3, H, W] in [-1, 1], -// which is the range the VAE encoder expects. -std::vector ReadPpmAsChw(const std::string& path, int64_t* out_h, int64_t* out_w) { - std::ifstream in(path, std::ios::binary); - if (!in) throw std::runtime_error("cannot open " + path); - std::string magic; - in >> magic; - if (magic != "P6") throw std::runtime_error(path + ": not a binary PPM (P6)"); - auto next_int = [&]() { - int v = 0; - while (in >> std::ws, in.peek() == '#') { std::string skip; std::getline(in, skip); } - in >> v; - return v; - }; - const int w = next_int(), h = next_int(), maxv = next_int(); - if (w <= 0 || h <= 0 || maxv <= 0) throw std::runtime_error(path + ": bad PPM header"); - in.get(); // the single whitespace byte before the payload - std::vector rgb(static_cast(w) * h * 3); - in.read(reinterpret_cast(rgb.data()), static_cast(rgb.size())); - if (!in) throw std::runtime_error(path + ": truncated PPM payload"); - std::vector chw(rgb.size()); - const int64_t plane = static_cast(w) * h; - for (int64_t i = 0; i < plane; ++i) { - for (int64_t c = 0; c < 3; ++c) { - chw[static_cast(c * plane + i)] = - static_cast(rgb[static_cast(i * 3 + c)]) / (maxv * 0.5f) - 1.0f; - } - } - if (out_h != nullptr) *out_h = h; - if (out_w != nullptr) *out_w = w; - return chw; -} - -// A binary PPM (P6) reader that returns HWC uint8 [0,255] -- the layout the shared -// Qwen3-VL image processor expects (it does its own rescale + 0.5/0.5 normalize). -std::vector ReadPpmAsHwcU8(const std::string& path, int64_t* out_h, int64_t* out_w) { - std::ifstream in(path, std::ios::binary); - if (!in) throw std::runtime_error("cannot open " + path); - std::string magic; - in >> magic; - if (magic != "P6") throw std::runtime_error(path + ": not a binary PPM (P6)"); - auto next_int = [&]() { - int v = 0; - while (in >> std::ws, in.peek() == '#') { std::string skip; std::getline(in, skip); } - in >> v; - return v; - }; - const int w = next_int(), h = next_int(), maxv = next_int(); - if (w <= 0 || h <= 0 || maxv <= 0) throw std::runtime_error(path + ": bad PPM header"); - in.get(); // the single whitespace byte before the payload - std::vector rgb(static_cast(w) * h * 3); - in.read(reinterpret_cast(rgb.data()), static_cast(rgb.size())); - if (!in) throw std::runtime_error(path + ": truncated PPM payload"); - if (out_h != nullptr) *out_h = h; - if (out_w != nullptr) *out_w = w; - return rgb; -} - -// The WAV reader lives in the LIBRARY (MiniMaxH3ReadWav), next to the writer and -// unit-gated with it; this only opens the file. Returns CHANNEL-MAJOR samples in -// [-1, 1], mono repeated up to kMiniMaxH3AudioChannels, and REFUSES a sample rate -// the audio VAE was not trained at rather than silently mis-encoding it. -std::vector ReadWavRef(const std::string& path, int64_t* out_channels, - int64_t* out_samples_per_channel) { - std::ifstream in(path, std::ios::binary); - if (!in) throw std::runtime_error("cannot open " + path); - const std::string bytes((std::istreambuf_iterator(in)), std::istreambuf_iterator()); - if (out_channels != nullptr) *out_channels = vllm::kMiniMaxH3AudioChannels; - return vllm::MiniMaxH3ReadWav(bytes, vllm::kMiniMaxH3AudioChannels, - vllm::kMiniMaxH3AudioSampleRate, out_samples_per_channel); -} - -std::vector ReadF32(const std::string& path) { - std::ifstream in(path, std::ios::binary | std::ios::ate); - if (!in) throw std::runtime_error("cannot open " + path); - const std::streamsize bytes = in.tellg(); - if (bytes % static_cast(sizeof(float)) != 0) { - throw std::runtime_error(path + ": size is not a whole number of f32 values"); +const char* Need(int argc, char** argv, int i, const char* flag) { + if (i >= argc) { + std::fprintf(stderr, "error: missing value for %s\n", flag); + std::exit(2); } - in.seekg(0); - std::vector out(static_cast(bytes) / sizeof(float)); - in.read(reinterpret_cast(out.data()), bytes); - return out; -} - -void WriteFile(const std::string& path, const std::string& bytes) { - std::ofstream out(path, std::ios::binary); - if (!out) throw std::runtime_error("cannot write " + path); - out.write(bytes.data(), static_cast(bytes.size())); -} - -bool EndsWith(const std::string& s, const std::string& suffix) { - return s.size() >= suffix.size() && s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0; -} - -std::string Need(int argc, char** argv, int i, const std::string& flag) { - if (i >= argc) throw std::runtime_error("missing value for " + flag); return argv[i]; } -// Encode `prompt` with the H3 text encoder; returns the [seq, hidden] f32 -// conditioning the DiT consumes. -// -// `encoder_path` is EITHER a ComfyUI-format GGUF (the shipped Q4_K_M tower) or a -// DIRECTORY holding the ORIGINAL bf16 release's safetensors shards plus -// model.safetensors.index.json. Everything after the weight bytes is shared: same -// tokenizer, same text-only M-RoPE positions, same -// MiniMaxH3EncoderTextForwardDevice, same f32 activations. That is exactly what -// makes "how much does quantizing the encoder change the conditioning?" a -// measurable question rather than an opinion — run both, diff the output. -// -// TEXT-ONLY, on purpose. This backs `--encoder-only`, which exists to produce a -// conditioning tensor for that A/B and nothing else. The normal run path keeps its -// own inline encoder block because that one also carries the VISION path -// (`--cond-image`: merged features masked_scatter'd into inputs_embeds plus the 3 -// DeepStack taps), which this helper deliberately does not duplicate. -std::vector EncodeH3Prompt(const std::string& encoder_path, const std::string& prompt, - const std::string& tokenizer_path, int64_t encoder_max_layers, - const std::string& device_name, int64_t* out_seq, - int64_t* out_hidden) { - const bool sharded = vllm::MiniMaxH3ShardedCheckpoint::IsShardedDir(encoder_path); - std::cerr << "loading encoder " << encoder_path - << (sharded ? " (bf16 shards)" : " (keep-quant GGUF)") << "\n"; - - // The queue is created BEFORE the weights are read: on a unified-memory box the - // CUDA context must exist first, or the driver's reservation lands on top of a - // pool the weights already filled. - vt::Device enc_dev{}; - if (device_name == "cuda") { - enc_dev = vt::GetBackend(vt::DeviceType::kCUDA).CreateQueue().device; - } - vt::Queue eq{enc_dev, nullptr}; - vt::Backend& eb = vt::GetBackend(enc_dev.type); - if (enc_dev.type != vt::DeviceType::kCPU) eq = eb.CreateQueue(); - - vllm::MiniMaxH3EncoderConfig ec; - std::vector ids; - std::vector embeds; - vllm::MiniMaxH3EncoderDeviceWeights staged; - const auto t0 = std::chrono::steady_clock::now(); - - if (sharded) { - const vllm::MiniMaxH3ShardedCheckpoint ckpt = - vllm::MiniMaxH3ShardedCheckpoint::Open(encoder_path); - std::cerr << " shards=" << ckpt.ShardCount() << " tensors=" << ckpt.Names().size() << "\n"; - ec = vllm::MiniMaxH3EncoderConfigFromShards(ckpt, encoder_max_layers); - if (tokenizer_path.empty()) { - throw std::runtime_error("--tokenizer is required with a safetensors-shard --encoder"); - } - const vllm::tok::Tokenizer tokenizer = vllm::tok::Tokenizer::FromHfJson(tokenizer_path); - ids = tokenizer.Encode(prompt); - VT_CHECK(!ids.empty(), "minimax-h3-gen: the prompt tokenized to nothing"); - // Gathered straight out of the mmap'd shard: the table is [151936, hidden] and - // the prompt touches a few dozen rows, so nothing is materialized. - embeds = vllm::MiniMaxH3EncoderEmbedTokensFromShards(ckpt, ids); - staged = vllm::StreamMiniMaxH3EncoderShardsToDevice(eq, ckpt, encoder_max_layers, &ec); - const vllm::MiniMaxH3EncoderShardStreamStats st = vllm::GetMiniMaxH3EncoderShardStreamStats(); - std::cerr << " streamed bf16 encoder -> device: layers=" << st.layers_streamed - << " tensors=" << st.tensors_streamed << " direct=" << st.direct_uploads - << " converted=" << st.converted_uploads << " fused=" << st.fused_groups - << " uploaded=" << (st.bytes_uploaded / (1024.0 * 1024.0 * 1024.0)) - << " GiB host_peak=" << (st.host_peak_bytes / (1024.0 * 1024.0)) << " MiB\n"; - } else { - const vllm::GgufFile ef = vllm::GgufFile::Open(encoder_path); - const vllm::MiniMaxH3EncoderQuantWeights enc = - vllm::LoadMiniMaxH3EncoderFromGguf(ef, encoder_max_layers); - ec = enc.config; - size_t quant_bytes = 0; - for (const auto& kv : enc.quant_storage) quant_bytes += kv.second.size(); - std::cerr << " encoder resident (keep-quant) = " - << (quant_bytes / (1024.0 * 1024.0 * 1024.0)) << " GiB\n"; - // The ComfyUI-style encoder GGUF is WEIGHTS ONLY — it carries no - // `tokenizer.ggml.*` metadata, unlike a llama.cpp export — so the vocab comes - // from the checkpoint's own tokenizer.json. - const vllm::tok::Tokenizer tokenizer = tokenizer_path.empty() - ? vllm::tok::Tokenizer::FromGguf(ef) - : vllm::tok::Tokenizer::FromHfJson(tokenizer_path); - ids = tokenizer.Encode(prompt); - VT_CHECK(!ids.empty(), "minimax-h3-gen: the prompt tokenized to nothing"); - embeds = vllm::MiniMaxH3EncoderEmbedTokens(enc, ids); - staged = vllm::StageMiniMaxH3EncoderWeights(eq, enc); - } - - std::cerr << " encoder layers=" << ec.num_hidden_layers << " hidden=" << ec.hidden_size - << " heads=" << ec.num_attention_heads << " kv_heads=" << ec.num_key_value_heads - << " head_dim=" << ec.head_dim << " ffn=" << ec.intermediate_size << "\n"; - std::cerr << " prompt tokens = " << ids.size() << " (load " - << std::chrono::duration(std::chrono::steady_clock::now() - t0).count() - << " s)\n"; - - // Text-only: all three M-RoPE axes are the token index. - const int64_t seq = static_cast(ids.size()); - std::vector pos(static_cast(3 * seq)); - for (int64_t a = 0; a < 3; ++a) { - for (int64_t s = 0; s < seq; ++s) pos[static_cast(a * seq + s)] = s; - } - std::cerr << " encoding prompt...\n"; - std::vector conditioning = - vllm::MiniMaxH3EncoderTextForwardDevice(eq, ec, staged, embeds, pos.data(), seq); - std::cerr << " conditioning = [" << seq << ", " << ec.hidden_size << "]\n"; - if (out_seq != nullptr) *out_seq = seq; - if (out_hidden != nullptr) *out_hidden = ec.hidden_size; - return conditioning; -} - -void WriteEmbeds(const std::string& path, const std::vector& values) { - std::ofstream out(path, std::ios::binary); - if (!out) throw std::runtime_error("cannot write " + path); - out.write(reinterpret_cast(values.data()), - static_cast(values.size() * sizeof(float))); - std::cerr << " saved conditioning -> " << path << "\n"; +[[noreturn]] void Usage(int code) { + std::fprintf( + stderr, + "usage: minimax-h3-gen --dit --video-vae --audio-vae " + "--partition fl2va|ref2va\n" + " (--prompt-embeds | --encoder " + "--prompt [--tokenizer ])\n" + " --workdir DIR [--out ] [--ffmpeg PATH]\n" + " [--video-vae-config ] [--audio-vae-config ]\n" + " [--steps N] [--frames N] [--height N] [--width N] [--seed N]\n" + " [--device cpu|cuda] [--dequant-bf16 | --keep-quant] " + "[--fp4-resident]\n" + " [--first-frame f.ppm] [--last-frame f.ppm] [--noise-aug A]\n" + " [--ref-image f.ppm] [--ref-video DIR] [--ref-audio f.wav]\n"); + std::exit(code); } } // namespace int main(int argc, char** argv) { - std::string dit_path, video_vae_path, video_cfg_path, audio_vae_path, audio_cfg_path; - std::string embeds_path, out_path, workdir = "/tmp/minimax_h3_gen", ffmpeg = "ffmpeg"; - bool keep_quant = false, dry_run = false, dequant_bf16 = false, denoise_only = false; - bool dump_params = false, fp4_resident = false, encoder_only = false; - std::string device_name = "cpu"; - std::string encoder_path, prompt, tokenizer_path, save_embeds_path; - std::string first_frame_path, last_frame_path; - std::string decode_latent_path; // diagnostic: decode a dumped VAE-input latent - std::string roundtrip_path; // diagnostic: encode->decode a real image - std::string prompt_image_path; // diagnostic: run an image through the vision tower - std::string cond_image_path; // route an image through the ENCODER vision path - // (merged scatter into prompt_embeds + DeepStack) - std::vector ref_image_paths; - std::string ref_video_prefix, ref_audio_path; - // The served checkpoint PARTITION. Community GGUF/NVFP4 files strip the release - // model_index.json `_minimax_h3` block, and the FL2VA/Ref2VA DiTs are structurally - // identical, so it cannot be inferred from the weights — it must be DECLARED. Empty - // => the guard refuses a full render and tells the user to pass it (the #77 catch). - std::string partition_flag; - double imgvid_noise_aug = 1.0; - int64_t encoder_max_layers = 0; - int64_t steps = 0, frames = 0, height = 0, width = 0; - - try { - for (int i = 1; i < argc; ++i) { - const std::string f = argv[i]; - if (f == "--dit") dit_path = Need(argc, argv, ++i, f); - else if (f == "--video-vae") video_vae_path = Need(argc, argv, ++i, f); - else if (f == "--video-vae-config") video_cfg_path = Need(argc, argv, ++i, f); - else if (f == "--audio-vae") audio_vae_path = Need(argc, argv, ++i, f); - else if (f == "--audio-vae-config") audio_cfg_path = Need(argc, argv, ++i, f); - else if (f == "--prompt-embeds") embeds_path = Need(argc, argv, ++i, f); - else if (f == "--out") out_path = Need(argc, argv, ++i, f); - else if (f == "--workdir") workdir = Need(argc, argv, ++i, f); - else if (f == "--ffmpeg") ffmpeg = Need(argc, argv, ++i, f); - else if (f == "--keep-quant") keep_quant = true; - else if (f == "--dequant-bf16") dequant_bf16 = true; - else if (f == "--fp4-resident") fp4_resident = true; - else if (f == "--dry-run") dry_run = true; - else if (f == "--denoise-only") denoise_only = true; - else if (f == "--dump-params") dump_params = true; - else if (f == "--encoder-only") encoder_only = true; - else if (f == "--decode-latent") decode_latent_path = Need(argc, argv, ++i, f); - else if (f == "--roundtrip") roundtrip_path = Need(argc, argv, ++i, f); - else if (f == "--prompt-image") prompt_image_path = Need(argc, argv, ++i, f); - else if (f == "--cond-image") cond_image_path = Need(argc, argv, ++i, f); - else if (f == "--device") device_name = Need(argc, argv, ++i, f); - else if (f == "--encoder") encoder_path = Need(argc, argv, ++i, f); - else if (f == "--prompt") prompt = Need(argc, argv, ++i, f); - else if (f == "--tokenizer") tokenizer_path = Need(argc, argv, ++i, f); - else if (f == "--save-embeds") save_embeds_path = Need(argc, argv, ++i, f); - else if (f == "--first-frame") first_frame_path = Need(argc, argv, ++i, f); - else if (f == "--last-frame") last_frame_path = Need(argc, argv, ++i, f); - else if (f == "--ref-image") ref_image_paths.push_back(Need(argc, argv, ++i, f)); - else if (f == "--ref-video") ref_video_prefix = Need(argc, argv, ++i, f); - else if (f == "--ref-audio") ref_audio_path = Need(argc, argv, ++i, f); - else if (f == "--partition") partition_flag = Need(argc, argv, ++i, f); - else if (f == "--noise-aug") imgvid_noise_aug = std::stod(Need(argc, argv, ++i, f)); - else if (f == "--encoder-max-layers") encoder_max_layers = std::stoll(Need(argc, argv, ++i, f)); - else if (f == "--steps") steps = std::stoll(Need(argc, argv, ++i, f)); - else if (f == "--frames") frames = std::stoll(Need(argc, argv, ++i, f)); - else if (f == "--height") height = std::stoll(Need(argc, argv, ++i, f)); - else if (f == "--width") width = std::stoll(Need(argc, argv, ++i, f)); - else throw std::runtime_error("unknown argument: " + f); - } - // --denoise-only stops after the DiT step loop, so it needs neither VAE nor an - // output path -- and, just as importantly, does not spend their memory. On a - // unified-memory box that headroom is the difference between a run and a reboot. - // --dump-params reads the manifest and exits, so it needs NOTHING but --dit: - // no VAEs, no conditioning, no output path. Requiring them would make the one - // tool that works on a checkpoint too large to load unusable on exactly that - // checkpoint. - // --prompt-image runs the vision tower ONLY (from --encoder); it needs no DiT/VAE/out. - const bool vision_probe = !prompt_image_path.empty(); - const bool diag_vae_only = !decode_latent_path.empty() || !roundtrip_path.empty(); - const bool need_vaes = !denoise_only && !dump_params && !encoder_only && !diag_vae_only; - const bool need_cond = !dump_params && !encoder_only && !diag_vae_only; - // --decode-latent / --roundtrip / --prompt-image need NO DiT and NO conditioning - // (their own blocks validate their inputs); the shared check below would otherwise - // reject --dit. - // --encoder-only needs NO DiT either, and that is the point: the DiT is loaded - // FIRST in the normal path, so asking for conditioning alone used to cost the - // DiT's residency on top of the tower's. On a 122 GiB UNIFIED pool that is the - // difference between ~49 GiB and ~96 GiB peak — i.e. between a run and an OOM - // reboot — and it is faster besides. - if (!diag_vae_only && !vision_probe && !encoder_only && - (dit_path.empty() || (need_vaes && (video_vae_path.empty() || audio_vae_path.empty())) || - (need_vaes && out_path.empty()) || - (need_cond && embeds_path.empty() && (encoder_path.empty() || prompt.empty())))) { - std::cerr << "usage: minimax-h3-gen --dit --video-vae --audio-vae " - "--prompt-embeds --out [--video-vae-config ] " - "[--audio-vae-config ] [--keep-quant] [--steps N] [--frames N] " - "[--height N] [--width N] [--device cpu|cuda] [--workdir DIR] [--ffmpeg PATH] " - "[--dry-run] [--denoise-only] [--dump-params] " - "[--first-frame f.ppm] [--last-frame f.ppm] [--noise-aug A] " - "[--ref-image f.ppm ...] [--ref-video DIR] [--ref-audio f.wav] " - "[--partition fl2va|ref2va]\n" - " or: minimax-h3-gen --encoder-only --encoder " - "--prompt --tokenizer --save-embeds " - "[--encoder-max-layers N] [--device cpu|cuda]\n"; - return 2; - } - - // --encoder-only: run the text tower, write its conditioning, exit. No DiT, no - // VAEs, no output path. This is what makes the encoder A/B affordable — and it - // is the tool for "produce conditioning once, reuse it across renders". - if (encoder_only) { - if (encoder_path.empty() || prompt.empty() || save_embeds_path.empty()) { - throw std::runtime_error( - "--encoder-only needs --encoder, --prompt and --save-embeds"); - } - int64_t seq = 0, hidden = 0; - const std::vector conditioning = EncodeH3Prompt( - encoder_path, prompt, tokenizer_path, encoder_max_layers, device_name, &seq, &hidden); - WriteEmbeds(save_embeds_path, conditioning); - std::cout << "tokens=" << seq << "\nhidden=" << hidden << "\n"; - return 0; - } - - // --dump-params reads the MANIFEST ONLY -- names and shapes, no payload -- and - // prints the geometry those shapes imply. That makes it safe on a checkpoint - // whose weights do not fit (the NVFP4 reference loader is ~132 GB of host f32), - // and it is the right tool for asking "do two checkpoints agree on geometry?" - // without running either. - if (dump_params) { - vllm::MiniMaxH3DitParams pr; - if (EndsWith(dit_path, ".gguf")) { - const vllm::GgufFile gf = vllm::GgufFile::Open(dit_path); - pr = vllm::ParseMiniMaxH3DitParamsFromGgufManifest(vllm::EnumerateMiniMaxH3GgufTensors(gf)); - } else if (vllm::MiniMaxH3ShardedCheckpoint::IsShardedDir(dit_path)) { - // A DIRECTORY of shards + index: the original bf16 release. Manifest only, - // so this answers "do the 13 shards agree on the geometry the GGUF arm - // derives?" on a 66.3 GB checkpoint without reading a single weight byte. - const vllm::MiniMaxH3ShardedCheckpoint ckpt = - vllm::MiniMaxH3ShardedCheckpoint::Open(dit_path); - std::cerr << " " << ckpt.ShardCount() << " shard(s), " << ckpt.Names().size() - << " tensors, index " << ckpt.IndexPath() << "\n"; - pr = vllm::ParseMiniMaxH3DitParamsFromGgufManifest( - vllm::EnumerateMiniMaxH3ShardedTensors(ckpt)); - } else { - const vllm::SafetensorsFile sf = vllm::SafetensorsFile::Open(dit_path); - std::vector manifest; - for (const std::string& name : sf.Names()) { - if ((name.size() > 12 && name.compare(name.size() - 12, 12, "weight_scale") == 0) || - (name.size() > 14 && name.compare(name.size() - 14, 14, "weight_scale_2") == 0)) { - continue; - } - const vllm::StTensor& st = sf.Get(name); - vllm::MiniMaxH3TensorSpec spec; - spec.name = name; - spec.shape = st.shape; - if (st.dtype == "U8") spec.shape = {st.shape[0], st.shape[1] * 2}; - manifest.push_back(std::move(spec)); - } - pr = vllm::ParseMiniMaxH3DitParamsFromGgufManifest(manifest); - } - std::cout << "num_layers=" << pr.num_layers - << "\ntoken_refiner_num_layers=" << pr.token_refiner_num_layers - << "\nhidden_size=" << pr.hidden_size - << "\nnum_attention_heads=" << pr.num_attention_heads - << "\nattention_head_dim=" << pr.attention_head_dim - << "\nffn_hidden_size=" << pr.ffn_hidden_size - << "\nlatents_dim=" << pr.latents_dim - << "\naudio_latents_dim=" << pr.audio_latents_dim - << "\npatch_size_t=" << pr.patch_size_t - << "\npatch_size_h=" << pr.patch_size_h - << "\npatch_size_w=" << pr.patch_size_w - << "\ntext_dim=" << pr.text_dim - << "\ntimestep_input_dim=" << pr.timestep_input_dim - << "\ntime_embed_hidden_size=" << pr.time_embed_hidden_size - << "\ntime_embed_dim=" << pr.time_embed_dim - << "\nadaln_out_features=" << pr.adaln_out_features - << "\nfinal_adaln_out_features=" << pr.final_adaln_out_features - << "\nrope_inv_freq_len=" << pr.rope_inv_freq_len - << "\nvideo_row_width=" << pr.video_row_width() - << "\nrope_rot_dim=" << pr.rope_rot_dim() << "\n"; - return 0; - } - - // --decode-latent DIAGNOSTIC: decode a dumped VAE-input latent - // (VT_H3_DUMP_DIR/vae_input_video_latent.f32) directly, with NO DiT and NO - // conditioning, on either device. Lets the device ViT3D decoder be compared - // against the scalar CPU reference (gated vs upstream at 8.9e-8) on the SAME - // real latent -- the VAE-branch oracle test for the render-coherence bisection. - if (!decode_latent_path.empty()) { - if (video_cfg_path.empty() || video_vae_path.empty() || out_path.empty()) { - throw std::runtime_error( - "--decode-latent needs --video-vae, --video-vae-config, --out and --width/--height/--frames"); - } - vllm::MiniMaxH3LatentStats vstats; - vllm::MiniMaxH3VideoVaeDecoderConfig vcfg = - vllm::ParseMiniMaxH3VideoVaeDecoderConfig(ReadJson(video_cfg_path), &vstats); - vllm::SafetensorsFile vfile = vllm::SafetensorsFile::Open(video_vae_path); - vllm::MiniMaxH3AudioVaeWeights vweights = vllm::LoadMiniMaxH3VideoVaeDecoderWeights(vfile); - const vllm::MiniMaxH3ShapePlan plan = vllm::MiniMaxH3ResolveShape( - "t2va", 0.0, frames, height, width, 0, 0); - const int64_t lt = plan.latent_t, lh = plan.height / vllm::kMiniMaxH3VaeRatio, - lw = plan.width / vllm::kMiniMaxH3VaeRatio, ch = vcfg.in_channels; - const int64_t need = ch * lt * lh * lw; - std::ifstream lf(decode_latent_path, std::ios::binary); - if (!lf) throw std::runtime_error("cannot open --decode-latent file"); - std::vector latent(static_cast(need)); - lf.read(reinterpret_cast(latent.data()), need * static_cast(sizeof(float))); - if (!lf) throw std::runtime_error("--decode-latent file too small for [C,T,H,W]"); - std::cerr << "decode-latent: [" << ch << "," << lt << "," << lh << "," << lw << "] on " - << device_name << "\n"; - vllm::MiniMaxH3T2vaResult result; - if (device_name == "cuda") { - vt::Device dev = vt::GetBackend(vt::DeviceType::kCUDA).CreateQueue().device; - vt::Queue vq = vt::GetBackend(dev.type).CreateQueue(); - const vllm::MiniMaxH3VideoVaeDeviceWeights staged = - vllm::StageMiniMaxH3VideoVaeWeights(vq, vcfg, vweights); - result.frames = vllm::MiniMaxH3VideoVaeDecodeTemporalDevice( - dev, vcfg, staged, latent, lt, lh, lw, plan.num_frames, &result.frame_shape); - } else { - result.frames = vllm::MiniMaxH3VideoVaeDecode(vcfg, vweights, latent, lt, lh, lw, - &result.frame_shape); - } - vllm::MiniMaxH3VideoDenormalizePixels( - result.frames, result.frame_shape.channels, - result.frame_shape.t * result.frame_shape.h * result.frame_shape.w); - std::string mkc = "mkdir -p '" + workdir + "'"; - if (std::system(mkc.c_str()) != 0) throw std::runtime_error("cannot create " + workdir); - for (int64_t fr = 0; fr < result.frame_shape.t; ++fr) { - char nm[512]; - std::snprintf(nm, sizeof(nm), "%s/frame_%06lld.ppm", workdir.c_str(), - static_cast(fr)); - WriteFile(nm, vllm::MiniMaxH3WritePpmFrame(result.frames, result.frame_shape, fr)); - } - std::cerr << "decode-latent: wrote " << result.frame_shape.t << " frames to " << workdir - << "\n"; - return 0; - } - - // --roundtrip DIAGNOSTIC: encode a real image through the video VAE encoder, - // apply post_quant_conv, and decode -- a DiT-independent gold-standard test of - // the decoder. A coherent round-trip proves the decoder works and localizes the - // render bug to the DiT-produced latent; a grid proves the decoder itself. - if (!roundtrip_path.empty()) { - if (video_cfg_path.empty() || video_vae_path.empty()) { - throw std::runtime_error("--roundtrip needs --video-vae and --video-vae-config"); - } - vllm::MiniMaxH3LatentStats vstats; - vllm::MiniMaxH3VideoVaeDecoderConfig vcfg = - vllm::ParseMiniMaxH3VideoVaeDecoderConfig(ReadJson(video_cfg_path), &vstats); - vllm::SafetensorsFile vfile = vllm::SafetensorsFile::Open(video_vae_path); - vllm::MiniMaxH3AudioVaeWeights dec_w = vllm::LoadMiniMaxH3VideoVaeDecoderWeights(vfile); - vllm::MiniMaxH3AudioVaeWeights enc_w = vllm::LoadMiniMaxH3VideoVaeEncoderWeights(vfile); - int64_t ih = 0, iw = 0; - std::vector chw = ReadPpmAsChw(roundtrip_path, &ih, &iw); // [3,H,W] in [0,1] - vllm::MiniMaxH3VideoNormalizePixels(chw, 3, ih * iw); // -> imagenet space - vllm::MiniMaxH3EncoderFcn3dConfig enc_cfg; - enc_cfg.z_channels = 2 * vcfg.in_channels; // moments (mean|logvar) - enc_cfg.t = 1; enc_cfg.h = ih; enc_cfg.w = iw; - vllm::MiniMaxH3VideoFrameShape ls{}; - std::vector z = vllm::MiniMaxH3VideoVaeEncodeToLatent(enc_cfg, enc_w, chw, &ls); - std::cerr << "roundtrip: encoded [" << vcfg.in_channels << "," << ls.t << "," << ls.h << "," - << ls.w << "]\n"; - const int64_t per = ls.t * ls.h * ls.w; - // per-channel stats of the ENCODED latent (the in-distribution reference) - { double s2 = 0; for (float v : z) s2 += double(v) * v; - std::cerr << "roundtrip: encoded-latent rms=" << std::sqrt(s2 / z.size()) << "\n"; } - if (const char* dd = std::getenv("VT_H3_DUMP_DIR")) { - std::string p = std::string(dd) + "/encoder_latent.f32"; - if (std::FILE* fp = std::fopen(p.c_str(), "wb")) { - std::fwrite(z.data(), sizeof(float), z.size(), fp); std::fclose(fp); - std::cerr << "roundtrip: dumped encoder latent [" << vcfg.in_channels << "," << ls.t - << "," << ls.h << "," << ls.w << "] to " << p << "\n"; - } - } - if (dec_w.Has("post_quant_conv.weight")) { - z = vllm::MiniMaxH3VideoVaePostQuantConv(dec_w, z, vcfg.in_channels, per); - } - vllm::MiniMaxH3T2vaResult result; - if (device_name == "cuda") { - vt::Device dev = vt::GetBackend(vt::DeviceType::kCUDA).CreateQueue().device; - vt::Queue vq = vt::GetBackend(dev.type).CreateQueue(); - const vllm::MiniMaxH3VideoVaeDeviceWeights staged = - vllm::StageMiniMaxH3VideoVaeWeights(vq, vcfg, dec_w); - result.frames = vllm::MiniMaxH3VideoVaeDecodeTemporalDevice( - dev, vcfg, staged, z, ls.t, ls.h, ls.w, 1, &result.frame_shape); - } else { - result.frames = vllm::MiniMaxH3VideoVaeDecode(vcfg, dec_w, z, ls.t, ls.h, ls.w, - &result.frame_shape); - } - vllm::MiniMaxH3VideoDenormalizePixels( - result.frames, result.frame_shape.channels, - result.frame_shape.t * result.frame_shape.h * result.frame_shape.w); - std::string mkc = "mkdir -p '" + workdir + "'"; - if (std::system(mkc.c_str()) != 0) throw std::runtime_error("cannot create " + workdir); - WriteFile(workdir + "/roundtrip.ppm", - vllm::MiniMaxH3WritePpmFrame(result.frames, result.frame_shape, 0)); - std::cerr << "roundtrip: wrote " << workdir << "/roundtrip.ppm (" - << result.frame_shape.w << "x" << result.frame_shape.h << ")\n"; - return 0; - } - - // --- --prompt-image DIAGNOSTIC: route a real image through the shared Qwen3-VL - // image processor + the encoder's VISION tower, loaded from the encoder GGUF's - // visual.* tensors (the piece the record reconciliation found missing, spec §8.8). - // Reports the conditioning feature stats -- the REAL-WEIGHTS proof that the vision - // tower now runs. Scattering the merged/deepstack features into the DiT-conditioning - // path (merge into prompt_embeds + DeepStack inject into the device text tower) is - // the tracked residual; this probe stops after the tower. - if (!prompt_image_path.empty()) { - if (encoder_path.empty()) - throw std::runtime_error("--prompt-image needs --encoder (the vision weights live in it)"); - if (device_name != "cuda") - throw std::runtime_error( - "--prompt-image needs --device cuda (the vision tower is device-resident)"); - std::cerr << "loading encoder vision tower from " << encoder_path << "\n"; - const vllm::GgufFile ef = vllm::GgufFile::Open(encoder_path); - const vllm::multimodal::Qwen3VLVisionConfig vcfg = vllm::MiniMaxH3EncoderVisionConfig(); - const vllm::multimodal::Qwen3VLVisionWeights vw = vllm::LoadQwen3VLVisionFromGguf(ef, vcfg); - std::cerr << " vision weights: depth=" << vcfg.depth << " hidden=" << vcfg.hidden_size - << " heads=" << vcfg.num_heads << " out=" << vcfg.out_hidden_size - << " deepstack=" << vw.deepstack_mergers.size() << "\n"; - - int64_t ih = 0, iw = 0; - const std::vector rgb = ReadPpmAsHwcU8(prompt_image_path, &ih, &iw); - std::cerr << " image " << iw << "x" << ih << "\n"; - vllm::multimodal::Qwen3VLProcessorConfig pcfg; // patch16/temporal2/merge2/0.5-norm - pcfg.merge_size = static_cast(vcfg.spatial_merge_size); - const vllm::multimodal::Qwen3VLImageProcessor proc(pcfg); - const vllm::multimodal::ImageKwargs kw = proc.ProcessImage(rgb.data(), ih, iw); - const std::array grid = kw.image_grid_thw; - const int64_t tokens = grid[0] * grid[1] * grid[2]; - const int64_t merge = vcfg.spatial_merge_size * vcfg.spatial_merge_size; - std::cerr << " grid_thw=[" << grid[0] << "," << grid[1] << "," << grid[2] - << "] tokens=" << tokens << " merged=" << (tokens / merge) << "\n"; - - vt::Backend& backend = vt::GetBackend(vt::DeviceType::kCUDA); - vllm::multimodal::Qwen3VLVisionCapture cap; - const std::vector tower = vllm::multimodal::Qwen3VLVisionForward( - kw.pixel_values_bf16, grid, vw, vcfg, backend, &cap); - const int64_t nm = tokens / merge; - const int64_t width = - vcfg.out_hidden_size * - (1 + static_cast(vcfg.deepstack_visual_indexes.size())); - auto stats = [&](const std::vector& v, const char* tag) { - double s = 0, s2 = 0, mx = 0; - bool fin = true; - for (float f : v) { - if (!std::isfinite(f)) fin = false; - s += f; - s2 += double(f) * f; - mx = std::max(mx, std::fabs(static_cast(f))); - } - const double n = v.empty() ? 1.0 : static_cast(v.size()); - std::cerr << " " << tag << ": n=" << v.size() << " finite=" << (fin ? "yes" : "NO") - << " mean=" << (s / n) << " rms=" << std::sqrt(s2 / n) << " maxabs=" << mx - << "\n"; - }; - std::cerr << " tower out = [" << nm << ", " << width << "]\n"; - stats(tower, "tower_concat"); - stats(cap.merger_out, "merged"); - for (size_t d = 0; d < cap.deepstack_out.size(); ++d) - stats(cap.deepstack_out[d], ("deepstack_" + std::to_string(d)).c_str()); - std::cout << "prompt-image: vision tower RAN on real weights; merged=[" << nm << "," - << vcfg.out_hidden_size << "] + " << cap.deepstack_out.size() << " deepstack\n"; - return 0; - } - - // --- 1. DiT --- - std::cerr << "loading DiT " << dit_path << (keep_quant ? " (keep-quant)" : "") << "\n"; - vllm::MiniMaxH3GgufDit dit; - vllm::MiniMaxH3DitDeviceWeights streamed; - bool have_streamed = false; - if (EndsWith(dit_path, ".gguf")) { - const vllm::GgufFile f = vllm::GgufFile::Open(dit_path); - // --dequant-bf16 loads STRAIGHT to bf16 (~33 GB). Keeping blocks would leave - // the AdaLN projections ineligible (K=2688 is not a whole number of 256-element - // Q3_K blocks) and dequantize them to ~52 GB of f32 — which is what does not fit. - if (dequant_bf16 && device_name == "cuda") { - // STREAM straight onto the device: dequantize + upload one tensor at a time - // so the host copy never accumulates. Peak is what kills a unified-memory - // box, and load-then-stage holds ~33 GB twice. - vt::Queue sq = vt::GetBackend(vt::DeviceType::kCUDA).CreateQueue(); - const auto t0 = std::chrono::steady_clock::now(); - streamed = vllm::StreamMiniMaxH3DitToDeviceBf16(sq, f, &dit.params); - have_streamed = true; - std::cerr << " streamed DiT -> device (bf16) in " - << std::chrono::duration(std::chrono::steady_clock::now() - t0).count() - << " s\n"; - } else { - dit = dequant_bf16 ? vllm::LoadMiniMaxH3DitFromGgufBf16(f) - : vllm::LoadMiniMaxH3DitFromGguf(f, keep_quant); - } - } else if (vllm::MiniMaxH3ShardedCheckpoint::IsShardedDir(dit_path)) { - // The ORIGINAL bf16 release: a DIRECTORY of shards plus - // model.safetensors.index.json. Every other --dit form is a single file and - // keeps working unchanged; this is the only one that can open the - // full-precision DiT, which is what makes the quantization-quality question - // answerable at all. - const vllm::MiniMaxH3ShardedCheckpoint ckpt = - vllm::MiniMaxH3ShardedCheckpoint::Open(dit_path); - std::cerr << " " << ckpt.ShardCount() << " shard(s), " << ckpt.Names().size() - << " tensors (index " << ckpt.IndexPath() << ")\n"; - if (device_name == "cuda") { - // STREAM. 66.3 GB cannot be materialized on the host and then staged: the - // pool is UNIFIED, so that holds the model twice against 122 GiB. One - // tensor at a time, and a bf16 shard tensor is uploaded straight out of - // the mmap with no host buffer at all. - vt::Queue sq = vt::GetBackend(vt::DeviceType::kCUDA).CreateQueue(); - const auto t0 = std::chrono::steady_clock::now(); - streamed = vllm::StreamMiniMaxH3ShardedToDeviceBf16(sq, ckpt, &dit.params); - have_streamed = true; - const vllm::MiniMaxH3ShardStreamStats st = vllm::GetMiniMaxH3ShardStreamStats(); - std::cerr << " streamed sharded bf16 DiT -> device in " - << std::chrono::duration(std::chrono::steady_clock::now() - t0).count() - << " s (" << st.tensors_streamed << " tensors, " << st.direct_uploads - << " direct, " << st.converted_uploads << " converted, peak host buffer " - << (st.host_peak_bytes / (1024.0 * 1024.0)) << " MiB)\n"; - } else { - // Host f32 reference path. ~132 GB on the real release — usable only on a - // reduced checkpoint; a real run wants --device cuda. - dit = vllm::LoadMiniMaxH3DitFromShards(ckpt); - } - } else { - const vllm::SafetensorsFile f = vllm::SafetensorsFile::Open(dit_path); - if (device_name == "cuda") { - // Stream NVFP4 straight to the device. The host-f32 reference loader is - // ~132 GB for this checkpoint and gets OOM-killed during load on a - // unified-memory box; streaming keeps peak at the device copy plus one - // tensor. - vt::Queue sq = vt::GetBackend(vt::DeviceType::kCUDA).CreateQueue(); - const auto t0 = std::chrono::steady_clock::now(); - // --fp4-resident keeps the packed FP4 on device (~1/4 the bf16 footprint, - // ~16 GB vs ~66 GB) and routes every quantized projection through the - // Marlin W4A16 GEMM on sm_121a; the default bf16 stream dequantizes to bf16. - streamed = fp4_resident - ? vllm::StreamMiniMaxH3Nvfp4ToDeviceFp4(sq, f, &dit.params) - : vllm::StreamMiniMaxH3Nvfp4ToDeviceBf16(sq, f, &dit.params); - have_streamed = true; - std::cerr << " streamed NVFP4 DiT -> device (" << (fp4_resident ? "fp4-resident" : "bf16") - << ") in " - << std::chrono::duration(std::chrono::steady_clock::now() - t0).count() - << " s\n"; - } else { - dit = vllm::LoadMiniMaxH3DitFromNvfp4(f); - } - } - std::cerr << " layers=" << dit.params.num_layers << " hidden=" << dit.params.hidden_size - << " heads=" << dit.params.num_attention_heads << "\n"; - - // --- 1b. optional encoder run. Same helper --encoder-only uses, so the - // conditioning a render consumes and the conditioning the A/B measures come - // out of ONE code path. --- - std::vector encoded_prompt; - if (!encoder_path.empty()) { - std::cerr << "loading encoder " << encoder_path << " (keep-quant)\n"; - const vllm::GgufFile ef = vllm::GgufFile::Open(encoder_path); - const vllm::MiniMaxH3EncoderQuantWeights enc = - vllm::LoadMiniMaxH3EncoderFromGguf(ef, encoder_max_layers); - vllm::MiniMaxH3EncoderConfig ec = enc.config; - std::cerr << " encoder layers=" << ec.num_hidden_layers << " hidden=" << ec.hidden_size - << " heads=" << ec.num_attention_heads << " kv_heads=" << ec.num_key_value_heads - << " head_dim=" << ec.head_dim << " ffn=" << ec.intermediate_size << "\n"; - size_t quant_bytes = 0; - for (const auto& kv : enc.quant_storage) quant_bytes += kv.second.size(); - std::cerr << " encoder resident (keep-quant) = " << (quant_bytes / (1024.0 * 1024.0 * 1024.0)) - << " GiB\n"; - - if (!prompt.empty()) { - // The ComfyUI-style encoder GGUF is WEIGHTS ONLY — it carries no - // `tokenizer.ggml.*` metadata, unlike a llama.cpp export — so the vocab - // comes from the checkpoint's own tokenizer.json. `--tokenizer` is - // therefore required with `--prompt` unless the GGUF happens to embed one. - const vllm::tok::Tokenizer tokenizer = - tokenizer_path.empty() ? vllm::tok::Tokenizer::FromGguf(ef) - : vllm::tok::Tokenizer::FromHfJson(tokenizer_path); - std::vector ids = tokenizer.Encode(prompt); - VT_CHECK(!ids.empty(), "minimax-h3-gen: the prompt tokenized to nothing"); - std::cerr << " prompt tokens = " << ids.size() << "\n"; - - // --cond-image routes a reference image through the ENCODER VISION PATH - // (upstream _encode, encoder.py:1064-1101): the vision tower's MERGED features - // masked_scatter into inputs_embeds at the image-pad positions, and its 3 - // DeepStack blocks inject into the first N text layers. This is the residual - // #86 left open — the vision features now REACH the DiT via prompt_embeds. - std::vector visual_mask; // 1 at each image-pad row - std::vector> deepstack; // 3 x [nm, hidden] taps - std::vector merged; // [nm, hidden] merged features - std::array vgrid{1, 0, 0}; - int64_t vsmerge = 2; - const bool have_vision = !cond_image_path.empty(); - if (have_vision) { - VT_CHECK(device_name == "cuda", - "minimax-h3-gen: --cond-image needs --device cuda (the vision tower is " - "device-resident)"); - const vllm::multimodal::Qwen3VLVisionConfig vcfg = vllm::MiniMaxH3EncoderVisionConfig(); - const vllm::multimodal::Qwen3VLVisionWeights vw = vllm::LoadQwen3VLVisionFromGguf(ef, vcfg); - vsmerge = vcfg.spatial_merge_size; - int64_t cih = 0, ciw = 0; - const std::vector rgb = ReadPpmAsHwcU8(cond_image_path, &cih, &ciw); - std::cerr << " cond-image " << ciw << "x" << cih << "\n"; - vllm::multimodal::Qwen3VLProcessorConfig pcfg; // patch16/temporal2/merge2/0.5-norm - pcfg.merge_size = static_cast(vsmerge); - const vllm::multimodal::Qwen3VLImageProcessor proc(pcfg); - const vllm::multimodal::ImageKwargs kw = proc.ProcessImage(rgb.data(), cih, ciw); - vgrid = kw.image_grid_thw; - vt::Backend& vbackend = vt::GetBackend(vt::DeviceType::kCUDA); - vllm::multimodal::Qwen3VLVisionCapture cap; - (void)vllm::multimodal::Qwen3VLVisionForward(kw.pixel_values_bf16, vgrid, vw, vcfg, - vbackend, &cap); - merged = std::move(cap.merger_out); - deepstack = std::move(cap.deepstack_out); - const int64_t nm = vgrid[0] * vgrid[1] * vgrid[2] / (vsmerge * vsmerge); - std::cerr << " cond-image: grid=[" << vgrid[0] << "," << vgrid[1] << "," << vgrid[2] - << "] merged=" << nm << " deepstack=" << deepstack.size() << "\n"; - // Prepend a vision block , then expand - // the single image_pad into nm image_token_id copies (the mm processor). - std::vector pre = {pcfg.vision_start_token_id, pcfg.image_token_id, - pcfg.vision_end_token_id}; - pre.insert(pre.end(), ids.begin(), ids.end()); - std::vector> placeholders; - ids = vllm::multimodal::ExpandImagePlaceholders( - pre, pcfg.image_token_id, static_cast(vsmerge), - {{vgrid[0], vgrid[1], vgrid[2]}}, &placeholders); - VT_CHECK(!placeholders.empty(), - "minimax-h3-gen: cond-image placeholder expansion produced none"); - } - - // DIAGNOSTIC (env-gated): dump the (possibly expanded) token ids so the - // tokenization can be diffed against upstream. A template mismatch shifts - // every text row and feeds the 32B tower a different string. - if (const char* dd = std::getenv("VT_H3_DUMP_INPUTS")) { - if (std::FILE* fp = std::fopen((std::string(dd) + "/prompt_token_ids.i32").c_str(), "wb")) { - std::fwrite(ids.data(), sizeof(int32_t), ids.size(), fp); - std::fclose(fp); - std::cerr << " [h3-dump-inputs] prompt_token_ids.i32 (" << ids.size() << " ids): "; - for (size_t k = 0; k < ids.size() && k < 64; ++k) std::cerr << ids[k] << " "; - std::cerr << "\n"; - } - } - - const int64_t seq = static_cast(ids.size()); - std::vector embeds = vllm::MiniMaxH3EncoderEmbedTokens(enc, ids); - std::vector pos(static_cast(3 * seq)); - if (!have_vision) { - // Text-only: all three M-RoPE axes are the token index. - for (int64_t a = 0; a < 3; ++a) { - for (int64_t s = 0; s < seq; ++s) pos[static_cast(a * seq + s)] = s; - } - } else { - // masked_scatter the merged features into the image-pad rows + build the - // visual position mask (upstream `inputs_embeds.masked_scatter(image_mask, - // image_embeds)` + `_get_placeholder_mask`). - const int32_t IMG = vllm::multimodal::Qwen3VLProcessorConfig{}.image_token_id; - visual_mask.assign(static_cast(seq), 0); - const int64_t hidden = ec.hidden_size; - int64_t vi = 0, off = -1; - for (int64_t s = 0; s < seq; ++s) { - if (ids[static_cast(s)] != IMG) continue; - if (off < 0) off = s; - visual_mask[static_cast(s)] = 1; - VT_CHECK((vi + 1) * hidden <= static_cast(merged.size()), - "minimax-h3-gen: merged vision rows < image-pad tokens"); - for (int64_t i = 0; i < hidden; ++i) - embeds[static_cast(s * hidden + i)] = - merged[static_cast(vi * hidden + i)]; - ++vi; - } - VT_CHECK(vi * hidden == static_cast(merged.size()), - "minimax-h3-gen: merged vision rows != image-pad tokens"); - // M-RoPE positions. Qwen3VLGetRopeIndex is byte-equivalent to H3's own - // _get_rope_index for a single-frame image (t==1): text runs sequentially, - // the image block takes the 3D vision grid, and the next text advances by - // max(llm_h, llm_w). Layout is [3, seq] flattened as [axis*seq + s]. - vllm::multimodal::MmImageSpan span{off, {vgrid[0], vgrid[1], vgrid[2]}}; - int64_t delta = 0; - const std::vector p3 = - vllm::multimodal::Qwen3VLGetRopeIndex(ids, {span}, vsmerge, &delta); - VT_CHECK(static_cast(p3.size()) == 3 * seq, - "minimax-h3-gen: rope index produced the wrong count"); - for (size_t i = 0; i < pos.size(); ++i) pos[i] = static_cast(p3[i]); - } - vt::Device enc_dev{}; - if (device_name == "cuda") { - enc_dev = vt::GetBackend(vt::DeviceType::kCUDA).CreateQueue().device; - } - vt::Queue eq{enc_dev, nullptr}; - vt::Backend& eb = vt::GetBackend(enc_dev.type); - if (enc_dev.type != vt::DeviceType::kCPU) eq = eb.CreateQueue(); - const vllm::MiniMaxH3EncoderDeviceWeights staged = - vllm::StageMiniMaxH3EncoderWeights(eq, enc); - std::cerr << " encoding prompt" << (have_vision ? " (vision-conditioned)" : "") << "...\n"; - encoded_prompt = vllm::MiniMaxH3EncoderTextForwardDevice( - eq, ec, staged, embeds, pos.data(), seq, - have_vision ? visual_mask.data() : nullptr, deepstack); - std::cerr << " conditioning = [" << seq << ", " << ec.hidden_size << "]\n"; - // Persisting the conditioning makes a checkpoint A/B CONTROLLED: two DiTs - // can then be compared on byte-identical text conditioning instead of two - // separate encoder runs. It also lets the second run skip the 13 GB tower - // entirely, which is the difference between fitting and not on a - // unified-memory box. - if (!save_embeds_path.empty()) { - std::ofstream out(save_embeds_path, std::ios::binary); - if (!out) throw std::runtime_error("cannot write " + save_embeds_path); - out.write(reinterpret_cast(encoded_prompt.data()), - static_cast(encoded_prompt.size() * sizeof(float))); - std::cerr << " saved conditioning -> " << save_embeds_path << "\n"; - } - } - } - - // --- 2. VAEs + their configs (the configs carry the latent statistics) --- - vllm::MiniMaxH3VideoVaeDecoderConfig video_cfg; - vllm::MiniMaxH3LatentStats video_stats; - if (!video_cfg_path.empty()) { - video_cfg = vllm::ParseMiniMaxH3VideoVaeDecoderConfig(ReadJson(video_cfg_path), &video_stats); - } - vllm::MiniMaxH3AudioVaeConfig audio_cfg; - vllm::MiniMaxH3LatentStats audio_stats; - if (!audio_cfg_path.empty()) { - audio_cfg = vllm::ParseMiniMaxH3AudioVaeConfig(ReadJson(audio_cfg_path), &audio_stats); - } - // The files stay in scope alongside the weights: the loaders read through the - // mapping, so releasing it early would leave the weights pointing at nothing. - std::optional video_file, audio_file; - vllm::MiniMaxH3AudioVaeWeights video_weights, audio_weights; - if (need_vaes) { - std::cerr << "loading video VAE " << video_vae_path << "\n"; - video_file = vllm::SafetensorsFile::Open(video_vae_path); - video_weights = vllm::LoadMiniMaxH3VideoVaeDecoderWeights(*video_file); - std::cerr << "loading audio VAE " << audio_vae_path << "\n"; - audio_file = vllm::SafetensorsFile::Open(audio_vae_path); - audio_weights = vllm::LoadMiniMaxH3AudioVaeWeights(*audio_file); - } - - // --- 3. request shape --- - vllm::MiniMaxH3T2vaRequest request; - request.text_len = 0; // set from the embeddings below - if (steps > 0) request.num_steps = steps; - // `_resolve_shape` decides frames/canvas/latent_t/audio_t; the latent GRID is - // that canvas divided by the VAE's spatial ratio (prod(space_down) = 16). - const vllm::MiniMaxH3ShapePlan plan = vllm::MiniMaxH3ResolveShape( - "t2va", /*duration_seconds=*/0.0, frames, height, width, - /*image_width=*/0, /*image_height=*/0); - request.latent_t = plan.latent_t; - request.num_frames = plan.num_frames; - request.latent_h = plan.height / vllm::kMiniMaxH3VaeRatio; - request.latent_w = plan.width / vllm::kMiniMaxH3VaeRatio; - request.audio_t = plan.audio_t; - request.audio_channel = vllm::kMiniMaxH3AudioChannels; - request.video_latents_mean = video_stats.mean; - request.video_latents_std = video_stats.std_dev; - request.audio_latents_mean = audio_stats.mean; - request.audio_latents_std = audio_stats.std_dev; - // Declare the served partition so MiniMaxH3GenerateT2va can refuse a task the - // checkpoint does not serve (the #77 catch: t2va on the Ref2VA arm). An empty - // --partition is declared-but-unknown; the guard then names the recipe lines and - // asks for fl2va|ref2va rather than silently rendering the wrong combination. - request.partition = vllm::MiniMaxH3PartitionFromFlag(partition_flag); - - // --- ref2va VIDEO reference: a CLIP prepended to the sequence. Reads - // DIR/frame_%06d.ppm, which is exactly what this example WRITES, so a previous - // run's workdir can be handed straight back in as a reference. - if (!ref_video_prefix.empty()) { - VT_CHECK(ref_image_paths.empty() && first_frame_path.empty() && last_frame_path.empty(), - "minimax-h3-gen: --ref-video is exclusive with --ref-image and --first/--last-frame"); - VT_CHECK(!video_vae_path.empty(), "minimax-h3-gen: --ref-video needs --video-vae"); - const vllm::SafetensorsFile vf = vllm::SafetensorsFile::Open(video_vae_path); - const vllm::MiniMaxH3AudioVaeWeights enc_w = vllm::LoadMiniMaxH3VideoVaeEncoderWeights(vf); - vllm::MiniMaxH3EncoderFcn3dConfig enc_cfg; - enc_cfg.z_channels = 2 * dit.params.latents_dim; - - std::vector clip; - int64_t ft = 0, fh = 0, fw = 0; - for (int64_t k = 0;; ++k) { - char nm[512]; - std::snprintf(nm, sizeof(nm), "%s/frame_%06lld.ppm", ref_video_prefix.c_str(), - static_cast(k)); - std::ifstream probe(nm, std::ios::binary); - if (!probe) break; - probe.close(); - int64_t h2 = 0, w2 = 0; - const std::vector f = ReadPpmAsChw(nm, &h2, &w2); - if (ft == 0) { fh = h2; fw = w2; } - VT_CHECK(h2 == fh && w2 == fw, - "minimax-h3-gen: every --ref-video frame must have the same size"); - // [C,H,W] per frame -> [C,T,H,W]: append per channel, so build channel-major. - clip.insert(clip.end(), f.begin(), f.end()); - ++ft; - } - VT_CHECK(ft > 0, "minimax-h3-gen: --ref-video found no frame_%06d.ppm files"); - // Re-lay the per-frame [C,H,W] stack into [C,T,H,W]. - std::vector chw(clip.size()); - const int64_t plane = fh * fw; - for (int64_t c = 0; c < 3; ++c) { - for (int64_t k = 0; k < ft; ++k) { - for (int64_t e = 0; e < plane; ++e) { - chw[static_cast((c * ft + k) * plane + e)] = - clip[static_cast(k * 3 * plane + c * plane + e)]; - } - } - } - vllm::MiniMaxH3RefBlock vb{}; - request.keyframe_cond_rows = vllm::MiniMaxH3EncodeReferenceVideo( - enc_cfg, enc_w, dit.params, chw, ft, fh, fw, &vb); - request.ref_blocks = {vb}; - std::cerr << " ref2va: reference VIDEO " << ft << " frames at " << fw << "x" << fh - << " (silent)\n"; - } - - // --- ref2va REFERENCES: whole reference images prepended to the sequence, - // as opposed to fl2va which pins frames OF THE OUTPUT. Mutually exclusive. - if (!ref_image_paths.empty()) { - VT_CHECK(first_frame_path.empty() && last_frame_path.empty(), - "minimax-h3-gen: --ref-image (ref2va) and --first/--last-frame (fl2va) are exclusive"); - VT_CHECK(!video_vae_path.empty(), "minimax-h3-gen: --ref-image needs --video-vae"); - const vllm::SafetensorsFile vf = vllm::SafetensorsFile::Open(video_vae_path); - const vllm::MiniMaxH3AudioVaeWeights enc_w = vllm::LoadMiniMaxH3VideoVaeEncoderWeights(vf); - vllm::MiniMaxH3EncoderFcn3dConfig enc_cfg; - enc_cfg.z_channels = 2 * dit.params.latents_dim; - - std::vector> imgs; - int64_t ih = 0, iw = 0; - for (const std::string& rp : ref_image_paths) { - int64_t h2 = 0, w2 = 0; - std::cerr << "loading reference image " << rp << "\n"; - imgs.push_back(ReadPpmAsChw(rp, &h2, &w2)); - if (ih == 0) { ih = h2; iw = w2; } - VT_CHECK(h2 == ih && w2 == iw, - "minimax-h3-gen: every --ref-image must have the same size"); - } - std::vector blocks; - request.keyframe_cond_rows = vllm::MiniMaxH3EncodeReferenceImages( - enc_cfg, enc_w, dit.params, imgs, ih, iw, &blocks); - request.ref_blocks = blocks; - std::cerr << " ref2va: " << blocks.size() << " reference image(s) at " << iw << "x" << ih - << "\n"; - } - - // --- ref2va REFERENCE AUDIO: a waveform prepended to the sequence, through - // the audio VAE's ENCODER half. It ATTACHES to a video reference when there is - // one (one kVideoAudio block carrying both, which is the layout - // packed_sequence.py builds), and otherwise stands alone as a kAudio block. - if (!ref_audio_path.empty()) { - VT_CHECK(first_frame_path.empty() && last_frame_path.empty(), - "minimax-h3-gen: --ref-audio (ref2va) and --first/--last-frame (fl2va) are " - "exclusive"); - VT_CHECK(!audio_vae_path.empty(), - "minimax-h3-gen: --ref-audio needs --audio-vae (the ENCODER half)"); - VT_CHECK(!audio_cfg_path.empty(), - "minimax-h3-gen: --ref-audio needs --audio-vae-config (it carries the latent " - "statistics the reference rows are normalized by)"); - std::cerr << "loading reference audio " << ref_audio_path << "\n"; - int64_t wav_channels = 0, wav_samples = 0; - const std::vector wave = ReadWavRef(ref_audio_path, &wav_channels, &wav_samples); - - // The audio VAE file is opened again for its ENCODER half; the decoder - // loader deliberately skips it, so the two halves are separate weight sets. - const vllm::SafetensorsFile af = vllm::SafetensorsFile::Open(audio_vae_path); - const vllm::MiniMaxH3AudioVaeWeights aenc = vllm::LoadMiniMaxH3AudioVaeEncoderWeights(af); - vllm::MiniMaxH3AudioVaeEncoderConfig aenc_cfg; // the shipped geometry - aenc_cfg.vae_latent_channels = dit.params.audio_latents_dim; - - vllm::MiniMaxH3RefBlock ab{}; - const std::vector arows = vllm::MiniMaxH3EncodeReferenceAudio( - aenc_cfg, aenc, wave, wav_channels, wav_samples, audio_stats.mean, audio_stats.std_dev, - /*noise_aug=*/1.0, {}, &ab); - request.audio_ref_rows = arows; - if (request.ref_blocks.size() == 1 && - request.ref_blocks[0].kind == vllm::MiniMaxH3RefBlock::Kind::kVideoAudio) { - // A video reference that now has SOUND: same block, non-zero ref_audio_t. - request.ref_blocks[0].ref_audio_t = ab.ref_audio_t; - std::cerr << " ref2va: reference video now carries audio, " << ab.ref_audio_t - << " latent frames\n"; - } else { - request.ref_blocks.push_back(ab); - std::cerr << " ref2va: reference AUDIO " << wav_samples << " samples/channel -> " - << ab.ref_audio_t << " latent frames\n"; - } - } - - // --- fl2va KEYFRAMES: encode the supplied frame(s) into pinned conditioning. - // Upstream allows exactly {0}, {-1} or {0, -1}: first, last, or both. - if (!first_frame_path.empty() || !last_frame_path.empty()) { - VT_CHECK(!video_vae_path.empty(), - "minimax-h3-gen: --first-frame/--last-frame need --video-vae (the ENCODER half)"); - const vllm::SafetensorsFile vf = vllm::SafetensorsFile::Open(video_vae_path); - const vllm::MiniMaxH3AudioVaeWeights enc_w = vllm::LoadMiniMaxH3VideoVaeEncoderWeights(vf); - vllm::MiniMaxH3EncoderFcn3dConfig enc_cfg; - enc_cfg.z_channels = 2 * dit.params.latents_dim; // moments: mean | logvar - - std::vector> imgs; - std::vector idx; - int64_t ih = 0, iw = 0; - if (!first_frame_path.empty()) { - std::cerr << "loading first frame " << first_frame_path << "\n"; - imgs.push_back(ReadPpmAsChw(first_frame_path, &ih, &iw)); - idx.push_back(0); - } - if (!last_frame_path.empty()) { - int64_t h2 = 0, w2 = 0; - std::cerr << "loading last frame " << last_frame_path << "\n"; - imgs.push_back(ReadPpmAsChw(last_frame_path, &h2, &w2)); - if (ih == 0) { ih = h2; iw = w2; } - VT_CHECK(h2 == ih && w2 == iw, - "minimax-h3-gen: the first and last frames must have the same size"); - idx.push_back(-1); - } - request.keyframe_frame_indices = idx; - request.imgvid_noise_aug = imgvid_noise_aug; - request.keyframe_cond_rows = vllm::MiniMaxH3EncodeKeyframeCondRows( - enc_cfg, enc_w, dit.params, imgs, ih, iw, request.latent_t, imgvid_noise_aug, {}); - std::cerr << " keyframe conditioning: " << imgs.size() << " frame(s) at " << iw << "x" << ih - << " -> " << request.keyframe_cond_rows.size() << " floats\n"; - } - - // Real conditioning when a prompt was encoded; otherwise the supplied file. - const std::vector prompt_embeds = - !encoded_prompt.empty() ? encoded_prompt : ReadF32(embeds_path); - if (dit.params.text_dim <= 0 || prompt_embeds.size() % static_cast(dit.params.text_dim) != 0) { - throw std::runtime_error("--prompt-embeds size is not a multiple of text_dim"); - } - request.text_len = static_cast(prompt_embeds.size()) / dit.params.text_dim; - std::cerr << " text_len=" << request.text_len << " latent=" << request.latent_t << "x" - << request.latent_h << "x" << request.latent_w << " steps=" << request.num_steps - << "\n"; - - if (dry_run) { - std::cerr << "--dry-run: everything LOADED and planned; stopping before generation\n"; - return 0; - } - - // --- 4. generate --- - // NOISE IS AN INPUT (upstream seeds a torch CPU generator, and reproducing its - // RNG bit-exactly decides WHICH sample you get, not whether the pipeline is - // right), so it is drawn here from the shared deterministic stream. - const int64_t frame_rows = (request.latent_h / dit.params.patch_size_h) * - (request.latent_w / dit.params.patch_size_w); - const int64_t video_rows = request.latent_t * frame_rows; - const int64_t audio_rows = request.audio_t * request.audio_channel; - // A splitmix64 stream, seeded so a run is REPRODUCIBLE. This deliberately does - // NOT reproduce torch's RNG: matching it bit-for-bit decides WHICH sample you - // get, not whether the pipeline is correct, and pretending otherwise would - // invite comparing our sample against upstream's as if they should match. - // A flow-matching model is trained with GAUSSIAN N(0,1) noise at sigma=1 - // (torch.randn); feeding uniform[-1,1] (std 0.577) is out-of-distribution. - // Upstream draws torch.randn — Gaussian is the DEFAULT (mirror policy); - // VT_H3_GAUSSIAN_NOISE=0 keeps the legacy uniform draw for the A/B. - const char* gn = std::getenv("VT_H3_GAUSSIAN_NOISE"); - const bool gaussian = !(gn != nullptr && gn[0] == '0'); - auto fill = [gaussian](std::vector& out, uint64_t seed) { - uint64_t x = seed; - auto u01 = [&x]() { - x += 0x9E3779B97F4A7C15ULL; - uint64_t z = x; - z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL; - z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL; - z ^= z >> 31; - return (z >> 11) * 0x1.0p-53; // [0,1) - }; - for (size_t i = 0; i < out.size(); ++i) { - if (gaussian) { - double u1 = u01(), u2 = u01(); - if (u1 < 1e-12) u1 = 1e-12; - out[i] = static_cast(std::sqrt(-2.0 * std::log(u1)) * - std::cos(2.0 * 3.14159265358979323846 * u2)); - } else { - out[i] = static_cast(u01() * 2.0 - 1.0); // uniform [-1,1] - } - } - }; - std::vector noise_video( - static_cast(video_rows * dit.params.video_row_width())); - std::vector noise_audio( - static_cast(audio_rows * dit.params.audio_latents_dim)); - fill(noise_video, 0x5EED1234ULL); - fill(noise_audio, 0x5EED5678ULL); - - std::cerr << "generating (" << request.num_steps << " steps)...\n"; - // On a device, the denoise loop stages the DiT weights ONCE and runs every step - // device-resident. On CPU it uses the portable reference forward, which is a - // correctness path, not a throughput one. - vt::Device device{}; - if (device_name == "cuda") { - device = vt::GetBackend(vt::DeviceType::kCUDA).CreateQueue().device; - } else if (device_name != "cpu") { - throw std::runtime_error("--device must be cpu or cuda"); - } - // Stage ONCE here rather than inside the denoise loop. --dequant-bf16 trades - // memory (DiT ~33 GB bf16 vs 15.6 GB kept-quant) for GEMM throughput: the - // keep-quant path measured ~103 GFLOP/s, which is what makes a full-quality - // render a multi-day job. - vllm::MiniMaxH3DitDeviceWeights staged; - const vllm::MiniMaxH3DitDeviceWeights* prestaged = nullptr; - if (have_streamed) { - prestaged = &streamed; - } else if (device.type != vt::DeviceType::kCPU) { - vt::Queue sq = vt::GetBackend(device.type).CreateQueue(); - const auto t0 = std::chrono::steady_clock::now(); - staged = vllm::StageMiniMaxH3DitWeights(sq, dit.params, dit.weights, vt::DType::kBF16); - std::cerr << " staged DiT (" << (dequant_bf16 ? "dequant-bf16" : "keep-quant") << ") in " - << std::chrono::duration(std::chrono::steady_clock::now() - t0).count() - << " s\n"; - prestaged = &staged; - } - if (denoise_only) { - // Time the DiT step loop by itself. Reported as an AVERAGE over the requested - // steps rather than a single step: the first step pays one-off costs (RoPE - // caches, allocator warm-up) that do not recur, so a one-step run overstates - // the steady-state cost. - const auto t0 = std::chrono::steady_clock::now(); - const vllm::MiniMaxH3DenoiseResult denoised = vllm::MiniMaxH3DenoiseT2va( - device, request, dit.params, dit.weights, prompt_embeds, noise_video, noise_audio, - vt::DType::kBF16, prestaged); - const double elapsed = - std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); - const int64_t seq = static_cast(request.text_len) + video_rows + audio_rows; - // A flow-matching schedule of N sigmas runs N-1 DiT FORWARDS (each step maps - // sigma[i] -> sigma[i+1]), which is exactly what the `[h3] step i/N-1` trace - // prints. Dividing the elapsed time by `num_steps` therefore UNDER-reports the - // per-forward cost by N/(N-1) -- 1.5x at the 3-step setting used for kernel - // A/Bs, which is large enough to make two measurements look like a regression. - // Report the per-FORWARD cost, which is what the trace and any kernel - // comparison are denominated in. - const int64_t forwards = request.num_steps > 1 ? request.num_steps - 1 : 1; - std::cout << "denoise-only: seq=" << seq << " steps=" << request.num_steps - << " forwards=" << forwards << " total=" << elapsed - << " s per_forward=" << (elapsed / double(forwards)) << " s\n"; - // Touch the result so the loop cannot be optimized away, and so an all-NaN - // forward shows up here instead of passing as a fast run. - double checksum = 0.0; - for (const float v : denoised.video_rows) checksum += v; - std::cout << " video_rows=" << denoised.video_rows.size() << " checksum=" << checksum - << "\n"; - return 0; + vllm_video_model_params mp = vllm_video_model_params_default(); + vllm_video_params vp = vllm_video_params_default(); + std::string workdir = "/tmp/minimax_h3_gen", out_path, ffmpeg = "ffmpeg", device = "cpu"; + + for (int i = 1; i < argc; ++i) { + const std::string f = argv[i]; + if (f == "--dit") mp.dit_path = Need(argc, argv, ++i, "--dit"); + else if (f == "--encoder") mp.encoder_path = Need(argc, argv, ++i, "--encoder"); + else if (f == "--tokenizer") mp.tokenizer_path = Need(argc, argv, ++i, "--tokenizer"); + else if (f == "--video-vae") mp.video_vae_path = Need(argc, argv, ++i, "--video-vae"); + else if (f == "--video-vae-config") mp.video_vae_config_path = Need(argc, argv, ++i, f.c_str()); + else if (f == "--audio-vae") mp.audio_vae_path = Need(argc, argv, ++i, "--audio-vae"); + else if (f == "--audio-vae-config") mp.audio_vae_config_path = Need(argc, argv, ++i, f.c_str()); + else if (f == "--prompt-embeds") mp.prompt_embeds_path = Need(argc, argv, ++i, f.c_str()); + else if (f == "--partition") mp.partition = Need(argc, argv, ++i, "--partition"); + else if (f == "--device") device = Need(argc, argv, ++i, "--device"); + else if (f == "--keep-quant") mp.dequant_bf16 = 0; // the default arm, named + else if (f == "--dequant-bf16") mp.dequant_bf16 = 1; + else if (f == "--fp4-resident") mp.fp4_resident = 1; + else if (f == "--prompt") vp.prompt = Need(argc, argv, ++i, "--prompt"); + else if (f == "--steps") vp.steps = std::atoi(Need(argc, argv, ++i, "--steps")); + else if (f == "--frames") vp.num_frames = std::atoi(Need(argc, argv, ++i, "--frames")); + else if (f == "--height") vp.height = std::atoi(Need(argc, argv, ++i, "--height")); + else if (f == "--width") vp.width = std::atoi(Need(argc, argv, ++i, "--width")); + else if (f == "--seed") { + vp.seed = static_cast(std::strtoull(Need(argc, argv, ++i, "--seed"), nullptr, 10)); + vp.has_seed = 1; + } else if (f == "--first-frame") vp.first_frame = Need(argc, argv, ++i, "--first-frame"); + else if (f == "--last-frame") vp.last_frame = Need(argc, argv, ++i, "--last-frame"); + else if (f == "--ref-image") vp.ref_image = Need(argc, argv, ++i, "--ref-image"); + else if (f == "--ref-video") vp.ref_video = Need(argc, argv, ++i, "--ref-video"); + else if (f == "--ref-audio") vp.ref_audio = Need(argc, argv, ++i, "--ref-audio"); + else if (f == "--noise-aug") vp.noise_aug = std::strtof(Need(argc, argv, ++i, "--noise-aug"), nullptr); + else if (f == "--workdir") workdir = Need(argc, argv, ++i, "--workdir"); + else if (f == "--out") out_path = Need(argc, argv, ++i, "--out"); + else if (f == "--ffmpeg") ffmpeg = Need(argc, argv, ++i, "--ffmpeg"); + else if (f == "--help" || f == "-h") Usage(0); + else { + std::fprintf(stderr, "error: unknown argument: %s\n", f.c_str()); + Usage(2); } + } + if (mp.dit_path == nullptr) Usage(2); + if (device == "cuda") mp.device = 1; + else if (device != "cpu") { + std::fprintf(stderr, "error: --device must be cpu or cuda\n"); + return 2; + } + vp.output_dir = workdir.c_str(); - const vllm::MiniMaxH3T2vaResult result = vllm::MiniMaxH3GenerateT2va( - device, request, dit.params, dit.weights, video_cfg, video_weights, audio_cfg, - audio_weights, prompt_embeds, noise_video, noise_audio, vt::DType::kBF16, prestaged); + vllm_video_engine* engine = nullptr; + if (vllm_video_engine_load(&mp, &engine) != VLLM_OK) { + std::fprintf(stderr, "error: %s\n", vllm_last_error()); + return 1; + } - // --- 5. artifacts (the LIBRARY builds these; nothing spawns) --- - std::string mkdir_cmd = "mkdir -p '" + workdir + "'"; - if (std::system(mkdir_cmd.c_str()) != 0) throw std::runtime_error("cannot create " + workdir); - for (int64_t f = 0; f < result.frame_shape.t; ++f) { - char name[512]; - std::snprintf(name, sizeof(name), "%s/frame_%06lld.ppm", workdir.c_str(), - static_cast(f)); - WriteFile(name, vllm::MiniMaxH3WritePpmFrame(result.frames, result.frame_shape, f)); - } - const std::string wav_path = workdir + "/audio.wav"; - WriteFile(wav_path, vllm::MiniMaxH3WriteWav(result.waveform, result.audio_channels, - result.audio_samples_per_channel, - result.sample_rate)); - std::cerr << " wrote " << result.frame_shape.t << " frames + " << wav_path << "\n"; + vllm_video_result out; + if (vllm_video_generate(engine, &vp, &out) != VLLM_OK) { + std::fprintf(stderr, "error: %s\n", vllm_last_error()); + vllm_video_engine_free(engine); + return 1; + } + std::fprintf(stderr, " wrote %d frames (%dx%d @ %d fps) + %s (%d Hz)\n", out.frame_count, + out.width, out.height, out.fps, out.audio_path, out.sample_rate); - // --- 6. mux (the ONE process spawn, and it is in examples/ by decision) --- - vllm::MiniMaxH3MuxRequest mux; - mux.frame_pattern = workdir + "/frame_%06d.ppm"; - mux.audio_path = wav_path; - mux.output_path = out_path; - std::vector args = vllm::MiniMaxH3BuildMp4MuxArgs(mux); + int status = 0; + if (!out_path.empty()) { + // Mux to the REQUESTED path: same composer the result's own argv used, + // pointed at --out, with --ffmpeg substituted for argv[0]. + const std::string pattern = std::string(out.frame_dir) + "/frame_%06d.ppm"; + vllm_video_mux_params mx = vllm_video_mux_params_default(); + mx.frames = pattern.c_str(); + mx.audio_path = out.audio_path; + mx.output_path = out_path.c_str(); + char** mux_argv = nullptr; + int32_t mux_argc = 0; + if (vllm_video_mux_argv(&mx, &mux_argv, &mux_argc) != VLLM_OK) { + std::fprintf(stderr, "error: %s\n", vllm_last_error()); + vllm_video_result_free(&out); + vllm_video_engine_free(engine); + return 1; + } + std::vector args(mux_argv, mux_argv + mux_argc); if (!args.empty()) args[0] = ffmpeg; - const int status = RunFfmpeg(args); - if (status != 0) { - std::cerr << "ffmpeg exited " << status << "\n"; - return status; + vllm_video_mux_argv_free(mux_argv, mux_argc); + status = RunFfmpeg(args); + if (status == 0) { + std::printf("wrote %s\n", out_path.c_str()); + } else { + std::fprintf(stderr, "ffmpeg exited %d\n", status); } - std::cout << "wrote " << out_path << "\n"; - return 0; - } catch (const std::exception& e) { - std::cerr << "error: " << e.what() << "\n"; - return 1; } + + vllm_video_result_free(&out); + vllm_video_engine_free(engine); + return status; } diff --git a/examples/minimax_h3_mux/main.cpp b/examples/minimax_h3_mux/main.cpp index 081feeb98..bf8c99710 100644 --- a/examples/minimax_h3_mux/main.cpp +++ b/examples/minimax_h3_mux/main.cpp @@ -1,18 +1,18 @@ // minimax-h3-mux: muxes a MiniMax-H3 clip (PPM frames + WAV) into an MP4 by -// INVOKING ffmpeg. +// INVOKING ffmpeg — a THIN CLIENT of the public C ABI (include/vllm.h) and +// nothing else, per the ONE SURFACE directive (ARCH-ONE-SURFACE ROW 2). // -// THIS FILE IS THE RATIFIED HOME OF THE PROCESS SPAWN. The developer's decision -// (2026-08-03): "re: ffmpeg invocation, correct - let's keep in the examples -// only". So the split is deliberate and load-bearing: +// THIS FILE IS THE RATIFIED HOME OF THE PROCESS SPAWN. The developer's +// decision (2026-08-03): "re: ffmpeg invocation, correct - let's keep in the +// examples only". So the split is deliberate and load-bearing: // -// src/vllm/ builds the ARTIFACTS (MiniMaxH3WritePpmFrame, MiniMaxH3WriteWav) -// and the ARGV (MiniMaxH3BuildMp4MuxArgs) -- and spawns NOTHING. -// examples/ (here) performs the invocation. +// the LIBRARY writes the artifacts and composes the ARGV — reachable here +// through vllm_video_mux_argv (and, for a whole generation, +// vllm_video_generate's result) — and spawns NOTHING. +// examples/ (here) performs the invocation. // -// That is also why `/v1/videos` takes a caller-supplied `VideoRunner` callback -// rather than muxing itself: RunFfmpeg below is precisely the piece a server -// embedder plugs into ApiServer::set_video_runner, and it lives outside the -// library on purpose. Keep it that way -- do not move fork/exec into src/vllm/. +// The argv printed below is byte-identical to the pre-fold binary's +// (--print-only golden, tests/vllm/models/fixtures/minimax_h3_video_fold). // // Usage: // minimax-h3-mux --frames --out [--audio ] @@ -26,15 +26,13 @@ #include #include -#include +#include #include -#include -#include -#include +#include #include #include -#include "vllm/model_executor/models/minimax_h3.h" +#include "vllm.h" namespace { @@ -49,87 +47,100 @@ int RunFfmpeg(const std::vector& args) { c_args.push_back(nullptr); const pid_t pid = fork(); - if (pid < 0) throw std::runtime_error("fork failed"); + if (pid < 0) { + std::fprintf(stderr, "error: fork failed\n"); + return -1; + } if (pid == 0) { execvp(c_args[0], c_args.data()); - // Only reached if exec failed; _exit (not exit) so the child never runs the - // parent's atexit handlers or flushes its buffers a second time. + // Only reached if exec failed; _exit (not exit) so the child never runs + // the parent's atexit handlers or flushes its buffers a second time. _exit(127); } int status = 0; - if (waitpid(pid, &status, 0) < 0) throw std::runtime_error("waitpid failed"); + if (waitpid(pid, &status, 0) < 0) { + std::fprintf(stderr, "error: waitpid failed\n"); + return -1; + } if (WIFSIGNALED(status)) { - throw std::runtime_error("ffmpeg died on signal " + - std::to_string(WTERMSIG(status))); + std::fprintf(stderr, "error: ffmpeg died on signal %d\n", WTERMSIG(status)); + return -1; } return WIFEXITED(status) ? WEXITSTATUS(status) : -1; } -std::string Need(int argc, char** argv, int i, const std::string& flag) { - if (i >= argc) throw std::runtime_error("missing value for " + flag); +const char* Need(int argc, char** argv, int i, const char* flag) { + if (i >= argc) { + std::fprintf(stderr, "error: missing value for %s\n", flag); + std::exit(2); + } return argv[i]; } } // namespace int main(int argc, char** argv) { - vllm::MiniMaxH3MuxRequest request; + vllm_video_mux_params request = vllm_video_mux_params_default(); std::string ffmpeg = "ffmpeg"; bool print_only = false; - try { - for (int i = 1; i < argc; ++i) { - const std::string flag = argv[i]; - if (flag == "--frames") { - request.frame_pattern = Need(argc, argv, ++i, flag); - } else if (flag == "--audio") { - request.audio_path = Need(argc, argv, ++i, flag); - } else if (flag == "--out") { - request.output_path = Need(argc, argv, ++i, flag); - } else if (flag == "--fps") { - request.fps = std::stoll(Need(argc, argv, ++i, flag)); - } else if (flag == "--crf") { - request.crf = std::stoll(Need(argc, argv, ++i, flag)); - } else if (flag == "--ffmpeg") { - ffmpeg = Need(argc, argv, ++i, flag); - } else if (flag == "--print-only") { - print_only = true; - } else { - throw std::runtime_error("unknown argument: " + flag); - } - } - if (request.frame_pattern.empty() || request.output_path.empty()) { - std::cerr << "usage: minimax-h3-mux --frames --out " - "[--audio ] [--fps N] [--crf N] [--ffmpeg ] " - "[--print-only]\n"; + for (int i = 1; i < argc; ++i) { + const std::string flag = argv[i]; + if (flag == "--frames") { + request.frames = Need(argc, argv, ++i, "--frames"); + } else if (flag == "--audio") { + request.audio_path = Need(argc, argv, ++i, "--audio"); + } else if (flag == "--out") { + request.output_path = Need(argc, argv, ++i, "--out"); + } else if (flag == "--fps") { + request.fps = std::atoi(Need(argc, argv, ++i, "--fps")); + } else if (flag == "--crf") { + request.crf = std::atoi(Need(argc, argv, ++i, "--crf")); + } else if (flag == "--ffmpeg") { + ffmpeg = Need(argc, argv, ++i, "--ffmpeg"); + } else if (flag == "--print-only") { + print_only = true; + } else { + std::fprintf(stderr, "error: unknown argument: %s\n", flag.c_str()); return 2; } + } + if (request.frames == nullptr || request.output_path == nullptr) { + std::fprintf(stderr, + "usage: minimax-h3-mux --frames --out " + "[--audio ] [--fps N] [--crf N] [--ffmpeg ] " + "[--print-only]\n"); + return 2; + } - // The LIBRARY decides the encoding contract (h264/yuv420p + AAC, -shortest, - // +faststart); this file only runs it. - std::vector args = vllm::MiniMaxH3BuildMp4MuxArgs(request); - if (!args.empty()) args[0] = ffmpeg; + // The LIBRARY decides the encoding contract (h264/yuv420p + AAC, -shortest, + // +faststart); this file only runs it. + char** mux_argv = nullptr; + int32_t mux_argc = 0; + if (vllm_video_mux_argv(&request, &mux_argv, &mux_argc) != VLLM_OK) { + std::fprintf(stderr, "error: %s\n", vllm_last_error()); + return 1; + } + std::vector args(mux_argv, mux_argv + mux_argc); + vllm_video_mux_argv_free(mux_argv, mux_argc); + if (!args.empty()) args[0] = ffmpeg; - for (size_t i = 0; i < args.size(); ++i) { - std::cout << (i == 0 ? "" : " ") << args[i]; - } - std::cout << "\n"; - if (print_only) return 0; + for (size_t i = 0; i < args.size(); ++i) { + std::printf("%s%s", i == 0 ? "" : " ", args[i].c_str()); + } + std::printf("\n"); + if (print_only) return 0; - const int status = RunFfmpeg(args); - if (status == 127) { - std::cerr << "failed to exec '" << ffmpeg - << "' — is ffmpeg installed and on PATH?\n"; - return 127; - } - if (status != 0) { - std::cerr << "ffmpeg exited " << status << "\n"; - return status; - } - std::cout << "wrote " << request.output_path << "\n"; - return 0; - } catch (const std::exception& e) { - std::cerr << "error: " << e.what() << "\n"; - return 1; + const int status = RunFfmpeg(args); + if (status == 127) { + std::fprintf(stderr, "failed to exec '%s' — is ffmpeg installed and on PATH?\n", + ffmpeg.c_str()); + return 127; + } + if (status != 0) { + std::fprintf(stderr, "ffmpeg exited %d\n", status); + return status; } + std::printf("wrote %s\n", request.output_path); + return 0; } diff --git a/examples/server/main.cpp b/examples/server/main.cpp index 0d0ab66c6..66777c48c 100644 --- a/examples/server/main.cpp +++ b/examples/server/main.cpp @@ -42,22 +42,21 @@ #include #include #include -#ifdef VT_BENCH_PROFILE_CONTROL #include +#include +#include +#ifdef VT_BENCH_PROFILE_CONTROL #include #include #include #include #include -#include #endif #include "vllm/config/kv_transfer.h" #include "vllm/config/scheduler.h" #include "vllm/entrypoints/chat_template.h" #include "vllm/entrypoints/model_loader.h" -#include "vllm/model_executor/models/minimax_h3.h" -#include "vllm/model_executor/model_loader/gguf_reader.h" #include #include "vllm/entrypoints/openai/api_server.h" #include "vllm/entrypoints/openai/chat_mm.h" @@ -70,6 +69,7 @@ #include "vllm/model_executor/models/qwen3_5_weights.h" #include "vllm/transformers_utils/hf_config.h" #include "vllm/model_executor/models/model_registry.h" +#include "vllm/multimodal/minimax_h3_video.h" #include "vllm/multimodal/parakeet_transcription.h" #include "vllm/tokenizer/tokenizer.h" #include "vllm/version.h" @@ -96,103 +96,30 @@ namespace fs = std::filesystem; using vllm::HfConfig; using vllm::Qwen3_5MoeWeights; -// Decode a binary PPM (P6) into [3, H, W] floats in [-1, 1] — the layout the H3 -// video-VAE encoder takes. It reads from BYTES rather than a path so one decoder -// serves both spellings of `input_reference` (a filesystem path and an inline -// data: URL). PPM is the only still-image container this tree can read: no PNG / -// JPEG codec is vendored, which is the same NAMED residual the chat multimodal -// path carries (see chat_mm.h), not a limitation of this endpoint. -std::vector DecodePpmChw(const std::string& bytes, int64_t* out_h, - int64_t* out_w) { - std::istringstream in(bytes, std::ios::binary); - std::string magic; - in >> magic; - if (magic != "P6") { - throw std::runtime_error( - "input_reference: not a binary PPM (P6); no PNG/JPEG codec is vendored, " - "so a reference image must be supplied as binary PPM"); +// Run an argv to completion and return its exit status — the ONE process +// spawn in the MiniMax-H3 path, and it lives HERE, in examples/, by the +// developer-ratified 2026-08-03 decision: the library (the +// MiniMaxH3VideoEngine seam behind /v1/videos) writes the artifacts and +// BUILDS this argv, and spawns nothing. +int RunFfmpegArgv(const std::vector& args) { + std::vector c_args; + c_args.reserve(args.size() + 1); + for (const std::string& arg : args) { + c_args.push_back(const_cast(arg.c_str())); } - auto next_int = [&]() { - int v = 0; - while (in >> std::ws, in.peek() == '#') { std::string skip; std::getline(in, skip); } - in >> v; - return v; - }; - const int w = next_int(), h = next_int(), maxv = next_int(); - if (w <= 0 || h <= 0 || maxv <= 0) { - throw std::runtime_error("input_reference: bad PPM header"); + c_args.push_back(nullptr); + const pid_t pid = fork(); + if (pid < 0) throw std::runtime_error("fork failed"); + if (pid == 0) { + execvp(c_args[0], c_args.data()); + _exit(127); // exec failed; never run the parent's atexit handlers } - in.get(); // the single whitespace byte before the payload - std::vector rgb(static_cast(w) * h * 3); - in.read(reinterpret_cast(rgb.data()), static_cast(rgb.size())); - if (!in) throw std::runtime_error("input_reference: truncated PPM payload"); - std::vector chw(rgb.size()); - const int64_t plane = static_cast(w) * h; - for (int64_t i = 0; i < plane; ++i) { - for (int64_t c = 0; c < 3; ++c) { - chw[static_cast(c * plane + i)] = - static_cast(rgb[static_cast(i * 3 + c)]) / (maxv * 0.5f) - 1.0f; - } + int status = 0; + if (waitpid(pid, &status, 0) < 0) throw std::runtime_error("waitpid failed"); + if (WIFSIGNALED(status)) { + throw std::runtime_error("ffmpeg died on signal " + std::to_string(WTERMSIG(status))); } - if (out_h != nullptr) *out_h = h; - if (out_w != nullptr) *out_w = w; - return chw; -} - -// The bytes behind one reference: the parser hands us either a path or the -// already-decoded payload of a data: URL. -std::string ReadReferenceBytes(const std::string& field, const std::string& path, - const std::vector& inline_bytes) { - if (!inline_bytes.empty()) return std::string(inline_bytes.begin(), inline_bytes.end()); - std::ifstream in(path, std::ios::binary); - if (!in) throw std::runtime_error(field + ": cannot open " + path); - return std::string((std::istreambuf_iterator(in)), std::istreambuf_iterator()); -} - -// A ref2va VIDEO reference: DIR/frame_%06d.ppm, the exact layout minimax_h3_gen -// and this server's own muxer WRITE, so one run's frames chain straight into the -// next request. Returns [C, T, H, W] in [-1, 1] (the encoder's clip layout); -// no container demuxer is vendored, which is why this is a frame directory. -std::vector ReadReferenceClipChw(const std::string& dir, int64_t* out_t, - int64_t* out_h, int64_t* out_w) { - std::vector per_frame; // frame-major [T][C,H,W] - int64_t frames = 0, fh = 0, fw = 0; - for (int64_t k = 0;; ++k) { - char name[512]; - std::snprintf(name, sizeof(name), "%s/frame_%06lld.ppm", dir.c_str(), - static_cast(k)); - std::ifstream probe(name, std::ios::binary); - if (!probe) break; - const std::string bytes((std::istreambuf_iterator(probe)), - std::istreambuf_iterator()); - int64_t h = 0, w = 0; - const std::vector frame = DecodePpmChw(bytes, &h, &w); - if (frames == 0) { fh = h; fw = w; } - if (h != fh || w != fw) { - throw std::runtime_error( - "metadata.input_reference_video: every frame_%06d.ppm must have the same size"); - } - per_frame.insert(per_frame.end(), frame.begin(), frame.end()); - ++frames; - } - if (frames == 0) { - throw std::runtime_error("metadata.input_reference_video: no frame_%06d.ppm files in " + dir); - } - // [T][C,H,W] -> [C,T,H,W], the causal 3-D encoder's layout. - std::vector chw(per_frame.size()); - const int64_t plane = fh * fw; - for (int64_t c = 0; c < 3; ++c) { - for (int64_t k = 0; k < frames; ++k) { - for (int64_t e = 0; e < plane; ++e) { - chw[static_cast((c * frames + k) * plane + e)] = - per_frame[static_cast(k * 3 * plane + c * plane + e)]; - } - } - } - *out_t = frames; - *out_h = fh; - *out_w = fw; - return chw; + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; } struct Args { @@ -214,7 +141,10 @@ struct Args { int video_encoder_max_layers = 50; std::string video_ffmpeg = "ffmpeg", video_device = "cuda"; std::string video_partition; // served partition (fl2va|ref2va); see the #77 guard - bool video_keep_quant = false; + // Keep-quant is the library seam's DEFAULT arm; --video-dequant-bf16 selects + // the bf16 dequant/stream arm (the throughput trade the gen example ships). + // --video-keep-quant is still accepted (it names the default). + bool video_dequant_bf16 = false; int cuda_profile_graph_replays = 0; // trace-only diagnostic build seam. int cuda_profile_graph_batch = 0; // 0 => accepted c16 trace contract. std::string benchmark_shutdown_fifo; // paired trace-only control path. @@ -353,7 +283,10 @@ Args ParseArgs(int argc, char** argv) { } else if (flag == "--video-partition") { a.video_partition = NextArg(argc, argv, i, argv[0]); } else if (flag == "--video-keep-quant") { - a.video_keep_quant = true; + // the seam's default arm; accepted for pre-fold CLI compatibility + a.video_dequant_bf16 = false; + } else if (flag == "--video-dequant-bf16") { + a.video_dequant_bf16 = true; } else if (flag == "--enable-server-dev-mode") { a.enable_server_dev_mode = true; } else if (flag == "--enable-prefix-caching" || @@ -737,122 +670,38 @@ int main(int argc, char** argv) { // /reset_prefix_cache stay unwired (no live backing on the AsyncLLM path) — // see ConfigureUtilityEndpoints + specs/{utility,admin}-endpoints.md. ──────── // ── MiniMax-H3 video generation. OPT-IN: with no --video-dit the routes are - // never registered and the server is byte-identical to before. The runner is a - // CALLBACK because src/vllm/ must not spawn processes — the ffmpeg invocation - // is allowed here, in examples/, by the developer's ratified decision. ─────── - struct VideoState { - vllm::MiniMaxH3GgufDit dit; - vllm::MiniMaxH3VideoVaeDecoderConfig video_cfg; - vllm::MiniMaxH3AudioVaeConfig audio_cfg; - vllm::MiniMaxH3LatentStats video_stats, audio_stats; - vllm::MiniMaxH3AudioVaeWeights video_weights, audio_weights; - std::vector prompt_embeds; // fallback when no encoder is configured - // Encoder staged ONCE: staging the 32B tower costs ~162 s, so per-request - // staging would dominate every generation. - bool has_encoder = false; - vllm::MiniMaxH3EncoderConfig enc_config; - vllm::MiniMaxH3EncoderQuantWeights enc_host; - vllm::MiniMaxH3EncoderDeviceWeights enc_staged; - std::unique_ptr tokenizer; - vt::Queue enc_queue{}; - std::string workdir, ffmpeg; - // The served partition (fl2va|ref2va). Resolved ONCE at load from - // --video-partition; the per-request guard in MiniMaxH3GenerateT2va refuses a - // task this partition does not serve (the #77 follow-up). - vllm::MiniMaxH3PartitionInfo partition_info; - vt::Device device; - std::atomic counter{0}; - // The two VAEs' ENCODER halves, for the reference modalities: the video VAE - // encodes an `input_reference` image and a `metadata.input_reference_video` - // clip, the audio VAE encodes a `metadata.input_reference_audio` waveform. - // Loaded LAZILY and ONCE each: a text-to-video server must not pay for - // weights it never uses, and a server that does use them must not reload - // per request. - std::string video_vae_path, audio_vae_path; - std::mutex encoder_mutex; - bool video_encoder_loaded = false, audio_encoder_loaded = false; - vllm::MiniMaxH3AudioVaeWeights video_encoder_weights, audio_encoder_weights; - vllm::MiniMaxH3EncoderFcn3dConfig video_encoder_cfg; - vllm::MiniMaxH3AudioVaeEncoderConfig audio_encoder_cfg; - }; - std::shared_ptr video; + // never registered and the server is byte-identical to before. The whole + // pipeline lives in the LIBRARY seam (vllm::multimodal::MiniMaxH3VideoEngine, + // ARCH-ONE-SURFACE ROW 2) — the SAME entry point the C ABI's vllm_video_* + // and the minimax-h3-gen example drive, so HTTP and FFI cannot drift. This + // file keeps exactly what an example may own: flag plumbing, the job + // directory, and the ONE process spawn (ffmpeg, ratified 2026-08-03 — the + // library builds the argv and spawns nothing). ──────────────────────────── + std::shared_ptr video_engine; if (!args.video_dit.empty()) { - video = std::make_shared(); std::cerr << "server: loading MiniMax-H3 video checkpoints...\n"; - if (args.video_dit.size() > 5 && - args.video_dit.compare(args.video_dit.size() - 5, 5, ".gguf") == 0) { - const vllm::GgufFile f = vllm::GgufFile::Open(args.video_dit); - video->dit = vllm::LoadMiniMaxH3DitFromGguf(f, args.video_keep_quant); - } else { - const vllm::SafetensorsFile f = vllm::SafetensorsFile::Open(args.video_dit); - video->dit = vllm::LoadMiniMaxH3DitFromNvfp4(f); - } - if (!args.video_vae_config.empty()) { - std::ifstream in(args.video_vae_config); - nlohmann::json j; - in >> j; - video->video_cfg = vllm::ParseMiniMaxH3VideoVaeDecoderConfig(j, &video->video_stats); - } - if (!args.audio_vae_config.empty()) { - std::ifstream in(args.audio_vae_config); - nlohmann::json j; - in >> j; - video->audio_cfg = vllm::ParseMiniMaxH3AudioVaeConfig(j, &video->audio_stats); - } - { - const vllm::SafetensorsFile f = vllm::SafetensorsFile::Open(args.video_vae); - video->video_weights = vllm::LoadMiniMaxH3VideoVaeDecoderWeights(f); - } - { - const vllm::SafetensorsFile f = vllm::SafetensorsFile::Open(args.audio_vae); - video->audio_weights = vllm::LoadMiniMaxH3AudioVaeWeights(f); - } - if (!args.video_prompt_embeds.empty()) { - std::ifstream in(args.video_prompt_embeds, std::ios::binary | std::ios::ate); - const std::streamsize n = in.tellg(); - in.seekg(0); - video->prompt_embeds.resize(static_cast(n) / sizeof(float)); - in.read(reinterpret_cast(video->prompt_embeds.data()), n); - } - if (!args.video_encoder.empty()) { - std::cerr << "server: loading MiniMax-H3 encoder (keep-quant)...\n"; - const vllm::GgufFile ef = vllm::GgufFile::Open(args.video_encoder); - video->enc_host = vllm::LoadMiniMaxH3EncoderFromGguf(ef, args.video_encoder_max_layers); - video->enc_config = video->enc_host.config; - // The ComfyUI-style encoder export is WEIGHTS ONLY, so the vocab comes from - // the checkpoint tokenizer.json unless the GGUF happens to embed one. - video->tokenizer = std::make_unique( - args.video_tokenizer.empty() - ? vllm::tok::Tokenizer::FromGguf(ef) - : vllm::tok::Tokenizer::FromHfJson(args.video_tokenizer)); - video->enc_queue = vt::Queue{video->device, nullptr}; - if (video->device.type != vt::DeviceType::kCPU) { - video->enc_queue = vt::GetBackend(video->device.type).CreateQueue(); - } - std::cerr << "server: staging encoder to device (once)...\n"; - video->enc_staged = vllm::StageMiniMaxH3EncoderWeights(video->enc_queue, video->enc_host); - video->has_encoder = true; - std::cerr << "server: encoder ready (layers=" << video->enc_config.num_hidden_layers - << ", hidden=" << video->enc_config.hidden_size << ")\n"; - } - video->workdir = args.video_workdir; - video->ffmpeg = args.video_ffmpeg; - video->partition_info = vllm::MiniMaxH3PartitionFromFlag(args.video_partition); - video->video_vae_path = args.video_vae; - video->audio_vae_path = args.audio_vae; - if (args.video_device == "cuda") { - video->device = vt::GetBackend(vt::DeviceType::kCUDA).CreateQueue().device; - } - std::cerr << "server: /v1/videos on (dit layers=" << video->dit.params.num_layers - << ", device=" << args.video_device - << (args.video_keep_quant ? ", keep-quant" : "") << ")\n"; - // HONEST LIMIT, stated at startup rather than buried: turning a PROMPT into - // conditioning needs the H3-Encoder (a 32B tower + tokenizer), which is not - // wired here. Until it is, every request is conditioned on the SAME supplied - // embeddings, so the prompt text does NOT steer the output. - if (video->has_encoder) { + vllm::multimodal::MiniMaxH3VideoModelParams vmp; + vmp.dit_path = args.video_dit; + vmp.encoder_path = args.video_encoder; + vmp.tokenizer_path = args.video_tokenizer; + vmp.video_vae_path = args.video_vae; + vmp.video_vae_config_path = args.video_vae_config; + vmp.audio_vae_path = args.audio_vae; + vmp.audio_vae_config_path = args.audio_vae_config; + vmp.prompt_embeds_path = args.video_prompt_embeds; + vmp.partition = args.video_partition; + vmp.device = args.video_device == "cuda" ? 1 : 0; + vmp.dequant_bf16 = args.video_dequant_bf16 ? 1 : 0; + vmp.encoder_max_layers = args.video_encoder_max_layers; + video_engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(vmp); + std::cerr << "server: /v1/videos on (device=" << args.video_device + << (args.video_dequant_bf16 ? ", dequant-bf16" : ", keep-quant") << ")\n"; + // HONEST LIMIT, stated at startup rather than buried: turning a PROMPT + // into conditioning needs the H3-Encoder; without one every request is + // conditioned on the SAME supplied embeddings. + if (video_engine->has_encoder()) { std::cerr << "server: /v1/videos conditions on the request PROMPT\n"; - } else if (video->prompt_embeds.empty()) { + } else if (!video_engine->has_prompt_embeds()) { std::cerr << "server: WARNING /v1/videos has neither --video-encoder nor " "--video-prompt-embeds; requests will be REJECTED\n"; } else { @@ -860,238 +709,25 @@ int main(int argc, char** argv) { "--video-encoder to condition on it\n"; } - server.set_video_runner([video](const vllm::openai::VideoRequest& req) -> std::string { - // REAL text conditioning when an encoder is configured: tokenize the - // request prompt, gather its rows from the block-quant table, and run the - // already-staged tower. - std::vector conditioning; - if (video->has_encoder) { - const std::vector ids = video->tokenizer->Encode(req.prompt); - if (ids.empty()) throw std::runtime_error("the prompt tokenized to nothing"); - const std::vector embeds = - vllm::MiniMaxH3EncoderEmbedTokens(video->enc_host, ids); - const int64_t eseq = static_cast(ids.size()); - std::vector epos(static_cast(3 * eseq)); - for (int64_t a = 0; a < 3; ++a) { - for (int64_t s = 0; s < eseq; ++s) epos[static_cast(a * eseq + s)] = s; - } - vllm::MiniMaxH3EncoderConfig ec = video->enc_config; - conditioning = vllm::MiniMaxH3EncoderTextForwardDevice( - video->enc_queue, ec, video->enc_staged, embeds, epos.data(), eseq); - } else if (!video->prompt_embeds.empty()) { - conditioning = video->prompt_embeds; - } else { - throw std::runtime_error( - "video generation needs conditioning: start the server with " - "--video-encoder (to condition on the prompt) or --video-prompt-embeds"); - } - // OpenAI `input_reference` -> fl2va FIRST-FRAME conditioning. - // - // WHY fl2va and not ref2va: OpenAI documents input_reference as the image - // the generated video STARTS FROM (image-to-video), which is exactly what - // fl2va expresses — MiniMaxH3EncodeKeyframeCondRows pins frame 0 OF THE - // OUTPUT to the supplied image. ref2va - // (MiniMaxH3EncodeReferenceImages) means something else: whole reference - // images PREPENDED to the sequence as their own blocks, i.e. subject or - // style guidance that never becomes a frame of the result. Mapping - // input_reference there would silently change what the API promises. The - // ref2va modalities OpenAI has no slot for enter through `metadata` - // (input_reference_video / input_reference_audio) instead; the parser has - // already refused the combinations the pipeline forbids. - std::vector reference_chw; - int64_t reference_h = 0, reference_w = 0; - if (req.has_input_reference()) { - reference_chw = DecodePpmChw( - ReadReferenceBytes("input_reference", req.input_reference_path, - req.input_reference_bytes), - &reference_h, &reference_w); - } - std::vector reference_clip; - int64_t clip_t = 0, clip_h = 0, clip_w = 0; - if (req.has_input_reference_video()) { - reference_clip = ReadReferenceClipChw(req.input_reference_video_dir, &clip_t, - &clip_h, &clip_w); - } - - const vllm::MiniMaxH3DitParams& p = video->dit.params; - vllm::MiniMaxH3T2vaRequest r; - r.partition = video->partition_info; // #77 guard: MiniMaxH3GenerateT2va - // refuses a task this partition can't serve. - // With a reference image and no explicit task, the task IS fl2va; a - // metadata reference means ref2va. The image aspect also drives the - // default resolution (_resolve_shape). - const bool has_ref2va = - req.has_input_reference_video() || req.has_input_reference_audio(); - const std::string task = - !req.task.empty() - ? req.task - : (req.has_input_reference() ? "fl2va" : (has_ref2va ? "ref2va" : "t2va")); - const vllm::MiniMaxH3ShapePlan plan = vllm::MiniMaxH3ResolveShape( - task, req.duration_seconds, req.num_frames, req.height, req.width, - reference_w, reference_h); - r.latent_t = plan.latent_t; - r.num_frames = plan.num_frames; - r.latent_h = plan.height / vllm::kMiniMaxH3VaeRatio; - r.latent_w = plan.width / vllm::kMiniMaxH3VaeRatio; - r.audio_t = plan.audio_t; - r.audio_channel = vllm::kMiniMaxH3AudioChannels; - r.num_steps = req.num_inference_steps; - r.video_shift = req.flow_shift; - r.audio_shift = req.audio_flow_shift; - r.video_latents_mean = video->video_stats.mean; - r.video_latents_std = video->video_stats.std_dev; - r.audio_latents_mean = video->audio_stats.mean; - r.audio_latents_std = video->audio_stats.std_dev; - r.text_len = static_cast(conditioning.size()) / p.text_dim; - - // Both VAE encoder halves load at most once, under one lock, whichever - // reference modality asks for them first. - auto ensure_video_encoder = [&]() { - if (video->video_vae_path.empty()) { - throw std::runtime_error( - "a video/image reference needs the video VAE ENCODER half: start " - "the server with --video-vae"); - } - if (video->video_encoder_loaded) return; - const vllm::SafetensorsFile vf = vllm::SafetensorsFile::Open(video->video_vae_path); - video->video_encoder_weights = vllm::LoadMiniMaxH3VideoVaeEncoderWeights(vf); - video->video_encoder_cfg = vllm::MiniMaxH3EncoderFcn3dConfig{}; - video->video_encoder_cfg.z_channels = 2 * p.latents_dim; // mean | logvar - video->video_encoder_loaded = true; - }; - auto ensure_audio_encoder = [&]() { - if (video->audio_vae_path.empty()) { - throw std::runtime_error( - "metadata.input_reference_audio needs the audio VAE ENCODER half: " - "start the server with --audio-vae"); - } - if (video->audio_encoder_loaded) return; - const vllm::SafetensorsFile af = vllm::SafetensorsFile::Open(video->audio_vae_path); - video->audio_encoder_weights = vllm::LoadMiniMaxH3AudioVaeEncoderWeights(af); - video->audio_encoder_cfg = vllm::MiniMaxH3AudioVaeEncoderConfig{}; - video->audio_encoder_cfg.vae_latent_channels = p.audio_latents_dim; - video->audio_encoder_loaded = true; - }; - - if (req.has_input_reference()) { - if (reference_h != plan.height || reference_w != plan.width) { - // No image resampler is vendored, and a mis-sized keyframe would - // either abort deep in the denoise or pin the wrong latent rows. Say - // so up front, with the geometry we resolved. - throw std::runtime_error( - "input_reference is " + std::to_string(reference_w) + "x" + - std::to_string(reference_h) + " but this request resolved to " + - std::to_string(plan.width) + "x" + std::to_string(plan.height) + - "; supply the reference at the output size (or pass a matching " - "`size`): no image resampler is vendored"); - } - std::lock_guard guard(video->encoder_mutex); - ensure_video_encoder(); - r.keyframe_frame_indices = {0}; // FIRST frame; upstream also allows {-1}/{0,-1} - r.imgvid_noise_aug = 1.0; // pin the frame exactly - r.keyframe_cond_rows = vllm::MiniMaxH3EncodeKeyframeCondRows( - video->video_encoder_cfg, video->video_encoder_weights, p, {reference_chw}, - reference_h, reference_w, r.latent_t, r.imgvid_noise_aug, {}); - } else if (has_ref2va) { - // ── ref2va REFERENCE BLOCKS. Exclusive with the fl2va branch above - // (minimax_h3_pipeline.cpp:251), which the parser already enforced. ── - std::lock_guard guard(video->encoder_mutex); - std::vector blocks; - if (req.has_input_reference_video()) { - ensure_video_encoder(); - vllm::MiniMaxH3RefBlock block{}; - r.keyframe_cond_rows = vllm::MiniMaxH3EncodeReferenceVideo( - video->video_encoder_cfg, video->video_encoder_weights, p, reference_clip, - clip_t, clip_h, clip_w, &block); - // SILENT by construction: MiniMaxH3EncodeReferenceVideo emits - // ref_audio_t == 0 because the clip's own soundtrack would need the - // audio VAE encoder run over it. An audio reference below ATTACHES to - // this block, which is the layout packed_sequence.py builds. - blocks.push_back(block); - } - if (req.has_input_reference_audio()) { - ensure_audio_encoder(); - const std::string wav_bytes = ReadReferenceBytes( - "metadata.input_reference_audio", req.input_reference_audio_path, - req.input_reference_audio_bytes); - int64_t samples_per_channel = 0; - const std::vector waveform = vllm::MiniMaxH3ReadWav( - wav_bytes, vllm::kMiniMaxH3AudioChannels, vllm::kMiniMaxH3AudioSampleRate, - &samples_per_channel); - vllm::MiniMaxH3RefBlock audio_block{}; - r.audio_ref_rows = vllm::MiniMaxH3EncodeReferenceAudio( - video->audio_encoder_cfg, video->audio_encoder_weights, waveform, - vllm::kMiniMaxH3AudioChannels, samples_per_channel, video->audio_stats.mean, - video->audio_stats.std_dev, /*noise_aug=*/1.0, {}, &audio_block); - if (!blocks.empty() && - blocks[0].kind == vllm::MiniMaxH3RefBlock::Kind::kVideoAudio) { - // The reference video now HAS sound: one kVideoAudio block carries - // both, so its ref_audio_t must claim exactly the rows just encoded. - blocks[0].ref_audio_t = audio_block.ref_audio_t; - } else { - blocks.push_back(audio_block); - } - } - r.ref_blocks = blocks; - } - - const int64_t frame_rows = - (r.latent_h / p.patch_size_h) * (r.latent_w / p.patch_size_w); - std::vector nv(static_cast(r.latent_t * frame_rows * - p.video_row_width())); - std::vector na(static_cast(r.audio_t * r.audio_channel * - p.audio_latents_dim)); - uint64_t x = req.has_seed ? static_cast(req.seed) : 0x5EED1234ULL; - auto fill = [&x](std::vector& o) { - for (float& value : o) { - x += 0x9E3779B97F4A7C15ULL; - uint64_t z = x; - z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL; - z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL; - z ^= z >> 31; - value = static_cast((z >> 11) * 0x1.0p-53 * 2.0 - 1.0); - } - }; - fill(nv); - fill(na); - - const vllm::MiniMaxH3T2vaResult out = vllm::MiniMaxH3GenerateT2va( - video->device, r, p, video->dit.weights, video->video_cfg, video->video_weights, - video->audio_cfg, video->audio_weights, conditioning, nv, na, - vt::DType::kBF16); - - const int64_t id = video->counter.fetch_add(1); - const std::string dir = video->workdir + "/job" + std::to_string(id); - if (std::system(("mkdir -p '" + dir + "'").c_str()) != 0) { - throw std::runtime_error("cannot create " + dir); - } - for (int64_t f = 0; f < out.frame_shape.t; ++f) { - char name[512]; - std::snprintf(name, sizeof(name), "%s/frame_%06lld.ppm", dir.c_str(), - static_cast(f)); - std::ofstream fo(name, std::ios::binary); - const std::string bytes = - vllm::MiniMaxH3WritePpmFrame(out.frames, out.frame_shape, f); - fo.write(bytes.data(), static_cast(bytes.size())); - } - const std::string wav = dir + "/audio.wav"; - { - std::ofstream fo(wav, std::ios::binary); - const std::string bytes = vllm::MiniMaxH3WriteWav( - out.waveform, out.audio_channels, out.audio_samples_per_channel, - out.sample_rate); - fo.write(bytes.data(), static_cast(bytes.size())); + auto counter = std::make_shared>(0); + const std::string workdir = args.video_workdir; + const std::string ffmpeg = args.video_ffmpeg; + server.set_video_runner([video_engine, counter, workdir, + ffmpeg](const vllm::openai::VideoRequest& req) -> std::string { + const int64_t id = counter->fetch_add(1); + const std::string dir = workdir + "/job" + std::to_string(id); + // The library-owned request mapping + generation: conditioning, task + // resolution, the #77 partition guard, reference encoding, artifacts. + const vllm::multimodal::MiniMaxH3VideoResult out = video_engine->Generate( + vllm::multimodal::MiniMaxH3VideoGenParamsFromRequest(req, dir)); + // The ONE process spawn: exec the argv the library composed. + std::vector argv_mux = out.mux_argv; + if (!argv_mux.empty()) argv_mux[0] = ffmpeg; + const int status = RunFfmpegArgv(argv_mux); + if (status != 0) { + throw std::runtime_error("ffmpeg exited " + std::to_string(status)); } - vllm::MiniMaxH3MuxRequest mux; - mux.frame_pattern = dir + "/frame_%06d.ppm"; - mux.audio_path = wav; - mux.output_path = dir + "/video.mp4"; - std::vector argv_mux = vllm::MiniMaxH3BuildMp4MuxArgs(mux); - if (!argv_mux.empty()) argv_mux[0] = video->ffmpeg; - std::string cmd; - for (const std::string& a : argv_mux) cmd += "'" + a + "' "; - if (std::system(cmd.c_str()) != 0) throw std::runtime_error("ffmpeg failed"); - return mux.output_path; + return out.mux_output_path; }); } diff --git a/include/vllm.h b/include/vllm.h index 9cf643e2e..7d3c415c3 100644 --- a/include/vllm.h +++ b/include/vllm.h @@ -82,6 +82,18 @@ extern "C" { * default, so zero-filling the struct growth keeps a v9 engine byte-identical. * Scheduler policy (incl. SGLang's cache-aware LPM) is selected through the v9 * string field .scheduling_policy = "lpm" — there is no separate int knob. + * v12: VIDEO+AUDIO GENERATION (ARCH-ONE-SURFACE ROW 2, MiniMax-H3) — the + * ratified video slice: an opaque vllm_video_engine loaded from the H3 + * checkpoint set (vllm_video_engine_load/free, vllm_video_model_params + + * _default), one blocking vllm_video_generate (vllm_video_params + _default) + * producing a vllm_video_result (frame dir + WAV + geometry + the ffmpeg argv + * the CALLER may exec — the library spawns nothing), vllm_video_result_free, + * and the standalone vllm_video_mux_argv(+_free) composer the mux example + * client uses. Appended so zero values preserve behaviour; a pre-v12 caller + * that never touches the video symbols is byte-identical. Text and video + * handles refuse each other's checkpoints LOUDLY: vllm_video_engine_load on a + * text-model directory names vllm_engine_load, and vllm_engine_load on an H3 + * checkpoint directory keeps failing exactly as at v11 (no config.json). * v11: AUDIO TRANSCRIPTION (ARCH-ONE-SURFACE fold #4) — vllm_transcribe / * vllm_transcription_params(_default) / vllm_transcription(_free), appended so * zero values preserve behaviour. vllm_engine_load now RESOLVES a @@ -91,7 +103,7 @@ extern "C" { * message instead of serving, and vllm_transcribe on a TEXT handle does the * same. A pre-v11 caller that never loads a Parakeet directory is * byte-identical. */ -#define VLLM_ABI_VERSION 11 +#define VLLM_ABI_VERSION 12 /* ── Export macro ───────────────────────────────────────────────────────────── * Marks the symbols that make up the stable ABI. Default visibility now; Task 3 @@ -515,6 +527,121 @@ VLLM_API vllm_status vllm_transcribe(vllm_engine* engine, * struct itself is caller storage. NULL is a no-op. */ VLLM_API void vllm_transcription_free(vllm_transcription* out); + +/* ── Video+audio generation (ABI v12, MiniMax-H3) ──────────────────────────── + * The video slice of the ONE-SURFACE fold: the SAME library pipeline the + * bundled server's /v1/videos routes and the minimax-h3-gen example drive + * (vllm::multimodal::MiniMaxH3VideoEngine), reachable by any embedder. + * + * A video engine is loaded from the H3 checkpoint SET (the release ships the + * DiT, the text encoder and the two VAEs as separate artifacts), not from one + * model directory — which is why this is a separate handle from vllm_engine. + * Loading a TEXT checkpoint here fails with VLLM_ERR_MODEL_LOAD naming + * vllm_engine_load; loading an H3 checkpoint with vllm_engine_load keeps + * failing exactly as before v12 (its directory carries no config.json). + * + * THE PROCESS BOUNDARY: the library writes frames + WAV and COMPOSES the + * ffmpeg argv, and spawns nothing. The caller execs mux_argv to get an MP4. */ +typedef struct vllm_video_engine vllm_video_engine; + +/* All paths; NULL/empty means "not supplied". Borrowed for the load call. */ +typedef struct vllm_video_model_params { + const char* dit_path; /* GGUF | NVFP4 safetensors | bf16 shard dir */ + const char* encoder_path; /* H3-Encoder GGUF or bf16 shard dir */ + const char* tokenizer_path; /* tokenizer.json (with an encoder) */ + const char* video_vae_path; + const char* video_vae_config_path; + const char* audio_vae_path; + const char* audio_vae_config_path; + /* Fallback conditioning when no encoder is supplied: rows of text_dim, + * little-endian f32 (the pre-fold --prompt-embeds file). */ + const char* prompt_embeds_path; + /* The served checkpoint PARTITION: "fl2va" (serves t2va+fl2va) or "ref2va". + * Community GGUF/NVFP4 files strip the release metadata and the two DiTs + * are byte-structurally identical, so it must be DECLARED; NULL/empty makes + * every generate refuse with the guidance (the #77 guard). */ + const char* partition; + int32_t device; /* 0 cpu, 1 cuda */ + int32_t dequant_bf16; /* 0 keep-quant, 1 dequant/stream bf16 */ + int32_t fp4_resident; /* NVFP4+cuda: keep FP4 packed, Marlin W4A16 GEMM */ +} vllm_video_model_params; + +typedef struct vllm_video_params { + const char* prompt; /* encoded when the engine has an encoder */ + int32_t width, height; /* <= 0 => aspect-derived default canvas */ + int32_t num_frames; /* <= 1 => per-task default */ + int32_t steps; /* <= 0 => H3 default (50) */ + uint64_t seed; + int32_t has_seed; /* 0 => the fixed default noise streams */ + const char* first_frame; /* fl2va keyframes: binary PPM (P6) paths; */ + const char* last_frame; /* pin frame 0 / the last frame OF THE OUTPUT */ + const char* ref_image; /* ref2va: ONE whole reference image (PPM) */ + const char* ref_video; /* ref2va: a DIRECTORY of frame_%06d.ppm */ + const char* ref_audio; /* ref2va: a 16-bit PCM WAV path */ + float noise_aug; /* keyframe pinning strength; <= 0 => 1.0 */ + /* Where frame_%06d.ppm + audio.wav land (created if absent). REQUIRED. */ + const char* output_dir; +} vllm_video_params; + +/* One finished generation. OWNERSHIP: every member is library-allocated; + * free the whole struct's members via vllm_video_result_free(out). */ +typedef struct vllm_video_result { + char* frame_dir; /* holds frame_%06d.ppm */ + char* audio_path; /* 16-bit PCM WAV */ + int32_t frame_count, width, height, fps, sample_rate; + /* The ffmpeg argv the CALLER may exec to mux /video.mp4; + * argv[0] is "ffmpeg" (substitute a custom binary before exec'ing). + * mux_argv[mux_argc] is NULL, so it is execvp-ready. */ + char** mux_argv; + int32_t mux_argc; +} vllm_video_result; + +/* Zero-initialized params (every path NULL, cpu, keep-quant). */ +VLLM_API vllm_video_model_params vllm_video_model_params_default(void); +/* Default generation params (all defaults resolved engine-side). */ +VLLM_API vllm_video_params vllm_video_params_default(void); + +/* Load the H3 checkpoint set and stage its weights once. On VLLM_OK, *out is + * a handle the caller frees via vllm_video_engine_free. On error, *out is + * NULL and vllm_last_error() carries the detail (a text-model directory maps + * to VLLM_ERR_MODEL_LOAD naming vllm_engine_load). */ +VLLM_API vllm_status vllm_video_engine_load(const vllm_video_model_params* params, + vllm_video_engine** out); +VLLM_API void vllm_video_engine_free(vllm_video_engine* engine); + +/* Run one BLOCKING generation, filling *out. Serialized per engine handle. + * VLLM_ERR_INVALID_ARGUMENT for a missing output_dir / illegal reference + * combination; VLLM_ERR_RUNTIME when the pipeline refuses (undeclared or + * mismatched partition, missing conditioning, unreadable references) or the + * forward fails. On any non-OK status *out is zeroed. */ +VLLM_API vllm_status vllm_video_generate(vllm_video_engine* engine, + const vllm_video_params* params, + vllm_video_result* out); + +/* Free the owned members of a result and zero the struct. The struct itself + * is caller storage. NULL is a no-op. */ +VLLM_API void vllm_video_result_free(vllm_video_result* out); + +/* ── Standalone MP4 mux-argv composer ───────────────────────────────────────── + * The encoding contract (h264/yuv420p + AAC, -shortest, +faststart) is the + * library's; the caller execs. This is the engine-free entry the + * minimax-h3-mux example client uses to mux EXISTING frames + WAV. */ +typedef struct vllm_video_mux_params { + const char* frames; /* printf-style pattern, e.g. dir/frame_%06d.ppm */ + const char* audio_path; /* NULL/empty => a silent clip */ + const char* output_path; /* the .mp4 to write */ + int32_t fps; /* <= 0 => the H3 default (24) */ + int32_t crf; /* <= 0 => the library default (18) */ +} vllm_video_mux_params; + +VLLM_API vllm_video_mux_params vllm_video_mux_params_default(void); + +/* Compose the argv (argv[0] "ffmpeg"; *out_argv[*out_argc] is NULL so it is + * execvp-ready). The caller frees via vllm_video_mux_argv_free. */ +VLLM_API vllm_status vllm_video_mux_argv(const vllm_video_mux_params* params, + char*** out_argv, int32_t* out_argc); +VLLM_API void vllm_video_mux_argv_free(char** argv, int32_t argc); + /* ── Memory helpers ─────────────────────────────────────────────────────────── * Free a heap string returned by the library. NULL is a no-op. */ VLLM_API void vllm_string_free(char* s); diff --git a/include/vllm/multimodal/minimax_h3_video.h b/include/vllm/multimodal/minimax_h3_video.h new file mode 100644 index 000000000..6c459fa4e --- /dev/null +++ b/include/vllm/multimodal/minimax_h3_video.h @@ -0,0 +1,144 @@ +// MiniMax-H3 video+audio generation seam — the ONE library entry point that +// composes checkpoint loading (DiT GGUF/NVFP4/sharded-bf16, both VAEs, the +// optional H3-Encoder text tower) -> task resolution + the #77 partition guard +// -> conditioning (prompt encode / prompt-embeds file / fl2va keyframes / +// ref2va references) -> the deterministic noise draw -> MiniMaxH3GenerateT2va +// -> artifacts (frame_%06d.ppm + audio.wav) + the ffmpeg argv, and that every +// consumer (C ABI `vllm_video_*`, the OpenAI server's /v1/videos, the +// `minimax-h3-gen` example) drives. +// +// ARCH-ONE-SURFACE ROW 2 (video+audio generation). This file ABSORBS the +// assembly pipeline `examples/minimax_h3_gen/main.cpp` owned privately +// (pre-refactor main.cpp:687-1288 @ fc636c76: loader-arm dispatch, encoder +// conditioning, reference encoding, the splitmix64 noise streams, artifact +// writing, mux-argv assembly) and the twin copy `examples/server/main.cpp` +// carried for /v1/videos (pre-refactor main.cpp:743-1096). Per the ONE SURFACE +// directive the examples keep argv parsing, printing and the PROCESS SPAWN +// only; the capability lives here, reachable by any embedder. +// +// THE PROCESS BOUNDARY (developer-ratified 2026-08-03) stands: this seam +// WRITES artifacts and BUILDS the mux argv (MiniMaxH3BuildMp4MuxArgs) and +// spawns NOTHING — the caller execs `MiniMaxH3VideoResult::mux_argv`. +// +// Upstream mirror shape: vLLM-Omni serves H3 through +// vllm_omni/diffusion/models/minimax_h3/pipeline_minimax_h3.py (the pipeline +// object owns checkpoints + does per-request _resolve_task/_resolve_shape); +// this seam is the C++ engine-side equivalent of that pipeline object. +// +// Byte-identity contract: on the committed fold fixture the CPU t2va render is +// byte-identical to the PRE-fold `minimax-h3-gen` binary at fc636c76 +// (tests/vllm/models/test_minimax_h3_video_fold.cpp, three-arm gate). +#pragma once + +#include +#include +#include +#include + +namespace vllm::openai { +struct VideoRequest; // entrypoints/openai/video_api.h +} + +namespace vllm::multimodal { + +// ── Load-time parameters (the checkpoint set; the C ABI mirror is +// vllm_video_model_params). Empty string == "not supplied". ───────────────── +struct MiniMaxH3VideoModelParams { + std::string dit_path; // GGUF | NVFP4 safetensors | bf16 shard DIR + std::string encoder_path; // H3-Encoder GGUF or bf16 shard DIR + std::string tokenizer_path; // tokenizer.json (encoder conditioning) + std::string video_vae_path, video_vae_config_path; + std::string audio_vae_path, audio_vae_config_path; + // Fallback conditioning when no encoder is configured: rows of text_dim, + // little-endian f32 (the pre-fold --prompt-embeds / --video-prompt-embeds). + std::string prompt_embeds_path; + // The served checkpoint PARTITION ("fl2va" | "ref2va"). Community GGUF/NVFP4 + // files strip the release model_index.json `_minimax_h3` block and the two + // DiTs are byte-structurally identical, so it must be DECLARED; empty is + // declared-but-unknown and the #77 guard refuses every full render + // (MiniMaxH3PartitionFromFlag / MiniMaxH3CheckTaskPartition). + std::string partition; + int32_t device = 0; // 0 cpu, 1 cuda + int32_t dequant_bf16 = 0; // 0 keep-quant, 1 dequant/stream bf16 + // NVFP4 + cuda only: keep the packed FP4 resident and route the quantized + // projections through the Marlin W4A16 GEMM (the pre-fold --fp4-resident). + int32_t fp4_resident = 0; + int64_t encoder_max_layers = 0; // 0 => all layers +}; + +// ── Per-generation parameters (the C ABI mirror is vllm_video_params). ────── +struct MiniMaxH3VideoGenParams { + std::string prompt; // encoded when the engine has an encoder + // "" => resolved from the references/partition (upstream _resolve_task): + // a keyframe => fl2va, any ref2va reference => ref2va, else t2va. + std::string task; + double duration_seconds = 0.0; // <= 0 => per-task default + int64_t num_frames = 0; // <= 1 => per-task default + int64_t height = 0, width = 0; // <= 0 => aspect-derived default + int64_t steps = 0; // <= 0 => H3 default (50) + double flow_shift = 0.0; // <= 0 => H3 default (12.0) + double audio_flow_shift = 0.0; // <= 0 => H3 default (3.0) + uint64_t seed = 0; + bool has_seed = false; // false => the pre-fold fixed default streams + + // fl2va KEYFRAMES: binary PPM (P6), as a path or in-memory bytes (exactly + // one spelling per frame). Pins frame 0 / the last frame OF THE OUTPUT. + std::string first_frame_path, last_frame_path; + std::string first_frame_ppm; // in-memory alternative (server data: URLs) + double noise_aug = 1.0; // condition-noise augmentation (1.0 pins) + + // ref2va REFERENCES (exclusive with keyframes, minimax_h3_pipeline.cpp:251): + std::vector ref_image_paths; // whole reference images (PPM) + std::string ref_video_dir; // DIR of frame_%06d.ppm + std::string ref_audio_path; // 16-bit PCM WAV path... + std::string ref_audio_wav; // ...or its bytes + + // Where frame_%06d.ppm + audio.wav land (created if absent). REQUIRED. + std::string output_dir; +}; + +// ── One finished generation (the C ABI mirror is vllm_video_result). ──────── +struct MiniMaxH3VideoResult { + std::string frame_dir; // holds frame_%06d.ppm + std::string audio_path; // 16-bit PCM WAV + int64_t frame_count = 0, width = 0, height = 0; + int64_t fps = 0, sample_rate = 0; + // The ffmpeg argv the CALLER may exec to mux /video.mp4 + // (argv[0] is "ffmpeg"; substitute a custom binary before exec'ing). + std::vector mux_argv; + std::string mux_output_path; // the -o target mux_argv names +}; + +// A loaded H3 video checkpoint set, weights staged once, ready to generate. +// Construction throws std::runtime_error naming the problem on any mismatch. +class MiniMaxH3VideoEngine { + public: + static std::unique_ptr Load(const MiniMaxH3VideoModelParams& params); + + MiniMaxH3VideoEngine(MiniMaxH3VideoEngine&&) noexcept; + MiniMaxH3VideoEngine& operator=(MiniMaxH3VideoEngine&&) noexcept; + ~MiniMaxH3VideoEngine(); + + // True when an encoder tower is loaded (the request PROMPT conditions the + // render); false => prompt_embeds_path conditioning (or Generate refuses). + bool has_encoder() const; + bool has_prompt_embeds() const; + + // Run one blocking generation. Serialized internally (the staged weights are + // shared state); throws std::runtime_error to fail the request. + MiniMaxH3VideoResult Generate(const MiniMaxH3VideoGenParams& params); + + private: + MiniMaxH3VideoEngine(); + struct Impl; + std::unique_ptr impl_; +}; + +// The ONE mapping from a parsed /v1/videos request onto the seam's params — +// library-owned so the HTTP route and the FFI cannot drift (the pre-fold +// server carried this as a private lambda). `output_dir` is the job directory +// the artifacts land in. +MiniMaxH3VideoGenParams MiniMaxH3VideoGenParamsFromRequest( + const ::vllm::openai::VideoRequest& request, const std::string& output_dir); + +} // namespace vllm::multimodal diff --git a/scripts/abi-capability-allowlist.txt b/scripts/abi-capability-allowlist.txt index c26279d12..98d28c824 100644 --- a/scripts/abi-capability-allowlist.txt +++ b/scripts/abi-capability-allowlist.txt @@ -20,5 +20,4 @@ # Format: ` | fold= | `. embeddings / pooling | fold=ARCH-ONE-SURFACE | Engine-side pooler landed (ENG-POOLER-SEQ: pooler/{dispatch_pooler,methods,heads,poolers}.cpp + pool/pooling_runner) but PoolingRunner is never invoked by the live step, no arch sets is_pooling_model, /v1/embeddings is a run_batch "does not support endpoint" residual, and vllm.h has no embed/pool symbol. Fold: grow ABI pooling entry point, wire the live route, register a pooling arch -video+audio generation (minimax-h3) | fold=ARCH-ONE-SURFACE | The H3 diffusion pipeline serves over /v1/videos ONLY via an example-injected VideoRunner (set_video_runner, api_server.h:167; the library never sets it); vllm.h has no video symbol. Fold: the ARCH-ONE-SURFACE video ABI slice (order-of-work item 3), then rewrite minimax-h3-gen/server as clients multimodal input (image/audio/video) | fold=ARCH-ONE-SURFACE | vllm_chat/vllm_chat_stream are text-only (vllm.h:440,451); the C-ABI's EnsureChatServing never sets the mm seam (src/capi/vllm_c.cpp), and even the server's Qwen3-VL image seam does not run the vision tower in the live engine step (MM-SERVE-E2E residual). Fold: grow ABI multimodal-input entry point + the engine mm-forward diff --git a/scripts/check-surface-coverage.py b/scripts/check-surface-coverage.py index cc6e5f2b5..9c3cf5428 100644 --- a/scripts/check-surface-coverage.py +++ b/scripts/check-surface-coverage.py @@ -103,14 +103,16 @@ # as folds land (developer-directed 2026-08-07: no permanent exemptions, the allowlist # shrinks). A new internal-reacher cannot be added without consciously RAISING this # ceiling — a reviewable red flag, which is the point. Lower it as each example is folded -# onto the public ABI. 11 since the ROW 1 Parakeet fold (2026-08-07: -# parakeet_transcribe became a clean vllm.h client; before it, 12 = server + the 4 -# spec-named drivers + minimax_h3_mux + parakeet_transcribe + the 5 dev/diagnostic -# tools; examples/cli is the clean baseline). +# onto the public ABI. 9 since the ROW 2 MiniMax-H3 video fold (2026-08-08: +# minimax_h3_gen AND minimax_h3_mux became clean vllm.h clients of the v12 +# vllm_video_* slice); 11 after the ROW 1 Parakeet fold (2026-08-07: +# parakeet_transcribe); before it, 12 = server + the 4 spec-named drivers + +# minimax_h3_mux + parakeet_transcribe + the 5 dev/diagnostic tools; +# examples/cli is the clean baseline. # COUPLED: the ratchet claims in .agents/specs/surface-coverage-2026-08-07.md and the # state log, and the equality pin in tests/scripts/test_check_surface_coverage.py, must # move in the SAME change as this constant. -MAX_INTERNAL_REACHING = 11 +MAX_INTERNAL_REACHING = 9 # An include INTO the internal C++ tree: `#include "vllm/..."` (the engine/model/layer # headers under include/vllm/, NOT the flat public "vllm.h"), `#include "vt/..."` (the diff --git a/scripts/example-abi-allowlist.txt b/scripts/example-abi-allowlist.txt index c77161639..f56a38397 100644 --- a/scripts/example-abi-allowlist.txt +++ b/scripts/example-abi-allowlist.txt @@ -25,8 +25,6 @@ examples/deepseek_v4_gen | fold=ARCH-ONE-SURFACE | DeepSeek-V4-Flash keep-quant GGUF greedy decode (DeepseekV4ForwardGguf(Cached) + DeepseekV4KvCache) is CLI-only; the registered DeepseekV4ForCausalLM forward is a W3-W8 stub (deepseek_v4_registry.cpp:22). Grow ABI keep-quant GGUF load+decode, rewrite as ABI client, delete the bespoke forward examples/laguna_gen | fold=ARCH-ONE-SURFACE | Laguna-S-2.1 keep-quant GGUF (multi-shard) + NVFP4 W4A4 device-resident decode (LagunaForwardGguf(Cached), Marlin residents, fp4-shared) is CLI-only; registered LagunaForCausalLM forward VT_CHECK(false)s on non-bf16 (laguna.cpp:156). Grow ABI, rewrite as client, delete bespoke forward examples/kimi_linear_gen | fold=ARCH-ONE-SURFACE | Kimi-Linear-48B bf16-resident streaming loader + §18/§19 paged-incremental decode (ForwardPrefillIncremental/ForwardDecodeStepIncremental/KimiDecodeCache) is CLI-only (the recompute forward IS shared). Grow ABI incremental-decode path, rewrite as client, delete -examples/minimax_h3_gen | fold=ARCH-ONE-SURFACE | The ENTIRE MiniMax-H3 video+audio diffusion pipeline (DiT + text encoder + VAEs + fl2va/ref2va + vision tower + ffmpeg) is CLI-only; MiniMaxH3 is off-registry (no REGISTER_VLLM_MODEL). Grow the video-gen ABI slice (ARCH-ONE-SURFACE order-of-work), rewrite as client, delete the driver glue -examples/minimax_h3_mux | fold=ARCH-ONE-SURFACE | H3 PPM+WAV->MP4 ffmpeg mux helper (MiniMaxH3BuildMp4MuxArgs + fork/execvp) reaches internal minimax_h3 headers; the video-gen ABI slice should expose the assembled artifact so the mux can be a client (or fold into the video ABI). Grow ABI, rewrite, delete examples/server | fold=ARCH-ONE-SURFACE | The reference OpenAI server constructs the engine, metrics, video_runner and mm seam directly from internal C++ headers (LoadedEngine/AsyncLLM, minimax_h3.h, chat_mm). It should stand on the public surface (C ABI, or a curated public C++ API the ABI wraps). Grow the surface, rewrite the server against it # --- Dev / diagnostic tools: NO permanent exemption (developer-directed 2026-08-07) --- diff --git a/src/capi/vllm_c.cpp b/src/capi/vllm_c.cpp index 600e8d550..995cb62ac 100644 --- a/src/capi/vllm_c.cpp +++ b/src/capi/vllm_c.cpp @@ -41,7 +41,9 @@ #include "vllm/entrypoints/openai/reasoning_parsers/abstract.h" // get_reasoning_parser #include "vllm/entrypoints/openai/reasoning_parsers/detect.h" // DetectReasoningParser #include "vllm/model_executor/models/model_registry.h" // refuse-by-task (v11) +#include "vllm/model_executor/models/minimax_h3.h" // mux argv (v12) #include "vllm/multimodal/parakeet_transcription.h" // vllm_transcribe (v11) +#include "vllm/multimodal/minimax_h3_video.h" // vllm_video_* (v12) #include "vllm/outputs.h" #include "vllm/sampling_params.h" #include "vllm/transformers_utils/hf_config.h" // PeekHfArchitectures (v11) @@ -1112,6 +1114,223 @@ VLLM_API void vllm_transcription_free(vllm_transcription* out) { out->has_text = 0; } +// ── Video+audio generation (ABI v12, MiniMax-H3) ──────────────────────────── +// Thin C wrappers over the ONE library seam +// (vllm::multimodal::MiniMaxH3VideoEngine) the server's /v1/videos routes and +// the minimax-h3-gen example drive — see include/vllm.h for the contract. + +VLLM_API vllm_video_model_params vllm_video_model_params_default(void) { + vllm_video_model_params p; + std::memset(&p, 0, sizeof(p)); + return p; +} + +VLLM_API vllm_video_params vllm_video_params_default(void) { + vllm_video_params p; + std::memset(&p, 0, sizeof(p)); + return p; +} + +namespace { + +std::string OrEmpty(const char* s) { return s == nullptr ? std::string() : std::string(s); } + +} // namespace + +// The opaque video handle: owns the loaded checkpoint set + staged weights. +struct vllm_video_engine { + std::unique_ptr engine; +}; + +VLLM_API vllm_status vllm_video_engine_load(const vllm_video_model_params* params, + vllm_video_engine** out) { + if (out == nullptr) { + SetError("vllm_video_engine_load: out handle pointer is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + *out = nullptr; + if (params == nullptr || params->dit_path == nullptr || params->dit_path[0] == '\0') { + SetError("vllm_video_engine_load: params or params->dit_path is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + try { + vllm::multimodal::MiniMaxH3VideoModelParams mp; + mp.dit_path = OrEmpty(params->dit_path); + mp.encoder_path = OrEmpty(params->encoder_path); + mp.tokenizer_path = OrEmpty(params->tokenizer_path); + mp.video_vae_path = OrEmpty(params->video_vae_path); + mp.video_vae_config_path = OrEmpty(params->video_vae_config_path); + mp.audio_vae_path = OrEmpty(params->audio_vae_path); + mp.audio_vae_config_path = OrEmpty(params->audio_vae_config_path); + mp.prompt_embeds_path = OrEmpty(params->prompt_embeds_path); + mp.partition = OrEmpty(params->partition); + mp.device = params->device; + mp.dequant_bf16 = params->dequant_bf16; + mp.fp4_resident = params->fp4_resident; + auto handle = std::make_unique(); + handle->engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(mp); + *out = handle.release(); + ClearError(); + return VLLM_OK; + } catch (const std::exception& e) { + SetError(std::string("vllm_video_engine_load: ") + e.what()); + return VLLM_ERR_MODEL_LOAD; + } catch (...) { + SetError("vllm_video_engine_load: unknown error"); + return VLLM_ERR_UNKNOWN; + } +} + +VLLM_API void vllm_video_engine_free(vllm_video_engine* engine) { delete engine; } + +namespace { + +// Copy an argv into the malloc'd, NULL-terminated shape the ABI promises. +// Returns false on OOM (with everything allocated so far freed). +bool DupArgv(const std::vector& argv, char*** out_argv, int32_t* out_argc) { + char** arr = static_cast(std::calloc(argv.size() + 1, sizeof(char*))); + if (arr == nullptr) return false; + for (size_t i = 0; i < argv.size(); ++i) { + arr[i] = DupString(argv[i]); + if (arr[i] == nullptr) { + for (size_t k = 0; k < i; ++k) std::free(arr[k]); + std::free(arr); + return false; + } + } + arr[argv.size()] = nullptr; // execvp-ready + *out_argv = arr; + *out_argc = static_cast(argv.size()); + return true; +} + +} // namespace + +VLLM_API vllm_status vllm_video_generate(vllm_video_engine* engine, + const vllm_video_params* params, + vllm_video_result* out) { + if (out == nullptr) { + SetError("vllm_video_generate: out is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + std::memset(out, 0, sizeof(*out)); + if (engine == nullptr || params == nullptr) { + SetError("vllm_video_generate: engine or params is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + if (params->output_dir == nullptr || params->output_dir[0] == '\0') { + SetError("vllm_video_generate: output_dir is required (frames + WAV land there)"); + return VLLM_ERR_INVALID_ARGUMENT; + } + try { + vllm::multimodal::MiniMaxH3VideoGenParams gen; + gen.prompt = OrEmpty(params->prompt); + gen.width = params->width; + gen.height = params->height; + gen.num_frames = params->num_frames; + gen.steps = params->steps; + gen.seed = params->seed; + gen.has_seed = params->has_seed != 0; + gen.first_frame_path = OrEmpty(params->first_frame); + gen.last_frame_path = OrEmpty(params->last_frame); + // A zeroed struct must preserve behaviour: <= 0 resolves to the exact-pin + // default (1.0), the value every pre-v12 consumer used. + gen.noise_aug = params->noise_aug > 0.0f ? static_cast(params->noise_aug) : 1.0; + if (params->ref_image != nullptr && params->ref_image[0] != '\0') { + gen.ref_image_paths.push_back(params->ref_image); + } + gen.ref_video_dir = OrEmpty(params->ref_video); + gen.ref_audio_path = OrEmpty(params->ref_audio); + gen.output_dir = params->output_dir; + + const vllm::multimodal::MiniMaxH3VideoResult result = engine->engine->Generate(gen); + + vllm_video_result r; + std::memset(&r, 0, sizeof(r)); + r.frame_dir = DupString(result.frame_dir); + r.audio_path = DupString(result.audio_path); + const bool argv_ok = DupArgv(result.mux_argv, &r.mux_argv, &r.mux_argc); + if (r.frame_dir == nullptr || r.audio_path == nullptr || !argv_ok) { + vllm_video_result_free(&r); + SetError("vllm_video_generate: out-of-memory copying the result"); + return VLLM_ERR_RUNTIME; + } + r.frame_count = static_cast(result.frame_count); + r.width = static_cast(result.width); + r.height = static_cast(result.height); + r.fps = static_cast(result.fps); + r.sample_rate = static_cast(result.sample_rate); + *out = r; + ClearError(); + return VLLM_OK; + } catch (const std::exception& e) { + SetError(std::string("vllm_video_generate: ") + e.what()); + return VLLM_ERR_RUNTIME; + } catch (...) { + SetError("vllm_video_generate: unknown error"); + return VLLM_ERR_UNKNOWN; + } +} + +VLLM_API void vllm_video_result_free(vllm_video_result* out) { + if (out == nullptr) return; + std::free(out->frame_dir); + std::free(out->audio_path); + if (out->mux_argv != nullptr) { + for (int32_t i = 0; i < out->mux_argc; ++i) std::free(out->mux_argv[i]); + std::free(out->mux_argv); + } + std::memset(out, 0, sizeof(*out)); +} + +VLLM_API vllm_video_mux_params vllm_video_mux_params_default(void) { + vllm_video_mux_params p; + std::memset(&p, 0, sizeof(p)); + return p; +} + +VLLM_API vllm_status vllm_video_mux_argv(const vllm_video_mux_params* params, + char*** out_argv, int32_t* out_argc) { + if (out_argv == nullptr || out_argc == nullptr) { + SetError("vllm_video_mux_argv: out_argv/out_argc is null"); + return VLLM_ERR_INVALID_ARGUMENT; + } + *out_argv = nullptr; + *out_argc = 0; + if (params == nullptr || params->frames == nullptr || params->frames[0] == '\0' || + params->output_path == nullptr || params->output_path[0] == '\0') { + SetError("vllm_video_mux_argv: frames (a printf-style pattern) and output_path are required"); + return VLLM_ERR_INVALID_ARGUMENT; + } + try { + vllm::MiniMaxH3MuxRequest request; // the library's encoding contract + request.frame_pattern = params->frames; + request.audio_path = OrEmpty(params->audio_path); + request.output_path = params->output_path; + if (params->fps > 0) request.fps = params->fps; + if (params->crf > 0) request.crf = params->crf; + const std::vector argv = vllm::MiniMaxH3BuildMp4MuxArgs(request); + if (!DupArgv(argv, out_argv, out_argc)) { + SetError("vllm_video_mux_argv: out-of-memory copying the argv"); + return VLLM_ERR_RUNTIME; + } + ClearError(); + return VLLM_OK; + } catch (const std::exception& e) { + SetError(std::string("vllm_video_mux_argv: ") + e.what()); + return VLLM_ERR_RUNTIME; + } catch (...) { + SetError("vllm_video_mux_argv: unknown error"); + return VLLM_ERR_UNKNOWN; + } +} + +VLLM_API void vllm_video_mux_argv_free(char** argv, int32_t argc) { + if (argv == nullptr) return; + for (int32_t i = 0; i < argc; ++i) std::free(argv[i]); + std::free(argv); +} + VLLM_API void vllm_string_free(char* s) { std::free(s); } VLLM_API void vllm_completion_free(vllm_completion* out) { diff --git a/src/vllm/multimodal/minimax_h3_video.cpp b/src/vllm/multimodal/minimax_h3_video.cpp new file mode 100644 index 000000000..c3ae3a644 --- /dev/null +++ b/src/vllm/multimodal/minimax_h3_video.cpp @@ -0,0 +1,730 @@ +// MiniMax-H3 video+audio generation seam — implementation. See the header for +// the contract; every stage below is the ABSORBED pre-fold assembly +// (examples/minimax_h3_gen/main.cpp:687-1288 and the /v1/videos twin +// examples/server/main.cpp:743-1096 @ fc636c76), byte-gated on the committed +// fold fixture (tests/vllm/models/test_minimax_h3_video_fold.cpp). +// +// The library SPAWNS NOTHING here: directories are created with +// std::filesystem (the pre-fold examples shelled out to `mkdir -p`, which a +// library must not), artifacts are written with plain file IO, and the ffmpeg +// invocation stays with the caller as `mux_argv`. +#include "vllm/multimodal/minimax_h3_video.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "vllm/entrypoints/openai/video_api.h" +#include "vllm/model_executor/model_loader/gguf_reader.h" +#include "vllm/model_executor/model_loader/safetensors_reader.h" +#include "vllm/model_executor/models/minimax_h3.h" +#include "vllm/tokenizer/tokenizer.h" +#include "vt/backend.h" +#include "vt/dtype.h" + +namespace vllm::multimodal { +namespace { + +nlohmann::json ReadJson(const std::string& path) { + std::ifstream in(path); + if (!in) throw std::runtime_error("minimax_h3 video: cannot open " + path); + nlohmann::json j; + in >> j; + return j; +} + +std::string ReadFileBytes(const std::string& field, const std::string& path) { + std::ifstream in(path, std::ios::binary); + if (!in) throw std::runtime_error(field + ": cannot open " + path); + return std::string((std::istreambuf_iterator(in)), std::istreambuf_iterator()); +} + +std::vector ReadF32File(const std::string& path) { + std::ifstream in(path, std::ios::binary | std::ios::ate); + if (!in) throw std::runtime_error("minimax_h3 video: cannot open " + path); + const std::streamsize bytes = in.tellg(); + if (bytes % static_cast(sizeof(float)) != 0) { + throw std::runtime_error(path + ": size is not a whole number of f32 values"); + } + in.seekg(0); + std::vector out(static_cast(bytes) / sizeof(float)); + in.read(reinterpret_cast(out.data()), bytes); + return out; +} + +void WriteFileBytes(const std::string& path, const std::string& bytes) { + std::ofstream out(path, std::ios::binary); + if (!out) throw std::runtime_error("minimax_h3 video: cannot write " + path); + out.write(bytes.data(), static_cast(bytes.size())); + if (!out) throw std::runtime_error("minimax_h3 video: short write " + path); +} + +bool EndsWith(const std::string& s, const std::string& suffix) { + return s.size() >= suffix.size() && + s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +// Decode a binary PPM (P6) into [3, H, W] floats in [-1, 1] — the layout the +// H3 video-VAE encoder takes. From BYTES so one decoder serves a path and an +// inline data: URL (absorbed from the pre-fold server's DecodePpmChw; the +// pre-fold example's ReadPpmAsChw was the file-path twin). PPM is the only +// still-image container this tree reads — no PNG/JPEG codec is vendored, the +// same NAMED residual the chat multimodal path carries. +std::vector DecodePpmChw(const std::string& field, const std::string& bytes, + int64_t* out_h, int64_t* out_w) { + std::istringstream in(bytes, std::ios::binary); + std::string magic; + in >> magic; + if (magic != "P6") { + throw std::runtime_error(field + + ": not a binary PPM (P6); no PNG/JPEG codec is vendored, " + "so a reference image must be supplied as binary PPM"); + } + auto next_int = [&]() { + int v = 0; + while (in >> std::ws, in.peek() == '#') { + std::string skip; + std::getline(in, skip); + } + in >> v; + return v; + }; + const int w = next_int(), h = next_int(), maxv = next_int(); + if (w <= 0 || h <= 0 || maxv <= 0) throw std::runtime_error(field + ": bad PPM header"); + in.get(); // the single whitespace byte before the payload + std::vector rgb(static_cast(w) * h * 3); + in.read(reinterpret_cast(rgb.data()), static_cast(rgb.size())); + if (!in) throw std::runtime_error(field + ": truncated PPM payload"); + std::vector chw(rgb.size()); + const int64_t plane = static_cast(w) * h; + for (int64_t i = 0; i < plane; ++i) { + for (int64_t c = 0; c < 3; ++c) { + chw[static_cast(c * plane + i)] = + static_cast(rgb[static_cast(i * 3 + c)]) / (maxv * 0.5f) - 1.0f; + } + } + if (out_h != nullptr) *out_h = h; + if (out_w != nullptr) *out_w = w; + return chw; +} + +std::vector ReadPpm(const std::string& field, const std::string& path, + const std::string& inline_bytes, int64_t* out_h, int64_t* out_w) { + if (!inline_bytes.empty()) return DecodePpmChw(field, inline_bytes, out_h, out_w); + return DecodePpmChw(field, ReadFileBytes(field, path), out_h, out_w); +} + +// A ref2va VIDEO reference: DIR/frame_%06d.ppm — the exact layout this seam +// WRITES, so one run's frames chain into the next request. Returns +// [C, T, H, W] in [-1, 1] (absorbed from both pre-fold copies). +std::vector ReadReferenceClipChw(const std::string& dir, int64_t* out_t, int64_t* out_h, + int64_t* out_w) { + std::vector per_frame; // frame-major [T][C,H,W] + int64_t frames = 0, fh = 0, fw = 0; + for (int64_t k = 0;; ++k) { + char name[512]; + std::snprintf(name, sizeof(name), "%s/frame_%06lld.ppm", dir.c_str(), + static_cast(k)); + std::ifstream probe(name, std::ios::binary); + if (!probe) break; + const std::string bytes((std::istreambuf_iterator(probe)), + std::istreambuf_iterator()); + int64_t h = 0, w = 0; + const std::vector frame = DecodePpmChw("reference video frame", bytes, &h, &w); + if (frames == 0) { + fh = h; + fw = w; + } + if (h != fh || w != fw) { + throw std::runtime_error( + "reference video: every frame_%06d.ppm must have the same size"); + } + per_frame.insert(per_frame.end(), frame.begin(), frame.end()); + ++frames; + } + if (frames == 0) { + throw std::runtime_error("reference video: no frame_%06d.ppm files in " + dir); + } + std::vector chw(per_frame.size()); + const int64_t plane = fh * fw; + for (int64_t c = 0; c < 3; ++c) { + for (int64_t k = 0; k < frames; ++k) { + for (int64_t e = 0; e < plane; ++e) { + chw[static_cast((c * frames + k) * plane + e)] = + per_frame[static_cast(k * 3 * plane + c * plane + e)]; + } + } + } + *out_t = frames; + *out_h = fh; + *out_w = fw; + return chw; +} + +uint64_t SplitMix64(uint64_t x) { + x += 0x9E3779B97F4A7C15ULL; + uint64_t z = x; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL; + z ^= z >> 31; + return z; +} + +// The deterministic noise draw — BYTE-EXACT to the pre-fold example +// (main.cpp:1159-1195): a splitmix64 stream per modality, Box-Muller Gaussian +// by default (upstream draws torch.randn; mirror policy), VT_H3_GAUSSIAN_NOISE=0 +// keeps the legacy uniform draw for the A/B. This deliberately does NOT +// reproduce torch's RNG bit-exactly: matching it decides WHICH sample you get, +// not whether the pipeline is right. +void FillNoise(std::vector& out, uint64_t seed) { + const char* gn = std::getenv("VT_H3_GAUSSIAN_NOISE"); + const bool gaussian = !(gn != nullptr && gn[0] == '0'); + uint64_t x = seed; + auto u01 = [&x]() { + x += 0x9E3779B97F4A7C15ULL; + uint64_t z = x; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL; + z ^= z >> 31; + return (z >> 11) * 0x1.0p-53; // [0,1) + }; + for (size_t i = 0; i < out.size(); ++i) { + if (gaussian) { + double u1 = u01(), u2 = u01(); + if (u1 < 1e-12) u1 = 1e-12; + out[i] = static_cast(std::sqrt(-2.0 * std::log(u1)) * + std::cos(2.0 * 3.14159265358979323846 * u2)); + } else { + out[i] = static_cast(u01() * 2.0 - 1.0); // uniform [-1,1] + } + } +} + +} // namespace + +// ── the engine ─────────────────────────────────────────────────────────────── + +struct MiniMaxH3VideoEngine::Impl { + MiniMaxH3VideoModelParams params; + vt::Device device{}; + MiniMaxH3GgufDit dit; // .params always; .weights only for host arms + MiniMaxH3DitDeviceWeights streamed; + MiniMaxH3DitDeviceWeights staged; + const MiniMaxH3DitDeviceWeights* prestaged = nullptr; + + MiniMaxH3VideoVaeDecoderConfig video_cfg; + MiniMaxH3LatentStats video_stats; + MiniMaxH3AudioVaeConfig audio_cfg; + MiniMaxH3LatentStats audio_stats; + MiniMaxH3AudioVaeWeights video_weights, audio_weights; + + std::vector prompt_embeds; // fallback conditioning + MiniMaxH3PartitionInfo partition_info; + + // Encoder tower (optional), staged ONCE — staging the 32B tower costs + // ~162 s, so per-request staging would dominate every generation. + bool has_encoder = false; + MiniMaxH3EncoderConfig enc_config; + MiniMaxH3EncoderQuantWeights enc_host; // GGUF arm + std::optional enc_shards; // bf16-shard arm + MiniMaxH3EncoderDeviceWeights enc_staged; + std::unique_ptr tokenizer; + vt::Queue enc_queue{}; + + // The two VAEs' ENCODER halves, for the reference modalities. Loaded LAZILY + // and ONCE each: a text-to-video engine must not pay for weights it never + // uses, and one that does must not reload per request. + bool video_encoder_loaded = false, audio_encoder_loaded = false; + MiniMaxH3AudioVaeWeights video_encoder_weights, audio_encoder_weights; + MiniMaxH3EncoderFcn3dConfig video_encoder_cfg; + MiniMaxH3AudioVaeEncoderConfig audio_encoder_cfg; + + // Generate() is serialized: the staged weights and lazy encoder halves are + // shared state, and one H3 render saturates the device anyway. + std::mutex mutex; + + std::vector EncodePrompt(const std::string& prompt); + void EnsureVideoEncoder(); + void EnsureAudioEncoder(); +}; + +MiniMaxH3VideoEngine::MiniMaxH3VideoEngine() = default; +MiniMaxH3VideoEngine::MiniMaxH3VideoEngine(MiniMaxH3VideoEngine&&) noexcept = default; +MiniMaxH3VideoEngine& MiniMaxH3VideoEngine::operator=(MiniMaxH3VideoEngine&&) noexcept = default; +MiniMaxH3VideoEngine::~MiniMaxH3VideoEngine() = default; + +bool MiniMaxH3VideoEngine::has_encoder() const { return impl_->has_encoder; } +bool MiniMaxH3VideoEngine::has_prompt_embeds() const { return !impl_->prompt_embeds.empty(); } + +std::unique_ptr MiniMaxH3VideoEngine::Load( + const MiniMaxH3VideoModelParams& params) { + if (params.dit_path.empty()) { + throw std::runtime_error("minimax_h3 video: dit_path is required"); + } + if (params.device != 0 && params.device != 1) { + throw std::runtime_error("minimax_h3 video: device must be 0 (cpu) or 1 (cuda)"); + } + auto engine = std::unique_ptr(new MiniMaxH3VideoEngine()); + engine->impl_ = std::make_unique(); + Impl& im = *engine->impl_; + im.params = params; + + // The CUDA context is created BEFORE any weight is read: on a unified-memory + // box the driver's reservation must land before the pool fills (the GB10 + // load recipe). This throws — loudly — when no CUDA backend is registered. + vt::Queue stream_queue{}; + if (params.device == 1) { + im.device = vt::GetBackend(vt::DeviceType::kCUDA).CreateQueue().device; + stream_queue = vt::GetBackend(vt::DeviceType::kCUDA).CreateQueue(); + } + + // ── 1. DiT: the four loader arms of the pre-fold driver ──────────────────── + // Refuse a TEXT-model directory LOUDLY before any loader guesses: an H3 DiT + // is a GGUF file, an NVFP4 safetensors file, or the original bf16 release's + // shard DIRECTORY (model.safetensors.index.json). A config.json directory is + // a text/transcription checkpoint and belongs to vllm_engine_load. + if (std::filesystem::is_directory(params.dit_path) && + !MiniMaxH3ShardedCheckpoint::IsShardedDir(params.dit_path)) { + throw std::runtime_error( + "minimax_h3 video: '" + params.dit_path + + "' is a directory without model.safetensors.index.json — not an H3 DiT " + "checkpoint. Expected a GGUF file, an NVFP4 safetensors file, or the bf16 " + "shard directory; a text-generation model loads through vllm_engine_load " + "instead"); + } + const bool cuda = im.device.type != vt::DeviceType::kCPU; + bool have_streamed = false; + if (EndsWith(params.dit_path, ".gguf")) { + const GgufFile f = GgufFile::Open(params.dit_path); + if (params.dequant_bf16 != 0 && cuda) { + // STREAM straight onto the device: dequantize + upload one tensor at a + // time so the host copy never accumulates (peak kills unified memory). + im.streamed = StreamMiniMaxH3DitToDeviceBf16(stream_queue, f, &im.dit.params); + have_streamed = true; + } else if (params.dequant_bf16 != 0) { + im.dit = LoadMiniMaxH3DitFromGgufBf16(f); + } else { + im.dit = LoadMiniMaxH3DitFromGguf(f, /*keep_quant=*/true); + } + } else if (MiniMaxH3ShardedCheckpoint::IsShardedDir(params.dit_path)) { + const MiniMaxH3ShardedCheckpoint ckpt = MiniMaxH3ShardedCheckpoint::Open(params.dit_path); + if (cuda) { + im.streamed = StreamMiniMaxH3ShardedToDeviceBf16(stream_queue, ckpt, &im.dit.params); + have_streamed = true; + } else { + // Host f32 reference path — usable only on a reduced checkpoint. + im.dit = LoadMiniMaxH3DitFromShards(ckpt); + } + } else { + const SafetensorsFile f = SafetensorsFile::Open(params.dit_path); + if (cuda) { + // fp4_resident keeps the packed FP4 on device (~1/4 the bf16 footprint) + // and routes quantized projections through the Marlin W4A16 GEMM; the + // default bf16 stream dequantizes on the way up. + im.streamed = params.fp4_resident != 0 + ? StreamMiniMaxH3Nvfp4ToDeviceFp4(stream_queue, f, &im.dit.params) + : StreamMiniMaxH3Nvfp4ToDeviceBf16(stream_queue, f, &im.dit.params); + have_streamed = true; + } else { + im.dit = LoadMiniMaxH3DitFromNvfp4(f); + } + } + + // ── 2. VAEs + their configs (the configs carry the latent statistics) ────── + if (!params.video_vae_config_path.empty()) { + im.video_cfg = + ParseMiniMaxH3VideoVaeDecoderConfig(ReadJson(params.video_vae_config_path), &im.video_stats); + } + if (!params.audio_vae_config_path.empty()) { + im.audio_cfg = ParseMiniMaxH3AudioVaeConfig(ReadJson(params.audio_vae_config_path), &im.audio_stats); + } + if (params.video_vae_path.empty() || params.audio_vae_path.empty()) { + throw std::runtime_error( + "minimax_h3 video: video_vae_path and audio_vae_path are required (a full " + "render decodes both modalities)"); + } + { + const SafetensorsFile f = SafetensorsFile::Open(params.video_vae_path); + im.video_weights = LoadMiniMaxH3VideoVaeDecoderWeights(f); + } + { + const SafetensorsFile f = SafetensorsFile::Open(params.audio_vae_path); + im.audio_weights = LoadMiniMaxH3AudioVaeWeights(f); + } + + // ── 3. fallback conditioning + partition guard input ─────────────────────── + if (!params.prompt_embeds_path.empty()) { + im.prompt_embeds = ReadF32File(params.prompt_embeds_path); + } + im.partition_info = MiniMaxH3PartitionFromFlag(params.partition); + + // ── 4. the optional H3-Encoder text tower, staged once ───────────────────── + if (!params.encoder_path.empty()) { + im.enc_queue = vt::Queue{im.device, nullptr}; + if (im.device.type != vt::DeviceType::kCPU) { + im.enc_queue = vt::GetBackend(im.device.type).CreateQueue(); + } + if (MiniMaxH3ShardedCheckpoint::IsShardedDir(params.encoder_path)) { + // The ORIGINAL bf16 release's shards: stream to the device, keep the + // checkpoint open for the per-request embedding gather. + im.enc_shards.emplace(MiniMaxH3ShardedCheckpoint::Open(params.encoder_path)); + if (params.tokenizer_path.empty()) { + throw std::runtime_error( + "minimax_h3 video: tokenizer_path is required with a safetensors-shard encoder"); + } + im.tokenizer = std::make_unique( + tok::Tokenizer::FromHfJson(params.tokenizer_path)); + im.enc_staged = StreamMiniMaxH3EncoderShardsToDevice( + im.enc_queue, *im.enc_shards, params.encoder_max_layers, &im.enc_config); + } else { + const GgufFile ef = GgufFile::Open(params.encoder_path); + im.enc_host = LoadMiniMaxH3EncoderFromGguf(ef, params.encoder_max_layers); + im.enc_config = im.enc_host.config; + // The ComfyUI-style encoder GGUF is WEIGHTS ONLY — no tokenizer.ggml.* + // metadata — so the vocab comes from the checkpoint's own tokenizer.json + // unless the GGUF happens to embed one. + im.tokenizer = std::make_unique( + params.tokenizer_path.empty() ? tok::Tokenizer::FromGguf(ef) + : tok::Tokenizer::FromHfJson(params.tokenizer_path)); + im.enc_staged = StageMiniMaxH3EncoderWeights(im.enc_queue, im.enc_host); + } + im.has_encoder = true; + } + + // ── 5. stage the DiT once (device arms) ──────────────────────────────────── + if (have_streamed) { + im.prestaged = &im.streamed; + } else if (cuda) { + vt::Queue sq = vt::GetBackend(im.device.type).CreateQueue(); + im.staged = StageMiniMaxH3DitWeights(sq, im.dit.params, im.dit.weights, vt::DType::kBF16); + im.prestaged = &im.staged; + } + return engine; +} + +// Encode `prompt` with the H3 text tower (text-only M-RoPE: all three axes are +// the token index) — the absorbed EncodeH3Prompt/server-lambda conditioning. +std::vector MiniMaxH3VideoEngine::Impl::EncodePrompt(const std::string& prompt) { + const std::vector ids = tokenizer->Encode(prompt); + if (ids.empty()) throw std::runtime_error("minimax_h3 video: the prompt tokenized to nothing"); + // DIAGNOSTIC (env-gated): dump the token ids so the tokenization can be + // diffed against upstream — the pre-fold driver's VT_H3_DUMP_INPUTS knob. + if (const char* dd = std::getenv("VT_H3_DUMP_INPUTS")) { + if (std::FILE* fp = std::fopen((std::string(dd) + "/prompt_token_ids.i32").c_str(), "wb")) { + std::fwrite(ids.data(), sizeof(int32_t), ids.size(), fp); + std::fclose(fp); + } + } + std::vector embeds = enc_shards.has_value() + ? MiniMaxH3EncoderEmbedTokensFromShards(*enc_shards, ids) + : MiniMaxH3EncoderEmbedTokens(enc_host, ids); + const int64_t seq = static_cast(ids.size()); + std::vector pos(static_cast(3 * seq)); + for (int64_t a = 0; a < 3; ++a) { + for (int64_t s = 0; s < seq; ++s) pos[static_cast(a * seq + s)] = s; + } + return MiniMaxH3EncoderTextForwardDevice(enc_queue, enc_config, enc_staged, embeds, pos.data(), + seq); +} + +void MiniMaxH3VideoEngine::Impl::EnsureVideoEncoder() { + if (video_encoder_loaded) return; + const SafetensorsFile vf = SafetensorsFile::Open(params.video_vae_path); + video_encoder_weights = LoadMiniMaxH3VideoVaeEncoderWeights(vf); + video_encoder_cfg = MiniMaxH3EncoderFcn3dConfig{}; + video_encoder_cfg.z_channels = 2 * dit.params.latents_dim; // moments: mean | logvar + video_encoder_loaded = true; +} + +void MiniMaxH3VideoEngine::Impl::EnsureAudioEncoder() { + if (audio_encoder_loaded) return; + if (params.audio_vae_config_path.empty()) { + throw std::runtime_error( + "minimax_h3 video: an audio reference needs audio_vae_config_path (it carries " + "the latent statistics the reference rows are normalized by)"); + } + const SafetensorsFile af = SafetensorsFile::Open(params.audio_vae_path); + audio_encoder_weights = LoadMiniMaxH3AudioVaeEncoderWeights(af); + audio_encoder_cfg = MiniMaxH3AudioVaeEncoderConfig{}; + audio_encoder_cfg.vae_latent_channels = dit.params.audio_latents_dim; + audio_encoder_loaded = true; +} + +MiniMaxH3VideoResult MiniMaxH3VideoEngine::Generate(const MiniMaxH3VideoGenParams& gen) { + Impl& im = *impl_; + std::lock_guard guard(im.mutex); + const MiniMaxH3DitParams& p = im.dit.params; + + if (gen.output_dir.empty()) { + throw std::runtime_error("minimax_h3 video: output_dir is required"); + } + + // ── conditioning ─────────────────────────────────────────────────────────── + std::vector conditioning; + if (im.has_encoder && !gen.prompt.empty()) { + conditioning = im.EncodePrompt(gen.prompt); + } else if (!im.prompt_embeds.empty()) { + conditioning = im.prompt_embeds; + } else { + throw std::runtime_error( + "minimax_h3 video: generation needs conditioning — load with an encoder (to " + "condition on the prompt) or with prompt_embeds_path"); + } + if (p.text_dim <= 0 || + conditioning.size() % static_cast(p.text_dim) != 0) { + throw std::runtime_error( + "minimax_h3 video: conditioning size is not a multiple of text_dim"); + } + + // ── reference exclusivity (the packed layout's rule, checked up front) ───── + const bool has_keyframes = !gen.first_frame_path.empty() || !gen.first_frame_ppm.empty() || + !gen.last_frame_path.empty(); + const bool has_ref2va = !gen.ref_image_paths.empty() || !gen.ref_video_dir.empty() || + !gen.ref_audio_path.empty() || !gen.ref_audio_wav.empty(); + if (has_keyframes && has_ref2va) { + throw std::runtime_error( + "minimax_h3 video: fl2va keyframes and ref2va references are exclusive " + "(minimax_h3_pipeline.cpp:251)"); + } + if (!gen.ref_video_dir.empty() && !gen.ref_image_paths.empty()) { + throw std::runtime_error( + "minimax_h3 video: a reference video is exclusive with reference images"); + } + + // ── the keyframe image (fl2va), read before the plan so its aspect can + // drive the default canvas (upstream _resolve_shape) ─────────────────────── + std::vector first_chw, last_chw; + int64_t ref_h = 0, ref_w = 0; + if (!gen.first_frame_path.empty() || !gen.first_frame_ppm.empty()) { + first_chw = ReadPpm("first_frame", gen.first_frame_path, gen.first_frame_ppm, &ref_h, &ref_w); + } + if (!gen.last_frame_path.empty()) { + int64_t h2 = 0, w2 = 0; + last_chw = ReadPpm("last_frame", gen.last_frame_path, "", &h2, &w2); + if (ref_h == 0) { + ref_h = h2; + ref_w = w2; + } else if (h2 != ref_h || w2 != ref_w) { + throw std::runtime_error( + "minimax_h3 video: the first and last frames must have the same size"); + } + } + + // ── task + shape plan (upstream _resolve_task/_resolve_shape) ────────────── + const std::string task = + !gen.task.empty() ? gen.task + : (has_keyframes ? "fl2va" : (has_ref2va ? "ref2va" : "t2va")); + const MiniMaxH3ShapePlan plan = MiniMaxH3ResolveShape( + task, gen.duration_seconds, gen.num_frames, gen.height, gen.width, ref_w, ref_h); + + MiniMaxH3T2vaRequest request; + request.partition = im.partition_info; // #77 guard: GenerateT2va refuses a + // task this partition cannot serve. + request.latent_t = plan.latent_t; + request.num_frames = plan.num_frames; + request.latent_h = plan.height / kMiniMaxH3VaeRatio; + request.latent_w = plan.width / kMiniMaxH3VaeRatio; + request.audio_t = plan.audio_t; + request.audio_channel = kMiniMaxH3AudioChannels; + if (gen.steps > 0) request.num_steps = gen.steps; + if (gen.flow_shift > 0.0) request.video_shift = gen.flow_shift; + if (gen.audio_flow_shift > 0.0) request.audio_shift = gen.audio_flow_shift; + request.video_latents_mean = im.video_stats.mean; + request.video_latents_std = im.video_stats.std_dev; + request.audio_latents_mean = im.audio_stats.mean; + request.audio_latents_std = im.audio_stats.std_dev; + request.text_len = static_cast(conditioning.size()) / p.text_dim; + + // ── fl2va KEYFRAME CONDITIONING ──────────────────────────────────────────── + if (has_keyframes) { + if (ref_h != plan.height || ref_w != plan.width) { + // No image resampler is vendored, and a mis-sized keyframe would either + // abort deep in the denoise or pin the wrong latent rows. Say so up + // front, with the geometry we resolved. + throw std::runtime_error( + "minimax_h3 video: the keyframe is " + std::to_string(ref_w) + "x" + + std::to_string(ref_h) + " but this request resolved to " + + std::to_string(plan.width) + "x" + std::to_string(plan.height) + + "; supply it at the output size (no image resampler is vendored)"); + } + im.EnsureVideoEncoder(); + std::vector> imgs; + std::vector idx; + if (!first_chw.empty()) { + imgs.push_back(std::move(first_chw)); + idx.push_back(0); + } + if (!last_chw.empty()) { + imgs.push_back(std::move(last_chw)); + idx.push_back(-1); + } + request.keyframe_frame_indices = idx; + request.imgvid_noise_aug = gen.noise_aug; + request.keyframe_cond_rows = + MiniMaxH3EncodeKeyframeCondRows(im.video_encoder_cfg, im.video_encoder_weights, p, imgs, + ref_h, ref_w, request.latent_t, gen.noise_aug, {}); + } + + // ── ref2va REFERENCE BLOCKS ──────────────────────────────────────────────── + if (!gen.ref_video_dir.empty()) { + im.EnsureVideoEncoder(); + int64_t ct = 0, ch = 0, cw = 0; + const std::vector clip = ReadReferenceClipChw(gen.ref_video_dir, &ct, &ch, &cw); + MiniMaxH3RefBlock block{}; + request.keyframe_cond_rows = MiniMaxH3EncodeReferenceVideo( + im.video_encoder_cfg, im.video_encoder_weights, p, clip, ct, ch, cw, &block); + // SILENT by construction (ref_audio_t == 0): an audio reference below + // ATTACHES to this block — the layout packed_sequence.py builds. + request.ref_blocks = {block}; + } else if (!gen.ref_image_paths.empty()) { + im.EnsureVideoEncoder(); + std::vector> imgs; + int64_t ih = 0, iw = 0; + for (const std::string& rp : gen.ref_image_paths) { + int64_t h2 = 0, w2 = 0; + imgs.push_back(ReadPpm("ref_image", rp, "", &h2, &w2)); + if (ih == 0) { + ih = h2; + iw = w2; + } else if (h2 != ih || w2 != iw) { + throw std::runtime_error( + "minimax_h3 video: every reference image must have the same size"); + } + } + std::vector blocks; + request.keyframe_cond_rows = MiniMaxH3EncodeReferenceImages( + im.video_encoder_cfg, im.video_encoder_weights, p, imgs, ih, iw, &blocks); + request.ref_blocks = blocks; + } + if (!gen.ref_audio_path.empty() || !gen.ref_audio_wav.empty()) { + im.EnsureAudioEncoder(); + const std::string wav_bytes = !gen.ref_audio_wav.empty() + ? gen.ref_audio_wav + : ReadFileBytes("ref_audio", gen.ref_audio_path); + int64_t samples_per_channel = 0; + const std::vector waveform = MiniMaxH3ReadWav( + wav_bytes, kMiniMaxH3AudioChannels, kMiniMaxH3AudioSampleRate, &samples_per_channel); + MiniMaxH3RefBlock audio_block{}; + request.audio_ref_rows = MiniMaxH3EncodeReferenceAudio( + im.audio_encoder_cfg, im.audio_encoder_weights, waveform, kMiniMaxH3AudioChannels, + samples_per_channel, im.audio_stats.mean, im.audio_stats.std_dev, + /*noise_aug=*/1.0, {}, &audio_block); + if (!request.ref_blocks.empty() && + request.ref_blocks[0].kind == MiniMaxH3RefBlock::Kind::kVideoAudio) { + // A video reference that now HAS sound: one kVideoAudio block carries + // both, so its ref_audio_t must claim exactly the rows just encoded. + request.ref_blocks[0].ref_audio_t = audio_block.ref_audio_t; + } else { + request.ref_blocks.push_back(audio_block); + } + } + + // ── the deterministic noise draw ─────────────────────────────────────────── + const int64_t frame_rows = + (request.latent_h / p.patch_size_h) * (request.latent_w / p.patch_size_w); + const int64_t video_rows = request.latent_t * frame_rows; + const int64_t audio_rows = request.audio_t * request.audio_channel; + std::vector noise_video(static_cast(video_rows * p.video_row_width())); + std::vector noise_audio(static_cast(audio_rows * p.audio_latents_dim)); + // Unseeded: the pre-fold driver's fixed per-modality streams (byte-identical + // default). Seeded: the video stream takes the seed, the audio stream a + // splitmix64 derivation of it, so the two modalities stay independent. + const uint64_t video_seed = gen.has_seed ? gen.seed : 0x5EED1234ULL; + const uint64_t audio_seed = gen.has_seed ? SplitMix64(gen.seed) : 0x5EED5678ULL; + FillNoise(noise_video, video_seed); + FillNoise(noise_audio, audio_seed); + + // ── generate ─────────────────────────────────────────────────────────────── + const MiniMaxH3T2vaResult out = MiniMaxH3GenerateT2va( + im.device, request, p, im.dit.weights, im.video_cfg, im.video_weights, im.audio_cfg, + im.audio_weights, conditioning, noise_video, noise_audio, vt::DType::kBF16, im.prestaged); + + // ── artifacts (the library WRITES these, spawns nothing) ─────────────────── + std::error_code ec; + std::filesystem::create_directories(gen.output_dir, ec); + if (ec) { + throw std::runtime_error("minimax_h3 video: cannot create " + gen.output_dir + ": " + + ec.message()); + } + MiniMaxH3VideoResult result; + result.frame_dir = gen.output_dir; + for (int64_t f = 0; f < out.frame_shape.t; ++f) { + char name[512]; + std::snprintf(name, sizeof(name), "%s/frame_%06lld.ppm", gen.output_dir.c_str(), + static_cast(f)); + WriteFileBytes(name, MiniMaxH3WritePpmFrame(out.frames, out.frame_shape, f)); + } + result.audio_path = gen.output_dir + "/audio.wav"; + WriteFileBytes(result.audio_path, + MiniMaxH3WriteWav(out.waveform, out.audio_channels, + out.audio_samples_per_channel, out.sample_rate)); + result.frame_count = out.frame_shape.t; + result.width = out.frame_shape.w; + result.height = out.frame_shape.h; + result.fps = kMiniMaxH3Fps; + result.sample_rate = out.sample_rate; + + // ── the mux argv the CALLER may exec (the ratified ffmpeg boundary) ──────── + MiniMaxH3MuxRequest mux; + mux.frame_pattern = gen.output_dir + "/frame_%06d.ppm"; + mux.audio_path = result.audio_path; + mux.output_path = gen.output_dir + "/video.mp4"; + result.mux_argv = MiniMaxH3BuildMp4MuxArgs(mux); + result.mux_output_path = mux.output_path; + return result; +} + +// ── the /v1/videos request mapping (library-owned so HTTP and FFI cannot +// drift; absorbed from the pre-fold server lambda) ─────────────────────────── +MiniMaxH3VideoGenParams MiniMaxH3VideoGenParamsFromRequest( + const ::vllm::openai::VideoRequest& request, const std::string& output_dir) { + MiniMaxH3VideoGenParams gen; + gen.prompt = request.prompt; + gen.task = request.task; + gen.duration_seconds = request.duration_seconds; + gen.num_frames = request.num_frames; + gen.height = request.height; + gen.width = request.width; + gen.steps = request.num_inference_steps; + gen.flow_shift = request.flow_shift; + gen.audio_flow_shift = request.audio_flow_shift; + gen.seed = static_cast(request.seed); + gen.has_seed = request.has_seed; + // OpenAI `input_reference` -> fl2va FIRST-FRAME conditioning: OpenAI + // documents it as the image the video STARTS FROM (image-to-video), which is + // exactly what a frame-0 keyframe expresses. The ref2va modalities OpenAI + // has no slot for enter through `metadata` instead; ParseVideoRequest has + // already refused the combinations the pipeline forbids. + if (!request.input_reference_bytes.empty()) { + gen.first_frame_ppm.assign(request.input_reference_bytes.begin(), + request.input_reference_bytes.end()); + } else { + gen.first_frame_path = request.input_reference_path; + } + gen.noise_aug = 1.0; // pin the frame exactly (the pre-fold server's choice) + gen.ref_video_dir = request.input_reference_video_dir; + if (!request.input_reference_audio_bytes.empty()) { + gen.ref_audio_wav.assign(request.input_reference_audio_bytes.begin(), + request.input_reference_audio_bytes.end()); + } else { + gen.ref_audio_path = request.input_reference_audio_path; + } + gen.output_dir = output_dir; + return gen; +} + +} // namespace vllm::multimodal diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3bcdb6c38..4f0610afa 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -50,6 +50,12 @@ vllm_cpp_add_test(test_gguf_keep_quant vllm/test_gguf_keep_quant.cpp) vllm_cpp_add_test(test_model_loader_gguf vllm/test_model_loader_gguf.cpp) vllm_cpp_add_test(test_model_registry vllm/models/test_model_registry.cpp) vllm_cpp_add_test(test_minimax_h3 vllm/models/test_minimax_h3.cpp) +# ARCH-ONE-SURFACE ROW 2 fold gate: the video seam == the replicated pre-fold +# pipeline == the goldens the PRE-fold minimax-h3-gen binary rendered at the +# branch base (frames + WAV byte-identical, mux argv pinned). +vllm_cpp_add_test(test_minimax_h3_video_fold vllm/models/test_minimax_h3_video_fold.cpp) +target_compile_definitions(test_minimax_h3_video_fold PRIVATE + MINIMAX_H3_VIDEO_FOLD_FIXTURE_DIR="${CMAKE_SOURCE_DIR}/tests/vllm/models/fixtures/minimax_h3_video_fold") vllm_cpp_add_test(test_video_api vllm/entrypoints/openai/test_video_api.cpp) target_include_directories(test_model_registry PRIVATE ${CMAKE_SOURCE_DIR}/src) vllm_cpp_add_test(test_platform vllm/platforms/test_platform.cpp) @@ -666,7 +672,8 @@ target_include_directories(test_bench PRIVATE ${CMAKE_SOURCE_DIR}/examples/bench vllm_cpp_add_test(test_capi capi/test_capi.cpp) target_include_directories(test_capi PRIVATE ${CMAKE_SOURCE_DIR}/src) target_compile_definitions(test_capi PRIVATE - PARAKEET_E2E_FIXTURE_DIR="${CMAKE_SOURCE_DIR}/tests/vllm/models/fixtures/parakeet_e2e") + PARAKEET_E2E_FIXTURE_DIR="${CMAKE_SOURCE_DIR}/tests/vllm/models/fixtures/parakeet_e2e" + MINIMAX_H3_VIDEO_FOLD_FIXTURE_DIR="${CMAKE_SOURCE_DIR}/tests/vllm/models/fixtures/minimax_h3_video_fold") vllm_cpp_add_test(test_chat_prompt capi/test_chat_prompt.cpp) target_include_directories(test_chat_prompt PRIVATE ${CMAKE_SOURCE_DIR}/src) diff --git a/tests/capi/test_capi.cpp b/tests/capi/test_capi.cpp index 9f216e999..ffa0e2215 100644 --- a/tests/capi/test_capi.cpp +++ b/tests/capi/test_capi.cpp @@ -1224,10 +1224,11 @@ TEST_CASE("capi: version and abi-version are exposed") { CHECK(vllm_abi_version() == VLLM_ABI_VERSION); // The engine-config growth (max_num_batched_tokens / scheduling_policy / // kv_transfer_config) is ABI v9; the jump-forward toggle is ABI v10; the - // transcription slice (vllm_transcribe) is ABI v11. The >= pin is the one - // check that can catch a WRONG bump: the == VLLM_ABI_VERSION assertions here - // and in test_dlopen compare against the same macro and move with it. - CHECK(vllm_abi_version() >= 11); + // transcription slice (vllm_transcribe) is ABI v11; the video-generation + // slice (vllm_video_*) is ABI v12. The >= pin is the one check that can + // catch a WRONG bump: the == VLLM_ABI_VERSION assertions here and in + // test_dlopen compare against the same macro and move with it. + CHECK(vllm_abi_version() >= 12); } // ─── ABI v11: audio transcription (ARCH-ONE-SURFACE ROW 1) ─────────────────── @@ -1369,3 +1370,261 @@ TEST_CASE("capi v11: vllm_transcribe argument contract") { vllm_transcription_free(nullptr); vllm_engine_free(eng); } + +// ─── ABI v12: video+audio generation (ARCH-ONE-SURFACE ROW 2) ──────────────── +// The video slice gated THROUGH the public ABI on the fold fixture: the same +// tiny checkpoint set + goldens the PRE-fold minimax-h3-gen binary rendered at +// the branch base (tests/vllm/models/fixtures/minimax_h3_video_fold). What the +// library-seam gate (test_minimax_h3_video_fold) proves for the C++ entry +// point, this proves for the C marshalling on top of it. + +#include "../vllm/models/minimax_h3_video_fold_fixture.h" + +namespace { + +std::string ReadAllBytes(const std::string& path) { + std::ifstream in(path, std::ios::binary); + REQUIRE_MESSAGE(in.good(), "cannot open ", path); + return std::string((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); +} + +// A pid-unique fixture + output workspace, torn down with the test. +struct VideoFoldWorkspace { + std::string root, fixture; + VideoFoldWorkspace() { + static int counter = 0; + root = "/tmp/vllm_capi_video_" + std::to_string(::getpid()) + "_" + + std::to_string(counter++); + std::filesystem::create_directories(root); + fixture = root + "/fixture"; + minimax_h3_fold::WriteFoldFixture(fixture); + } + ~VideoFoldWorkspace() { + std::error_code ec; + std::filesystem::remove_all(root, ec); + } +}; + +// Owned strings so the borrowed const char* fields stay alive per call. +struct VideoFixtureParams { + std::string dit, vvae, vcfg, avae, acfg, embeds; + vllm_video_model_params mp; + explicit VideoFixtureParams(const std::string& dir) + : dit(dir + "/dit.gguf"), + vvae(dir + "/video_vae.safetensors"), + vcfg(dir + "/video_vae_config.json"), + avae(dir + "/audio_vae.safetensors"), + acfg(dir + "/audio_vae_config.json"), + embeds(dir + "/prompt_embeds.f32") { + mp = vllm_video_model_params_default(); + mp.dit_path = dit.c_str(); + mp.video_vae_path = vvae.c_str(); + mp.video_vae_config_path = vcfg.c_str(); + mp.audio_vae_path = avae.c_str(); + mp.audio_vae_config_path = acfg.c_str(); + mp.prompt_embeds_path = embeds.c_str(); + mp.partition = "fl2va"; + } +}; + +} // namespace + +TEST_CASE("capi v12: the zero-value/default contract") { + // Zero values must preserve behaviour: the _default() constructors return + // fully zeroed structs (cpu, keep-quant, no paths, unseeded, and — via the + // documented <=0 mapping — noise_aug 1.0). The golden e2e case below runs + // on exactly these defaults (t2va), which pins every zero-value resolution + // EXCEPT noise_aug: that mapping only fires on a keyframe render, which the + // tiny fixture cannot express (the fl2va VAE-encoder half is the real + // 128-channel geometry). It is pinned at the seam level instead + // (MiniMaxH3VideoGenParams.noise_aug defaults to 1.0; the capi maps <=0 + // onto it) — an honest, disclosed edge of this contract test. + const vllm_video_model_params mp = vllm_video_model_params_default(); + CHECK(mp.dit_path == nullptr); + CHECK(mp.encoder_path == nullptr); + CHECK(mp.tokenizer_path == nullptr); + CHECK(mp.video_vae_path == nullptr); + CHECK(mp.video_vae_config_path == nullptr); + CHECK(mp.audio_vae_path == nullptr); + CHECK(mp.audio_vae_config_path == nullptr); + CHECK(mp.prompt_embeds_path == nullptr); + CHECK(mp.partition == nullptr); + CHECK(mp.device == 0); + CHECK(mp.dequant_bf16 == 0); + CHECK(mp.fp4_resident == 0); + + const vllm_video_params vp = vllm_video_params_default(); + CHECK(vp.prompt == nullptr); + CHECK(vp.width == 0); + CHECK(vp.height == 0); + CHECK(vp.num_frames == 0); + CHECK(vp.steps == 0); + CHECK(vp.seed == 0); + CHECK(vp.has_seed == 0); + CHECK(vp.first_frame == nullptr); + CHECK(vp.last_frame == nullptr); + CHECK(vp.ref_image == nullptr); + CHECK(vp.ref_video == nullptr); + CHECK(vp.ref_audio == nullptr); + CHECK(vp.noise_aug == 0.0f); // <= 0 resolves engine-side to the 1.0 pin + CHECK(vp.output_dir == nullptr); + + const vllm_video_mux_params mx = vllm_video_mux_params_default(); + CHECK(mx.frames == nullptr); + CHECK(mx.audio_path == nullptr); + CHECK(mx.output_path == nullptr); + CHECK(mx.fps == 0); // <= 0 resolves to the H3 default 24 + CHECK(mx.crf == 0); // <= 0 resolves to the library default 18 +} + +TEST_CASE("capi v12: vllm_video_generate reproduces the pre-fold goldens") { + VideoFoldWorkspace ws; + VideoFixtureParams fp(ws.fixture); + vllm_video_engine* eng = nullptr; + REQUIRE_MESSAGE(vllm_video_engine_load(&fp.mp, &eng) == VLLM_OK, + vllm_last_error()); + REQUIRE(eng != nullptr); + + const std::string out_dir = ws.root + "/out"; + vllm_video_params vp = vllm_video_params_default(); + vp.num_frames = 5; + vp.height = 32; + vp.width = 32; + vp.steps = 3; + vp.output_dir = out_dir.c_str(); + + vllm_video_result out; + REQUIRE_MESSAGE(vllm_video_generate(eng, &vp, &out) == VLLM_OK, + vllm_last_error()); + CHECK(std::string(out.frame_dir) == out_dir); + CHECK(std::string(out.audio_path) == out_dir + "/audio.wav"); + CHECK(out.frame_count == 8); + CHECK(out.width == 32); + CHECK(out.height == 32); + CHECK(out.fps == 24); + CHECK(out.sample_rate == 32000); + + const std::string golden_dir = MINIMAX_H3_VIDEO_FOLD_FIXTURE_DIR; + for (int f = 0; f < 8; ++f) { + char name[64]; + std::snprintf(name, sizeof(name), "/frame_%06d.ppm", f); + INFO("frame ", f); + CHECK(ReadAllBytes(out_dir + name) == ReadAllBytes(golden_dir + name)); + } + CHECK(ReadAllBytes(out_dir + "/audio.wav") == + ReadAllBytes(golden_dir + "/audio.wav")); + + // The mux argv is execvp-ready (NULL-terminated) and byte-matches the + // pre-fold `minimax-h3-mux --print-only` capture. + REQUIRE(out.mux_argv != nullptr); + REQUIRE(out.mux_argc > 0); + CHECK(out.mux_argv[out.mux_argc] == nullptr); + std::string joined; + for (int32_t i = 0; i < out.mux_argc; ++i) { + joined += (i == 0 ? "" : " ") + std::string(out.mux_argv[i]); + } + std::string golden_argv = ReadAllBytes(golden_dir + "/golden_mux_argv.txt"); + while (!golden_argv.empty() && + (golden_argv.back() == '\n' || golden_argv.back() == '\r')) { + golden_argv.pop_back(); + } + size_t pos = 0; + while ((pos = golden_argv.find("W/", pos)) != std::string::npos) { + golden_argv.replace(pos, 1, out_dir); + pos += out_dir.size() + 1; + } + CHECK(joined == golden_argv); + + vllm_video_result_free(&out); + CHECK(out.frame_dir == nullptr); + CHECK(out.mux_argv == nullptr); + vllm_video_result_free(&out); // double-free is a safe no-op + vllm_video_result_free(nullptr); + vllm_video_engine_free(eng); +} + +TEST_CASE("capi v12: text and video engines refuse each other's checkpoints") { + // Direction 1: vllm_video_engine_load on a TEXT/transcription checkpoint + // directory fails LOUDLY, naming vllm_engine_load as the right entry point. + VideoFoldWorkspace ws; + const std::string text_dir = ParakeetFixture("ctc"); + vllm_video_model_params mp = vllm_video_model_params_default(); + mp.dit_path = text_dir.c_str(); + vllm_video_engine* eng = reinterpret_cast(0x1); + CHECK(vllm_video_engine_load(&mp, &eng) == VLLM_ERR_MODEL_LOAD); + CHECK(eng == nullptr); + CHECK(std::string(vllm_last_error()).find("vllm_engine_load") != + std::string::npos); + + // Direction 2: vllm_engine_load on the H3 checkpoint directory keeps + // failing EXACTLY as it did at v11 (captured at the branch base): status 2 + // with the missing-config.json cause — the fold changed nothing here. + vllm_model_params tmp = vllm_model_params_default(); + tmp.model_path = ws.fixture.c_str(); + vllm_engine* text_eng = nullptr; + CHECK(vllm_engine_load(&tmp, &text_eng) == VLLM_ERR_MODEL_LOAD); + CHECK(text_eng == nullptr); + CHECK(std::string(vllm_last_error()).find("config.json") != std::string::npos); +} + +TEST_CASE("capi v12: video argument contract") { + VideoFoldWorkspace ws; + + // Null/missing load arguments. + vllm_video_engine* eng = nullptr; + CHECK(vllm_video_engine_load(nullptr, &eng) == VLLM_ERR_INVALID_ARGUMENT); + vllm_video_model_params empty = vllm_video_model_params_default(); + CHECK(vllm_video_engine_load(&empty, &eng) == VLLM_ERR_INVALID_ARGUMENT); + CHECK(vllm_video_engine_load(&empty, nullptr) == VLLM_ERR_INVALID_ARGUMENT); + // A missing VAE is a load error with the cause named. + VideoFixtureParams no_vae(ws.fixture); + no_vae.mp.video_vae_path = nullptr; + CHECK(vllm_video_engine_load(&no_vae.mp, &eng) == VLLM_ERR_MODEL_LOAD); + CHECK(std::string(vllm_last_error()).find("video_vae") != std::string::npos); + + // Generate-side contract on a good engine. + VideoFixtureParams fp(ws.fixture); + REQUIRE(vllm_video_engine_load(&fp.mp, &eng) == VLLM_OK); + vllm_video_result out; + vllm_video_params vp = vllm_video_params_default(); + CHECK(vllm_video_generate(nullptr, &vp, &out) == VLLM_ERR_INVALID_ARGUMENT); + CHECK(vllm_video_generate(eng, nullptr, &out) == VLLM_ERR_INVALID_ARGUMENT); + CHECK(vllm_video_generate(eng, &vp, nullptr) == VLLM_ERR_INVALID_ARGUMENT); + // output_dir is required. + CHECK(vllm_video_generate(eng, &vp, &out) == VLLM_ERR_INVALID_ARGUMENT); + CHECK(std::string(vllm_last_error()).find("output_dir") != std::string::npos); + // An illegal reference combination surfaces as a runtime refusal. + const std::string out_dir = ws.root + "/out"; + vp.output_dir = out_dir.c_str(); + vp.first_frame = "/nonexistent.ppm"; + vp.ref_video = ws.fixture.c_str(); + CHECK(vllm_video_generate(eng, &vp, &out) == VLLM_ERR_RUNTIME); + CHECK(std::string(vllm_last_error()).find("exclusive") != std::string::npos); + vllm_video_engine_free(eng); + vllm_video_engine_free(nullptr); // no-op + + // The standalone mux composer: contract + golden byte-match. + char** argv = nullptr; + int32_t argc = 0; + CHECK(vllm_video_mux_argv(nullptr, &argv, &argc) == VLLM_ERR_INVALID_ARGUMENT); + vllm_video_mux_params mx = vllm_video_mux_params_default(); + CHECK(vllm_video_mux_argv(&mx, &argv, &argc) == VLLM_ERR_INVALID_ARGUMENT); + mx.frames = "frames_%06d.ppm"; + mx.output_path = "silent.mp4"; + REQUIRE(vllm_video_mux_argv(&mx, &argv, &argc) == VLLM_OK); + REQUIRE(argv != nullptr); + REQUIRE(argc > 0); + CHECK(argv[argc] == nullptr); // execvp-ready + std::string joined; + for (int32_t i = 0; i < argc; ++i) joined += (i == 0 ? "" : " ") + std::string(argv[i]); + std::string golden = ReadAllBytes(std::string(MINIMAX_H3_VIDEO_FOLD_FIXTURE_DIR) + + "/golden_mux_argv_silent.txt"); + while (!golden.empty() && (golden.back() == '\n' || golden.back() == '\r')) { + golden.pop_back(); + } + CHECK(joined == golden); + vllm_video_mux_argv_free(argv, argc); + vllm_video_mux_argv_free(nullptr, 3); // no-op +} + diff --git a/tests/scripts/test_check_surface_coverage.py b/tests/scripts/test_check_surface_coverage.py index a8565a83b..cdb484d34 100644 --- a/tests/scripts/test_check_surface_coverage.py +++ b/tests/scripts/test_check_surface_coverage.py @@ -312,12 +312,14 @@ def test_boundary_green(self) -> None: self.assertNotIn("examples/cli", allow) # The known CLI-only capability drivers ARE internal-reachers. # (parakeet_transcribe left this list when the ROW 1 fold made it a - # clean vllm.h client — asserted below instead.) - for unit in ("examples/laguna_gen", "examples/deepseek_v4_gen", - "examples/minimax_h3_gen"): + # clean vllm.h client; minimax_h3_gen + minimax_h3_mux left it when the + # ROW 2 video fold did the same — asserted below instead.) + for unit in ("examples/laguna_gen", "examples/deepseek_v4_gen"): self.assertIn(unit, reaching) - self.assertNotIn("examples/parakeet_transcribe", reaching) - self.assertNotIn("examples/parakeet_transcribe", allow) + for unit in ("examples/parakeet_transcribe", "examples/minimax_h3_gen", + "examples/minimax_h3_mux"): + self.assertNotIn(unit, reaching) + self.assertNotIn(unit, allow) def test_public_surface_pinned_green(self) -> None: self.assertTrue(mod.public_surface_pinned(mod.read(mod.CMAKELISTS))) @@ -328,10 +330,10 @@ def test_reaching_count_within_ratchet(self) -> None: reaching |= set(mod.internal_include_dir_grant_units(mod.read(mod.EXAMPLES_CMAKE))) self.assertLessEqual(len(reaching), mod.MAX_INTERNAL_REACHING) - def test_ratchet_ceiling_pinned_at_11(self) -> None: + def test_ratchet_ceiling_pinned_at_9(self) -> None: # EQUALITY pin: a ceiling bump (up OR down) must move this line + the ratchet claims in # the spec/state, so the change is test-visible and reviewed, never silent. - self.assertEqual(mod.MAX_INTERNAL_REACHING, 11) + self.assertEqual(mod.MAX_INTERNAL_REACHING, 9) def test_capability_green(self) -> None: cap_allow, allow_errors = mod.parse_allowlist(mod.read(mod.CAP_ALLOWLIST)) diff --git a/tests/vllm/models/fixtures/minimax_h3_video_fold/audio.wav b/tests/vllm/models/fixtures/minimax_h3_video_fold/audio.wav new file mode 100644 index 0000000000000000000000000000000000000000..4f99043259b856f5c9a95543f46a439a11ba1fd4 GIT binary patch literal 108 zcmWIYbaP8#U|dGIyZ+z$ HfBQcG�fhy�it�qzsN�kPu�yvsP}ow\�m�sys{tbX}�v{gT�zhavlikyvOk{vlla�ya|�dwxjyTvlu�llkt�qyw���r�dpqkmw�i�widw�xcqmq|tmt[f�obvYb~zgz}Yx�`�iS|]d�{u�|Z�d^�\jysjfWXrmn�rz�awzgn|fyc_zjkm�prpf�qy]vo�zhstv�om�~ux�{�nuvi_�e~^wbutWo�gbXtwv�hhZ�pz|�v�fm�zh~nu�a�fl��Vnnh��qb}n�r}�xfw�h{�nt��h�b~�Z�ohverzg�u|reZppx�sUxqZ�vf�{d�rvvjn}wp�lngra��j�Y��d�soz�y}xi��dunxxsYvl~��o��Qwqa|ffvs�dpkzg~ur��za{�ocym]pts�o^tp^�]i�|^unVwom�x`kz��shzZt�dR�mltip�ptpx�fVl�`lR��kj�vwwrt|xqkhQ�dq��b~sQ{lpZdi�iksKf��]r[i}_��iqm�vd�wbso�s�tq�h��z�pwo~ymrmXfhxgwYo}]zyey�e��f^dpmwqs�oc{l�u]�afqu�}wbemk�wY�nU��{pvvqgdvcgtT�vp}�[mwf�ln�~�ezmnho~qk�oqbxQq�fy{r�~l�ze�a[�Ybo^�mb|{hk~cgdwo`md�escmp[p|]�ynrml}quxib�xYdrV�]�vW�o^�o{mj{�eo��ci}v|o\wwg}w}�xf}�^�sT�bSh�}{jqquY�~ierm��Pyxajy\�o_t�fs�`�rkvtn}�g�yZ�lw}}kw�K�sXvzaisc��n�km{X��rrt`T�Po\o�v�ns�c}ys��[puzhfl��v�six�Myqcu�ldyf�}m�imk~tYzn{}^uwtt}c�}j�ou�xm�ra�goqpq}gY��apyoy�drtu�pf�Xsuis{~T�|\Wss�ci��`�`n�pT�t�v�~devw|u`s�mcsma��i}}^hXX~�p�rl�xg�edqir�ka||Z�fC�cp�ckvxty�g�rN�vdz|{p�VyZ�{^�[d{qVnvbfln��m�eo��k�bn�qqyhs�rjnse�ske]ewv��}q�jQ�kt}j]ro{{wowrfqi\��X�}i�Porq{�hvjZ�s���moyg�g�wSt�lymq�oh�r`vko�cc�j^{Prnr��u��c�qrzw�xxttqqt�o��Zxwdyg[\r�c��jjrw��fu�[zvjvkc�jowxQonm�{u�c^�uas�d�{{{{]�u^�QzzmlxXylu~uk�vVz�Ps�bqjc]pQ�`Oosvfgs�ho�}T�wY�ifw�d]zq�i^�ln��u�mm�|Zztj�r|ngw~q_�g`w�X�kfucL{n`iZgwrIhpsuxQ}xgu{b�rbbnl�qxuv_}�guk�uZpogd�hbk`X^ay�jay�an�jzew�t��_}{iztg�\�va~}g|yeviT|w��l��kqnu�g_��p�t��|\�wq�|fn^gvik}�o��T|hpynn�xlqm�wtrnxh�`eyegxu|pdozjcqdq{mv�twu_\�{n�olrzH�}�tktjg�bs~Ty��{�i�jR�nNu�nosXj}�ycniU{xphcRm�Ppvc�qf�jO�qqu�Y��qo�^��x��Y�af{ji~yW��hw[f��y_Z�k`x�]��getp�u{vS�u^m{n�lsl�l�xqrsS�f[��Y�eq�pk�^p�o]�fexYRsgj�kW��x}�i��e�dasgY}ays�q�r^{�Xc|Z��de�diigy�\�nep�drmVssc�s[sw\~}p�vc{sR�bG�^elm�v|Yt�_~yZt�ev_~vO~[X�fe�`c��Z~�i�lkwZy|uny]�zZw�ojsiy�j�h\��pot`l{�qyawkXrcxird|ki��PqQ�m�qhaq`�lR�pa^cj��svnfxsa~hw�il{}M�hw�bur][ow~fds\mroicJ]�s`np]{srfW��Q{t[�ul�th�udmol��yz�td{qw\f�|G�}om�tt�omePn�y�~j��^}|Xnuuo~��Rq�}`l^{ya��w]yn{zh�lV�tR�x]k�bbbiSwQ�oe�q`�qt|k�fulxmZfYd�]dqbVe~|�osasT}]Q�oq�x{xad��Y}wg{eZ�^Tt]g}�R�qO�]W��m�ur�zl�vczq[�gZo{Yq�l�j`�pym�e{m`�khq�NyyW�}Yt�]`mXfO��Rln��mx�i|nfh��t^zte�g{w~W�id�wr�te�{]�n{o_d|qnl{kallzm_�ok{iS�oqbkc�mt�motk~�bVtjs|we�qXs���]erp\wph�}ZsxF�htctpsj��gxja�{Nuo��cv�q�eu�m]ou��hx�xwitZhoU��Y�sTf|okRb�`m�u�pyZ�hhTXd�d�su�sev�c|�k�sqkva�]q��|owV||er\ssdO�bp�fRsdc��nxybsiOb\}pneuve��rmh\�}sv�UwmjqjifrW�{l�y\|�Tosuwkj}s]�sVy�qtkw�af~�Pjkykxxs�a�sia�g]yenqd�\j�v~�xavt[crg�ohkvYz\{�qs}ub[uioVhdY�}diz{�|Z{~ps�`xlP�zVi{{t�g�[Mp{i�p[o}r�eQ��w�fnu{ripbU�eznc�s_or�ulgqhc�rq|j]�~\�q_�qe�hot�c�hry`gfimy_y�sw�Spv[�kwcpkl{cwuVmfpmevevr�t_�qQy�u�qr�q\c{mrw_]�yvc]tcbxf|t�]fXYpcWadkgz^�jVmolgcj�qc{�P�w��ftrruoV�f^ig�nil��p�p]{sYylaiwv�op{gR�d�x]qpmn�ij�hbspsx�Yzyv}sstjh}bRobYu�F�neucisclyyg�ilhnS��gfwk�pV|jkrzd}gav�W�yr�x\|vt�ymol�ffxdz��x�o��{u�du{zfq�p�hhtji�gr \ No newline at end of file diff --git a/tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000006.ppm b/tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000006.ppm new file mode 100644 index 000000000..475a2306a --- /dev/null +++ b/tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000006.ppm @@ -0,0 +1,4 @@ +P6 +32 32 +255 +�dK�visfz�mkqge�`}]r�zz�e_|^_�sHz�{�pa�}euZ��xcxnpyafzsp�ZoVo~h�iy�xzhVchb�wf��gP�bx�o��cu�btz{mmp�{kohnufW�sy�cjo�`�nPuq[��d�xT�yr�oa��Nx�dhk�{i�_v��^rkk�[s��p�tgotj{TztWrlHwqi\su��umuequ�j��rdlxo`pipahd^~ad�Zs�tm�ut�{]|]zyyh�`a��s�pb�t^~�^��m�~W�vu�r^i�]w�j�mu{wwu�ixzx��bvnhrfdsyX�kwjxgy�e�hTqold�`v�ukpls�q�un�aY�`grwq�cQ|tV�sqhaw~dRew_�jgm�[nuqofo�`K�mNevu��\�wiWekhq�~t�lSqu^wno�u{_zg~st�oc�kSz�w|f��ktXdg[w}|uutkeJs�T�ob�~a|b\oqnddsq�N�aXei�sogddmNp�rU{v[|~\thg��gj|f�oj�eo��vz�ww�Tj[]��haySy�U�}emj]i�ef_pvs��ti`k�^||dn�wo_r`��W�sg�gq�skx|crqnmsdv�jw}�jVzzZ[hu�wi�u|�lz~u}sv��x�frjqji�x�qglks�t[}yWotRkojzkt�z[oe_�nikcyh\�pg�x\kkp�|h�{ex|a�l{�plsipqo�nw�pm�\k�f��o�khyi��gt�a�yftld�rT�wpvmmzvV�u`x��pxc|h]}}{�xg~|�z|Vx}[d�n~p\rbK{[y�ll�ir��`�Znp\uViyohsmYgwltfg�r��]�ul�vgr~XoyZ�xnwfuyoa�li�hJcuo{zaz[a~ucmxs��S}���eZl�`�u\|_mnxnzzRgQdoc�~qupa�r`�uS�|Bnt^y�wxpw�o]jmWtiU�sq{gYv���_ovov�`brfk�pr�q^z}\~|ir=�q]yrn�xj�vfhxzt�s[�fs�ii{rU��r�szpln{pm��_~x_�s\cth{�fpy��XN�u�ox�`xutfx�hs`as�j�ho�mx]ol|vk�va�er�c�n[sj`f�lstpl�[�qx�erRjsmw~cYo}Q��]xt�|l[jk{nIugsy�H~lc�sa�|n�}gzda|qm{gzWd�jmavWi[i�xYzm^��o�m\s�^�qx}uh�m~vlT~hg�q�shnr�bdn[�e�ly��jxfbWx{h~kmja_zrg�rjzvPx`icjas�^haut~yZsa`brx`kxqg��Yxwnqde~�c}vlvmK��q�yr{�j�knhw^v�j|q^qzd�iJgeXl]w_du��~gxcvr�\�QYyo_}�b\uypzqpb|spa|�d{|��ie��lnlslhjpxU��jujaxpdz�V�rS�bw�w�tc]�gV�pst�Z��N�lftzT{WqfZnw`�v]z�L�nr�kw�ub|�Z��m}rq{tr�jR�w\yvm�m�fu�mnu�^h�kl{{w��jq�ghxPpsQ{�U�Sc�tvq�i�RUxvrxqX�kL�gyy�`iz\hnmyc\ge|�j]�dWxzs��q��h��Z�of^sJ��b�|���Xoj�xaW��ohoXxqY�{o|[\��sx_FvvO�qen�kshHl`ywna�sZ�mx�}]tht�xV�drs�Zj|[gfT��T�h}j{Ipgm��vz�bugu�zr~ymlynstFoxsWisw^�bX�Xh�_�^_nik�fm�yU��`xyb��o}mFz�jssl�^bpyl{pV{�b��s�t\e�W�ekkvq�w~xH�n^wfqyn`ic{�pg��[��omypx}fbuXv�v�Jw�bp�s^�cgwqQjr`tyj�~fz�d��j�sY�is~fErzo�vc��w}}rfihz^xoR�{n�qXk�k~�c�sYl|�j�tiiU}mkvtV��m�qhv�mgztwwjoTvii|ww|�X�z]~\Ur�o�go��l�lX�lurzo�s^yiK��l�yr�b`s�vs}cv�lh�ufjpt���g�M��x��e|�m�n�rcdj�b]~Z��njz]o�h��m|�a�mYzsk~j_|Qq|n�]^|uUp�tk��x�awp`~|p��u�}}�kl�uiw~vyYs�topo~��S{i�|��rgzpxu~qyuP�to�t^��Qslbpyns�x�z�qx�~Pxogl\��^}g�|skmp�xow[\gkT��lwWnkiM�bLrZ^�sJq�k��aiyg�zRrk�ilX��myql�}a�gO�}vuq]rl`�iL{id�{wccturlh�Tr_��jsvdlja}�I�bg�|j�n[pkV��y�vx�qfbtW�mm��Z|j`n|crrM�hgjl`}vjsvp|�dz�u��J~nLuztl~gw�~w~z�{f�cnqWazuX�jz�nx�Tlx{z�{t{rdoyX|wxssulx[[cc�pcxijt�zd~d�cZ{n�iMx�_��_dti�yg�r^rkn�t\z`gv|b�zt�vg�mq�[�m�Xv{eup|`ftym�x}Z}wwu�u��c�}flqo�eY�efqyhrqaxO_�rnx�w�{M�hH{]P|jR}sa{tljxQb}Vy{}�|uz�^fz�hl��emhx�otkjqu�{w�q~w�c}fityZy�W��hpxer�l�`uu�g�ss�vpuqb�ne||L�m\xzp�f{�ngzikuXmniVp{l�ykmr^�~QnlHxnmwlnun_}kwvgawacb|x�rO}a;~}Zvnizlg�wn��{�g�gj�ycw{vk�pvZv�f_|z�ic`�A��R��n�ojoyb��a�vzs�cp}j{dl�`chp}y�b�tm��b�Nzmf[ma[Xu\q�us�a�nbfrh�u`rtM��X��[yr^�yds_VgvV�mdhj_}�b��}ilWalXy{[�y\�_d�bl�v}�qt��\oyh�pnxu]~�[�rv�qlwnb|sqyivdv�UjVzr~hv�cyl`�nVqh�fc�wOmv^���wtiq�j{\blge{eM��Q��q�~\}vp|m�elY�v�n|�hu�s[qutzPseg}�kwesmvIl�d}ci�fqvzT}�f�uuqqn�~geu_��W�yEywg{�p�zap`Yq�_rpert_�igzjlybjz}enx|li}{btlg|lxz~gPz|Zml{b�y]g{afwi�d�pi}Xdtddjas��uWt\rkA�X��}Gso~hbzy�nrNw����sm}q�w|wO�quo�a�i�x�w}{nejKnwj��igiw}v \ No newline at end of file diff --git a/tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000007.ppm b/tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000007.ppm new file mode 100644 index 000000000..0d54e4ac2 --- /dev/null +++ b/tests/vllm/models/fixtures/minimax_h3_video_fold/frame_000007.ppm @@ -0,0 +1,4 @@ +P6 +32 32 +255 +��xY�r�Rjx�^{cjz{dzvfqlO�pU��m~vk~�ixn[�f`{�U�nn{gZs�P�hwr_{go`�icz[X�xI�r\�hmqjuxivpx�q^|u]{g[�kIz�k��fj�[�xm|`h}id�v[w�`|~ypow�o\�~\fuS�cm��ju�Rcqcjom�rixxqqnY|�xzjlyp]�nX|oe|~t�dlgjcor{}zH��osxcooF��z~�fuy���Xh�T{�|�takrn�jOv�t�qyv�a�ncnshskX�~~o`h{eqxxmi�o�k���vybY|�p�}{�mZn}uudZ�`^��glmw�|gukUqWj^��{af�eam�S~zn�ytn\�j[uyb�xevt`~jkvq`�|^ub`�lT�n`|a��tbv}�r��yM|�i�]k�mi|mh�t��z^�~e�diX|c�uYq�o�ui�mf�fq��X�sX��k}l`uwe|tj�tbn}]q�k��j�yatmW�wpg�Xyd�~ek|�j�sM{lu|uag�W��w��h�icprk�fc|ffsyW_pguah_�xhhw�fvtg�zteei�wr�p�x�kzlZ}�xxe_�va��ltuZxjW��pzvv_ySiig~�`�nTn�o]zR�zydmaxrh�be�sC��v�{Vjxj��l�fr�kmn�oug�w�PszYydi�op{mh�_su�tdxfzk\zs�vRikU�}OxyRgzq�{n|m]���qnh�ia�pz|�i|mzr_lvs�px�f}pjbzto��fvrk{trjwm~yP�bi�ox�v~^�n�Y�}Wuv_noduqnpsHjbY�vn�kb|}Sgubr�P�~_{w}xht�gd|y^yrfmxw}_woxgr�hxmpfvkx|r�hpps}y�x�ya~rLxzjz^^yx]ia�tS{pn{s�qh|zlwg_rte~�i�sctoh�v��e}{To{}��b�nj�lPbhq�iu��[j�{y�jr�Rn�^z�|{|plZ|w�t�h�r�]�{Up�q{�vz|m�eYnU�{hl�pf|ot�\m}mn{�m�xc{pd�pj�ah�Tju|t��jf{oznY{nn�yO�lu�{Xzrepp]jq��vf}Zt}t���u|jZtxk�`]g�d~jfwba�r[w�K�\��z\y|x|�o�sahaeis]~o_ayy��^�nj�sm}mEsj��ae��el�`�`�yq�fe_}]k�ll�n{tbu]i|rU{tf��k�ma�w^�wj�Wqwm\dpu�_Y��Q�pU�~�sonz}Y_dqs}`�~e�td�{g�xp�r~~�bjlT~y`�pqujg�vb�ut�r���{|d[pi`�eQsfrvwu�k[�t�zndxwnpz_�[k|ri�tnu\��ys|~a�g~��vYutzla�wT�gzlhiu�Zs�jj~j�pscfY�sn�y\tr{��[�{q�id�sow}p�lu�}Y�v\p|^|q[���}_spzr�lr��}nx�gWoam~{W�mhors~}io�e\V~�i��i��n�t]jok�xjn�m�txt_P�~T~akprj�}^~x_ygi}�qv�pvtn�t`wmW�v\�YYthO�qrvzpqz�|fv�dgqj�{j��_eya�\Y�fe�{m�qhrwy}�a�_u�zku�j�ucoeYpnhzxfw�o�md��WjmVz_p�jexl�}�[sq�qzr�fqngd��`x\s�xU�louc_~v�n�^�Wc�}mi�isx��xT�jk~wr�ks��e�la��mbvw��{km~wz�y`vtu]nd�jTlpfpps{v�Zw�wgs�f�zR�vhuZrtekbJi�rx�rfqlibmw`u�V�`_w��p��ty`�qXsQl}`��rgxZg�hpegjhtzik�josjd�zph�i�is�h\yp\guZ��s|sUv�r�[QsmV�qS��qwr^rlSijo|{a��gm�lplm����jex{Sjoy`Sskt�wowgcw{n�cmx�x{Xp�pi�xkj�nx�Y�`o���t�r�|xl�ztou{^q�yNtzpkq�pdt�q[�_mmfjnphqenk[��tp{kigcg�g~mS�hf�bS�yx�lzqyq�mbrujroX�`c�}X�nmi[e�~|�zfzsa�io�sZy�Uz�_�uj��sq�jvoY}fz�vhrt_j]J�djsg~{hm�o�{�Dp�~`Gfsro}mO{hn�ikqrjzrcxqevZe�z|fspeu\tkf�sbkdojkT�~gn�Vtk[k|]�to��{}i`�y�tabyp�ghmig�u}��ryrY�dsmv�lhhv]��l{aV{uR�gZrrm�o\�uw�sUsk[�ncvq{szo�i^�|i��v~Rl}�rxjI�ih�od{wgv�k�[Yz�{�srqsbyyp}b`]|y�dj�XfjvQyZrshlw�\q�i~gV��iy{v�d���r�pT�ylz�f��ycdQw�Q��q��^�u�hl�rmjwb]pE�g`�p~ncux�zn�R�|��tg[s`��pyvN�so�[zvzZjjH}gc�aumeyy�~wl��o^�e]nxs��my�[�kt�emrln�ezgjpt�g�kZz�ervlrhb�s\q_��Zk]XrwT�vu�hw�d�qZ��apqYhxo}sLoTgzx}sj`xqqn�`���vx]orr|�iv�\a}pj�Wz�YsaTz�tq^nek|}�|dp�}|hW\oby�b|lSyjk�Yh�tZ}jl]d{�c�wY�]WjYg_lX�ycl�k�ku��N��|z{�rn�z|T~`l|�c}ujc�k�ghfc[zsfy{z�b\}vs�}D�zz�nasaquvWts\hu{�{X�xMcp`��o��st_j|zm�V{T|nlWpyvfprhyw]]k�[�dTsp]��jsSxqfv~hh�gjpl[j�qf�hgzZ~Ygkt��mY^�oxwb�lgyVol|r~sb�wj�oVmnl��d}�k|�syzq{~d�c^��blivbp`|�_~p`�pk�ky�b^`^]j�Qvlk{}q�`ph�q�]�tg^�S{w]tpu}�b|umd�pr�fnlskm��qxyu~Y]�uoqsT�ybltP�tlsrh�zZ~�qjzB~rc~nG|z��zkkiFsbe~pUWjL�lejfa�u�tfj}|�h��mjog[t�`�xq�`a�ivsr_��ge�fnY�zh�l�ngpzrgxlK|�xrwyt��lhMef~n��kQnZ^lgvov\ps��fgyt_lr^m�U��rnhUmgflhioxd�fb�Z��[�dg~Z`tpi{�m�vs�ze�a�ozhmS[{ch��fr�e�gO�^�upYq�o]yR��_�la�ft�u\�ogyxcsZqzlh{�zp�V|�H�pu{{Kye_ \ No newline at end of file diff --git a/tests/vllm/models/fixtures/minimax_h3_video_fold/golden_mux_argv.txt b/tests/vllm/models/fixtures/minimax_h3_video_fold/golden_mux_argv.txt new file mode 100644 index 000000000..eba1b3d6f --- /dev/null +++ b/tests/vllm/models/fixtures/minimax_h3_video_fold/golden_mux_argv.txt @@ -0,0 +1 @@ +ffmpeg -y -loglevel error -framerate 24 -i W/frame_%06d.ppm -i W/audio.wav -c:v libx264 -pix_fmt yuv420p -crf 18 -c:a aac -b:a 192k -shortest -movflags +faststart W/video.mp4 diff --git a/tests/vllm/models/fixtures/minimax_h3_video_fold/golden_mux_argv_silent.txt b/tests/vllm/models/fixtures/minimax_h3_video_fold/golden_mux_argv_silent.txt new file mode 100644 index 000000000..1b8d39200 --- /dev/null +++ b/tests/vllm/models/fixtures/minimax_h3_video_fold/golden_mux_argv_silent.txt @@ -0,0 +1 @@ +ffmpeg -y -loglevel error -framerate 24 -i frames_%06d.ppm -c:v libx264 -pix_fmt yuv420p -crf 18 -movflags +faststart silent.mp4 diff --git a/tests/vllm/models/minimax_h3_video_fold_fixture.h b/tests/vllm/models/minimax_h3_video_fold_fixture.h new file mode 100644 index 000000000..2a850eb93 --- /dev/null +++ b/tests/vllm/models/minimax_h3_video_fold_fixture.h @@ -0,0 +1,342 @@ +// MiniMax-H3 VIDEO FOLD fixture — the deterministic tiny checkpoint set the +// ARCH-ONE-SURFACE ROW 2 fold gate runs the WHOLE t2va assembly on (CPU-feasible +// by construction: reduced dims, random-but-deterministic weights). +// +// This header is deliberately SELF-CONTAINED file assembly (gguf_builder.h + +// raw safetensors/json writing, no vllm library dependency), because it is used +// on BOTH sides of the fold: +// 1. at the branch BASE it generated the inputs the PRE-fold +// `minimax-h3-gen` binary rendered, whose frames + WAV are committed as +// tests/vllm/models/fixtures/minimax_h3_video_fold/ goldens; +// 2. the fold gate (test_minimax_h3_video_fold.cpp) regenerates the SAME +// bytes at runtime and requires the library seam, the replicated old +// pipeline, and the rewritten thin-client binary to reproduce those +// goldens byte-identically. +// Every value is a pure function of the tensor name (FNV-1a -> splitmix64), so +// the generated files are byte-stable across platforms and compilers. +// +// Geometry: the tiny ComfyUI-GGUF DiT of test_minimax_h3.cpp:3786 (2 layers, +// hidden 64, latents 8/6) + a video-VAE whose PARSED config reproduces the +// reduced ViT3D decoder of the "WHOLE t2va path composes end to end" case, and +// the same reduced BigVGAN audio VAE. Render request: --partition fl2va +// --frames 5 --height 32 --width 32 --steps 3 (latent 2x2x2, audio_t 8). +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "../gguf_builder.h" + +namespace minimax_h3_fold { + +// ── deterministic parameter stream (FNV-1a name hash -> splitmix64) ────────── +inline std::vector Param(const std::string& name, int64_t count, double scale, + double offset = 0.0) { + uint64_t x = 1469598103934665603ULL; + for (const char c : name) { + x ^= static_cast(c); + x *= 1099511628211ULL; + } + std::vector out(static_cast(count)); + for (float& v : out) { + x += 0x9E3779B97F4A7C15ULL; + uint64_t z = x; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL; + z ^= z >> 31; + const double u = static_cast(z >> 11) * 0x1.0p-53; // [0, 1) + v = static_cast((u * 2.0 - 1.0) * scale + offset); + } + return out; +} + +inline void WriteFileBytes(const std::string& path, const std::string& bytes) { + std::FILE* f = std::fopen(path.c_str(), "wb"); + if (f == nullptr) throw std::runtime_error("fixture: cannot write " + path); + if (!bytes.empty() && std::fwrite(bytes.data(), 1, bytes.size(), f) != bytes.size()) { + std::fclose(f); + throw std::runtime_error("fixture: short write " + path); + } + std::fclose(f); +} + +// ── minimal safetensors writer (single file, F32 tensors) ──────────────────── +struct StEntry { + std::string name; + std::vector shape; + std::vector values; +}; + +inline void WriteSafetensors(const std::vector& entries, const std::string& path) { + std::string header = "{"; + size_t offset = 0; + bool first = true; + for (const StEntry& e : entries) { + if (!first) header += ","; + first = false; + header += "\"" + e.name + "\":{\"dtype\":\"F32\",\"shape\":["; + for (size_t i = 0; i < e.shape.size(); ++i) { + if (i != 0) header += ","; + header += std::to_string(e.shape[i]); + } + const size_t bytes = e.values.size() * sizeof(float); + header += "],\"data_offsets\":[" + std::to_string(offset) + "," + + std::to_string(offset + bytes) + "]}"; + offset += bytes; + } + header += "}"; + std::string file; + const uint64_t n = header.size(); + file.append(reinterpret_cast(&n), sizeof(n)); + file += header; + for (const StEntry& e : entries) { + file.append(reinterpret_cast(e.values.data()), e.values.size() * sizeof(float)); + } + WriteFileBytes(path, file); +} + +// ── the tiny DiT geometry (test_minimax_h3.cpp:3786) ───────────────────────── +struct FoldDitGeometry { + int64_t num_layers = 2; + int64_t token_refiner_num_layers = 1; + int64_t hidden_size = 64; + int64_t num_attention_heads = 4; + int64_t attention_head_dim = 16; + int64_t ffn_hidden_size = 128; + int64_t latents_dim = 8; + int64_t audio_latents_dim = 6; + int64_t text_dim = 24; + int64_t timestep_input_dim = 16; + int64_t time_embed_hidden_size = 64; + int64_t time_embed_dim = 32; + int64_t adaln_out_features = 18 * 64; + int64_t final_adaln_out_features = 2 * 64; + int64_t rope_inv_freq_len = 2; + // Derived (patch 1x2x2, the shipped patching): the packed video token width. + int64_t video_row_width() const { return latents_dim * 1 * 2 * 2; } +}; + +// The render request the goldens were captured with (the pre-fold binary's +// exact flags). text_len is the prompt_embeds row count. +struct FoldRenderRequest { + int64_t frames = 5; + int64_t height = 32; + int64_t width = 32; + int64_t steps = 3; + int64_t text_len = 4; + const char* partition = "fl2va"; // serves t2va + fl2va (the #77 guard) +}; + +// ── write the ComfyUI-format F32 GGUF DiT ──────────────────────────────────── +inline void WriteFoldDitGguf(const FoldDitGeometry& g, const std::string& path) { + gguf_test::GgufModelBuilder builder; + builder.AddKv(gguf_test::StrKv("general.architecture", "wan")); + // GGUF stores `ne` REVERSED vs torch: a logical [out, in] weight is written + // [in, out]. F32 everywhere — the fold gate is about the ASSEMBLY, not quant. + auto add = [&](const std::string& name, const std::vector& logical) { + int64_t numel = 1; + for (const int64_t d : logical) numel *= d; + const std::vector values = Param("fold.dit." + name, numel, 0.1); + std::string bytes(reinterpret_cast(values.data()), + values.size() * sizeof(float)); + std::vector ne; + for (auto it = logical.rbegin(); it != logical.rend(); ++it) { + ne.push_back(static_cast(*it)); + } + builder.AddTensor(name, ne, /*ggml_type=*/0 /*F32*/, bytes); + }; + + const int64_t inner = g.num_attention_heads * g.attention_head_dim; + const int64_t video_width = g.video_row_width(); + add("video_patch_proj.weight", {g.hidden_size, video_width}); + add("video_patch_proj.bias", {g.hidden_size}); + add("audio_patch_proj.weight", {g.hidden_size, g.audio_latents_dim}); + add("audio_patch_proj.bias", {g.hidden_size}); + add("condition_proj.weight", {g.hidden_size, g.text_dim}); + add("condition_proj.bias", {g.hidden_size}); + add("time_embedder.proj_in.weight", {g.time_embed_hidden_size, g.timestep_input_dim}); + add("time_embedder.proj_in.bias", {g.time_embed_hidden_size}); + add("time_embedder.proj_out.weight", {g.time_embed_dim, g.time_embed_hidden_size}); + add("time_embedder.proj_out.bias", {g.time_embed_dim}); + add("rope.inv_freq", {g.rope_inv_freq_len}); + auto add_block = [&](const std::string& prefix, bool with_adaln) { + add(prefix + ".norm1.weight", {g.hidden_size}); + add(prefix + ".norm2.weight", {g.hidden_size}); + add(prefix + ".attn.qkv_proj.weight", {3 * inner, g.hidden_size}); + add(prefix + ".attn.q_norm.weight", {g.attention_head_dim}); + add(prefix + ".attn.k_norm.weight", {g.attention_head_dim}); + add(prefix + ".attn.out_proj.weight", {g.hidden_size, inner}); + add(prefix + ".mlp.fc1.weight", {2 * g.ffn_hidden_size, g.hidden_size}); + add(prefix + ".mlp.fc2.weight", {g.hidden_size, g.ffn_hidden_size}); + if (with_adaln) { + add(prefix + ".adaln_proj.linear.weight", {g.adaln_out_features, g.time_embed_dim}); + add(prefix + ".adaln_proj.linear.bias", {g.adaln_out_features}); + } + }; + for (int64_t i = 0; i < g.token_refiner_num_layers; ++i) { + add_block("token_refiner.blocks." + std::to_string(i), false); + } + add("token_refiner.final_norm.weight", {g.hidden_size}); + for (int64_t i = 0; i < g.num_layers; ++i) { + add_block("blocks." + std::to_string(i), true); + } + add("final_layer.norm.weight", {g.hidden_size}); + add("final_layer.adaln_proj.linear.weight", {g.final_adaln_out_features, g.time_embed_dim}); + add("final_layer.adaln_proj.linear.bias", {g.final_adaln_out_features}); + add("final_layer.video_out.weight", {video_width, g.hidden_size}); + add("final_layer.video_out.bias", {video_width}); + add("final_layer.audio_out.weight", {g.audio_latents_dim, g.hidden_size}); + add("final_layer.audio_out.bias", {g.audio_latents_dim}); + + WriteFileBytes(path, builder.Build()); +} + +// ── the reduced ViT3D video-VAE decoder + its config.json ──────────────────── +// The config keys mirror what ParseMiniMaxH3VideoVaeDecoderConfig reads +// (minimax_h3_vae_loader.cpp:512): heads 2 x dim_head 8 => dim 16, ffn_mult 1, +// register tokens 2, rope_dim_ratio 0.75 => rope_apply_dim 6 — the reduced +// decoder of the e2e composition test. patch_size (16) and patch_size_t (4) are +// struct constants, not config keys, so the decoded canvas is latent*16 spatial +// and latent_t*4-(4-1) temporal (5 frames at latent_t 2). +inline void WriteFoldVideoVae(const FoldDitGeometry& g, const std::string& weights_path, + const std::string& config_path) { + const int64_t dim = 16, heads = 2, dim_head = 8, ff_inner = 16, num_register = 2; + std::vector entries; + auto put = [&](const std::string& n, const std::vector& shape, double sc, + double off = 0.0) { + int64_t numel = 1; + for (const int64_t d : shape) numel *= d; + entries.push_back({n, shape, Param("fold.vvae." + n, numel, sc, off)}); + }; + const int64_t inner = heads * dim_head; + put("decoder.x_embedder.weight", {dim, g.latents_dim}, 0.1); + put("decoder.x_embedder.bias", {dim}, 0.05); + put("decoder.register_tokens", {num_register, dim}, 0.1); + put("decoder.norm_out.weight", {dim}, 0.1, 1.0); + put("decoder.norm_out.bias", {dim}, 0.05); + const int64_t patch_dim = 3 * 4 * 16 * 16; // out_channels * pt * ps * ps + put("decoder.proj_out.weight", {patch_dim, dim}, 0.1); + put("decoder.proj_out.bias", {patch_dim}, 0.05); + const std::string b = "decoder.transformer_blocks.0."; + put(b + "norm1.weight", {dim}, 0.1, 1.0); + put(b + "norm2.weight", {dim}, 0.1, 1.0); + put(b + "scale1", {dim}, 0.1); + put(b + "scale2", {dim}, 0.1); + put(b + "attn.to_qkv.weight", {3 * inner, dim}, 0.1); + put(b + "attn.to_qkv.bias", {3 * inner}, 0.05); + put(b + "attn.to_out.weight", {dim, inner}, 0.1); + put(b + "attn.to_out.bias", {dim}, 0.05); + put(b + "ff.w1.weight", {2 * ff_inner, dim}, 0.1); + put(b + "ff.w1.bias", {2 * ff_inner}, 0.05); + put(b + "ff.w2.weight", {dim, ff_inner}, 0.1); + put(b + "ff.w2.bias", {dim}, 0.05); + WriteSafetensors(entries, weights_path); + + std::string cfg = "{\n"; + cfg += " \"decoder_num_layers\": 1,\n"; + cfg += " \"latent_channels\": " + std::to_string(g.latents_dim) + ",\n"; + cfg += " \"out_channels\": 3,\n"; + cfg += " \"decoder_num_register_tokens\": 2,\n"; + cfg += " \"decoder_num_attention_heads\": 2,\n"; + cfg += " \"decoder_attention_head_dim\": 8,\n"; + cfg += " \"decoder_ffn_mult\": 1,\n"; + cfg += " \"decoder_norm_eps\": 1e-5,\n"; + cfg += " \"decoder_rope_dim_ratio\": 0.75,\n"; + cfg += " \"decoder_rope_theta\": 100.0,\n"; + cfg += " \"latents_mean\": ["; + for (int64_t i = 0; i < g.latents_dim; ++i) { + cfg += (i ? ", " : "") + std::string("0.0"); + } + cfg += "],\n \"latents_std\": ["; + for (int64_t i = 0; i < g.latents_dim; ++i) { + cfg += (i ? ", " : "") + std::string("0.5"); + } + cfg += "]\n}\n"; + WriteFileBytes(config_path, cfg); +} + +// ── the reduced BigVGAN audio VAE + its config.json ────────────────────────── +// Mirrors ParseMiniMaxH3AudioVaeConfig (latent_dim => num_mels, decoder_dim, +// decoder_rates/kernel_sizes, resblock kernels/dilations) and the reduced +// weight set of the e2e composition test, under the checkpoint's `decoder.` / +// weight-norm naming the loader maps (minimax_h3_vae_loader.cpp:108). +inline void WriteFoldAudioVae(const FoldDitGeometry& g, const std::string& weights_path, + const std::string& config_path) { + const int64_t num_mels = 8, decoder_dim = 8, ch = decoder_dim / 2; + std::vector entries; + auto put = [&](const std::string& n, const std::vector& shape, double sc, + double off = 0.0) { + int64_t numel = 1; + for (const int64_t d : shape) numel *= d; + entries.push_back({n, shape, Param("fold.avae." + n, numel, sc, off)}); + }; + auto put_conv = [&](const std::string& prefix, int64_t oc, int64_t ic, int64_t k, bool bias) { + put(prefix + ".parametrizations.weight.original0", {oc, 1, 1}, 0.03, 0.15); + put(prefix + ".parametrizations.weight.original1", {oc, ic, k}, 0.08); + if (bias) put(prefix + ".bias", {oc}, 0.05); + }; + // dec_in_proj: audio_latents_dim -> num_mels (a PLAIN Conv1d k=1). + put("decoder.dec_in_proj.weight", {num_mels, g.audio_latents_dim, 1}, 0.1); + put("decoder.dec_in_proj.bias", {num_mels}, 0.05); + put_conv("decoder.conv_pre", decoder_dim, num_mels, 7, true); + put("decoder.ups.0.0.parametrizations.weight.original0", {decoder_dim, 1, 1}, 0.03, 0.15); + put("decoder.ups.0.0.parametrizations.weight.original1", {decoder_dim, ch, 4}, 0.08); + put("decoder.ups.0.0.bias", {ch}, 0.05); + put_conv("decoder.resblocks.0.convs1.0", ch, ch, 3, true); + put_conv("decoder.resblocks.0.convs2.0", ch, ch, 3, true); + for (const char* a : {"decoder.resblocks.0.activations.0", + "decoder.resblocks.0.activations.1", "decoder.activation_post"}) { + put(std::string(a) + ".act.alpha", {ch}, 0.2); + put(std::string(a) + ".act.beta", {ch}, 0.2); + } + put_conv("decoder.conv_post", 1, ch, 7, false); + WriteSafetensors(entries, weights_path); + + std::string cfg = "{\n"; + cfg += " \"latent_dim\": " + std::to_string(num_mels) + ",\n"; + cfg += " \"decoder_dim\": " + std::to_string(decoder_dim) + ",\n"; + cfg += " \"decoder_rates\": [2],\n"; + cfg += " \"decoder_kernel_sizes\": [4],\n"; + cfg += " \"resblock_kernel_sizes\": [3],\n"; + cfg += " \"resblock_dilation_sizes\": [[1]],\n"; + cfg += " \"latents_mean\": ["; + for (int64_t i = 0; i < g.audio_latents_dim; ++i) { + cfg += (i ? ", " : "") + std::string("0.0"); + } + cfg += "],\n \"latents_std\": ["; + for (int64_t i = 0; i < g.audio_latents_dim; ++i) { + cfg += (i ? ", " : "") + std::string("0.5"); + } + cfg += "]\n}\n"; + WriteFileBytes(config_path, cfg); +} + +// ── prompt embeddings (rows of text_dim, little-endian f32) ────────────────── +inline void WriteFoldPromptEmbeds(const FoldDitGeometry& g, const FoldRenderRequest& r, + const std::string& path) { + const std::vector values = + Param("fold.prompt_embeds", r.text_len * g.text_dim, 1.0); + std::string bytes(reinterpret_cast(values.data()), + values.size() * sizeof(float)); + WriteFileBytes(path, bytes); +} + +// Write the whole fixture set under `dir` (which must exist). +// Files: dit.gguf, video_vae.safetensors, video_vae_config.json, +// audio_vae.safetensors, audio_vae_config.json, prompt_embeds.f32. +inline void WriteFoldFixture(const std::string& dir) { + ::mkdir(dir.c_str(), 0755); + const FoldDitGeometry g; + const FoldRenderRequest r; + WriteFoldDitGguf(g, dir + "/dit.gguf"); + WriteFoldVideoVae(g, dir + "/video_vae.safetensors", dir + "/video_vae_config.json"); + WriteFoldAudioVae(g, dir + "/audio_vae.safetensors", dir + "/audio_vae_config.json"); + WriteFoldPromptEmbeds(g, r, dir + "/prompt_embeds.f32"); +} + +} // namespace minimax_h3_fold diff --git a/tests/vllm/models/test_minimax_h3_video_fold.cpp b/tests/vllm/models/test_minimax_h3_video_fold.cpp new file mode 100644 index 000000000..68054f589 --- /dev/null +++ b/tests/vllm/models/test_minimax_h3_video_fold.cpp @@ -0,0 +1,406 @@ +// ARCH-ONE-SURFACE ROW 2 fold gate — MiniMax-H3 video+audio generation. +// +// Three arms over the SAME deterministic tiny fixture +// (minimax_h3_video_fold_fixture.h), all held to the committed goldens in +// fixtures/minimax_h3_video_fold/ — which are the frames + WAV the PRE-fold +// `minimax-h3-gen` binary rendered at the branch base (fc636c76), BEFORE any +// refactor: +// +// A. the LIBRARY SEAM (MiniMaxH3VideoEngine::Load + Generate) — the entry +// point the C ABI, the server's /v1/videos and the thin-client example +// now drive — must reproduce the goldens BYTE-IDENTICALLY; +// B. the REPLICATED OLD PIPELINE — the pre-fold example's exact assembly +// (keep-quant GGUF load -> request build -> the dual-stream splitmix64 +// Gaussian noise -> MiniMaxH3GenerateT2va -> PPM/WAV serialization), +// re-stated here line for line — must also reproduce them, proving the +// goldens still describe the OLD pipeline on this tree; +// C. the committed goldens themselves. +// +// A == B == C means the fold moved the capability without moving a single +// output byte. The rewritten thin-client binary is held to the same goldens by +// the operator-run binary gate (recorded in the spec); the mux argv the seam +// composes is pinned against the pre-fold `minimax-h3-mux --print-only` +// capture the same way. +#include "vllm/multimodal/minimax_h3_video.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "vllm/entrypoints/openai/video_api.h" +#include "vllm/model_executor/model_loader/gguf_reader.h" +#include "vllm/model_executor/models/minimax_h3.h" +#include "minimax_h3_video_fold_fixture.h" + +namespace { + +std::string ReadAll(const std::string& path) { + std::ifstream in(path, std::ios::binary); + REQUIRE_MESSAGE(in.good(), "cannot open ", path); + return std::string((std::istreambuf_iterator(in)), std::istreambuf_iterator()); +} + +std::string GoldenDir() { return MINIMAX_H3_VIDEO_FOLD_FIXTURE_DIR; } + +// A fresh fixture + output workspace per test run (pid-unique under /tmp). +struct FoldWorkspace { + std::string root; + FoldWorkspace() { + static int counter = 0; + root = "/tmp/vllm_h3_video_fold_" + std::to_string(::getpid()) + "_" + + std::to_string(counter++); + ::mkdir(root.c_str(), 0755); + fixture = root + "/fixture"; + minimax_h3_fold::WriteFoldFixture(fixture); + } + ~FoldWorkspace() { + // Best-effort cleanup; a leftover /tmp dir on abort is diagnosable, not + // harmful. + const std::string cmd = "rm -rf '" + root + "'"; + const int rc = std::system(cmd.c_str()); + (void)rc; + } + std::string fixture; +}; + +vllm::multimodal::MiniMaxH3VideoModelParams FixtureModelParams(const std::string& dir) { + vllm::multimodal::MiniMaxH3VideoModelParams mp; + mp.dit_path = dir + "/dit.gguf"; + mp.video_vae_path = dir + "/video_vae.safetensors"; + mp.video_vae_config_path = dir + "/video_vae_config.json"; + mp.audio_vae_path = dir + "/audio_vae.safetensors"; + mp.audio_vae_config_path = dir + "/audio_vae_config.json"; + mp.prompt_embeds_path = dir + "/prompt_embeds.f32"; + mp.partition = "fl2va"; + mp.device = 0; // the CPU-feasible fold arm + mp.dequant_bf16 = 0; // keep-quant, the arm the goldens were captured on + return mp; +} + +vllm::multimodal::MiniMaxH3VideoGenParams FixtureGenParams(const std::string& out_dir) { + vllm::multimodal::MiniMaxH3VideoGenParams gen; + gen.num_frames = 5; + gen.height = 32; + gen.width = 32; + gen.steps = 3; + gen.output_dir = out_dir; + return gen; +} + +// The number of golden frames committed (the pre-fold binary wrote 8: latent_t +// 2 x the ViT3D decoder's fixed patch_size_t 4). +constexpr int kGoldenFrames = 8; + +void CheckAgainstGoldens(const std::string& out_dir) { + for (int f = 0; f < kGoldenFrames; ++f) { + char name[64]; + std::snprintf(name, sizeof(name), "/frame_%06d.ppm", f); + const std::string got = ReadAll(out_dir + name); + const std::string want = ReadAll(GoldenDir() + name); + INFO("frame ", f); + REQUIRE(got.size() == want.size()); + CHECK_MESSAGE(got == want, "frame ", f, " diverged from the pre-fold golden"); + } + const std::string got_wav = ReadAll(out_dir + "/audio.wav"); + const std::string want_wav = ReadAll(GoldenDir() + "/audio.wav"); + REQUIRE(got_wav.size() == want_wav.size()); + CHECK_MESSAGE(got_wav == want_wav, "audio.wav diverged from the pre-fold golden"); + // And there is no NINTH frame: a pipeline that silently rendered a different + // clip length would otherwise pass every per-frame compare above. + char extra[64]; + std::snprintf(extra, sizeof(extra), "/frame_%06d.ppm", kGoldenFrames); + std::ifstream ninth(out_dir + extra, std::ios::binary); + CHECK_MESSAGE(!ninth.good(), "the render produced more frames than the golden clip"); +} + +} // namespace + +// ─── ARM A: the library seam reproduces the pre-fold binary byte for byte ──── +TEST_CASE("minimax_h3 video fold: the library seam reproduces the pre-fold goldens") { + FoldWorkspace ws; + auto engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(FixtureModelParams(ws.fixture)); + CHECK(!engine->has_encoder()); + CHECK(engine->has_prompt_embeds()); + + const std::string out_dir = ws.root + "/seam_out"; + const vllm::multimodal::MiniMaxH3VideoResult result = + engine->Generate(FixtureGenParams(out_dir)); + + CHECK(result.frame_dir == out_dir); + CHECK(result.frame_count == kGoldenFrames); + CHECK(result.width == 32); + CHECK(result.height == 32); + CHECK(result.fps == 24); + CHECK(result.sample_rate == 32000); + CheckAgainstGoldens(out_dir); + + // The mux argv the seam composes must be EXACTLY what the pre-fold + // `minimax-h3-mux --print-only` printed (with W/ as the directory). + std::string joined; + for (size_t i = 0; i < result.mux_argv.size(); ++i) { + joined += (i == 0 ? "" : " ") + result.mux_argv[i]; + } + std::string golden_argv = ReadAll(GoldenDir() + "/golden_mux_argv.txt"); + while (!golden_argv.empty() && (golden_argv.back() == '\n' || golden_argv.back() == '\r')) { + golden_argv.pop_back(); + } + // The golden was captured with the placeholder directory `W`; substitute the + // real output dir to compare whole-argv. + size_t pos = 0; + while ((pos = golden_argv.find("W/", pos)) != std::string::npos) { + golden_argv.replace(pos, 1, out_dir); + pos += out_dir.size() + 1; + } + CHECK(joined == golden_argv); + CHECK(result.mux_output_path == out_dir + "/video.mp4"); + + // A SECOND generation from the same engine handle must be byte-identical + // too (staged state is reused, never mutated). + const std::string out2 = ws.root + "/seam_out2"; + (void)engine->Generate(FixtureGenParams(out2)); + CheckAgainstGoldens(out2); +} + +// ─── ARM B: the replicated OLD pipeline still produces the same bytes ──────── +TEST_CASE("minimax_h3 video fold: the replicated pre-fold pipeline matches the goldens") { + FoldWorkspace ws; + + // This block restates examples/minimax_h3_gen/main.cpp@fc636c76 (the + // pre-fold assembly) against the same fixture: any behavioural drift in the + // seam OR a stale golden makes exactly one of the two arms diverge. + const vllm::GgufFile f = vllm::GgufFile::Open(ws.fixture + "/dit.gguf"); + const vllm::MiniMaxH3GgufDit dit = vllm::LoadMiniMaxH3DitFromGguf(f, /*keep_quant=*/true); + + vllm::MiniMaxH3LatentStats video_stats, audio_stats; + vllm::MiniMaxH3VideoVaeDecoderConfig video_cfg; + vllm::MiniMaxH3AudioVaeConfig audio_cfg; + { + std::ifstream in(ws.fixture + "/video_vae_config.json"); + nlohmann::json j; + in >> j; + video_cfg = vllm::ParseMiniMaxH3VideoVaeDecoderConfig(j, &video_stats); + } + { + std::ifstream in(ws.fixture + "/audio_vae_config.json"); + nlohmann::json j; + in >> j; + audio_cfg = vllm::ParseMiniMaxH3AudioVaeConfig(j, &audio_stats); + } + const vllm::SafetensorsFile vfile = + vllm::SafetensorsFile::Open(ws.fixture + "/video_vae.safetensors"); + const vllm::MiniMaxH3AudioVaeWeights video_weights = + vllm::LoadMiniMaxH3VideoVaeDecoderWeights(vfile); + const vllm::SafetensorsFile afile = + vllm::SafetensorsFile::Open(ws.fixture + "/audio_vae.safetensors"); + const vllm::MiniMaxH3AudioVaeWeights audio_weights = + vllm::LoadMiniMaxH3AudioVaeWeights(afile); + + std::vector prompt_embeds; + { + std::ifstream in(ws.fixture + "/prompt_embeds.f32", std::ios::binary | std::ios::ate); + const std::streamsize bytes = in.tellg(); + in.seekg(0); + prompt_embeds.resize(static_cast(bytes) / sizeof(float)); + in.read(reinterpret_cast(prompt_embeds.data()), bytes); + } + + vllm::MiniMaxH3T2vaRequest request; + request.num_steps = 3; + const vllm::MiniMaxH3ShapePlan plan = + vllm::MiniMaxH3ResolveShape("t2va", 0.0, 5, 32, 32, 0, 0); + request.latent_t = plan.latent_t; + request.num_frames = plan.num_frames; + request.latent_h = plan.height / vllm::kMiniMaxH3VaeRatio; + request.latent_w = plan.width / vllm::kMiniMaxH3VaeRatio; + request.audio_t = plan.audio_t; + request.audio_channel = vllm::kMiniMaxH3AudioChannels; + request.video_latents_mean = video_stats.mean; + request.video_latents_std = video_stats.std_dev; + request.audio_latents_mean = audio_stats.mean; + request.audio_latents_std = audio_stats.std_dev; + request.partition = vllm::MiniMaxH3PartitionFromFlag("fl2va"); + request.text_len = static_cast(prompt_embeds.size()) / dit.params.text_dim; + + // The pre-fold noise draw, restated verbatim (splitmix64 + Box-Muller, + // Gaussian default, fixed per-modality seeds). + const int64_t frame_rows = (request.latent_h / dit.params.patch_size_h) * + (request.latent_w / dit.params.patch_size_w); + const int64_t video_rows = request.latent_t * frame_rows; + const int64_t audio_rows = request.audio_t * request.audio_channel; + auto fill = [](std::vector& out, uint64_t seed) { + uint64_t x = seed; + auto u01 = [&x]() { + x += 0x9E3779B97F4A7C15ULL; + uint64_t z = x; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ULL; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBULL; + z ^= z >> 31; + return (z >> 11) * 0x1.0p-53; + }; + for (size_t i = 0; i < out.size(); ++i) { + double u1 = u01(), u2 = u01(); + if (u1 < 1e-12) u1 = 1e-12; + out[i] = static_cast(std::sqrt(-2.0 * std::log(u1)) * + std::cos(2.0 * 3.14159265358979323846 * u2)); + } + }; + std::vector noise_video( + static_cast(video_rows * dit.params.video_row_width())); + std::vector noise_audio( + static_cast(audio_rows * dit.params.audio_latents_dim)); + fill(noise_video, 0x5EED1234ULL); + fill(noise_audio, 0x5EED5678ULL); + + const vllm::MiniMaxH3T2vaResult out = vllm::MiniMaxH3GenerateT2va( + vt::Device{}, request, dit.params, dit.weights, video_cfg, video_weights, audio_cfg, + audio_weights, prompt_embeds, noise_video, noise_audio, vt::DType::kBF16); + + REQUIRE(out.frame_shape.t == kGoldenFrames); + for (int64_t fr = 0; fr < out.frame_shape.t; ++fr) { + char name[64]; + std::snprintf(name, sizeof(name), "/frame_%06lld.ppm", static_cast(fr)); + const std::string want = ReadAll(GoldenDir() + name); + const std::string got = vllm::MiniMaxH3WritePpmFrame(out.frames, out.frame_shape, fr); + INFO("frame ", fr); + CHECK_MESSAGE(got == want, "replicated pre-fold pipeline diverged on frame ", fr); + } + const std::string want_wav = ReadAll(GoldenDir() + "/audio.wav"); + const std::string got_wav = vllm::MiniMaxH3WriteWav( + out.waveform, out.audio_channels, out.audio_samples_per_channel, out.sample_rate); + CHECK_MESSAGE(got_wav == want_wav, "replicated pre-fold pipeline diverged on audio.wav"); + + // And the silent mux argv is still the pre-fold `minimax-h3-mux` contract. + vllm::MiniMaxH3MuxRequest mux; + mux.frame_pattern = "frames_%06d.ppm"; + mux.output_path = "silent.mp4"; + const std::vector argv = vllm::MiniMaxH3BuildMp4MuxArgs(mux); + std::string joined; + for (size_t i = 0; i < argv.size(); ++i) joined += (i == 0 ? "" : " ") + argv[i]; + std::string golden = ReadAll(GoldenDir() + "/golden_mux_argv_silent.txt"); + while (!golden.empty() && (golden.back() == '\n' || golden.back() == '\r')) golden.pop_back(); + CHECK(joined == golden); +} + +// ─── the guards the seam must keep (the #77 partition catch + inputs) ──────── +TEST_CASE("minimax_h3 video fold: the seam honors the partition/task guard and refuses bad input") { + FoldWorkspace ws; + + SUBCASE("an undeclared partition refuses a full render, naming the fix") { + vllm::multimodal::MiniMaxH3VideoModelParams mp = FixtureModelParams(ws.fixture); + mp.partition = ""; // declared-but-unknown: the guard refuses every task + auto engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(mp); + try { + (void)engine->Generate(FixtureGenParams(ws.root + "/out")); + FAIL("an undeclared partition must refuse to render"); + } catch (const std::exception& e) { + const std::string msg = e.what(); + CHECK(msg.find("partition") != std::string::npos); + } + } + + SUBCASE("the ref2va partition refuses a t2va render") { + vllm::multimodal::MiniMaxH3VideoModelParams mp = FixtureModelParams(ws.fixture); + mp.partition = "ref2va"; + auto engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(mp); + try { + (void)engine->Generate(FixtureGenParams(ws.root + "/out")); + FAIL("the ref2va partition must refuse t2va"); + } catch (const std::exception& e) { + const std::string msg = e.what(); + CHECK(msg.find("ref2va") != std::string::npos); + } + } + + SUBCASE("an unknown partition string is refused at load") { + vllm::multimodal::MiniMaxH3VideoModelParams mp = FixtureModelParams(ws.fixture); + mp.partition = "nonsense"; + CHECK_THROWS(vllm::multimodal::MiniMaxH3VideoEngine::Load(mp)); + } + + SUBCASE("no conditioning is a loud refusal, not a garbage render") { + vllm::multimodal::MiniMaxH3VideoModelParams mp = FixtureModelParams(ws.fixture); + mp.prompt_embeds_path = ""; // no encoder either + auto engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(mp); + try { + (void)engine->Generate(FixtureGenParams(ws.root + "/out")); + FAIL("no conditioning must refuse"); + } catch (const std::exception& e) { + CHECK(std::string(e.what()).find("conditioning") != std::string::npos); + } + } + + SUBCASE("a missing output_dir is refused before any compute") { + auto engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(FixtureModelParams(ws.fixture)); + vllm::multimodal::MiniMaxH3VideoGenParams gen = FixtureGenParams(""); + CHECK_THROWS(engine->Generate(gen)); + } + + SUBCASE("keyframes and ref2va references are exclusive") { + auto engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(FixtureModelParams(ws.fixture)); + vllm::multimodal::MiniMaxH3VideoGenParams gen = FixtureGenParams(ws.root + "/out"); + gen.first_frame_path = ws.fixture + "/nonexistent.ppm"; + gen.ref_video_dir = ws.fixture; + CHECK_THROWS(engine->Generate(gen)); + } + + SUBCASE("a missing DiT path is refused at load") { + vllm::multimodal::MiniMaxH3VideoModelParams mp = FixtureModelParams(ws.fixture); + mp.dit_path = ""; + CHECK_THROWS(vllm::multimodal::MiniMaxH3VideoEngine::Load(mp)); + } +} + +// ─── the /v1/videos mapping (library-owned so HTTP and FFI cannot drift) ───── +TEST_CASE("minimax_h3 video fold: the VideoRequest mapping carries every field") { + vllm::openai::VideoRequest req; + req.prompt = "a cat"; + req.task = "t2va"; + req.duration_seconds = 2.5; + req.num_frames = 29; + req.height = 320; + req.width = 640; + req.num_inference_steps = 7; + req.flow_shift = 11.0; + req.audio_flow_shift = 2.5; + req.seed = 42; + req.has_seed = true; + req.input_reference_path = "/x/ref.ppm"; + req.input_reference_video_dir = "/x/clip"; + req.input_reference_audio_bytes = {1, 2, 3}; + + const vllm::multimodal::MiniMaxH3VideoGenParams gen = + vllm::multimodal::MiniMaxH3VideoGenParamsFromRequest(req, "/x/job0"); + CHECK(gen.prompt == "a cat"); + CHECK(gen.task == "t2va"); + CHECK(gen.duration_seconds == 2.5); + CHECK(gen.num_frames == 29); + CHECK(gen.height == 320); + CHECK(gen.width == 640); + CHECK(gen.steps == 7); + CHECK(gen.flow_shift == 11.0); + CHECK(gen.audio_flow_shift == 2.5); + CHECK(gen.seed == 42); + CHECK(gen.has_seed); + CHECK(gen.first_frame_path == "/x/ref.ppm"); + CHECK(gen.noise_aug == 1.0); // input_reference pins frame 0 exactly + CHECK(gen.ref_video_dir == "/x/clip"); + CHECK(gen.ref_audio_wav == std::string("\x01\x02\x03")); + CHECK(gen.output_dir == "/x/job0"); + + // The inline data:-URL spelling lands on the BYTES member, not the path. + vllm::openai::VideoRequest inline_req; + inline_req.input_reference_bytes = {'P', '6'}; + const vllm::multimodal::MiniMaxH3VideoGenParams inline_gen = + vllm::multimodal::MiniMaxH3VideoGenParamsFromRequest(inline_req, "/x/job1"); + CHECK(inline_gen.first_frame_ppm == "P6"); + CHECK(inline_gen.first_frame_path.empty()); +}