feat(minimax-h3): port the MiniMax-H3 omni-modal video+audio DiT (portable path complete; speed GPU-gated)

CMakeLists.txt

@@ -463,6 +463,25 @@ add_library(vllm STATIC
   src/vllm/model_executor/models/kimi_k3_registry.cpp
   src/vllm/model_executor/models/kimi_k3_weights.cpp
   src/vllm/model_executor/models/kimi_k3.cpp
+  src/vllm/model_executor/models/minimax_h3.cpp
+  src/vllm/model_executor/models/minimax_h3_packing.cpp
+  src/vllm/model_executor/models/minimax_h3_planner.cpp
+  src/vllm/model_executor/models/minimax_h3_gguf.cpp
+  src/vllm/model_executor/models/minimax_h3_nvfp4.cpp
+  src/vllm/model_executor/models/minimax_h3_audio_vae.cpp
+  src/vllm/model_executor/models/minimax_h3_video_vae.cpp
+  src/vllm/model_executor/models/minimax_h3_video_vae_device.cpp
+  src/vllm/model_executor/models/minimax_h3_vae_cnn.cpp
+  src/vllm/model_executor/models/minimax_h3_encoder.cpp
+  src/vllm/model_executor/models/minimax_h3_pipeline.cpp
+  src/vllm/model_executor/models/minimax_h3_wav.cpp
+  src/vllm/model_executor/models/minimax_h3_mux.cpp
+  src/vllm/model_executor/models/minimax_h3_device_resolve.cpp
+  src/vllm/model_executor/models/minimax_h3_device.cpp
+  src/vllm/model_executor/models/minimax_h3_vae_loader.cpp
+  src/vllm/model_executor/models/minimax_h3_encoder_gguf.cpp
+  src/vllm/model_executor/models/minimax_h3_encoder_device.cpp
+  src/vllm/entrypoints/openai/video_api.cpp
   src/vllm/model_executor/models/kimi_kda.cpp
   src/vllm/model_executor/models/glm4_registry.cpp
   src/vllm/model_executor/models/glm4_weights.cpp
@@ -716,6 +735,7 @@ add_library(vllm STATIC
   src/vt/cpu/cpu_backend.cpp
   src/vt/cpu/cpu_threadpool.cpp
   src/vt/cpu/cpu_ops.cpp
+  src/vt/cpu/cpu_minimax_h3.cpp
   src/vt/cpu/cpu_matmul_elem.cpp
   src/vt/cpu/cpu_quant_dequant.cpp
   src/vt/cpu/cpu_quant_act.cpp
@@ -888,7 +908,8 @@ if(VLLM_CPP_CUDA)
     src/vt/cuda/cuda_combine_tokens.cu
     src/vt/cuda/cuda_sample.cu
     src/vt/cuda/cuda_deepseek_v4.cu
-    src/vt/cuda/cuda_laguna.cu)
+    src/vt/cuda/cuda_laguna.cu
+    src/vt/cuda/cuda_minimax_h3.cu)
   find_package(CUDAToolkit REQUIRED)
   # cublasLt is linked now so the Task 4 matmul lands without a build change.
   target_link_libraries(vllm PUBLIC CUDA::cudart CUDA::cublasLt)

docs/ENVIRONMENT.md

@@ -85,6 +85,7 @@ Read-only observability; none change output.
 | `VT_OP_PROVIDER_DISABLE` | (none) | Comma-separated provider names to disable, forcing fallback (diagnostic) |
 | `VT_GDN_VALIDATE` | off | Run the GDN validation/cross-check path (slower; for kernel debugging) |
 | `VT_FP4_AUTOTUNE_VERBOSE` | off | Log the NVFP4 GEMM autotuner's tactic selection |
+| `VT_H3_PROGRESS` | unset | Trace the MiniMax-H3 denoise loop's phases to stderr: which forward path was taken (device vs the CPU reference), how long the ONE-TIME device weight staging took, and per-step forward seconds with the sequence length. A real-checkpoint run spends its minutes in exactly one of those phases, and this says which without guessing — it was added after GPU-utilization counters proved unreliable on Tegra-class boards |
 | `VT_POOL_BYPASS` | off | `=1` makes every device-scratch pool allocation an exact-size driver `Alloc` and every release a real `Free`, so `compute-sanitizer` can see tensor boundaries and use-after-free that the caching, size-class-rounding pool hides. DEBUGGING ONLY: it reinstates the per-op `cudaMalloc`/`cudaFree` device-sync storm the pool exists to remove, so it is never a timing configuration |
 
 ## Kernel-internal knobs (deferred)

examples/CMakeLists.txt

@@ -55,3 +55,17 @@ if(VLLM_CPP_SERVER)
       PASS_REGULAR_EXPRESSION "--max-num-seqs N.*--max-num-batched-tokens N.*--cuda-profile-graph-replays N.*--cuda-profile-graph-batch N.*--benchmark-shutdown-fifo F.*--enable-force-include-usage.*--\\[no-\\]enable-prefix-caching")
   endif()
 endif()
+
+# minimax-h3-mux: muxes MiniMax-H3 PPM frames + WAV into an MP4 by invoking
+# ffmpeg. This is the RATIFIED home of that process spawn (developer decision
+# 2026-08-03): the library builds the artifacts and the argv, examples/ runs it.
+add_executable(minimax-h3-mux minimax_h3_mux/main.cpp)
+target_link_libraries(minimax-h3-mux PRIVATE vllm::vllm)
+vllm_cpp_set_warnings(minimax-h3-mux)
+
+# minimax-h3-gen: the ASSEMBLY driver — opens the real checkpoints, runs the whole
+# t2va path and writes an MP4. Lives here because it invokes ffmpeg (the ratified
+# boundary) and because it is a driver, not library surface.
+add_executable(minimax-h3-gen minimax_h3_gen/main.cpp)
+target_link_libraries(minimax-h3-gen PRIVATE vllm::vllm)
+vllm_cpp_set_warnings(minimax-h3-gen)

examples/minimax_h3_mux/main.cpp

@@ -0,0 +1,135 @@
+// minimax-h3-mux: muxes a MiniMax-H3 clip (PPM frames + WAV) into an MP4 by
+// INVOKING ffmpeg.
+//
+// THIS FILE IS THE RATIFIED HOME OF THE PROCESS SPAWN. The developer's decision
+// (2026-08-03): "re: ffmpeg invocation, correct - let's keep in the examples
+// only". So the split is deliberate and load-bearing:
+//
+//   src/vllm/  builds the ARTIFACTS (MiniMaxH3WritePpmFrame, MiniMaxH3WriteWav)
+//              and the ARGV (MiniMaxH3BuildMp4MuxArgs) -- and spawns NOTHING.
+//   examples/  (here) performs the invocation.
+//
+// That is also why `/v1/videos` takes a caller-supplied `VideoRunner` callback
+// rather than muxing itself: RunFfmpeg below is precisely the piece a server
+// embedder plugs into ApiServer::set_video_runner, and it lives outside the
+// library on purpose. Keep it that way -- do not move fork/exec into src/vllm/.
+//
+// Usage:
+//   minimax-h3-mux --frames <pattern> --out <out.mp4> [--audio <in.wav>]
+//                  [--fps N] [--crf N] [--ffmpeg <path>] [--print-only]
+//
+//   --frames  printf-style pattern the library's PPM writer filled in,
+//             e.g. /tmp/h3/frame_%06d.ppm
+//   --audio   omitted => a silent clip
+//   --print-only  print the argv and exit WITHOUT spawning (lets the argv be
+//                 inspected, diffed or run by hand on a box with no ffmpeg).
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <cstdint>
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "vllm/model_executor/models/minimax_h3.h"
+
+namespace {
+
+// Run argv to completion and return its exit status. The ONLY process spawn in
+// the MiniMax-H3 path, and it is in examples/ by project decision.
+int RunFfmpeg(const std::vector<std::string>& args) {
+  std::vector<char*> c_args;
+  c_args.reserve(args.size() + 1);
+  for (const std::string& arg : args) {
+    c_args.push_back(const_cast<char*>(arg.c_str()));
+  }
+  c_args.push_back(nullptr);
+
+  const pid_t pid = fork();
+  if (pid < 0) throw std::runtime_error("fork failed");
+  if (pid == 0) {
+    execvp(c_args[0], c_args.data());
+    // Only reached if exec failed; _exit (not exit) so the child never runs the
+    // parent's atexit handlers or flushes its buffers a second time.
+    _exit(127);
+  }
+  int status = 0;
+  if (waitpid(pid, &status, 0) < 0) throw std::runtime_error("waitpid failed");
+  if (WIFSIGNALED(status)) {
+    throw std::runtime_error("ffmpeg died on signal " +
+                             std::to_string(WTERMSIG(status)));
+  }
+  return WIFEXITED(status) ? WEXITSTATUS(status) : -1;
+}
+
+std::string Need(int argc, char** argv, int i, const std::string& flag) {
+  if (i >= argc) throw std::runtime_error("missing value for " + flag);
+  return argv[i];
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  vllm::MiniMaxH3MuxRequest request;
+  std::string ffmpeg = "ffmpeg";
+  bool print_only = false;
+
+  try {
+    for (int i = 1; i < argc; ++i) {
+      const std::string flag = argv[i];
+      if (flag == "--frames") {
+        request.frame_pattern = Need(argc, argv, ++i, flag);
+      } else if (flag == "--audio") {
+        request.audio_path = Need(argc, argv, ++i, flag);
+      } else if (flag == "--out") {
+        request.output_path = Need(argc, argv, ++i, flag);
+      } else if (flag == "--fps") {
+        request.fps = std::stoll(Need(argc, argv, ++i, flag));
+      } else if (flag == "--crf") {
+        request.crf = std::stoll(Need(argc, argv, ++i, flag));
+      } else if (flag == "--ffmpeg") {
+        ffmpeg = Need(argc, argv, ++i, flag);
+      } else if (flag == "--print-only") {
+        print_only = true;
+      } else {
+        throw std::runtime_error("unknown argument: " + flag);
+      }
+    }
+    if (request.frame_pattern.empty() || request.output_path.empty()) {
+      std::cerr << "usage: minimax-h3-mux --frames <pattern> --out <out.mp4> "
+                   "[--audio <in.wav>] [--fps N] [--crf N] [--ffmpeg <path>] "
+                   "[--print-only]\n";
+      return 2;
+    }
+
+    // The LIBRARY decides the encoding contract (h264/yuv420p + AAC, -shortest,
+    // +faststart); this file only runs it.
+    std::vector<std::string> args = vllm::MiniMaxH3BuildMp4MuxArgs(request);
+    if (!args.empty()) args[0] = ffmpeg;
+
+    for (size_t i = 0; i < args.size(); ++i) {
+      std::cout << (i == 0 ? "" : " ") << args[i];
+    }
+    std::cout << "\n";
+    if (print_only) return 0;
+
+    const int status = RunFfmpeg(args);
+    if (status == 127) {
+      std::cerr << "failed to exec '" << ffmpeg
+                << "' — is ffmpeg installed and on PATH?\n";
+      return 127;
+    }
+    if (status != 0) {
+      std::cerr << "ffmpeg exited " << status << "\n";
+      return status;
+    }
+    std::cout << "wrote " << request.output_path << "\n";
+    return 0;
+  } catch (const std::exception& e) {
+    std::cerr << "error: " << e.what() << "\n";
+    return 1;
+  }
+}