From 20c4bda715c28f3e99cc241ada40957049fc822d Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 9 Aug 2026 07:50:14 +0000 Subject: [PATCH] =?UTF-8?q?arch(ARCH-ONE-SURFACE):=20the=20OpenAI=20server?= =?UTF-8?q?=20becomes=20a=20THIN=20ABI=20CLIENT=20=E2=80=94=20vllm=5Fserve?= =?UTF-8?q?r=5Fmain,=20ABI=20v17?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit examples/server was the DEEPEST breach of the ONE SURFACE directive (.agents/specs/one-surface-abi.md): 1046 lines reaching into 36 internal headers to construct the engine, the serving layers, metrics, the video seam and the ASR seam by hand. It carried the correspondingly largest entry in scripts/example-abi-allowlist.txt, whose fold order is fixed: GROW the public surface, REWRITE the example against it, DELETE the parallel implementation. Grown: `vllm_server_main(argc, argv)` at ABI v17 (purely additive). The construction moved into the library verbatim as vllm::entrypoints::openai::VllmServerMain (src/vllm/entrypoints/openai/server_main.{h,cpp}), and src/capi wraps it so nothing throws across the C boundary. argv rather than a params STRUCT is deliberate. The server takes ~57 flags and gains more with every serving feature; mirroring them into a C struct would put that churn in the ABI, where every field is permanent. The flag surface already has a contract — vLLM's cli_args.py, which this mirrors — so argv IS the stable interface. Embedders wanting programmatic control keep the granular entry points (vllm_engine_load / vllm_chat / vllm_video_generate / vllm_transcribe). Rewritten: examples/server/main.cpp is 23 lines, `#include "vllm.h"` and nothing else, and now links vllm::shared rather than the internal vllm::vllm target — so it builds exactly as an out-of-tree consumer does, which is what actually tests the boundary. HTTP and FFI cannot drift: the server the ABI runs IS the server this binary runs. Gate closed: the examples/server line is removed from scripts/example-abi-allowlist.txt. That file reds on a STALE entry for a now-clean example, so removing it is the enforcement closing rather than bookkeeping. Eight examples remain tracked. Verified on Thor (sm_110, clean configure+build from this branch): BUILD_RC=0, `./server --help` runs through the ABI and prints the full flag surface, EXIT=0. The binary drops from 32,780,560 to 70,304 bytes — it no longer statically carries the engine, which is the fold in one number. FOLLOWING_AGENTS_PROTOCOL Assisted-by: Claude Code:claude-opus-5 [ClaudeCode] --- CMakeLists.txt | 4 + examples/CMakeLists.txt | 6 +- examples/server/main.cpp | 1061 +------------------ include/vllm.h | 33 +- scripts/example-abi-allowlist.txt | 1 - src/capi/vllm_c.cpp | 17 + src/vllm/entrypoints/openai/server_main.cpp | 1055 ++++++++++++++++++ src/vllm/entrypoints/openai/server_main.h | 35 + 8 files changed, 1166 insertions(+), 1046 deletions(-) create mode 100644 src/vllm/entrypoints/openai/server_main.cpp create mode 100644 src/vllm/entrypoints/openai/server_main.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e89e81d3..5fbbc096f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1811,6 +1811,10 @@ endif() if(VLLM_CPP_SERVER) find_package(Threads REQUIRED) target_sources(vllm PRIVATE src/vllm/entrypoints/openai/api_server.cpp) + # ARCH-ONE-SURFACE: the server ENTRY POINT (flag parsing + engine/serving + # construction) lives in the library so examples/server can be a thin client of + # the C ABI's vllm_server_main (v17). It shares api_server.cpp's httplib gate. + target_sources(vllm PRIVATE src/vllm/entrypoints/openai/server_main.cpp) target_compile_definitions(vllm PUBLIC VLLM_CPP_SERVER) target_link_libraries(vllm PUBLIC Threads::Threads) # third_party/httplib/httplib.h is reached as (third_party diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 62b8b0f77..92275e965 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -67,7 +67,11 @@ endif() # The OpenAI HTTP server example is gated on the vendored cpp-httplib transport. if(VLLM_CPP_SERVER) add_executable(server server/main.cpp) - target_link_libraries(server PRIVATE vllm::vllm) + # ARCH-ONE-SURFACE: server/main.cpp is a THIN CLIENT of the public C ABI + # (vllm_server_main, v17) -- it includes vllm.h and nothing else, so it links the + # PACKAGED shared library exactly as an out-of-tree consumer would, not the + # internal C++ target. + target_link_libraries(server PRIVATE vllm::shared) vllm_cpp_set_warnings(server) if(VLLM_CPP_BUILD_TESTS) add_test(NAME test_server_help COMMAND $ --help) diff --git a/examples/server/main.cpp b/examples/server/main.cpp index 4c9c26edc..61c0c5aac 100644 --- a/examples/server/main.cpp +++ b/examples/server/main.cpp @@ -1,1046 +1,23 @@ -// server: an OpenAI-compatible HTTP server over the vllm.cpp LLMEngine (M3.1 -// Task 4). Loads a supported model (safetensors or GGUF weights + tokenizer + a KV-cache config → -// LLMEngine), constructs the OpenAI serving handlers (chat wired with the real -// chat template via MakeChatTemplatePromptFn(LoadChatTemplateFromConfig(...))) -// and serves /v1/completions, /v1/chat/completions, /v1/models, /health, -// /version. +// vllm.cpp original. `vllm-server` — the OpenAI-compatible server, as a THIN +// CLIENT of the public C ABI. // -// server --model [--host 0.0.0.0] [--port 8000] -// [--tokenizer-config ] -// [--served-model-name ] -// [--block-size N] [--num-blocks N] [--max-model-len N] -// [--gpu-memory-utilization F] [--kv-cache-memory BYTES] -// [--max-num-seqs N] [--max-num-batched-tokens N] -// [--enable-force-include-usage] -// [--[no-]enable-prefix-caching] -// [--scheduling-policy fcfs|priority] -// [--tool-call-parser |auto|none] -// [--reasoning-parser |auto|none] -// [--kv-transfer-config ''] +// ARCH-ONE-SURFACE (.agents/specs/one-surface-abi.md, developer-directed +// 2026-08-07): every unit under examples/ is a thin client of the public surface +// — the flat C ABI `vllm.h`, the ONLY header `make install` ships — never of the +// internal C++ tree. This file used to be the DEEPEST breach of that rule: 1046 +// lines reaching into 36 internal headers to construct the engine, the serving +// layers, metrics, the video seam and the ASR seam by hand, which is why it +// carried an entry in scripts/example-abi-allowlist.txt. // -// A directory holds config.json, tokenizer.json and supported safetensors -// shards. A supported GGUF file is also accepted and supplies model metadata -// plus embedded vocabulary. If --tokenizer-config is omitted for a directory it -// defaults to /tokenizer_config.json; when that file has no chat_template -// the chat endpoint falls back to the simple role-join prompt. +// The construction moved into the library verbatim +// (src/vllm/entrypoints/openai/server_main.cpp, declared in +// vllm/entrypoints/openai/server_main.h) and is published as `vllm_server_main` +// at ABI v17. HTTP and FFI therefore cannot drift: the server the ABI runs IS +// the server this binary runs. // -// NOTE: loading the real 35B checkpoint is a GPU/dgx concern; on the CPU CI box -// this binary is only built + smoke-tested against a synthetic engine (see -// tests/vllm/entrypoints/openai/test_api_server.cpp). The wiring below is the -// same either way. -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef VT_BENCH_PROFILE_CONTROL -#include -#include -#include -#include -#include -#endif +// Flag parsing lives in the library with the construction it feeds, because the +// two are one contract (vLLM's cli_args.py). Mirroring ~57 flags into a C struct +// would have put that churn in the ABI, where every field is permanent. +#include "vllm.h" -#include "vllm/config/device.h" -#include "vllm/config/kv_transfer.h" -#include "vllm/config/scheduler.h" -#include "vllm/entrypoints/chat_template.h" -#include "vllm/entrypoints/model_loader.h" -#include -#include "vllm/entrypoints/openai/api_server.h" -#include "vllm/entrypoints/openai/chat_mm.h" -#include "vllm/entrypoints/openai/request_logger.h" -#include "vllm/entrypoints/openai/serving_chat.h" -#include "vllm/entrypoints/openai/serving_completion.h" -#include "vllm/entrypoints/openai/serving_models.h" -#include "vllm/v1/metrics/loggers.h" -#include "vllm/entrypoints/openai/reasoning_parsers/detect.h" -#include "vllm/entrypoints/openai/tool_parsers/detect.h" -#include "vllm/model_executor/model_loader/safetensors_reader.h" -#include "vllm/model_executor/models/qwen3_5_weights.h" -#include "vllm/transformers_utils/hf_config.h" -#include "vllm/model_executor/models/model_registry.h" -#include "vllm/multimodal/minimax_h3_video.h" -#include "vllm/multimodal/parakeet_transcription.h" -#include "vllm/tokenizer/tokenizer.h" -#include "vllm/version.h" -#include "vllm/v1/core/kv_cache_utils.h" -#include "vllm/v1/core/sched/scheduler.h" -#include "vllm/v1/engine/core.h" -#include "vllm/v1/engine/input_processor.h" -#include "vllm/v1/engine/llm_engine.h" -#include "vllm/v1/engine/output_processor.h" -#include "vllm/v1/executor/executor.h" -#include "vllm/v1/kv_cache_interface.h" -#include "vllm/v1/kv_offload/kv_connector.h" -#include "vllm/v1/worker/gpu/runner.h" -#include "vt/backend.h" -#ifdef VT_BENCH_PROFILE_CONTROL -#include "vt/cuda/cuda_profiler_control.h" -#endif -#include "vt/dtype.h" -#include "vt/tensor.h" - -namespace { - -namespace fs = std::filesystem; -using vllm::HfConfig; -using vllm::Qwen3_5MoeWeights; - -// Run an argv to completion and return its exit status — the ONE process -// spawn in the MiniMax-H3 path, and it lives HERE, in examples/, by the -// developer-ratified 2026-08-03 decision: the library (the -// MiniMaxH3VideoEngine seam behind /v1/videos) writes the artifacts and -// BUILDS this argv, and spawns nothing. -int RunFfmpegArgv(const std::vector& args) { - std::vector c_args; - c_args.reserve(args.size() + 1); - for (const std::string& arg : args) { - c_args.push_back(const_cast(arg.c_str())); - } - c_args.push_back(nullptr); - const pid_t pid = fork(); - if (pid < 0) throw std::runtime_error("fork failed"); - if (pid == 0) { - execvp(c_args[0], c_args.data()); - _exit(127); // exec failed; never run the parent's atexit handlers - } - int status = 0; - if (waitpid(pid, &status, 0) < 0) throw std::runtime_error("waitpid failed"); - if (WIFSIGNALED(status)) { - throw std::runtime_error("ffmpeg died on signal " + std::to_string(WTERMSIG(status))); - } - return WIFEXITED(status) ? WEXITSTATUS(status) : -1; -} - -struct Args { - std::string model_dir; - std::string host = "0.0.0.0"; - int port = 8000; - std::string tokenizer_config; // default: /tokenizer_config.json - std::string served_model_name; // default: the model dir name - int block_size = 32; - // --num-blocks is the KV block-count OVERRIDE (0 => auto: sized by the knobs - // below, else the 256-block fallback). ROAD-V1-MEM M1. - int num_blocks = 0; - // --gpu-memory-utilization: fraction of free device memory for the whole - // engine (needs the M3 profile run; inert until then). --kv-cache-memory: an - // absolute KV-pool size in bytes that sizes the block count directly (0 => - // unset). - double gpu_memory_utilization = 0.92; - long long kv_cache_memory_bytes = 0; - int max_model_len = 0; // 0 => config.max_position_embeddings - int max_num_seqs = 8; - int max_num_batched_tokens = 0; // 0 => per-architecture default. - // --device: explicit device selection for the TEXT engine (ARCH-ONE-SURFACE - // ROW 8), the vLLM DeviceConfig.device names this build serves: "auto" - // (default — the accelerator-first probe, byte-identical to before the flag), - // "cpu" (force the CPU queue), "cuda" (require CUDA; an absent device fails - // startup LOUDLY, never a silent fallback). The video engine keeps its own - // --video-device below: the two engines are loaded from different - // checkpoints and may legitimately serve on different devices. - std::string device = "auto"; - // --- MiniMax-H3 video generation (opt-in; absent => /v1/videos is unregistered - // and the server behaves exactly as before). --- - std::string video_dit, video_vae, video_vae_config, audio_vae, audio_vae_config; - std::string video_prompt_embeds, video_workdir = "/tmp/vllm_h3_videos"; - std::string video_encoder, video_tokenizer; - int video_encoder_max_layers = 50; - std::string video_ffmpeg = "ffmpeg", video_device = "cuda"; - std::string video_partition; // served partition (fl2va|ref2va); see the #77 guard - // Keep-quant is the library seam's DEFAULT arm; --video-dequant-bf16 selects - // the bf16 dequant/stream arm (the throughput trade the gen example ships). - // --video-keep-quant is still accepted (it names the default). - bool video_dequant_bf16 = false; - int cuda_profile_graph_replays = 0; // trace-only diagnostic build seam. - int cuda_profile_graph_batch = 0; // 0 => accepted c16 trace contract. - std::string benchmark_shutdown_fifo; // paired trace-only control path. - std::optional enable_prefix_caching = std::nullopt; - bool enable_force_include_usage = false; - // GET /tokenizer_info gate. Mirrors vLLM's --enable-tokenizer-info-endpoint - // (entrypoints/openai/cli_args.py:140, default False; the route is registered - // only when serve/tokenize/api_router.py:95 sees the flag). Default off → the - // route 404s, byte-identical to before. - bool enable_tokenizer_info_endpoint = false; - // Dev/admin endpoint gate. Mirrors vLLM's VLLM_SERVER_DEV_MODE env - // (envs.py:157, default 0): build_app registers the dev/rlhf + dev/cache - // routers only under `if envs.VLLM_SERVER_DEV_MODE` (api_server.py:238). Off by - // default → /abort_requests 404s. Enables the /abort_requests production wiring. - bool enable_server_dev_mode = false; - bool verbose = false; - // Gemma4 HF/vLLM: --default-chat-template-kwargs enable_thinking (default OFF). - bool enable_thinking = false; - // Request logging (Python vLLM --enable-log-requests parity). Default ON. - bool enable_log_requests = true; - bool enable_log_outputs = false; - int max_log_len = 256; - // Attach Prometheus logger + GET /metrics (default ON for solid Hermes serve). - bool enable_metrics = true; - // Scheduling policy: "fcfs" (default), "priority" (mirrors vLLM's - // --scheduling-policy / SchedulerConfig.policy), or "lpm" (SGLang's - // cache-aware longest-prefix-match admission ordering, ENG-SGLANG-BEHAVIOR-FLAG; - // opt-in, output-neutral, resolves to fcfs when prefix caching is off). - // --schedule-policy is accepted as an SGLang-compatible alias. - std::string scheduling_policy = "fcfs"; - // Jump-forward decoding (ENG-SGLANG-BEHAVIOR-FLAG SW3): tri-state, mirrors the - // C-ABI vllm_model_params.enable_jump_forward. Unset (default) => OFF unless - // VT_ENABLE_JUMP_FORWARD is set; --enable-jump-forward forces on, - // --disable-jump-forward forces off (the env var still overrides). The - // token-unique forced-run subset only; see .agents/specs/sglang-enablement.md. - std::optional enable_jump_forward = std::nullopt; - // Tool-call / reasoning dialect selection (mirrors vLLM's --tool-call-parser - // and --reasoning-parser). THE DEFAULTS ARE TODAY'S HARDCODED BEHAVIOUR: - // "hermes" is exactly what OpenAIServingChat was constructed with before this - // flag existed, and "none" is the empty reasoning-parser name it passed. An - // invocation that names neither flag is therefore unchanged, byte for byte. - // "auto" opts into the chat-template detection the C ABI uses. - std::string tool_call_parser = "hermes"; - std::string reasoning_parser = "none"; - // vLLM's --kv-transfer-config: the external KV connector selection, as the - // same JSON object vLLM takes. Empty (default) == no connector == the inert - // production path. See docs/KV-OFFLOAD.md. - std::string kv_transfer_config; - // vLLM's --speculative-config: the speculative-decoding selection, as the same - // JSON object vLLM takes (e.g. '{"method":"mtp","num_speculative_tokens":1}'). - // Empty (default) == no speculation == the inert production path (SPEC-MTP I5d). - std::string speculative_config; -}; - -[[noreturn]] void Usage(const char* argv0, int code) { - std::cerr - << "usage: " << argv0 - << " --model [--host H] [--port P] [--tokenizer-config F]\n" - " [--served-model-name N] [--block-size N] " - "[--num-blocks N] [--max-model-len N]\n" - " [--gpu-memory-utilization F] " - "[--kv-cache-memory BYTES]\n" - " [--max-num-seqs N] " - "[--max-num-batched-tokens N]\n" - " [--device auto|cpu|cuda]\n" - " [--cuda-profile-graph-replays N]\n" - " [--cuda-profile-graph-batch N]\n" - " [--benchmark-shutdown-fifo F]\n" - " [--enable-force-include-usage]\n" - " [--enable-tokenizer-info-endpoint]\n" - " [--enable-server-dev-mode]\n" - " [--verbose]\n" - " [--enable-thinking|--no-enable-thinking]\n" - " [--enable-log-requests|--disable-log-requests]\n" - " [--enable-log-outputs] [--max-log-len N]\n" - " [--enable-metrics|--disable-metrics]\n" - " [--[no-]enable-prefix-caching]\n" - " [--[no-]enable-radix-attention]\n" - " [--scheduling-policy fcfs|priority|lpm]\n" - " [--[enable|disable]-jump-forward]\n" - " [--tool-call-parser |auto|none]\n" - " [--reasoning-parser |auto|none]\n" - " [--kv-transfer-config '']\n" - " [--speculative-config '']\n"; - std::exit(code); -} - -std::string NextArg(int argc, char** argv, int& i, const char* argv0) { - if (i + 1 >= argc) Usage(argv0, 2); - return argv[++i]; -} - -Args ParseArgs(int argc, char** argv) { - Args a; - for (int i = 1; i < argc; ++i) { - const std::string flag = argv[i]; - if (flag == "--model") { - a.model_dir = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--host") { - a.host = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--port") { - a.port = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--tokenizer-config") { - a.tokenizer_config = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--served-model-name") { - a.served_model_name = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--block-size") { - a.block_size = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--num-blocks") { - a.num_blocks = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--gpu-memory-utilization") { - a.gpu_memory_utilization = std::stod(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--kv-cache-memory") { - a.kv_cache_memory_bytes = std::stoll(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--max-model-len") { - a.max_model_len = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--max-num-seqs") { - a.max_num_seqs = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--max-num-batched-tokens") { - a.max_num_batched_tokens = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--device") { - // Text-engine device selection (mirrors vLLM's DeviceConfig.device - // names). Validated by vllm::DeviceFromString at engine construction; - // --video-device (below) stays the video engine's separate knob. - a.device = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--cuda-profile-graph-replays") { - a.cuda_profile_graph_replays = - std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--cuda-profile-graph-batch") { - a.cuda_profile_graph_batch = - std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--benchmark-shutdown-fifo") { - a.benchmark_shutdown_fifo = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--enable-force-include-usage") { - a.enable_force_include_usage = true; - } else if (flag == "--enable-tokenizer-info-endpoint") { - a.enable_tokenizer_info_endpoint = true; - } else if (flag == "--video-dit") { - a.video_dit = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-vae") { - a.video_vae = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-vae-config") { - a.video_vae_config = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--audio-vae") { - a.audio_vae = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--audio-vae-config") { - a.audio_vae_config = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-encoder") { - a.video_encoder = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-tokenizer") { - a.video_tokenizer = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-encoder-max-layers") { - a.video_encoder_max_layers = std::atoi(NextArg(argc, argv, i, argv[0]).c_str()); - } else if (flag == "--video-prompt-embeds") { - a.video_prompt_embeds = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-workdir") { - a.video_workdir = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-ffmpeg") { - a.video_ffmpeg = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-device") { - a.video_device = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-partition") { - a.video_partition = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-keep-quant") { - // the seam's default arm; accepted for pre-fold CLI compatibility - a.video_dequant_bf16 = false; - } else if (flag == "--video-dequant-bf16") { - a.video_dequant_bf16 = true; - } else if (flag == "--enable-server-dev-mode") { - a.enable_server_dev_mode = true; - } else if (flag == "--verbose" || flag == "-v") { - a.verbose = true; - } else if (flag == "--enable-thinking") { - a.enable_thinking = true; - } else if (flag == "--no-enable-thinking") { - a.enable_thinking = false; - } else if (flag == "--enable-log-requests") { - a.enable_log_requests = true; - } else if (flag == "--disable-log-requests") { - a.enable_log_requests = false; - } else if (flag == "--enable-log-outputs") { - a.enable_log_outputs = true; - } else if (flag == "--max-log-len") { - a.max_log_len = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--enable-metrics") { - a.enable_metrics = true; - } else if (flag == "--disable-metrics") { - a.enable_metrics = false; - } else if (flag == "--enable-prefix-caching" || - flag == "--no-enable-prefix-caching" || - flag == "--enable-radix-attention" || - flag == "--disable-radix-attention") { - // --[no-]enable-prefix-caching is vLLM's flag. --enable-radix-attention / - // --disable-radix-attention are SGLang-compatible ALIASES for the SAME - // toggle (RadixAttention is fused into our block-hash APC — there is no - // distinct radix code path; see .agents/specs/sglang-radixattention.md §1). - // They set the identical tri-state as the vLLM flag; last-wins is rejected - // (mirrors passing the vLLM flag twice) so a contradictory pair is caught. - if (a.enable_prefix_caching.has_value()) { - std::cerr << "server: prefix-caching flag (--[no-]enable-prefix-caching " - "/ --[disable|enable]-radix-attention) specified more than " - "once\n"; - Usage(argv[0], 2); - } - a.enable_prefix_caching = - flag == "--enable-prefix-caching" || flag == "--enable-radix-attention"; - } else if (flag == "--scheduling-policy" || flag == "--schedule-policy") { - // --scheduling-policy is vLLM's flag; --schedule-policy is SGLang's name, - // accepted as an alias. Both take fcfs|priority|lpm. - a.scheduling_policy = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--enable-jump-forward" || - flag == "--disable-jump-forward") { - // ENG-SGLANG-BEHAVIOR-FLAG SW3: opt into (or force off) jump-forward - // decoding — the token-unique grammar-speed subset (see - // .agents/specs/sglang-enablement.md). Absent => the default (OFF unless - // VT_ENABLE_JUMP_FORWARD is set). The env var, when set, still overrides. - if (a.enable_jump_forward.has_value()) { - std::cerr << "server: jump-forward flag " - "(--[enable|disable]-jump-forward) specified more than " - "once\n"; - Usage(argv[0], 2); - } - a.enable_jump_forward = flag == "--enable-jump-forward"; - } else if (flag == "--tool-call-parser") { - a.tool_call_parser = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--reasoning-parser") { - a.reasoning_parser = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--kv-transfer-config") { - a.kv_transfer_config = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--speculative-config") { - a.speculative_config = NextArg(argc, argv, i, argv[0]); - } else if (flag == "-h" || flag == "--help") { - Usage(argv[0], 0); - } else { - std::cerr << "server: unknown argument '" << flag << "'\n"; - Usage(argv[0], 2); - } - } - if (a.model_dir.empty()) { - std::cerr << "server: --model is required\n"; - Usage(argv[0], 2); - } - if (a.max_num_seqs <= 0 || a.max_num_batched_tokens < 0 || - a.cuda_profile_graph_replays < 0 || a.cuda_profile_graph_batch < 0) { - std::cerr << "server: scheduler capacities must be positive " - "(--max-num-batched-tokens may be 0 for auto)\n"; - Usage(argv[0], 2); - } - if ((a.cuda_profile_graph_replays > 0) != - !a.benchmark_shutdown_fifo.empty()) { - std::cerr << "server: --cuda-profile-graph-replays and " - "--benchmark-shutdown-fifo must be specified together\n"; - Usage(argv[0], 2); - } - if (a.cuda_profile_graph_replays == 0 && a.cuda_profile_graph_batch != 0) { - std::cerr << "server: --cuda-profile-graph-batch requires " - "--cuda-profile-graph-replays\n"; - Usage(argv[0], 2); - } - if (a.cuda_profile_graph_replays > 0 && a.cuda_profile_graph_batch == 0) { - a.cuda_profile_graph_batch = 16; - } - if (a.cuda_profile_graph_batch > a.max_num_seqs) { - std::cerr << "server: --cuda-profile-graph-batch exceeds --max-num-seqs\n"; - Usage(argv[0], 2); - } - // Validate a NAMED parser dialect here, before the (multi-GB) model load, so a - // typo costs a second rather than a full load. "auto" cannot be checked yet — - // it resolves against the chat template — but detection only ever returns - // registered names, so it cannot fail later either. - namespace oai = vllm::entrypoints::openai; - if (a.tool_call_parser != "auto") { - (void)oai::ResolveToolParserName(a.tool_call_parser, ""); - } - if (a.reasoning_parser != "auto") { - (void)oai::ResolveReasoningParserName(a.reasoning_parser, ""); - } - return a; -} - -} // namespace - -int main(int argc, char** argv) { - try { - const Args args = ParseArgs(argc, argv); - if (args.verbose) { - setenv("VT_SERVER_VERBOSE", "1", /*overwrite=*/1); - std::cerr << "server: verbose stage logging enabled (debug_stages)\n"; - } - { - vllm::entrypoints::openai::RequestLogConfig log_cfg; - log_cfg.enable_log_requests = args.enable_log_requests; - log_cfg.enable_log_outputs = args.enable_log_outputs || args.verbose; - log_cfg.max_log_len = args.max_log_len; - log_cfg.debug_stages = args.verbose || - (std::getenv("VT_SERVER_VERBOSE") && - std::getenv("VT_SERVER_VERBOSE")[0] == '1'); - vllm::entrypoints::openai::ConfigureRequestLogger(log_cfg); - std::cerr << "server: request logging " - << (log_cfg.enable_log_requests ? "ON" : "OFF") - << " outputs=" << (log_cfg.enable_log_outputs ? "ON" : "OFF") - << " max_log_len=" << log_cfg.max_log_len - << " debug_stages=" << (log_cfg.debug_stages ? "ON" : "OFF") - << "\n"; - } - - const fs::path dir(args.model_dir); - const std::string config_path = (dir / "config.json").string(); - const std::string tokenizer_path = (dir / "tokenizer.json").string(); - const std::string tokenizer_config_path = - args.tokenizer_config.empty() - ? (dir / "tokenizer_config.json").string() - : args.tokenizer_config; - const std::string served_model_name = - args.served_model_name.empty() - ? (dir.has_filename() ? dir.filename().string() - : dir.parent_path().filename().string()) - : args.served_model_name; - - // ── TASK DISPATCH (ARCH-ONE-SURFACE ROW 1): a model dir whose - // architectures resolve to a SupportsTranscription-ONLY registration - // (Parakeet CTC/RNNT/TDT) serves /v1/audio/transcriptions through the ONE - // library seam — the same ParakeetTranscriber vllm_transcribe drives — and - // registers NO generate routes (vLLM's task-conditional registration, - // api_server.py:255-265). Every other model takes the text path below, - // byte-identical to before. ──────────────────────────────────────────────── - { - bool transcription_only = false; - const std::vector archs = - vllm::PeekHfArchitectures(config_path); - if (!archs.empty()) { - try { - transcription_only = - vllm::ModelRegistry::Resolve(std::span(archs)) - .info.supports_transcription_only; - } catch (const std::exception&) { - transcription_only = false; // unknown arch: the text path diagnoses - } - } - // ── POOLING TASK DISPATCH (ARCH-ONE-SURFACE ROW 6): a model dir whose - // architectures resolve to a POOLING registration (is_pooling_model, - // e.g. "LlamaModel" — vLLM _EMBEDDING_MODELS registry.py:230) serves - // /v1/embeddings through the ONE engine path (LoadedEngine -> - // LLMEngine::embed -> registry forward -> PoolingRunner) — the same - // path vllm_embed drives — and registers NO generate routes (vLLM's - // task-conditional registration, api_server.py:255-265). ────────────── - bool pooling_model = false; - if (!archs.empty()) { - try { - pooling_model = - vllm::ModelRegistry::Resolve(std::span(archs)) - .info.is_pooling_model; - } catch (const std::exception&) { - pooling_model = false; // unknown arch: the text path diagnoses - } - } - if (pooling_model) { - std::cerr << "server: pooling (embedding) model (" << archs[0] - << "); serving /v1/embeddings\n"; - vllm::entrypoints::EngineParams embed_params; - embed_params.block_size = args.block_size; - embed_params.num_blocks = args.num_blocks; - embed_params.gpu_memory_utilization = args.gpu_memory_utilization; - embed_params.kv_cache_memory_bytes = args.kv_cache_memory_bytes; - embed_params.max_model_len = args.max_model_len; - embed_params.max_num_seqs = args.max_num_seqs; - embed_params.max_num_batched_tokens = args.max_num_batched_tokens; - embed_params.enable_prefix_caching = args.enable_prefix_caching; - auto loaded_embed = std::shared_ptr( - vllm::entrypoints::LoadedEngine::FromModelDir(args.model_dir, - embed_params)); - namespace oai = vllm::entrypoints::openai; - oai::OpenAIServingModels embed_models(served_model_name); - oai::ApiServer embed_server(embed_models, vllm::Version()); - auto embed_mutex = std::make_shared(); - auto embed_counter = std::make_shared>(0); - embed_server.set_embedder( - [loaded_embed, embed_mutex, embed_counter]( - const std::vector& inputs) { - // Serialize batches: the pooling path drives the SYNCHRONOUS - // LLMEngine (async scheduling resolves OFF for pooling models). - std::lock_guard lock(*embed_mutex); - oai::ApiServer::EmbeddingBatch batch; - for (const std::string& text : inputs) { - std::vector ids = - loaded_embed->tokenizer().EncodeWithSpecialTokens(text); - if (ids.empty()) { - throw std::runtime_error( - "input tokenized to an empty prompt"); - } - batch.prompt_tokens += static_cast(ids.size()); - vllm::RequestOutput ro = loaded_embed->engine().embed( - std::move(ids), vllm::PoolingParams{}, - "embd-" + std::to_string(embed_counter->fetch_add(1))); - if (!ro.finished || !ro.pooling_output.has_value()) { - throw std::runtime_error( - "engine produced no pooled output"); - } - batch.embeddings.push_back(std::move(*ro.pooling_output)); - } - return batch; - }); - std::cerr << "server: listening on http://" << args.host << ":" - << args.port << "\n"; - if (!embed_server.listen(args.host, args.port)) { - std::cerr << "server: failed to bind " << args.host << ":" - << args.port << "\n"; - return 1; - } - return 0; - } - - if (transcription_only) { - std::cerr << "server: transcription-only model (" << archs[0] - << "); serving /v1/audio/transcriptions\n"; - auto transcriber = - std::make_shared( - vllm::multimodal::ParakeetTranscriber::FromDir(args.model_dir)); - namespace oai = vllm::entrypoints::openai; - oai::OpenAIServingModels asr_models(served_model_name); - oai::ApiServer asr_server(asr_models, vllm::Version()); - asr_server.set_transcriber( - [transcriber](const uint8_t* wav, size_t n) { - return transcriber->TranscribeWavBytes(wav, n); - }); - std::cerr << "server: listening on http://" << args.host << ":" - << args.port << "\n"; - if (!asr_server.listen(args.host, args.port)) { - std::cerr << "server: failed to bind " << args.host << ":" - << args.port << "\n"; - return 1; - } - return 0; - } - } - - // ── Load the model + build the full engine stack via the shared loader - // (src/vllm/entrypoints/model_loader.cpp) — the same path the C ABI drives. - // It loads config.json + tokenizer.json + *.safetensors and wires the M1.8 - // LLMEngine over Scheduler + runner + KV + processors. ───────────────────── - std::cerr << "server: loading model from " << args.model_dir << " (config " - << config_path << ", tokenizer " << tokenizer_path << ")\n"; - vllm::entrypoints::EngineParams engine_params; - engine_params.block_size = args.block_size; - engine_params.num_blocks = args.num_blocks; - engine_params.gpu_memory_utilization = args.gpu_memory_utilization; - engine_params.kv_cache_memory_bytes = args.kv_cache_memory_bytes; - engine_params.max_model_len = args.max_model_len; // 0 => from config. - engine_params.max_num_seqs = args.max_num_seqs; - engine_params.max_num_batched_tokens = args.max_num_batched_tokens; - engine_params.enable_prefix_caching = args.enable_prefix_caching; - // --device: explicit device selection (ARCH-ONE-SURFACE ROW 8). "auto" - // (default) keeps the accelerator-first probe byte-identical; an unknown - // name throws HERE (a startup error), and an explicitly named ABSENT - // device fails FromModelDir loudly — never a silent fallback - // (vllm/config/device.py:61-66). - engine_params.device = vllm::DeviceFromString(args.device); - // Reject an unknown policy string (mirrors upstream SchedulingPolicy(value)). - engine_params.policy = vllm::SchedulerPolicyFromString(args.scheduling_policy); - // ENG-SGLANG-BEHAVIOR-FLAG (SW1): `lpm` needs prefix caching to have any - // cache to match against; with APC explicitly off it degrades to fcfs - // (the scheduler leaves arrival order intact). Warn once at load so the - // no-op is visible (mirrors the spec's lpm+cache-off resolution). - if (engine_params.policy == vllm::SchedulerPolicy::kLPM && - args.enable_prefix_caching.has_value() && - !args.enable_prefix_caching.value()) { - std::cerr << "server: --scheduling-policy lpm has no effect with prefix " - "caching disabled; falling back to fcfs admission order\n"; - } - // ENG-SGLANG-BEHAVIOR-FLAG SW3: jump-forward decoding. Unset => the default - // (env-resolved, OFF); --[enable|disable]-jump-forward forces it, and - // VT_ENABLE_JUMP_FORWARD still overrides at resolution time. - engine_params.enable_jump_forward = args.enable_jump_forward; - // --kv-transfer-config: the external KV connector, mirroring vLLM's own - // flag and JSON shape. Absent (default) leaves the optional unset, which is - // the inert no-connector path the server has always run. A malformed - // document, an unknown key/role, or a connector whose worker half cannot - // move bytes on this device (the D1 guard, inside LoadedEngine) all throw - // out of here and are reported at startup by the catch in main. - if (!args.kv_transfer_config.empty()) { - vllm::KVTransferConfig kv_cfg = - vllm::ParseKVTransferConfigJson(args.kv_transfer_config); - if (kv_cfg.kv_connector.has_value() && - !vllm::v1::kv_offload::KVConnectorFactory::IsRegistered( - *kv_cfg.kv_connector)) { - std::string msg = "unknown kv_connector \"" + *kv_cfg.kv_connector + - "\" (registered connectors: "; - const std::vector names = - vllm::v1::kv_offload::KVConnectorFactory::RegisteredNames(); - for (size_t n = 0; n < names.size(); ++n) { - if (n != 0) msg += ", "; - msg += names[n]; - } - msg += ")"; - throw std::invalid_argument(msg); - } - engine_params.kv_transfer_config = std::move(kv_cfg); - } - // --speculative-config: speculative decoding (SPEC-MTP I5d). Absent (default) - // leaves the optional unset — the byte-identical no-speculation path. The - // parse validates method/k here; n_predict + the resolved k are finalized in - // LoadedEngine once the checkpoint's mtp_num_hidden_layers is known. A - // malformed document or unsupported method throws and is reported at startup. - if (!args.speculative_config.empty()) { - engine_params.speculative_config = - vllm::ParseSpeculativeConfigJson(args.speculative_config); - } - std::unique_ptr loaded = - vllm::entrypoints::LoadedEngine::FromModelDir(args.model_dir, - engine_params); - std::cerr << "server: prefix caching " - << (loaded->prefix_caching_enabled() ? "enabled" : "disabled") - << "\n"; - // W2: the production server uses AsyncLLM over EngineCoreProc's dedicated - // engine thread. HTTP workers submit independently and stream from their - // per-request collectors; no server-wide engine mutex remains. - vllm::v1::AsyncLLM& engine = loaded->async_engine(); - const vllm::tok::Tokenizer& tokenizer = loaded->tokenizer(); - - if (args.cuda_profile_graph_replays > 0) { -#ifdef VT_BENCH_PROFILE_CONTROL - vt::cuda::ConfigureCudaGraphReplayProfiler( - static_cast(args.cuda_profile_graph_replays), - static_cast(args.cuda_profile_graph_batch)); - std::cerr << "[VT_CUDA_PROFILE] ready pid=" << getpid() - << " signal=SIGUSR2 target_replays=" - << args.cuda_profile_graph_replays << "\n"; -#else - throw std::invalid_argument( - "--cuda-profile-graph-replays requires " - "VLLM_CPP_BENCH_PROFILE_CONTROL=ON"); -#endif - } - - // ── OpenAI serving handlers. The chat handler is wired with the real chat - // template (Task 3) when tokenizer_config.json carries one; otherwise it - // keeps the default role-join fallback. ──────────────────────────────── - namespace oai = vllm::entrypoints::openai; - oai::OpenAIServingModels models(served_model_name); - oai::OpenAIServingCompletion completion( - engine, served_model_name, args.enable_force_include_usage); - - oai::ChatPromptFn chat_prompt_fn = oai::DefaultChatPromptFallback; - // Kept outside the try so the parser resolution below can sniff it when - // --tool-call-parser/--reasoning-parser are "auto"; empty when the model - // ships no template (auto then falls back to hermes / disabled). - std::string chat_template; - try { - chat_template = - vllm::entrypoints::LoadChatTemplateFromConfig(tokenizer_config_path); - const std::string bos = - tokenizer.BosId() >= 0 ? tokenizer.Decode({tokenizer.BosId()}) : ""; - const std::string eos = - tokenizer.EosId() >= 0 ? tokenizer.Decode({tokenizer.EosId()}) : ""; - chat_prompt_fn = - vllm::entrypoints::MakeChatTemplatePromptFn( - chat_template, bos, eos, args.enable_thinking); - std::cerr << "server: using chat template (" << chat_template.size() - << " chars) from " << tokenizer_config_path - << " or sibling chat_template.jinja" - << " enable_thinking=" - << (args.enable_thinking ? "true" : "false") << "\n"; - } catch (const std::exception& e) { - std::cerr << "server: no chat template (" << e.what() - << "); falling back to the simple role-join prompt\n"; - } - // Dialect selection. Defaults reproduce the previously hardcoded pair - // ("hermes", "") exactly; an unknown name throws std::invalid_argument - // listing every registered parser and aborts startup, rather than leaving - // tool/reasoning parsing silently off for the life of the process. - const std::string tool_parser_name = - oai::ResolveToolParserName(args.tool_call_parser, chat_template); - const std::string reasoning_parser_name = - oai::ResolveReasoningParserName(args.reasoning_parser, chat_template); - std::cerr << "server: tool-call parser " - << (tool_parser_name.empty() ? "disabled" : tool_parser_name) - << ", reasoning parser " - << (reasoning_parser_name.empty() ? "disabled" - : reasoning_parser_name) - << "\n"; - oai::OpenAIServingChat chat(engine, served_model_name, chat_prompt_fn, - tool_parser_name, reasoning_parser_name, - args.enable_force_include_usage); - - // SAMPLE-BEAM (C7): enable use_beam_search on the production AsyncLLM path. - // Both handlers need the tokenizer (prompt tok + per-beam detok) and the eos - // id (beam retirement); a use_beam_search request then routes through - // BeamSearchAsync (online.py) over the async engine. Without this, beam - // requests reject with "requires an engine and a tokenizer". - const std::optional beam_eos = - tokenizer.EosId() >= 0 - ? std::optional(tokenizer.EosId()) - : std::nullopt; - completion.set_beam_search_tokenizer(&tokenizer, beam_eos); - chat.set_beam_search_tokenizer(&tokenizer, beam_eos); - - // ── MM-SERVE-E2E: wire the multimodal chat seam for image-capable models. - // When the model dir carries a preprocessor_config.json the Qwen3-VL image - // processor loads, we construct the seam body (MakeQwen3VLImageChatFn) so an - // OpenAI image_url request renders the placeholder marker → tokenizes to the - // single image_pad id → EXPANDS to N image tokens + mm_features carried onto - // the engine request. A text-only model (no preprocessor_config.json) leaves - // the seam UNSET → the chat path is byte-identical. The container-format - // image codec (PNG/JPEG → RGB) is a NAMED residual: no codec is vendored, so - // the production codec rejects encoded images with a clear message (the M2c - // single-sequence gate consumes pre-decoded raw RGB). The mm FORWARD (vision - // tower + merge + MRoPE/DeepStack on the GPU worker consuming - // Request.mm_features) is the remaining MM-SERVE-E2E residual — the engine - // model runner has no mm-forward path yet. Kept alive for the server loop. - std::unique_ptr mm_image_proc; - const std::string preprocessor_config_path = - (dir / "preprocessor_config.json").string(); - if (fs::exists(preprocessor_config_path)) { - try { - vllm::multimodal::Qwen3VLProcessorConfig pcfg = - vllm::multimodal::LoadQwen3VLProcessorConfig( - preprocessor_config_path, config_path, served_model_name); - mm_image_proc = - std::make_unique(pcfg); - oai::ImageCodecFn codec = - [](const oai::DecodedMedia& media) -> oai::DecodedImageRgb { - // Raw-RGB passthrough (image/x-raw-rgb): the single-sequence e2e / - // gate fixture format. A square raw-RGB payload is decoded directly; - // any container format (PNG/JPEG) is the NAMED codec residual. - if (media.media_type == "image/x-raw-rgb") { - const std::size_t n = media.bytes.size(); - const std::size_t px = n / 3; - const auto side = - static_cast(std::llround(std::sqrt( - static_cast(px)))); - if (side <= 0 || static_cast(side * side * 3) != n) { - throw std::runtime_error( - "image/x-raw-rgb payload is not a square HxWx3 buffer"); - } - oai::DecodedImageRgb out; - out.rgb = media.bytes; - out.height = side; - out.width = side; - return out; - } - throw std::runtime_error( - "multimodal image: container-format decode (PNG/JPEG -> RGB) is a " - "named MM-SERVE residual; supply raw RGB (image/x-raw-rgb)"); - }; - chat.set_multimodal_chat_fn(oai::MakeQwen3VLImageChatFn( - *mm_image_proc, tokenizer, chat_prompt_fn, std::move(codec))); - std::cerr << "server: multimodal image seam wired (Qwen3-VL processor " - "from " - << preprocessor_config_path << ")\n"; - } catch (const std::exception& e) { - std::cerr << "server: no multimodal image seam (" << e.what() - << "); image requests fall back to the text path\n"; - } - } - - // Diagnostic opt-out exists only for same-binary attribution. Production - // defaults to the capacity-derived fixed pool. - const char* fixed_pool_env = std::getenv("VLLM_CPP_HTTP_FIXED_POOL"); - const auto worker_pool_mode = - fixed_pool_env != nullptr && std::string(fixed_pool_env) == "0" - ? oai::ApiServer::HttpWorkerPoolMode::kLegacyDynamic - : oai::ApiServer::HttpWorkerPoolMode::kCapacityFixed; - oai::ApiServer server(completion, chat, models, vllm::Version(), - static_cast(args.max_num_seqs), - worker_pool_mode); - - // ── C8 opt-in utility/admin endpoints (SERVE-UTILITY-ENDPOINTS / - // SERVE-ADMIN). Wire the setters from the LIVE engine + tokenizer through the - // single shared seam so the production server actually serves /tokenize, - // /detokenize, /tokenizer_info (flag) and /abort_requests (dev-mode flag), - // mirroring vLLM 0.26's per-endpoint default gating. /metrics and - // /reset_prefix_cache stay unwired (no live backing on the AsyncLLM path) — - // see ConfigureUtilityEndpoints + specs/{utility,admin}-endpoints.md. ──────── - // ── MiniMax-H3 video generation. OPT-IN: with no --video-dit the routes are - // never registered and the server is byte-identical to before. The whole - // pipeline lives in the LIBRARY seam (vllm::multimodal::MiniMaxH3VideoEngine, - // ARCH-ONE-SURFACE ROW 2) — the SAME entry point the C ABI's vllm_video_* - // and the minimax-h3-gen example drive, so HTTP and FFI cannot drift. This - // file keeps exactly what an example may own: flag plumbing, the job - // directory, and the ONE process spawn (ffmpeg, ratified 2026-08-03 — the - // library builds the argv and spawns nothing). ──────────────────────────── - std::shared_ptr video_engine; - if (!args.video_dit.empty()) { - std::cerr << "server: loading MiniMax-H3 video checkpoints...\n"; - vllm::multimodal::MiniMaxH3VideoModelParams vmp; - vmp.dit_path = args.video_dit; - vmp.encoder_path = args.video_encoder; - vmp.tokenizer_path = args.video_tokenizer; - vmp.video_vae_path = args.video_vae; - vmp.video_vae_config_path = args.video_vae_config; - vmp.audio_vae_path = args.audio_vae; - vmp.audio_vae_config_path = args.audio_vae_config; - vmp.prompt_embeds_path = args.video_prompt_embeds; - vmp.partition = args.video_partition; - vmp.device = args.video_device == "cuda" ? 1 : 0; - vmp.dequant_bf16 = args.video_dequant_bf16 ? 1 : 0; - vmp.encoder_max_layers = args.video_encoder_max_layers; - video_engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(vmp); - std::cerr << "server: /v1/videos on (device=" << args.video_device - << (args.video_dequant_bf16 ? ", dequant-bf16" : ", keep-quant") << ")\n"; - // HONEST LIMIT, stated at startup rather than buried: turning a PROMPT - // into conditioning needs the H3-Encoder; without one every request is - // conditioned on the SAME supplied embeddings. - if (video_engine->has_encoder()) { - std::cerr << "server: /v1/videos conditions on the request PROMPT\n"; - } else if (!video_engine->has_prompt_embeds()) { - std::cerr << "server: WARNING /v1/videos has neither --video-encoder nor " - "--video-prompt-embeds; requests will be REJECTED\n"; - } else { - std::cerr << "server: WARNING /v1/videos ignores the request PROMPT — pass " - "--video-encoder to condition on it\n"; - } - - auto counter = std::make_shared>(0); - const std::string workdir = args.video_workdir; - const std::string ffmpeg = args.video_ffmpeg; - server.set_video_runner([video_engine, counter, workdir, - ffmpeg](const vllm::openai::VideoRequest& req) -> std::string { - const int64_t id = counter->fetch_add(1); - const std::string dir = workdir + "/job" + std::to_string(id); - // The library-owned request mapping + generation: conditioning, task - // resolution, the #77 partition guard, reference encoding, artifacts. - const vllm::multimodal::MiniMaxH3VideoResult out = video_engine->Generate( - vllm::multimodal::MiniMaxH3VideoGenParamsFromRequest(req, dir)); - // The ONE process spawn: exec the argv the library composed. - std::vector argv_mux = out.mux_argv; - if (!argv_mux.empty()) argv_mux[0] = ffmpeg; - const int status = RunFfmpegArgv(argv_mux); - if (status != 0) { - throw std::runtime_error("ffmpeg exited " + std::to_string(status)); - } - return out.mux_output_path; - }); - } - - oai::UtilityEndpointOptions endpoint_opts; - endpoint_opts.enable_tokenizer_info_endpoint = - args.enable_tokenizer_info_endpoint; - endpoint_opts.enable_server_dev_mode = args.enable_server_dev_mode; - oai::ConfigureUtilityEndpoints(server, tokenizer, loaded->max_model_len(), - engine, endpoint_opts); - std::cerr << "server: utility endpoints: /tokenize /detokenize on" - << (args.enable_tokenizer_info_endpoint ? ", /tokenizer_info on" - : "") - << (args.enable_server_dev_mode ? ", /abort_requests on (dev-mode)" - : "") - << "\n"; - - // Prometheus /metrics (Python vLLM always-on family names). - std::unique_ptr prom_logger; - if (args.enable_metrics) { - prom_logger = std::make_unique( - served_model_name, loaded->max_model_len(), /*engine_index=*/0); - // Sync engine path records on step(); async may under-report until fully wired. - loaded->engine().set_stat_logger(prom_logger.get()); - server.set_metrics_logger(prom_logger.get()); - std::cerr << "server: GET /metrics enabled (PrometheusStatLogger)\n"; - } - - std::cerr << "server: listening on http://" << args.host << ":" << args.port - << " (model '" << served_model_name << "', HTTP worker pool "; - if (server.http_worker_count() == 0) { - std::cerr << "legacy-dynamic"; - } else { - std::cerr << server.http_worker_count() << " fixed"; - } - std::cerr << ")\n"; - -#ifdef VT_BENCH_PROFILE_CONTROL - std::atomic benchmark_shutdown_waiter_ready{false}; - std::atomic benchmark_shutdown_received{false}; - std::atomic benchmark_shutdown_failed{false}; - std::atomic benchmark_shutdown_cancelled{false}; - std::thread benchmark_shutdown_thread; - if (args.cuda_profile_graph_replays > 0) { - benchmark_shutdown_thread = std::thread([&]() { - const int shutdown_fd = - open(args.benchmark_shutdown_fifo.c_str(), - O_RDONLY | O_NONBLOCK | O_CLOEXEC | O_NOFOLLOW); - if (shutdown_fd < 0) { - const int status = errno; - std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=open status=" - << status << "\n"; - benchmark_shutdown_failed.store(true, std::memory_order_release); - return; - } - struct stat shutdown_stat {}; - const int stat_status = fstat(shutdown_fd, &shutdown_stat); - if (stat_status != 0 || !S_ISFIFO(shutdown_stat.st_mode)) { - const int status = stat_status != 0 ? errno : EINVAL; - std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=fstat status=" - << status << "\n"; - close(shutdown_fd); - benchmark_shutdown_failed.store(true, std::memory_order_release); - return; - } - benchmark_shutdown_waiter_ready.store(true, std::memory_order_release); - std::cerr << "[VT_BENCH_SHUTDOWN] ready pid=" << getpid() - << " control=fifo\n"; - while (!benchmark_shutdown_cancelled.load(std::memory_order_acquire)) { - char command = '\0'; - const ssize_t bytes = read(shutdown_fd, &command, 1); - if (bytes == 1) { - if (command == 'Q') { - benchmark_shutdown_received.store(true, - std::memory_order_release); - std::cerr - << "[VT_BENCH_SHUTDOWN] requested control=fifo\n"; - close(shutdown_fd); - server.stop(); - return; - } - std::cerr - << "[VT_BENCH_SHUTDOWN] failed operation=command status=" - << static_cast( - static_cast(command)) - << "\n"; - close(shutdown_fd); - benchmark_shutdown_failed.store(true, std::memory_order_release); - server.stop(); - return; - } - if (bytes < 0 && errno != EAGAIN && errno != EWOULDBLOCK && - errno != EINTR) { - const int status = errno; - std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=read status=" - << status << "\n"; - close(shutdown_fd); - benchmark_shutdown_failed.store(true, std::memory_order_release); - server.stop(); - return; - } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - close(shutdown_fd); - }); - while (!benchmark_shutdown_waiter_ready.load(std::memory_order_acquire) && - !benchmark_shutdown_failed.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - if (benchmark_shutdown_failed.load(std::memory_order_acquire)) { - benchmark_shutdown_cancelled.store(true, std::memory_order_release); - benchmark_shutdown_thread.join(); - return 1; - } - } -#endif - - const bool listen_ok = server.listen(args.host, args.port); - -#ifdef VT_BENCH_PROFILE_CONTROL - if (benchmark_shutdown_thread.joinable()) { - benchmark_shutdown_cancelled.store(true, std::memory_order_release); - benchmark_shutdown_thread.join(); - if (benchmark_shutdown_received.load(std::memory_order_acquire)) { - std::cerr << "[VT_BENCH_SHUTDOWN] completed control=fifo\n"; - } else { - if (!benchmark_shutdown_failed.load(std::memory_order_acquire)) { - std::cerr - << "[VT_BENCH_SHUTDOWN] failed operation=cancelled status=0\n"; - } - if (listen_ok) { - return 1; - } - } - } -#endif - - if (!listen_ok) { - std::cerr << "server: failed to bind " << args.host << ":" << args.port - << "\n"; - return 1; - } - return 0; - } catch (const std::exception& e) { - std::cerr << "server: fatal: " << e.what() << "\n"; - return 1; - } -} +int main(int argc, char** argv) { return vllm_server_main(argc, argv); } diff --git a/include/vllm.h b/include/vllm.h index 8028de31b..667477da8 100644 --- a/include/vllm.h +++ b/include/vllm.h @@ -129,8 +129,20 @@ extern "C" { * other's tasks LOUDLY: vllm_complete/vllm_chat on a pooling engine name * vllm_embed, and vllm_embed on a text engine names vllm_complete — the * SupportsTranscription-refusal precedent (v11) applied to the pooling task. - * Purely additive — no struct changed; zero values preserve behaviour. */ -#define VLLM_ABI_VERSION 16 + * Purely additive — no struct changed; zero values preserve behaviour. + * + * v17 — vllm_server_main: RUN THE OPENAI SERVER from the public surface. + * examples/server was the deepest ONE SURFACE breach (36 internal headers: the + * engine, the serving layers, metrics, the video and ASR seams), which is why it + * carried an example-abi-allowlist entry. That construction moved INTO the + * library (vllm/entrypoints/openai/server_main.h) and the example is now a thin + * client of this call. argv rather than a params struct is deliberate: the + * server takes ~57 flags and gains more with every serving feature, and a + * mirrored C struct would put that churn in the ABI where every field is + * permanent. The flag surface mirrors vLLM's cli_args.py, which is the real + * contract. Embedders wanting programmatic control keep the granular entry + * points. Purely additive. */ +#define VLLM_ABI_VERSION 17 /* ── Export macro ───────────────────────────────────────────────────────────── * Marks the symbols that make up the stable ABI. Default visibility now; Task 3 @@ -792,6 +804,23 @@ VLLM_API const char* vllm_last_error(void); * not free. */ VLLM_API const char* vllm_version(void); +/* ── OpenAI-compatible server ───────────────────────────────────────────────── + * Parse `argv` and RUN the OpenAI-compatible HTTP server until it exits, + * returning the process exit code (0 on clean shutdown). `--help` prints usage + * and returns 0; a bad argument or a startup failure prints the reason and + * returns non-zero. Never throws across this boundary. + * + * This is what `vllm-server` is: examples/server is a thin client of this call. + * It serves /v1/chat/completions, /v1/completions, /v1/models, /v1/embeddings, + * and — when the matching flags are supplied — /v1/videos (MiniMax-H3) and + * transcription, all through the SAME library seams the granular entry points + * below drive, so HTTP and FFI cannot drift. + * + * BLOCKS for the lifetime of the server. `argv` must hold `argc` NUL-terminated + * strings and stay valid for the duration; the library does not take ownership. + * The conventional argv[0] program name is expected at index 0. */ +VLLM_API int32_t vllm_server_main(int32_t argc, char** argv); + /* The ABI version the library was built with (compare against VLLM_ABI_VERSION). */ VLLM_API int32_t vllm_abi_version(void); diff --git a/scripts/example-abi-allowlist.txt b/scripts/example-abi-allowlist.txt index 2ce066a0a..9c4691e51 100644 --- a/scripts/example-abi-allowlist.txt +++ b/scripts/example-abi-allowlist.txt @@ -24,7 +24,6 @@ # --- Capability drivers whose FAST PATH is reachable ONLY here, not through the ABI --- examples/deepseek_v4_gen | fold=ARCH-ONE-SURFACE | DeepSeek-V4-Flash keep-quant GGUF greedy decode (DeepseekV4ForwardGguf(Cached) + DeepseekV4KvCache) is CLI-only; the registered DeepseekV4ForCausalLM forward is a W3-W8 stub (deepseek_v4_registry.cpp:22). Grow ABI keep-quant GGUF load+decode, rewrite as ABI client, delete the bespoke forward examples/laguna_gen | fold=ARCH-ONE-SURFACE | Laguna-S-2.1 keep-quant GGUF (multi-shard) + NVFP4 W4A4 device-resident decode (LagunaForwardGguf(Cached), Marlin residents, fp4-shared) is CLI-only; registered LagunaForCausalLM forward VT_CHECK(false)s on non-bf16 (laguna.cpp:156). Grow ABI, rewrite as client, delete bespoke forward -examples/server | fold=ARCH-ONE-SURFACE | The reference OpenAI server constructs the engine, metrics, video_runner and mm seam directly from internal C++ headers (LoadedEngine/AsyncLLM, minimax_h3.h, chat_mm). It should stand on the public surface (C ABI, or a curated public C++ API the ABI wraps). Grow the surface, rewrite the server against it # --- Dev / diagnostic tools: NO permanent exemption (developer-directed 2026-08-07) --- # Every one is a transition-tracker like the drivers above; the allowlist only shrinks. diff --git a/src/capi/vllm_c.cpp b/src/capi/vllm_c.cpp index 3d63ee56d..b42faa5f6 100644 --- a/src/capi/vllm_c.cpp +++ b/src/capi/vllm_c.cpp @@ -44,6 +44,7 @@ #include "vllm/model_executor/models/minimax_h3.h" // mux argv (v12) #include "vllm/multimodal/parakeet_transcription.h" // vllm_transcribe (v11) #include "vllm/multimodal/minimax_h3_video.h" // vllm_video_* (v12) +#include "vllm/entrypoints/openai/server_main.h" // vllm_server_main (v17) #include "vllm/outputs.h" #include "vllm/sampling_params.h" #include "vllm/transformers_utils/hf_config.h" // PeekHfArchitectures (v11) @@ -1615,6 +1616,22 @@ VLLM_API const char* vllm_version(void) { return kVersion.c_str(); } +VLLM_API int32_t vllm_server_main(int32_t argc, char** argv) { + // The server owns its own error reporting on stderr (it is a PROCESS entry + // point, not a request call), so this does not set vllm_last_error. What it + // must guarantee is that nothing throws across the C boundary. + try { + return static_cast( + vllm::entrypoints::openai::VllmServerMain(static_cast(argc), argv)); + } catch (const std::exception& e) { + std::fprintf(stderr, "vllm_server_main: %s\n", e.what()); + return 1; + } catch (...) { + std::fprintf(stderr, "vllm_server_main: unknown error\n"); + return 1; + } +} + VLLM_API int32_t vllm_abi_version(void) { return VLLM_ABI_VERSION; } } // extern "C" diff --git a/src/vllm/entrypoints/openai/server_main.cpp b/src/vllm/entrypoints/openai/server_main.cpp new file mode 100644 index 000000000..b72ba3ba5 --- /dev/null +++ b/src/vllm/entrypoints/openai/server_main.cpp @@ -0,0 +1,1055 @@ +// server: an OpenAI-compatible HTTP server over the vllm.cpp LLMEngine (M3.1 +// Task 4). Loads a supported model (safetensors or GGUF weights + tokenizer + a KV-cache config → +// LLMEngine), constructs the OpenAI serving handlers (chat wired with the real +// chat template via MakeChatTemplatePromptFn(LoadChatTemplateFromConfig(...))) +// and serves /v1/completions, /v1/chat/completions, /v1/models, /health, +// /version. +// +// server --model [--host 0.0.0.0] [--port 8000] +// [--tokenizer-config ] +// [--served-model-name ] +// [--block-size N] [--num-blocks N] [--max-model-len N] +// [--gpu-memory-utilization F] [--kv-cache-memory BYTES] +// [--max-num-seqs N] [--max-num-batched-tokens N] +// [--enable-force-include-usage] +// [--[no-]enable-prefix-caching] +// [--scheduling-policy fcfs|priority] +// [--tool-call-parser |auto|none] +// [--reasoning-parser |auto|none] +// [--kv-transfer-config ''] +// +// A directory holds config.json, tokenizer.json and supported safetensors +// shards. A supported GGUF file is also accepted and supplies model metadata +// plus embedded vocabulary. If --tokenizer-config is omitted for a directory it +// defaults to /tokenizer_config.json; when that file has no chat_template +// the chat endpoint falls back to the simple role-join prompt. +// +// NOTE: loading the real 35B checkpoint is a GPU/dgx concern; on the CPU CI box +// this binary is only built + smoke-tested against a synthetic engine (see +// tests/vllm/entrypoints/openai/test_api_server.cpp). The wiring below is the +// same either way. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef VT_BENCH_PROFILE_CONTROL +#include +#include +#include +#include +#include +#endif + +#include "vllm/config/device.h" +#include "vllm/config/kv_transfer.h" +#include "vllm/config/scheduler.h" +#include "vllm/entrypoints/chat_template.h" +#include "vllm/entrypoints/model_loader.h" +#include +#include "vllm/entrypoints/openai/server_main.h" +#include "vllm/entrypoints/openai/api_server.h" +#include "vllm/entrypoints/openai/chat_mm.h" +#include "vllm/entrypoints/openai/request_logger.h" +#include "vllm/entrypoints/openai/serving_chat.h" +#include "vllm/entrypoints/openai/serving_completion.h" +#include "vllm/entrypoints/openai/serving_models.h" +#include "vllm/v1/metrics/loggers.h" +#include "vllm/entrypoints/openai/reasoning_parsers/detect.h" +#include "vllm/entrypoints/openai/tool_parsers/detect.h" +#include "vllm/model_executor/model_loader/safetensors_reader.h" +#include "vllm/model_executor/models/qwen3_5_weights.h" +#include "vllm/transformers_utils/hf_config.h" +#include "vllm/model_executor/models/model_registry.h" +#include "vllm/multimodal/minimax_h3_video.h" +#include "vllm/multimodal/parakeet_transcription.h" +#include "vllm/tokenizer/tokenizer.h" +#include "vllm/version.h" +#include "vllm/v1/core/kv_cache_utils.h" +#include "vllm/v1/core/sched/scheduler.h" +#include "vllm/v1/engine/core.h" +#include "vllm/v1/engine/input_processor.h" +#include "vllm/v1/engine/llm_engine.h" +#include "vllm/v1/engine/output_processor.h" +#include "vllm/v1/executor/executor.h" +#include "vllm/v1/kv_cache_interface.h" +#include "vllm/v1/kv_offload/kv_connector.h" +#include "vllm/v1/worker/gpu/runner.h" +#include "vt/backend.h" +#ifdef VT_BENCH_PROFILE_CONTROL +#include "vt/cuda/cuda_profiler_control.h" +#endif +#include "vt/dtype.h" +#include "vt/tensor.h" + +namespace { + +namespace fs = std::filesystem; +using vllm::HfConfig; +using vllm::Qwen3_5MoeWeights; + +// Run an argv to completion and return its exit status — the ONE process +// spawn in the MiniMax-H3 path, and it lives HERE, in examples/, by the +// developer-ratified 2026-08-03 decision: the library (the +// MiniMaxH3VideoEngine seam behind /v1/videos) writes the artifacts and +// BUILDS this argv, and spawns nothing. +int RunFfmpegArgv(const std::vector& args) { + std::vector c_args; + c_args.reserve(args.size() + 1); + for (const std::string& arg : args) { + c_args.push_back(const_cast(arg.c_str())); + } + c_args.push_back(nullptr); + const pid_t pid = fork(); + if (pid < 0) throw std::runtime_error("fork failed"); + if (pid == 0) { + execvp(c_args[0], c_args.data()); + _exit(127); // exec failed; never run the parent's atexit handlers + } + int status = 0; + if (waitpid(pid, &status, 0) < 0) throw std::runtime_error("waitpid failed"); + if (WIFSIGNALED(status)) { + throw std::runtime_error("ffmpeg died on signal " + std::to_string(WTERMSIG(status))); + } + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; +} + +struct Args { + std::string model_dir; + std::string host = "0.0.0.0"; + int port = 8000; + std::string tokenizer_config; // default: /tokenizer_config.json + std::string served_model_name; // default: the model dir name + int block_size = 32; + // --num-blocks is the KV block-count OVERRIDE (0 => auto: sized by the knobs + // below, else the 256-block fallback). ROAD-V1-MEM M1. + int num_blocks = 0; + // --gpu-memory-utilization: fraction of free device memory for the whole + // engine (needs the M3 profile run; inert until then). --kv-cache-memory: an + // absolute KV-pool size in bytes that sizes the block count directly (0 => + // unset). + double gpu_memory_utilization = 0.92; + long long kv_cache_memory_bytes = 0; + int max_model_len = 0; // 0 => config.max_position_embeddings + int max_num_seqs = 8; + int max_num_batched_tokens = 0; // 0 => per-architecture default. + // --device: explicit device selection for the TEXT engine (ARCH-ONE-SURFACE + // ROW 8), the vLLM DeviceConfig.device names this build serves: "auto" + // (default — the accelerator-first probe, byte-identical to before the flag), + // "cpu" (force the CPU queue), "cuda" (require CUDA; an absent device fails + // startup LOUDLY, never a silent fallback). The video engine keeps its own + // --video-device below: the two engines are loaded from different + // checkpoints and may legitimately serve on different devices. + std::string device = "auto"; + // --- MiniMax-H3 video generation (opt-in; absent => /v1/videos is unregistered + // and the server behaves exactly as before). --- + std::string video_dit, video_vae, video_vae_config, audio_vae, audio_vae_config; + std::string video_prompt_embeds, video_workdir = "/tmp/vllm_h3_videos"; + std::string video_encoder, video_tokenizer; + int video_encoder_max_layers = 50; + std::string video_ffmpeg = "ffmpeg", video_device = "cuda"; + std::string video_partition; // served partition (fl2va|ref2va); see the #77 guard + // Keep-quant is the library seam's DEFAULT arm; --video-dequant-bf16 selects + // the bf16 dequant/stream arm (the throughput trade the gen example ships). + // --video-keep-quant is still accepted (it names the default). + bool video_dequant_bf16 = false; + int cuda_profile_graph_replays = 0; // trace-only diagnostic build seam. + int cuda_profile_graph_batch = 0; // 0 => accepted c16 trace contract. + std::string benchmark_shutdown_fifo; // paired trace-only control path. + std::optional enable_prefix_caching = std::nullopt; + bool enable_force_include_usage = false; + // GET /tokenizer_info gate. Mirrors vLLM's --enable-tokenizer-info-endpoint + // (entrypoints/openai/cli_args.py:140, default False; the route is registered + // only when serve/tokenize/api_router.py:95 sees the flag). Default off → the + // route 404s, byte-identical to before. + bool enable_tokenizer_info_endpoint = false; + // Dev/admin endpoint gate. Mirrors vLLM's VLLM_SERVER_DEV_MODE env + // (envs.py:157, default 0): build_app registers the dev/rlhf + dev/cache + // routers only under `if envs.VLLM_SERVER_DEV_MODE` (api_server.py:238). Off by + // default → /abort_requests 404s. Enables the /abort_requests production wiring. + bool enable_server_dev_mode = false; + bool verbose = false; + // Gemma4 HF/vLLM: --default-chat-template-kwargs enable_thinking (default OFF). + bool enable_thinking = false; + // Request logging (Python vLLM --enable-log-requests parity). Default ON. + bool enable_log_requests = true; + bool enable_log_outputs = false; + int max_log_len = 256; + // Attach Prometheus logger + GET /metrics (default ON for solid Hermes serve). + bool enable_metrics = true; + // Scheduling policy: "fcfs" (default), "priority" (mirrors vLLM's + // --scheduling-policy / SchedulerConfig.policy), or "lpm" (SGLang's + // cache-aware longest-prefix-match admission ordering, ENG-SGLANG-BEHAVIOR-FLAG; + // opt-in, output-neutral, resolves to fcfs when prefix caching is off). + // --schedule-policy is accepted as an SGLang-compatible alias. + std::string scheduling_policy = "fcfs"; + // Jump-forward decoding (ENG-SGLANG-BEHAVIOR-FLAG SW3): tri-state, mirrors the + // C-ABI vllm_model_params.enable_jump_forward. Unset (default) => OFF unless + // VT_ENABLE_JUMP_FORWARD is set; --enable-jump-forward forces on, + // --disable-jump-forward forces off (the env var still overrides). The + // token-unique forced-run subset only; see .agents/specs/sglang-enablement.md. + std::optional enable_jump_forward = std::nullopt; + // Tool-call / reasoning dialect selection (mirrors vLLM's --tool-call-parser + // and --reasoning-parser). THE DEFAULTS ARE TODAY'S HARDCODED BEHAVIOUR: + // "hermes" is exactly what OpenAIServingChat was constructed with before this + // flag existed, and "none" is the empty reasoning-parser name it passed. An + // invocation that names neither flag is therefore unchanged, byte for byte. + // "auto" opts into the chat-template detection the C ABI uses. + std::string tool_call_parser = "hermes"; + std::string reasoning_parser = "none"; + // vLLM's --kv-transfer-config: the external KV connector selection, as the + // same JSON object vLLM takes. Empty (default) == no connector == the inert + // production path. See docs/KV-OFFLOAD.md. + std::string kv_transfer_config; + // vLLM's --speculative-config: the speculative-decoding selection, as the same + // JSON object vLLM takes (e.g. '{"method":"mtp","num_speculative_tokens":1}'). + // Empty (default) == no speculation == the inert production path (SPEC-MTP I5d). + std::string speculative_config; +}; + +[[noreturn]] void Usage(const char* argv0, int code) { + std::cerr + << "usage: " << argv0 + << " --model [--host H] [--port P] [--tokenizer-config F]\n" + " [--served-model-name N] [--block-size N] " + "[--num-blocks N] [--max-model-len N]\n" + " [--gpu-memory-utilization F] " + "[--kv-cache-memory BYTES]\n" + " [--max-num-seqs N] " + "[--max-num-batched-tokens N]\n" + " [--device auto|cpu|cuda]\n" + " [--cuda-profile-graph-replays N]\n" + " [--cuda-profile-graph-batch N]\n" + " [--benchmark-shutdown-fifo F]\n" + " [--enable-force-include-usage]\n" + " [--enable-tokenizer-info-endpoint]\n" + " [--enable-server-dev-mode]\n" + " [--verbose]\n" + " [--enable-thinking|--no-enable-thinking]\n" + " [--enable-log-requests|--disable-log-requests]\n" + " [--enable-log-outputs] [--max-log-len N]\n" + " [--enable-metrics|--disable-metrics]\n" + " [--[no-]enable-prefix-caching]\n" + " [--[no-]enable-radix-attention]\n" + " [--scheduling-policy fcfs|priority|lpm]\n" + " [--[enable|disable]-jump-forward]\n" + " [--tool-call-parser |auto|none]\n" + " [--reasoning-parser |auto|none]\n" + " [--kv-transfer-config '']\n" + " [--speculative-config '']\n"; + std::exit(code); +} + +std::string NextArg(int argc, char** argv, int& i, const char* argv0) { + if (i + 1 >= argc) Usage(argv0, 2); + return argv[++i]; +} + +Args ParseArgs(int argc, char** argv) { + Args a; + for (int i = 1; i < argc; ++i) { + const std::string flag = argv[i]; + if (flag == "--model") { + a.model_dir = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--host") { + a.host = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--port") { + a.port = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--tokenizer-config") { + a.tokenizer_config = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--served-model-name") { + a.served_model_name = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--block-size") { + a.block_size = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--num-blocks") { + a.num_blocks = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--gpu-memory-utilization") { + a.gpu_memory_utilization = std::stod(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--kv-cache-memory") { + a.kv_cache_memory_bytes = std::stoll(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--max-model-len") { + a.max_model_len = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--max-num-seqs") { + a.max_num_seqs = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--max-num-batched-tokens") { + a.max_num_batched_tokens = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--device") { + // Text-engine device selection (mirrors vLLM's DeviceConfig.device + // names). Validated by vllm::DeviceFromString at engine construction; + // --video-device (below) stays the video engine's separate knob. + a.device = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--cuda-profile-graph-replays") { + a.cuda_profile_graph_replays = + std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--cuda-profile-graph-batch") { + a.cuda_profile_graph_batch = + std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--benchmark-shutdown-fifo") { + a.benchmark_shutdown_fifo = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--enable-force-include-usage") { + a.enable_force_include_usage = true; + } else if (flag == "--enable-tokenizer-info-endpoint") { + a.enable_tokenizer_info_endpoint = true; + } else if (flag == "--video-dit") { + a.video_dit = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-vae") { + a.video_vae = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-vae-config") { + a.video_vae_config = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--audio-vae") { + a.audio_vae = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--audio-vae-config") { + a.audio_vae_config = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-encoder") { + a.video_encoder = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-tokenizer") { + a.video_tokenizer = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-encoder-max-layers") { + a.video_encoder_max_layers = std::atoi(NextArg(argc, argv, i, argv[0]).c_str()); + } else if (flag == "--video-prompt-embeds") { + a.video_prompt_embeds = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-workdir") { + a.video_workdir = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-ffmpeg") { + a.video_ffmpeg = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-device") { + a.video_device = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-partition") { + a.video_partition = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-keep-quant") { + // the seam's default arm; accepted for pre-fold CLI compatibility + a.video_dequant_bf16 = false; + } else if (flag == "--video-dequant-bf16") { + a.video_dequant_bf16 = true; + } else if (flag == "--enable-server-dev-mode") { + a.enable_server_dev_mode = true; + } else if (flag == "--verbose" || flag == "-v") { + a.verbose = true; + } else if (flag == "--enable-thinking") { + a.enable_thinking = true; + } else if (flag == "--no-enable-thinking") { + a.enable_thinking = false; + } else if (flag == "--enable-log-requests") { + a.enable_log_requests = true; + } else if (flag == "--disable-log-requests") { + a.enable_log_requests = false; + } else if (flag == "--enable-log-outputs") { + a.enable_log_outputs = true; + } else if (flag == "--max-log-len") { + a.max_log_len = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--enable-metrics") { + a.enable_metrics = true; + } else if (flag == "--disable-metrics") { + a.enable_metrics = false; + } else if (flag == "--enable-prefix-caching" || + flag == "--no-enable-prefix-caching" || + flag == "--enable-radix-attention" || + flag == "--disable-radix-attention") { + // --[no-]enable-prefix-caching is vLLM's flag. --enable-radix-attention / + // --disable-radix-attention are SGLang-compatible ALIASES for the SAME + // toggle (RadixAttention is fused into our block-hash APC — there is no + // distinct radix code path; see .agents/specs/sglang-radixattention.md §1). + // They set the identical tri-state as the vLLM flag; last-wins is rejected + // (mirrors passing the vLLM flag twice) so a contradictory pair is caught. + if (a.enable_prefix_caching.has_value()) { + std::cerr << "server: prefix-caching flag (--[no-]enable-prefix-caching " + "/ --[disable|enable]-radix-attention) specified more than " + "once\n"; + Usage(argv[0], 2); + } + a.enable_prefix_caching = + flag == "--enable-prefix-caching" || flag == "--enable-radix-attention"; + } else if (flag == "--scheduling-policy" || flag == "--schedule-policy") { + // --scheduling-policy is vLLM's flag; --schedule-policy is SGLang's name, + // accepted as an alias. Both take fcfs|priority|lpm. + a.scheduling_policy = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--enable-jump-forward" || + flag == "--disable-jump-forward") { + // ENG-SGLANG-BEHAVIOR-FLAG SW3: opt into (or force off) jump-forward + // decoding — the token-unique grammar-speed subset (see + // .agents/specs/sglang-enablement.md). Absent => the default (OFF unless + // VT_ENABLE_JUMP_FORWARD is set). The env var, when set, still overrides. + if (a.enable_jump_forward.has_value()) { + std::cerr << "server: jump-forward flag " + "(--[enable|disable]-jump-forward) specified more than " + "once\n"; + Usage(argv[0], 2); + } + a.enable_jump_forward = flag == "--enable-jump-forward"; + } else if (flag == "--tool-call-parser") { + a.tool_call_parser = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--reasoning-parser") { + a.reasoning_parser = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--kv-transfer-config") { + a.kv_transfer_config = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--speculative-config") { + a.speculative_config = NextArg(argc, argv, i, argv[0]); + } else if (flag == "-h" || flag == "--help") { + Usage(argv[0], 0); + } else { + std::cerr << "server: unknown argument '" << flag << "'\n"; + Usage(argv[0], 2); + } + } + if (a.model_dir.empty()) { + std::cerr << "server: --model is required\n"; + Usage(argv[0], 2); + } + if (a.max_num_seqs <= 0 || a.max_num_batched_tokens < 0 || + a.cuda_profile_graph_replays < 0 || a.cuda_profile_graph_batch < 0) { + std::cerr << "server: scheduler capacities must be positive " + "(--max-num-batched-tokens may be 0 for auto)\n"; + Usage(argv[0], 2); + } + if ((a.cuda_profile_graph_replays > 0) != + !a.benchmark_shutdown_fifo.empty()) { + std::cerr << "server: --cuda-profile-graph-replays and " + "--benchmark-shutdown-fifo must be specified together\n"; + Usage(argv[0], 2); + } + if (a.cuda_profile_graph_replays == 0 && a.cuda_profile_graph_batch != 0) { + std::cerr << "server: --cuda-profile-graph-batch requires " + "--cuda-profile-graph-replays\n"; + Usage(argv[0], 2); + } + if (a.cuda_profile_graph_replays > 0 && a.cuda_profile_graph_batch == 0) { + a.cuda_profile_graph_batch = 16; + } + if (a.cuda_profile_graph_batch > a.max_num_seqs) { + std::cerr << "server: --cuda-profile-graph-batch exceeds --max-num-seqs\n"; + Usage(argv[0], 2); + } + // Validate a NAMED parser dialect here, before the (multi-GB) model load, so a + // typo costs a second rather than a full load. "auto" cannot be checked yet — + // it resolves against the chat template — but detection only ever returns + // registered names, so it cannot fail later either. + namespace oai = vllm::entrypoints::openai; + if (a.tool_call_parser != "auto") { + (void)oai::ResolveToolParserName(a.tool_call_parser, ""); + } + if (a.reasoning_parser != "auto") { + (void)oai::ResolveReasoningParserName(a.reasoning_parser, ""); + } + return a; +} + +} // namespace + +namespace vllm { +namespace entrypoints { +namespace openai { + +int VllmServerMain(int argc, char** argv) { + try { + const Args args = ParseArgs(argc, argv); + if (args.verbose) { + setenv("VT_SERVER_VERBOSE", "1", /*overwrite=*/1); + std::cerr << "server: verbose stage logging enabled (debug_stages)\n"; + } + { + vllm::entrypoints::openai::RequestLogConfig log_cfg; + log_cfg.enable_log_requests = args.enable_log_requests; + log_cfg.enable_log_outputs = args.enable_log_outputs || args.verbose; + log_cfg.max_log_len = args.max_log_len; + log_cfg.debug_stages = args.verbose || + (std::getenv("VT_SERVER_VERBOSE") && + std::getenv("VT_SERVER_VERBOSE")[0] == '1'); + vllm::entrypoints::openai::ConfigureRequestLogger(log_cfg); + std::cerr << "server: request logging " + << (log_cfg.enable_log_requests ? "ON" : "OFF") + << " outputs=" << (log_cfg.enable_log_outputs ? "ON" : "OFF") + << " max_log_len=" << log_cfg.max_log_len + << " debug_stages=" << (log_cfg.debug_stages ? "ON" : "OFF") + << "\n"; + } + + const fs::path dir(args.model_dir); + const std::string config_path = (dir / "config.json").string(); + const std::string tokenizer_path = (dir / "tokenizer.json").string(); + const std::string tokenizer_config_path = + args.tokenizer_config.empty() + ? (dir / "tokenizer_config.json").string() + : args.tokenizer_config; + const std::string served_model_name = + args.served_model_name.empty() + ? (dir.has_filename() ? dir.filename().string() + : dir.parent_path().filename().string()) + : args.served_model_name; + + // ── TASK DISPATCH (ARCH-ONE-SURFACE ROW 1): a model dir whose + // architectures resolve to a SupportsTranscription-ONLY registration + // (Parakeet CTC/RNNT/TDT) serves /v1/audio/transcriptions through the ONE + // library seam — the same ParakeetTranscriber vllm_transcribe drives — and + // registers NO generate routes (vLLM's task-conditional registration, + // api_server.py:255-265). Every other model takes the text path below, + // byte-identical to before. ──────────────────────────────────────────────── + { + bool transcription_only = false; + const std::vector archs = + vllm::PeekHfArchitectures(config_path); + if (!archs.empty()) { + try { + transcription_only = + vllm::ModelRegistry::Resolve(std::span(archs)) + .info.supports_transcription_only; + } catch (const std::exception&) { + transcription_only = false; // unknown arch: the text path diagnoses + } + } + // ── POOLING TASK DISPATCH (ARCH-ONE-SURFACE ROW 6): a model dir whose + // architectures resolve to a POOLING registration (is_pooling_model, + // e.g. "LlamaModel" — vLLM _EMBEDDING_MODELS registry.py:230) serves + // /v1/embeddings through the ONE engine path (LoadedEngine -> + // LLMEngine::embed -> registry forward -> PoolingRunner) — the same + // path vllm_embed drives — and registers NO generate routes (vLLM's + // task-conditional registration, api_server.py:255-265). ────────────── + bool pooling_model = false; + if (!archs.empty()) { + try { + pooling_model = + vllm::ModelRegistry::Resolve(std::span(archs)) + .info.is_pooling_model; + } catch (const std::exception&) { + pooling_model = false; // unknown arch: the text path diagnoses + } + } + if (pooling_model) { + std::cerr << "server: pooling (embedding) model (" << archs[0] + << "); serving /v1/embeddings\n"; + vllm::entrypoints::EngineParams embed_params; + embed_params.block_size = args.block_size; + embed_params.num_blocks = args.num_blocks; + embed_params.gpu_memory_utilization = args.gpu_memory_utilization; + embed_params.kv_cache_memory_bytes = args.kv_cache_memory_bytes; + embed_params.max_model_len = args.max_model_len; + embed_params.max_num_seqs = args.max_num_seqs; + embed_params.max_num_batched_tokens = args.max_num_batched_tokens; + embed_params.enable_prefix_caching = args.enable_prefix_caching; + auto loaded_embed = std::shared_ptr( + vllm::entrypoints::LoadedEngine::FromModelDir(args.model_dir, + embed_params)); + namespace oai = vllm::entrypoints::openai; + oai::OpenAIServingModels embed_models(served_model_name); + oai::ApiServer embed_server(embed_models, vllm::Version()); + auto embed_mutex = std::make_shared(); + auto embed_counter = std::make_shared>(0); + embed_server.set_embedder( + [loaded_embed, embed_mutex, embed_counter]( + const std::vector& inputs) { + // Serialize batches: the pooling path drives the SYNCHRONOUS + // LLMEngine (async scheduling resolves OFF for pooling models). + std::lock_guard lock(*embed_mutex); + oai::ApiServer::EmbeddingBatch batch; + for (const std::string& text : inputs) { + std::vector ids = + loaded_embed->tokenizer().EncodeWithSpecialTokens(text); + if (ids.empty()) { + throw std::runtime_error( + "input tokenized to an empty prompt"); + } + batch.prompt_tokens += static_cast(ids.size()); + vllm::RequestOutput ro = loaded_embed->engine().embed( + std::move(ids), vllm::PoolingParams{}, + "embd-" + std::to_string(embed_counter->fetch_add(1))); + if (!ro.finished || !ro.pooling_output.has_value()) { + throw std::runtime_error( + "engine produced no pooled output"); + } + batch.embeddings.push_back(std::move(*ro.pooling_output)); + } + return batch; + }); + std::cerr << "server: listening on http://" << args.host << ":" + << args.port << "\n"; + if (!embed_server.listen(args.host, args.port)) { + std::cerr << "server: failed to bind " << args.host << ":" + << args.port << "\n"; + return 1; + } + return 0; + } + + if (transcription_only) { + std::cerr << "server: transcription-only model (" << archs[0] + << "); serving /v1/audio/transcriptions\n"; + auto transcriber = + std::make_shared( + vllm::multimodal::ParakeetTranscriber::FromDir(args.model_dir)); + namespace oai = vllm::entrypoints::openai; + oai::OpenAIServingModels asr_models(served_model_name); + oai::ApiServer asr_server(asr_models, vllm::Version()); + asr_server.set_transcriber( + [transcriber](const uint8_t* wav, size_t n) { + return transcriber->TranscribeWavBytes(wav, n); + }); + std::cerr << "server: listening on http://" << args.host << ":" + << args.port << "\n"; + if (!asr_server.listen(args.host, args.port)) { + std::cerr << "server: failed to bind " << args.host << ":" + << args.port << "\n"; + return 1; + } + return 0; + } + } + + // ── Load the model + build the full engine stack via the shared loader + // (src/vllm/entrypoints/model_loader.cpp) — the same path the C ABI drives. + // It loads config.json + tokenizer.json + *.safetensors and wires the M1.8 + // LLMEngine over Scheduler + runner + KV + processors. ───────────────────── + std::cerr << "server: loading model from " << args.model_dir << " (config " + << config_path << ", tokenizer " << tokenizer_path << ")\n"; + vllm::entrypoints::EngineParams engine_params; + engine_params.block_size = args.block_size; + engine_params.num_blocks = args.num_blocks; + engine_params.gpu_memory_utilization = args.gpu_memory_utilization; + engine_params.kv_cache_memory_bytes = args.kv_cache_memory_bytes; + engine_params.max_model_len = args.max_model_len; // 0 => from config. + engine_params.max_num_seqs = args.max_num_seqs; + engine_params.max_num_batched_tokens = args.max_num_batched_tokens; + engine_params.enable_prefix_caching = args.enable_prefix_caching; + // --device: explicit device selection (ARCH-ONE-SURFACE ROW 8). "auto" + // (default) keeps the accelerator-first probe byte-identical; an unknown + // name throws HERE (a startup error), and an explicitly named ABSENT + // device fails FromModelDir loudly — never a silent fallback + // (vllm/config/device.py:61-66). + engine_params.device = vllm::DeviceFromString(args.device); + // Reject an unknown policy string (mirrors upstream SchedulingPolicy(value)). + engine_params.policy = vllm::SchedulerPolicyFromString(args.scheduling_policy); + // ENG-SGLANG-BEHAVIOR-FLAG (SW1): `lpm` needs prefix caching to have any + // cache to match against; with APC explicitly off it degrades to fcfs + // (the scheduler leaves arrival order intact). Warn once at load so the + // no-op is visible (mirrors the spec's lpm+cache-off resolution). + if (engine_params.policy == vllm::SchedulerPolicy::kLPM && + args.enable_prefix_caching.has_value() && + !args.enable_prefix_caching.value()) { + std::cerr << "server: --scheduling-policy lpm has no effect with prefix " + "caching disabled; falling back to fcfs admission order\n"; + } + // ENG-SGLANG-BEHAVIOR-FLAG SW3: jump-forward decoding. Unset => the default + // (env-resolved, OFF); --[enable|disable]-jump-forward forces it, and + // VT_ENABLE_JUMP_FORWARD still overrides at resolution time. + engine_params.enable_jump_forward = args.enable_jump_forward; + // --kv-transfer-config: the external KV connector, mirroring vLLM's own + // flag and JSON shape. Absent (default) leaves the optional unset, which is + // the inert no-connector path the server has always run. A malformed + // document, an unknown key/role, or a connector whose worker half cannot + // move bytes on this device (the D1 guard, inside LoadedEngine) all throw + // out of here and are reported at startup by the catch in main. + if (!args.kv_transfer_config.empty()) { + vllm::KVTransferConfig kv_cfg = + vllm::ParseKVTransferConfigJson(args.kv_transfer_config); + if (kv_cfg.kv_connector.has_value() && + !vllm::v1::kv_offload::KVConnectorFactory::IsRegistered( + *kv_cfg.kv_connector)) { + std::string msg = "unknown kv_connector \"" + *kv_cfg.kv_connector + + "\" (registered connectors: "; + const std::vector names = + vllm::v1::kv_offload::KVConnectorFactory::RegisteredNames(); + for (size_t n = 0; n < names.size(); ++n) { + if (n != 0) msg += ", "; + msg += names[n]; + } + msg += ")"; + throw std::invalid_argument(msg); + } + engine_params.kv_transfer_config = std::move(kv_cfg); + } + // --speculative-config: speculative decoding (SPEC-MTP I5d). Absent (default) + // leaves the optional unset — the byte-identical no-speculation path. The + // parse validates method/k here; n_predict + the resolved k are finalized in + // LoadedEngine once the checkpoint's mtp_num_hidden_layers is known. A + // malformed document or unsupported method throws and is reported at startup. + if (!args.speculative_config.empty()) { + engine_params.speculative_config = + vllm::ParseSpeculativeConfigJson(args.speculative_config); + } + std::unique_ptr loaded = + vllm::entrypoints::LoadedEngine::FromModelDir(args.model_dir, + engine_params); + std::cerr << "server: prefix caching " + << (loaded->prefix_caching_enabled() ? "enabled" : "disabled") + << "\n"; + // W2: the production server uses AsyncLLM over EngineCoreProc's dedicated + // engine thread. HTTP workers submit independently and stream from their + // per-request collectors; no server-wide engine mutex remains. + vllm::v1::AsyncLLM& engine = loaded->async_engine(); + const vllm::tok::Tokenizer& tokenizer = loaded->tokenizer(); + + if (args.cuda_profile_graph_replays > 0) { +#ifdef VT_BENCH_PROFILE_CONTROL + vt::cuda::ConfigureCudaGraphReplayProfiler( + static_cast(args.cuda_profile_graph_replays), + static_cast(args.cuda_profile_graph_batch)); + std::cerr << "[VT_CUDA_PROFILE] ready pid=" << getpid() + << " signal=SIGUSR2 target_replays=" + << args.cuda_profile_graph_replays << "\n"; +#else + throw std::invalid_argument( + "--cuda-profile-graph-replays requires " + "VLLM_CPP_BENCH_PROFILE_CONTROL=ON"); +#endif + } + + // ── OpenAI serving handlers. The chat handler is wired with the real chat + // template (Task 3) when tokenizer_config.json carries one; otherwise it + // keeps the default role-join fallback. ──────────────────────────────── + namespace oai = vllm::entrypoints::openai; + oai::OpenAIServingModels models(served_model_name); + oai::OpenAIServingCompletion completion( + engine, served_model_name, args.enable_force_include_usage); + + oai::ChatPromptFn chat_prompt_fn = oai::DefaultChatPromptFallback; + // Kept outside the try so the parser resolution below can sniff it when + // --tool-call-parser/--reasoning-parser are "auto"; empty when the model + // ships no template (auto then falls back to hermes / disabled). + std::string chat_template; + try { + chat_template = + vllm::entrypoints::LoadChatTemplateFromConfig(tokenizer_config_path); + const std::string bos = + tokenizer.BosId() >= 0 ? tokenizer.Decode({tokenizer.BosId()}) : ""; + const std::string eos = + tokenizer.EosId() >= 0 ? tokenizer.Decode({tokenizer.EosId()}) : ""; + chat_prompt_fn = + vllm::entrypoints::MakeChatTemplatePromptFn( + chat_template, bos, eos, args.enable_thinking); + std::cerr << "server: using chat template (" << chat_template.size() + << " chars) from " << tokenizer_config_path + << " or sibling chat_template.jinja" + << " enable_thinking=" + << (args.enable_thinking ? "true" : "false") << "\n"; + } catch (const std::exception& e) { + std::cerr << "server: no chat template (" << e.what() + << "); falling back to the simple role-join prompt\n"; + } + // Dialect selection. Defaults reproduce the previously hardcoded pair + // ("hermes", "") exactly; an unknown name throws std::invalid_argument + // listing every registered parser and aborts startup, rather than leaving + // tool/reasoning parsing silently off for the life of the process. + const std::string tool_parser_name = + oai::ResolveToolParserName(args.tool_call_parser, chat_template); + const std::string reasoning_parser_name = + oai::ResolveReasoningParserName(args.reasoning_parser, chat_template); + std::cerr << "server: tool-call parser " + << (tool_parser_name.empty() ? "disabled" : tool_parser_name) + << ", reasoning parser " + << (reasoning_parser_name.empty() ? "disabled" + : reasoning_parser_name) + << "\n"; + oai::OpenAIServingChat chat(engine, served_model_name, chat_prompt_fn, + tool_parser_name, reasoning_parser_name, + args.enable_force_include_usage); + + // SAMPLE-BEAM (C7): enable use_beam_search on the production AsyncLLM path. + // Both handlers need the tokenizer (prompt tok + per-beam detok) and the eos + // id (beam retirement); a use_beam_search request then routes through + // BeamSearchAsync (online.py) over the async engine. Without this, beam + // requests reject with "requires an engine and a tokenizer". + const std::optional beam_eos = + tokenizer.EosId() >= 0 + ? std::optional(tokenizer.EosId()) + : std::nullopt; + completion.set_beam_search_tokenizer(&tokenizer, beam_eos); + chat.set_beam_search_tokenizer(&tokenizer, beam_eos); + + // ── MM-SERVE-E2E: wire the multimodal chat seam for image-capable models. + // When the model dir carries a preprocessor_config.json the Qwen3-VL image + // processor loads, we construct the seam body (MakeQwen3VLImageChatFn) so an + // OpenAI image_url request renders the placeholder marker → tokenizes to the + // single image_pad id → EXPANDS to N image tokens + mm_features carried onto + // the engine request. A text-only model (no preprocessor_config.json) leaves + // the seam UNSET → the chat path is byte-identical. The container-format + // image codec (PNG/JPEG → RGB) is a NAMED residual: no codec is vendored, so + // the production codec rejects encoded images with a clear message (the M2c + // single-sequence gate consumes pre-decoded raw RGB). The mm FORWARD (vision + // tower + merge + MRoPE/DeepStack on the GPU worker consuming + // Request.mm_features) is the remaining MM-SERVE-E2E residual — the engine + // model runner has no mm-forward path yet. Kept alive for the server loop. + std::unique_ptr mm_image_proc; + const std::string preprocessor_config_path = + (dir / "preprocessor_config.json").string(); + if (fs::exists(preprocessor_config_path)) { + try { + vllm::multimodal::Qwen3VLProcessorConfig pcfg = + vllm::multimodal::LoadQwen3VLProcessorConfig( + preprocessor_config_path, config_path, served_model_name); + mm_image_proc = + std::make_unique(pcfg); + oai::ImageCodecFn codec = + [](const oai::DecodedMedia& media) -> oai::DecodedImageRgb { + // Raw-RGB passthrough (image/x-raw-rgb): the single-sequence e2e / + // gate fixture format. A square raw-RGB payload is decoded directly; + // any container format (PNG/JPEG) is the NAMED codec residual. + if (media.media_type == "image/x-raw-rgb") { + const std::size_t n = media.bytes.size(); + const std::size_t px = n / 3; + const auto side = + static_cast(std::llround(std::sqrt( + static_cast(px)))); + if (side <= 0 || static_cast(side * side * 3) != n) { + throw std::runtime_error( + "image/x-raw-rgb payload is not a square HxWx3 buffer"); + } + oai::DecodedImageRgb out; + out.rgb = media.bytes; + out.height = side; + out.width = side; + return out; + } + throw std::runtime_error( + "multimodal image: container-format decode (PNG/JPEG -> RGB) is a " + "named MM-SERVE residual; supply raw RGB (image/x-raw-rgb)"); + }; + chat.set_multimodal_chat_fn(oai::MakeQwen3VLImageChatFn( + *mm_image_proc, tokenizer, chat_prompt_fn, std::move(codec))); + std::cerr << "server: multimodal image seam wired (Qwen3-VL processor " + "from " + << preprocessor_config_path << ")\n"; + } catch (const std::exception& e) { + std::cerr << "server: no multimodal image seam (" << e.what() + << "); image requests fall back to the text path\n"; + } + } + + // Diagnostic opt-out exists only for same-binary attribution. Production + // defaults to the capacity-derived fixed pool. + const char* fixed_pool_env = std::getenv("VLLM_CPP_HTTP_FIXED_POOL"); + const auto worker_pool_mode = + fixed_pool_env != nullptr && std::string(fixed_pool_env) == "0" + ? oai::ApiServer::HttpWorkerPoolMode::kLegacyDynamic + : oai::ApiServer::HttpWorkerPoolMode::kCapacityFixed; + oai::ApiServer server(completion, chat, models, vllm::Version(), + static_cast(args.max_num_seqs), + worker_pool_mode); + + // ── C8 opt-in utility/admin endpoints (SERVE-UTILITY-ENDPOINTS / + // SERVE-ADMIN). Wire the setters from the LIVE engine + tokenizer through the + // single shared seam so the production server actually serves /tokenize, + // /detokenize, /tokenizer_info (flag) and /abort_requests (dev-mode flag), + // mirroring vLLM 0.26's per-endpoint default gating. /metrics and + // /reset_prefix_cache stay unwired (no live backing on the AsyncLLM path) — + // see ConfigureUtilityEndpoints + specs/{utility,admin}-endpoints.md. ──────── + // ── MiniMax-H3 video generation. OPT-IN: with no --video-dit the routes are + // never registered and the server is byte-identical to before. The whole + // pipeline lives in the LIBRARY seam (vllm::multimodal::MiniMaxH3VideoEngine, + // ARCH-ONE-SURFACE ROW 2) — the SAME entry point the C ABI's vllm_video_* + // and the minimax-h3-gen example drive, so HTTP and FFI cannot drift. This + // file keeps exactly what an example may own: flag plumbing, the job + // directory, and the ONE process spawn (ffmpeg, ratified 2026-08-03 — the + // library builds the argv and spawns nothing). ──────────────────────────── + std::shared_ptr video_engine; + if (!args.video_dit.empty()) { + std::cerr << "server: loading MiniMax-H3 video checkpoints...\n"; + vllm::multimodal::MiniMaxH3VideoModelParams vmp; + vmp.dit_path = args.video_dit; + vmp.encoder_path = args.video_encoder; + vmp.tokenizer_path = args.video_tokenizer; + vmp.video_vae_path = args.video_vae; + vmp.video_vae_config_path = args.video_vae_config; + vmp.audio_vae_path = args.audio_vae; + vmp.audio_vae_config_path = args.audio_vae_config; + vmp.prompt_embeds_path = args.video_prompt_embeds; + vmp.partition = args.video_partition; + vmp.device = args.video_device == "cuda" ? 1 : 0; + vmp.dequant_bf16 = args.video_dequant_bf16 ? 1 : 0; + vmp.encoder_max_layers = args.video_encoder_max_layers; + video_engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(vmp); + std::cerr << "server: /v1/videos on (device=" << args.video_device + << (args.video_dequant_bf16 ? ", dequant-bf16" : ", keep-quant") << ")\n"; + // HONEST LIMIT, stated at startup rather than buried: turning a PROMPT + // into conditioning needs the H3-Encoder; without one every request is + // conditioned on the SAME supplied embeddings. + if (video_engine->has_encoder()) { + std::cerr << "server: /v1/videos conditions on the request PROMPT\n"; + } else if (!video_engine->has_prompt_embeds()) { + std::cerr << "server: WARNING /v1/videos has neither --video-encoder nor " + "--video-prompt-embeds; requests will be REJECTED\n"; + } else { + std::cerr << "server: WARNING /v1/videos ignores the request PROMPT — pass " + "--video-encoder to condition on it\n"; + } + + auto counter = std::make_shared>(0); + const std::string workdir = args.video_workdir; + const std::string ffmpeg = args.video_ffmpeg; + server.set_video_runner([video_engine, counter, workdir, + ffmpeg](const vllm::openai::VideoRequest& req) -> std::string { + const int64_t id = counter->fetch_add(1); + const std::string dir = workdir + "/job" + std::to_string(id); + // The library-owned request mapping + generation: conditioning, task + // resolution, the #77 partition guard, reference encoding, artifacts. + const vllm::multimodal::MiniMaxH3VideoResult out = video_engine->Generate( + vllm::multimodal::MiniMaxH3VideoGenParamsFromRequest(req, dir)); + // The ONE process spawn: exec the argv the library composed. + std::vector argv_mux = out.mux_argv; + if (!argv_mux.empty()) argv_mux[0] = ffmpeg; + const int status = RunFfmpegArgv(argv_mux); + if (status != 0) { + throw std::runtime_error("ffmpeg exited " + std::to_string(status)); + } + return out.mux_output_path; + }); + } + + oai::UtilityEndpointOptions endpoint_opts; + endpoint_opts.enable_tokenizer_info_endpoint = + args.enable_tokenizer_info_endpoint; + endpoint_opts.enable_server_dev_mode = args.enable_server_dev_mode; + oai::ConfigureUtilityEndpoints(server, tokenizer, loaded->max_model_len(), + engine, endpoint_opts); + std::cerr << "server: utility endpoints: /tokenize /detokenize on" + << (args.enable_tokenizer_info_endpoint ? ", /tokenizer_info on" + : "") + << (args.enable_server_dev_mode ? ", /abort_requests on (dev-mode)" + : "") + << "\n"; + + // Prometheus /metrics (Python vLLM always-on family names). + std::unique_ptr prom_logger; + if (args.enable_metrics) { + prom_logger = std::make_unique( + served_model_name, loaded->max_model_len(), /*engine_index=*/0); + // Sync engine path records on step(); async may under-report until fully wired. + loaded->engine().set_stat_logger(prom_logger.get()); + server.set_metrics_logger(prom_logger.get()); + std::cerr << "server: GET /metrics enabled (PrometheusStatLogger)\n"; + } + + std::cerr << "server: listening on http://" << args.host << ":" << args.port + << " (model '" << served_model_name << "', HTTP worker pool "; + if (server.http_worker_count() == 0) { + std::cerr << "legacy-dynamic"; + } else { + std::cerr << server.http_worker_count() << " fixed"; + } + std::cerr << ")\n"; + +#ifdef VT_BENCH_PROFILE_CONTROL + std::atomic benchmark_shutdown_waiter_ready{false}; + std::atomic benchmark_shutdown_received{false}; + std::atomic benchmark_shutdown_failed{false}; + std::atomic benchmark_shutdown_cancelled{false}; + std::thread benchmark_shutdown_thread; + if (args.cuda_profile_graph_replays > 0) { + benchmark_shutdown_thread = std::thread([&]() { + const int shutdown_fd = + open(args.benchmark_shutdown_fifo.c_str(), + O_RDONLY | O_NONBLOCK | O_CLOEXEC | O_NOFOLLOW); + if (shutdown_fd < 0) { + const int status = errno; + std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=open status=" + << status << "\n"; + benchmark_shutdown_failed.store(true, std::memory_order_release); + return; + } + struct stat shutdown_stat {}; + const int stat_status = fstat(shutdown_fd, &shutdown_stat); + if (stat_status != 0 || !S_ISFIFO(shutdown_stat.st_mode)) { + const int status = stat_status != 0 ? errno : EINVAL; + std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=fstat status=" + << status << "\n"; + close(shutdown_fd); + benchmark_shutdown_failed.store(true, std::memory_order_release); + return; + } + benchmark_shutdown_waiter_ready.store(true, std::memory_order_release); + std::cerr << "[VT_BENCH_SHUTDOWN] ready pid=" << getpid() + << " control=fifo\n"; + while (!benchmark_shutdown_cancelled.load(std::memory_order_acquire)) { + char command = '\0'; + const ssize_t bytes = read(shutdown_fd, &command, 1); + if (bytes == 1) { + if (command == 'Q') { + benchmark_shutdown_received.store(true, + std::memory_order_release); + std::cerr + << "[VT_BENCH_SHUTDOWN] requested control=fifo\n"; + close(shutdown_fd); + server.stop(); + return; + } + std::cerr + << "[VT_BENCH_SHUTDOWN] failed operation=command status=" + << static_cast( + static_cast(command)) + << "\n"; + close(shutdown_fd); + benchmark_shutdown_failed.store(true, std::memory_order_release); + server.stop(); + return; + } + if (bytes < 0 && errno != EAGAIN && errno != EWOULDBLOCK && + errno != EINTR) { + const int status = errno; + std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=read status=" + << status << "\n"; + close(shutdown_fd); + benchmark_shutdown_failed.store(true, std::memory_order_release); + server.stop(); + return; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + close(shutdown_fd); + }); + while (!benchmark_shutdown_waiter_ready.load(std::memory_order_acquire) && + !benchmark_shutdown_failed.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + if (benchmark_shutdown_failed.load(std::memory_order_acquire)) { + benchmark_shutdown_cancelled.store(true, std::memory_order_release); + benchmark_shutdown_thread.join(); + return 1; + } + } +#endif + + const bool listen_ok = server.listen(args.host, args.port); + +#ifdef VT_BENCH_PROFILE_CONTROL + if (benchmark_shutdown_thread.joinable()) { + benchmark_shutdown_cancelled.store(true, std::memory_order_release); + benchmark_shutdown_thread.join(); + if (benchmark_shutdown_received.load(std::memory_order_acquire)) { + std::cerr << "[VT_BENCH_SHUTDOWN] completed control=fifo\n"; + } else { + if (!benchmark_shutdown_failed.load(std::memory_order_acquire)) { + std::cerr + << "[VT_BENCH_SHUTDOWN] failed operation=cancelled status=0\n"; + } + if (listen_ok) { + return 1; + } + } + } +#endif + + if (!listen_ok) { + std::cerr << "server: failed to bind " << args.host << ":" << args.port + << "\n"; + return 1; + } + return 0; + } catch (const std::exception& e) { + std::cerr << "server: fatal: " << e.what() << "\n"; + return 1; + } +} + +} // namespace openai +} // namespace entrypoints +} // namespace vllm diff --git a/src/vllm/entrypoints/openai/server_main.h b/src/vllm/entrypoints/openai/server_main.h new file mode 100644 index 000000000..7cfe00bf6 --- /dev/null +++ b/src/vllm/entrypoints/openai/server_main.h @@ -0,0 +1,35 @@ +// vllm.cpp original. The OpenAI server's ENTRY POINT, owned by the library. +// +// ARCH-ONE-SURFACE: `examples/server` used to construct the engine, the serving +// layers, metrics, the video seam and the ASR seam directly from 36 internal +// headers -- the deepest breach of the ONE SURFACE directive +// (.agents/specs/one-surface-abi.md), and the reason the example carried an +// entry in scripts/example-abi-allowlist.txt. The construction moved HERE +// verbatim; the example is now a thin client of the C ABI's `vllm_server_main`, +// which wraps this. +// +// argv rather than a params STRUCT is deliberate. The server takes ~57 flags and +// grows more with every serving feature; a mirrored C struct would put that +// churn in the ABI, where every field is permanent. The flag surface is already +// specified by vLLM's cli_args.py, which is the contract this mirrors, so argv +// IS the stable interface. Embedders that want programmatic control keep the +// granular entry points (vllm_engine_load / vllm_chat / vllm_video_generate / +// vllm_transcribe) -- this one exists to RUN THE SERVER. +#ifndef VLLM_ENTRYPOINTS_OPENAI_SERVER_MAIN_H_ +#define VLLM_ENTRYPOINTS_OPENAI_SERVER_MAIN_H_ + +namespace vllm { +namespace entrypoints { +namespace openai { + +// Parse `argv` and run the OpenAI-compatible server until it exits. Returns the +// process exit code (0 on clean shutdown). Prints usage and returns 0 for +// `--help`, and returns non-zero after printing the reason on a bad argument or +// a startup failure -- it does NOT throw across the C ABI boundary. +int VllmServerMain(int argc, char** argv); + +} // namespace openai +} // namespace entrypoints +} // namespace vllm + +#endif // VLLM_ENTRYPOINTS_OPENAI_SERVER_MAIN_H_