diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e89e81d3..5fbbc096f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1811,6 +1811,10 @@ endif() if(VLLM_CPP_SERVER) find_package(Threads REQUIRED) target_sources(vllm PRIVATE src/vllm/entrypoints/openai/api_server.cpp) + # ARCH-ONE-SURFACE: the server ENTRY POINT (flag parsing + engine/serving + # construction) lives in the library so examples/server can be a thin client of + # the C ABI's vllm_server_main (v17). It shares api_server.cpp's httplib gate. + target_sources(vllm PRIVATE src/vllm/entrypoints/openai/server_main.cpp) target_compile_definitions(vllm PUBLIC VLLM_CPP_SERVER) target_link_libraries(vllm PUBLIC Threads::Threads) # third_party/httplib/httplib.h is reached as (third_party diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 62b8b0f77..92275e965 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -67,7 +67,11 @@ endif() # The OpenAI HTTP server example is gated on the vendored cpp-httplib transport. if(VLLM_CPP_SERVER) add_executable(server server/main.cpp) - target_link_libraries(server PRIVATE vllm::vllm) + # ARCH-ONE-SURFACE: server/main.cpp is a THIN CLIENT of the public C ABI + # (vllm_server_main, v17) -- it includes vllm.h and nothing else, so it links the + # PACKAGED shared library exactly as an out-of-tree consumer would, not the + # internal C++ target. + target_link_libraries(server PRIVATE vllm::shared) vllm_cpp_set_warnings(server) if(VLLM_CPP_BUILD_TESTS) add_test(NAME test_server_help COMMAND $ --help) diff --git a/examples/server/main.cpp b/examples/server/main.cpp index 4c9c26edc..61c0c5aac 100644 --- a/examples/server/main.cpp +++ b/examples/server/main.cpp @@ -1,1046 +1,23 @@ -// server: an OpenAI-compatible HTTP server over the vllm.cpp LLMEngine (M3.1 -// Task 4). Loads a supported model (safetensors or GGUF weights + tokenizer + a KV-cache config → -// LLMEngine), constructs the OpenAI serving handlers (chat wired with the real -// chat template via MakeChatTemplatePromptFn(LoadChatTemplateFromConfig(...))) -// and serves /v1/completions, /v1/chat/completions, /v1/models, /health, -// /version. +// vllm.cpp original. `vllm-server` — the OpenAI-compatible server, as a THIN +// CLIENT of the public C ABI. // -// server --model [--host 0.0.0.0] [--port 8000] -// [--tokenizer-config ] -// [--served-model-name ] -// [--block-size N] [--num-blocks N] [--max-model-len N] -// [--gpu-memory-utilization F] [--kv-cache-memory BYTES] -// [--max-num-seqs N] [--max-num-batched-tokens N] -// [--enable-force-include-usage] -// [--[no-]enable-prefix-caching] -// [--scheduling-policy fcfs|priority] -// [--tool-call-parser |auto|none] -// [--reasoning-parser |auto|none] -// [--kv-transfer-config ''] +// ARCH-ONE-SURFACE (.agents/specs/one-surface-abi.md, developer-directed +// 2026-08-07): every unit under examples/ is a thin client of the public surface +// — the flat C ABI `vllm.h`, the ONLY header `make install` ships — never of the +// internal C++ tree. This file used to be the DEEPEST breach of that rule: 1046 +// lines reaching into 36 internal headers to construct the engine, the serving +// layers, metrics, the video seam and the ASR seam by hand, which is why it +// carried an entry in scripts/example-abi-allowlist.txt. // -// A directory holds config.json, tokenizer.json and supported safetensors -// shards. A supported GGUF file is also accepted and supplies model metadata -// plus embedded vocabulary. If --tokenizer-config is omitted for a directory it -// defaults to /tokenizer_config.json; when that file has no chat_template -// the chat endpoint falls back to the simple role-join prompt. +// The construction moved into the library verbatim +// (src/vllm/entrypoints/openai/server_main.cpp, declared in +// vllm/entrypoints/openai/server_main.h) and is published as `vllm_server_main` +// at ABI v17. HTTP and FFI therefore cannot drift: the server the ABI runs IS +// the server this binary runs. // -// NOTE: loading the real 35B checkpoint is a GPU/dgx concern; on the CPU CI box -// this binary is only built + smoke-tested against a synthetic engine (see -// tests/vllm/entrypoints/openai/test_api_server.cpp). The wiring below is the -// same either way. -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef VT_BENCH_PROFILE_CONTROL -#include -#include -#include -#include -#include -#endif +// Flag parsing lives in the library with the construction it feeds, because the +// two are one contract (vLLM's cli_args.py). Mirroring ~57 flags into a C struct +// would have put that churn in the ABI, where every field is permanent. +#include "vllm.h" -#include "vllm/config/device.h" -#include "vllm/config/kv_transfer.h" -#include "vllm/config/scheduler.h" -#include "vllm/entrypoints/chat_template.h" -#include "vllm/entrypoints/model_loader.h" -#include -#include "vllm/entrypoints/openai/api_server.h" -#include "vllm/entrypoints/openai/chat_mm.h" -#include "vllm/entrypoints/openai/request_logger.h" -#include "vllm/entrypoints/openai/serving_chat.h" -#include "vllm/entrypoints/openai/serving_completion.h" -#include "vllm/entrypoints/openai/serving_models.h" -#include "vllm/v1/metrics/loggers.h" -#include "vllm/entrypoints/openai/reasoning_parsers/detect.h" -#include "vllm/entrypoints/openai/tool_parsers/detect.h" -#include "vllm/model_executor/model_loader/safetensors_reader.h" -#include "vllm/model_executor/models/qwen3_5_weights.h" -#include "vllm/transformers_utils/hf_config.h" -#include "vllm/model_executor/models/model_registry.h" -#include "vllm/multimodal/minimax_h3_video.h" -#include "vllm/multimodal/parakeet_transcription.h" -#include "vllm/tokenizer/tokenizer.h" -#include "vllm/version.h" -#include "vllm/v1/core/kv_cache_utils.h" -#include "vllm/v1/core/sched/scheduler.h" -#include "vllm/v1/engine/core.h" -#include "vllm/v1/engine/input_processor.h" -#include "vllm/v1/engine/llm_engine.h" -#include "vllm/v1/engine/output_processor.h" -#include "vllm/v1/executor/executor.h" -#include "vllm/v1/kv_cache_interface.h" -#include "vllm/v1/kv_offload/kv_connector.h" -#include "vllm/v1/worker/gpu/runner.h" -#include "vt/backend.h" -#ifdef VT_BENCH_PROFILE_CONTROL -#include "vt/cuda/cuda_profiler_control.h" -#endif -#include "vt/dtype.h" -#include "vt/tensor.h" - -namespace { - -namespace fs = std::filesystem; -using vllm::HfConfig; -using vllm::Qwen3_5MoeWeights; - -// Run an argv to completion and return its exit status — the ONE process -// spawn in the MiniMax-H3 path, and it lives HERE, in examples/, by the -// developer-ratified 2026-08-03 decision: the library (the -// MiniMaxH3VideoEngine seam behind /v1/videos) writes the artifacts and -// BUILDS this argv, and spawns nothing. -int RunFfmpegArgv(const std::vector& args) { - std::vector c_args; - c_args.reserve(args.size() + 1); - for (const std::string& arg : args) { - c_args.push_back(const_cast(arg.c_str())); - } - c_args.push_back(nullptr); - const pid_t pid = fork(); - if (pid < 0) throw std::runtime_error("fork failed"); - if (pid == 0) { - execvp(c_args[0], c_args.data()); - _exit(127); // exec failed; never run the parent's atexit handlers - } - int status = 0; - if (waitpid(pid, &status, 0) < 0) throw std::runtime_error("waitpid failed"); - if (WIFSIGNALED(status)) { - throw std::runtime_error("ffmpeg died on signal " + std::to_string(WTERMSIG(status))); - } - return WIFEXITED(status) ? WEXITSTATUS(status) : -1; -} - -struct Args { - std::string model_dir; - std::string host = "0.0.0.0"; - int port = 8000; - std::string tokenizer_config; // default: /tokenizer_config.json - std::string served_model_name; // default: the model dir name - int block_size = 32; - // --num-blocks is the KV block-count OVERRIDE (0 => auto: sized by the knobs - // below, else the 256-block fallback). ROAD-V1-MEM M1. - int num_blocks = 0; - // --gpu-memory-utilization: fraction of free device memory for the whole - // engine (needs the M3 profile run; inert until then). --kv-cache-memory: an - // absolute KV-pool size in bytes that sizes the block count directly (0 => - // unset). - double gpu_memory_utilization = 0.92; - long long kv_cache_memory_bytes = 0; - int max_model_len = 0; // 0 => config.max_position_embeddings - int max_num_seqs = 8; - int max_num_batched_tokens = 0; // 0 => per-architecture default. - // --device: explicit device selection for the TEXT engine (ARCH-ONE-SURFACE - // ROW 8), the vLLM DeviceConfig.device names this build serves: "auto" - // (default — the accelerator-first probe, byte-identical to before the flag), - // "cpu" (force the CPU queue), "cuda" (require CUDA; an absent device fails - // startup LOUDLY, never a silent fallback). The video engine keeps its own - // --video-device below: the two engines are loaded from different - // checkpoints and may legitimately serve on different devices. - std::string device = "auto"; - // --- MiniMax-H3 video generation (opt-in; absent => /v1/videos is unregistered - // and the server behaves exactly as before). --- - std::string video_dit, video_vae, video_vae_config, audio_vae, audio_vae_config; - std::string video_prompt_embeds, video_workdir = "/tmp/vllm_h3_videos"; - std::string video_encoder, video_tokenizer; - int video_encoder_max_layers = 50; - std::string video_ffmpeg = "ffmpeg", video_device = "cuda"; - std::string video_partition; // served partition (fl2va|ref2va); see the #77 guard - // Keep-quant is the library seam's DEFAULT arm; --video-dequant-bf16 selects - // the bf16 dequant/stream arm (the throughput trade the gen example ships). - // --video-keep-quant is still accepted (it names the default). - bool video_dequant_bf16 = false; - int cuda_profile_graph_replays = 0; // trace-only diagnostic build seam. - int cuda_profile_graph_batch = 0; // 0 => accepted c16 trace contract. - std::string benchmark_shutdown_fifo; // paired trace-only control path. - std::optional enable_prefix_caching = std::nullopt; - bool enable_force_include_usage = false; - // GET /tokenizer_info gate. Mirrors vLLM's --enable-tokenizer-info-endpoint - // (entrypoints/openai/cli_args.py:140, default False; the route is registered - // only when serve/tokenize/api_router.py:95 sees the flag). Default off → the - // route 404s, byte-identical to before. - bool enable_tokenizer_info_endpoint = false; - // Dev/admin endpoint gate. Mirrors vLLM's VLLM_SERVER_DEV_MODE env - // (envs.py:157, default 0): build_app registers the dev/rlhf + dev/cache - // routers only under `if envs.VLLM_SERVER_DEV_MODE` (api_server.py:238). Off by - // default → /abort_requests 404s. Enables the /abort_requests production wiring. - bool enable_server_dev_mode = false; - bool verbose = false; - // Gemma4 HF/vLLM: --default-chat-template-kwargs enable_thinking (default OFF). - bool enable_thinking = false; - // Request logging (Python vLLM --enable-log-requests parity). Default ON. - bool enable_log_requests = true; - bool enable_log_outputs = false; - int max_log_len = 256; - // Attach Prometheus logger + GET /metrics (default ON for solid Hermes serve). - bool enable_metrics = true; - // Scheduling policy: "fcfs" (default), "priority" (mirrors vLLM's - // --scheduling-policy / SchedulerConfig.policy), or "lpm" (SGLang's - // cache-aware longest-prefix-match admission ordering, ENG-SGLANG-BEHAVIOR-FLAG; - // opt-in, output-neutral, resolves to fcfs when prefix caching is off). - // --schedule-policy is accepted as an SGLang-compatible alias. - std::string scheduling_policy = "fcfs"; - // Jump-forward decoding (ENG-SGLANG-BEHAVIOR-FLAG SW3): tri-state, mirrors the - // C-ABI vllm_model_params.enable_jump_forward. Unset (default) => OFF unless - // VT_ENABLE_JUMP_FORWARD is set; --enable-jump-forward forces on, - // --disable-jump-forward forces off (the env var still overrides). The - // token-unique forced-run subset only; see .agents/specs/sglang-enablement.md. - std::optional enable_jump_forward = std::nullopt; - // Tool-call / reasoning dialect selection (mirrors vLLM's --tool-call-parser - // and --reasoning-parser). THE DEFAULTS ARE TODAY'S HARDCODED BEHAVIOUR: - // "hermes" is exactly what OpenAIServingChat was constructed with before this - // flag existed, and "none" is the empty reasoning-parser name it passed. An - // invocation that names neither flag is therefore unchanged, byte for byte. - // "auto" opts into the chat-template detection the C ABI uses. - std::string tool_call_parser = "hermes"; - std::string reasoning_parser = "none"; - // vLLM's --kv-transfer-config: the external KV connector selection, as the - // same JSON object vLLM takes. Empty (default) == no connector == the inert - // production path. See docs/KV-OFFLOAD.md. - std::string kv_transfer_config; - // vLLM's --speculative-config: the speculative-decoding selection, as the same - // JSON object vLLM takes (e.g. '{"method":"mtp","num_speculative_tokens":1}'). - // Empty (default) == no speculation == the inert production path (SPEC-MTP I5d). - std::string speculative_config; -}; - -[[noreturn]] void Usage(const char* argv0, int code) { - std::cerr - << "usage: " << argv0 - << " --model [--host H] [--port P] [--tokenizer-config F]\n" - " [--served-model-name N] [--block-size N] " - "[--num-blocks N] [--max-model-len N]\n" - " [--gpu-memory-utilization F] " - "[--kv-cache-memory BYTES]\n" - " [--max-num-seqs N] " - "[--max-num-batched-tokens N]\n" - " [--device auto|cpu|cuda]\n" - " [--cuda-profile-graph-replays N]\n" - " [--cuda-profile-graph-batch N]\n" - " [--benchmark-shutdown-fifo F]\n" - " [--enable-force-include-usage]\n" - " [--enable-tokenizer-info-endpoint]\n" - " [--enable-server-dev-mode]\n" - " [--verbose]\n" - " [--enable-thinking|--no-enable-thinking]\n" - " [--enable-log-requests|--disable-log-requests]\n" - " [--enable-log-outputs] [--max-log-len N]\n" - " [--enable-metrics|--disable-metrics]\n" - " [--[no-]enable-prefix-caching]\n" - " [--[no-]enable-radix-attention]\n" - " [--scheduling-policy fcfs|priority|lpm]\n" - " [--[enable|disable]-jump-forward]\n" - " [--tool-call-parser |auto|none]\n" - " [--reasoning-parser |auto|none]\n" - " [--kv-transfer-config '']\n" - " [--speculative-config '']\n"; - std::exit(code); -} - -std::string NextArg(int argc, char** argv, int& i, const char* argv0) { - if (i + 1 >= argc) Usage(argv0, 2); - return argv[++i]; -} - -Args ParseArgs(int argc, char** argv) { - Args a; - for (int i = 1; i < argc; ++i) { - const std::string flag = argv[i]; - if (flag == "--model") { - a.model_dir = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--host") { - a.host = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--port") { - a.port = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--tokenizer-config") { - a.tokenizer_config = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--served-model-name") { - a.served_model_name = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--block-size") { - a.block_size = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--num-blocks") { - a.num_blocks = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--gpu-memory-utilization") { - a.gpu_memory_utilization = std::stod(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--kv-cache-memory") { - a.kv_cache_memory_bytes = std::stoll(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--max-model-len") { - a.max_model_len = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--max-num-seqs") { - a.max_num_seqs = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--max-num-batched-tokens") { - a.max_num_batched_tokens = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--device") { - // Text-engine device selection (mirrors vLLM's DeviceConfig.device - // names). Validated by vllm::DeviceFromString at engine construction; - // --video-device (below) stays the video engine's separate knob. - a.device = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--cuda-profile-graph-replays") { - a.cuda_profile_graph_replays = - std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--cuda-profile-graph-batch") { - a.cuda_profile_graph_batch = - std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--benchmark-shutdown-fifo") { - a.benchmark_shutdown_fifo = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--enable-force-include-usage") { - a.enable_force_include_usage = true; - } else if (flag == "--enable-tokenizer-info-endpoint") { - a.enable_tokenizer_info_endpoint = true; - } else if (flag == "--video-dit") { - a.video_dit = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-vae") { - a.video_vae = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-vae-config") { - a.video_vae_config = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--audio-vae") { - a.audio_vae = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--audio-vae-config") { - a.audio_vae_config = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-encoder") { - a.video_encoder = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-tokenizer") { - a.video_tokenizer = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-encoder-max-layers") { - a.video_encoder_max_layers = std::atoi(NextArg(argc, argv, i, argv[0]).c_str()); - } else if (flag == "--video-prompt-embeds") { - a.video_prompt_embeds = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-workdir") { - a.video_workdir = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-ffmpeg") { - a.video_ffmpeg = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-device") { - a.video_device = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-partition") { - a.video_partition = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--video-keep-quant") { - // the seam's default arm; accepted for pre-fold CLI compatibility - a.video_dequant_bf16 = false; - } else if (flag == "--video-dequant-bf16") { - a.video_dequant_bf16 = true; - } else if (flag == "--enable-server-dev-mode") { - a.enable_server_dev_mode = true; - } else if (flag == "--verbose" || flag == "-v") { - a.verbose = true; - } else if (flag == "--enable-thinking") { - a.enable_thinking = true; - } else if (flag == "--no-enable-thinking") { - a.enable_thinking = false; - } else if (flag == "--enable-log-requests") { - a.enable_log_requests = true; - } else if (flag == "--disable-log-requests") { - a.enable_log_requests = false; - } else if (flag == "--enable-log-outputs") { - a.enable_log_outputs = true; - } else if (flag == "--max-log-len") { - a.max_log_len = std::stoi(NextArg(argc, argv, i, argv[0])); - } else if (flag == "--enable-metrics") { - a.enable_metrics = true; - } else if (flag == "--disable-metrics") { - a.enable_metrics = false; - } else if (flag == "--enable-prefix-caching" || - flag == "--no-enable-prefix-caching" || - flag == "--enable-radix-attention" || - flag == "--disable-radix-attention") { - // --[no-]enable-prefix-caching is vLLM's flag. --enable-radix-attention / - // --disable-radix-attention are SGLang-compatible ALIASES for the SAME - // toggle (RadixAttention is fused into our block-hash APC — there is no - // distinct radix code path; see .agents/specs/sglang-radixattention.md §1). - // They set the identical tri-state as the vLLM flag; last-wins is rejected - // (mirrors passing the vLLM flag twice) so a contradictory pair is caught. - if (a.enable_prefix_caching.has_value()) { - std::cerr << "server: prefix-caching flag (--[no-]enable-prefix-caching " - "/ --[disable|enable]-radix-attention) specified more than " - "once\n"; - Usage(argv[0], 2); - } - a.enable_prefix_caching = - flag == "--enable-prefix-caching" || flag == "--enable-radix-attention"; - } else if (flag == "--scheduling-policy" || flag == "--schedule-policy") { - // --scheduling-policy is vLLM's flag; --schedule-policy is SGLang's name, - // accepted as an alias. Both take fcfs|priority|lpm. - a.scheduling_policy = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--enable-jump-forward" || - flag == "--disable-jump-forward") { - // ENG-SGLANG-BEHAVIOR-FLAG SW3: opt into (or force off) jump-forward - // decoding — the token-unique grammar-speed subset (see - // .agents/specs/sglang-enablement.md). Absent => the default (OFF unless - // VT_ENABLE_JUMP_FORWARD is set). The env var, when set, still overrides. - if (a.enable_jump_forward.has_value()) { - std::cerr << "server: jump-forward flag " - "(--[enable|disable]-jump-forward) specified more than " - "once\n"; - Usage(argv[0], 2); - } - a.enable_jump_forward = flag == "--enable-jump-forward"; - } else if (flag == "--tool-call-parser") { - a.tool_call_parser = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--reasoning-parser") { - a.reasoning_parser = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--kv-transfer-config") { - a.kv_transfer_config = NextArg(argc, argv, i, argv[0]); - } else if (flag == "--speculative-config") { - a.speculative_config = NextArg(argc, argv, i, argv[0]); - } else if (flag == "-h" || flag == "--help") { - Usage(argv[0], 0); - } else { - std::cerr << "server: unknown argument '" << flag << "'\n"; - Usage(argv[0], 2); - } - } - if (a.model_dir.empty()) { - std::cerr << "server: --model is required\n"; - Usage(argv[0], 2); - } - if (a.max_num_seqs <= 0 || a.max_num_batched_tokens < 0 || - a.cuda_profile_graph_replays < 0 || a.cuda_profile_graph_batch < 0) { - std::cerr << "server: scheduler capacities must be positive " - "(--max-num-batched-tokens may be 0 for auto)\n"; - Usage(argv[0], 2); - } - if ((a.cuda_profile_graph_replays > 0) != - !a.benchmark_shutdown_fifo.empty()) { - std::cerr << "server: --cuda-profile-graph-replays and " - "--benchmark-shutdown-fifo must be specified together\n"; - Usage(argv[0], 2); - } - if (a.cuda_profile_graph_replays == 0 && a.cuda_profile_graph_batch != 0) { - std::cerr << "server: --cuda-profile-graph-batch requires " - "--cuda-profile-graph-replays\n"; - Usage(argv[0], 2); - } - if (a.cuda_profile_graph_replays > 0 && a.cuda_profile_graph_batch == 0) { - a.cuda_profile_graph_batch = 16; - } - if (a.cuda_profile_graph_batch > a.max_num_seqs) { - std::cerr << "server: --cuda-profile-graph-batch exceeds --max-num-seqs\n"; - Usage(argv[0], 2); - } - // Validate a NAMED parser dialect here, before the (multi-GB) model load, so a - // typo costs a second rather than a full load. "auto" cannot be checked yet — - // it resolves against the chat template — but detection only ever returns - // registered names, so it cannot fail later either. - namespace oai = vllm::entrypoints::openai; - if (a.tool_call_parser != "auto") { - (void)oai::ResolveToolParserName(a.tool_call_parser, ""); - } - if (a.reasoning_parser != "auto") { - (void)oai::ResolveReasoningParserName(a.reasoning_parser, ""); - } - return a; -} - -} // namespace - -int main(int argc, char** argv) { - try { - const Args args = ParseArgs(argc, argv); - if (args.verbose) { - setenv("VT_SERVER_VERBOSE", "1", /*overwrite=*/1); - std::cerr << "server: verbose stage logging enabled (debug_stages)\n"; - } - { - vllm::entrypoints::openai::RequestLogConfig log_cfg; - log_cfg.enable_log_requests = args.enable_log_requests; - log_cfg.enable_log_outputs = args.enable_log_outputs || args.verbose; - log_cfg.max_log_len = args.max_log_len; - log_cfg.debug_stages = args.verbose || - (std::getenv("VT_SERVER_VERBOSE") && - std::getenv("VT_SERVER_VERBOSE")[0] == '1'); - vllm::entrypoints::openai::ConfigureRequestLogger(log_cfg); - std::cerr << "server: request logging " - << (log_cfg.enable_log_requests ? "ON" : "OFF") - << " outputs=" << (log_cfg.enable_log_outputs ? "ON" : "OFF") - << " max_log_len=" << log_cfg.max_log_len - << " debug_stages=" << (log_cfg.debug_stages ? "ON" : "OFF") - << "\n"; - } - - const fs::path dir(args.model_dir); - const std::string config_path = (dir / "config.json").string(); - const std::string tokenizer_path = (dir / "tokenizer.json").string(); - const std::string tokenizer_config_path = - args.tokenizer_config.empty() - ? (dir / "tokenizer_config.json").string() - : args.tokenizer_config; - const std::string served_model_name = - args.served_model_name.empty() - ? (dir.has_filename() ? dir.filename().string() - : dir.parent_path().filename().string()) - : args.served_model_name; - - // ── TASK DISPATCH (ARCH-ONE-SURFACE ROW 1): a model dir whose - // architectures resolve to a SupportsTranscription-ONLY registration - // (Parakeet CTC/RNNT/TDT) serves /v1/audio/transcriptions through the ONE - // library seam — the same ParakeetTranscriber vllm_transcribe drives — and - // registers NO generate routes (vLLM's task-conditional registration, - // api_server.py:255-265). Every other model takes the text path below, - // byte-identical to before. ──────────────────────────────────────────────── - { - bool transcription_only = false; - const std::vector archs = - vllm::PeekHfArchitectures(config_path); - if (!archs.empty()) { - try { - transcription_only = - vllm::ModelRegistry::Resolve(std::span(archs)) - .info.supports_transcription_only; - } catch (const std::exception&) { - transcription_only = false; // unknown arch: the text path diagnoses - } - } - // ── POOLING TASK DISPATCH (ARCH-ONE-SURFACE ROW 6): a model dir whose - // architectures resolve to a POOLING registration (is_pooling_model, - // e.g. "LlamaModel" — vLLM _EMBEDDING_MODELS registry.py:230) serves - // /v1/embeddings through the ONE engine path (LoadedEngine -> - // LLMEngine::embed -> registry forward -> PoolingRunner) — the same - // path vllm_embed drives — and registers NO generate routes (vLLM's - // task-conditional registration, api_server.py:255-265). ────────────── - bool pooling_model = false; - if (!archs.empty()) { - try { - pooling_model = - vllm::ModelRegistry::Resolve(std::span(archs)) - .info.is_pooling_model; - } catch (const std::exception&) { - pooling_model = false; // unknown arch: the text path diagnoses - } - } - if (pooling_model) { - std::cerr << "server: pooling (embedding) model (" << archs[0] - << "); serving /v1/embeddings\n"; - vllm::entrypoints::EngineParams embed_params; - embed_params.block_size = args.block_size; - embed_params.num_blocks = args.num_blocks; - embed_params.gpu_memory_utilization = args.gpu_memory_utilization; - embed_params.kv_cache_memory_bytes = args.kv_cache_memory_bytes; - embed_params.max_model_len = args.max_model_len; - embed_params.max_num_seqs = args.max_num_seqs; - embed_params.max_num_batched_tokens = args.max_num_batched_tokens; - embed_params.enable_prefix_caching = args.enable_prefix_caching; - auto loaded_embed = std::shared_ptr( - vllm::entrypoints::LoadedEngine::FromModelDir(args.model_dir, - embed_params)); - namespace oai = vllm::entrypoints::openai; - oai::OpenAIServingModels embed_models(served_model_name); - oai::ApiServer embed_server(embed_models, vllm::Version()); - auto embed_mutex = std::make_shared(); - auto embed_counter = std::make_shared>(0); - embed_server.set_embedder( - [loaded_embed, embed_mutex, embed_counter]( - const std::vector& inputs) { - // Serialize batches: the pooling path drives the SYNCHRONOUS - // LLMEngine (async scheduling resolves OFF for pooling models). - std::lock_guard lock(*embed_mutex); - oai::ApiServer::EmbeddingBatch batch; - for (const std::string& text : inputs) { - std::vector ids = - loaded_embed->tokenizer().EncodeWithSpecialTokens(text); - if (ids.empty()) { - throw std::runtime_error( - "input tokenized to an empty prompt"); - } - batch.prompt_tokens += static_cast(ids.size()); - vllm::RequestOutput ro = loaded_embed->engine().embed( - std::move(ids), vllm::PoolingParams{}, - "embd-" + std::to_string(embed_counter->fetch_add(1))); - if (!ro.finished || !ro.pooling_output.has_value()) { - throw std::runtime_error( - "engine produced no pooled output"); - } - batch.embeddings.push_back(std::move(*ro.pooling_output)); - } - return batch; - }); - std::cerr << "server: listening on http://" << args.host << ":" - << args.port << "\n"; - if (!embed_server.listen(args.host, args.port)) { - std::cerr << "server: failed to bind " << args.host << ":" - << args.port << "\n"; - return 1; - } - return 0; - } - - if (transcription_only) { - std::cerr << "server: transcription-only model (" << archs[0] - << "); serving /v1/audio/transcriptions\n"; - auto transcriber = - std::make_shared( - vllm::multimodal::ParakeetTranscriber::FromDir(args.model_dir)); - namespace oai = vllm::entrypoints::openai; - oai::OpenAIServingModels asr_models(served_model_name); - oai::ApiServer asr_server(asr_models, vllm::Version()); - asr_server.set_transcriber( - [transcriber](const uint8_t* wav, size_t n) { - return transcriber->TranscribeWavBytes(wav, n); - }); - std::cerr << "server: listening on http://" << args.host << ":" - << args.port << "\n"; - if (!asr_server.listen(args.host, args.port)) { - std::cerr << "server: failed to bind " << args.host << ":" - << args.port << "\n"; - return 1; - } - return 0; - } - } - - // ── Load the model + build the full engine stack via the shared loader - // (src/vllm/entrypoints/model_loader.cpp) — the same path the C ABI drives. - // It loads config.json + tokenizer.json + *.safetensors and wires the M1.8 - // LLMEngine over Scheduler + runner + KV + processors. ───────────────────── - std::cerr << "server: loading model from " << args.model_dir << " (config " - << config_path << ", tokenizer " << tokenizer_path << ")\n"; - vllm::entrypoints::EngineParams engine_params; - engine_params.block_size = args.block_size; - engine_params.num_blocks = args.num_blocks; - engine_params.gpu_memory_utilization = args.gpu_memory_utilization; - engine_params.kv_cache_memory_bytes = args.kv_cache_memory_bytes; - engine_params.max_model_len = args.max_model_len; // 0 => from config. - engine_params.max_num_seqs = args.max_num_seqs; - engine_params.max_num_batched_tokens = args.max_num_batched_tokens; - engine_params.enable_prefix_caching = args.enable_prefix_caching; - // --device: explicit device selection (ARCH-ONE-SURFACE ROW 8). "auto" - // (default) keeps the accelerator-first probe byte-identical; an unknown - // name throws HERE (a startup error), and an explicitly named ABSENT - // device fails FromModelDir loudly — never a silent fallback - // (vllm/config/device.py:61-66). - engine_params.device = vllm::DeviceFromString(args.device); - // Reject an unknown policy string (mirrors upstream SchedulingPolicy(value)). - engine_params.policy = vllm::SchedulerPolicyFromString(args.scheduling_policy); - // ENG-SGLANG-BEHAVIOR-FLAG (SW1): `lpm` needs prefix caching to have any - // cache to match against; with APC explicitly off it degrades to fcfs - // (the scheduler leaves arrival order intact). Warn once at load so the - // no-op is visible (mirrors the spec's lpm+cache-off resolution). - if (engine_params.policy == vllm::SchedulerPolicy::kLPM && - args.enable_prefix_caching.has_value() && - !args.enable_prefix_caching.value()) { - std::cerr << "server: --scheduling-policy lpm has no effect with prefix " - "caching disabled; falling back to fcfs admission order\n"; - } - // ENG-SGLANG-BEHAVIOR-FLAG SW3: jump-forward decoding. Unset => the default - // (env-resolved, OFF); --[enable|disable]-jump-forward forces it, and - // VT_ENABLE_JUMP_FORWARD still overrides at resolution time. - engine_params.enable_jump_forward = args.enable_jump_forward; - // --kv-transfer-config: the external KV connector, mirroring vLLM's own - // flag and JSON shape. Absent (default) leaves the optional unset, which is - // the inert no-connector path the server has always run. A malformed - // document, an unknown key/role, or a connector whose worker half cannot - // move bytes on this device (the D1 guard, inside LoadedEngine) all throw - // out of here and are reported at startup by the catch in main. - if (!args.kv_transfer_config.empty()) { - vllm::KVTransferConfig kv_cfg = - vllm::ParseKVTransferConfigJson(args.kv_transfer_config); - if (kv_cfg.kv_connector.has_value() && - !vllm::v1::kv_offload::KVConnectorFactory::IsRegistered( - *kv_cfg.kv_connector)) { - std::string msg = "unknown kv_connector \"" + *kv_cfg.kv_connector + - "\" (registered connectors: "; - const std::vector names = - vllm::v1::kv_offload::KVConnectorFactory::RegisteredNames(); - for (size_t n = 0; n < names.size(); ++n) { - if (n != 0) msg += ", "; - msg += names[n]; - } - msg += ")"; - throw std::invalid_argument(msg); - } - engine_params.kv_transfer_config = std::move(kv_cfg); - } - // --speculative-config: speculative decoding (SPEC-MTP I5d). Absent (default) - // leaves the optional unset — the byte-identical no-speculation path. The - // parse validates method/k here; n_predict + the resolved k are finalized in - // LoadedEngine once the checkpoint's mtp_num_hidden_layers is known. A - // malformed document or unsupported method throws and is reported at startup. - if (!args.speculative_config.empty()) { - engine_params.speculative_config = - vllm::ParseSpeculativeConfigJson(args.speculative_config); - } - std::unique_ptr loaded = - vllm::entrypoints::LoadedEngine::FromModelDir(args.model_dir, - engine_params); - std::cerr << "server: prefix caching " - << (loaded->prefix_caching_enabled() ? "enabled" : "disabled") - << "\n"; - // W2: the production server uses AsyncLLM over EngineCoreProc's dedicated - // engine thread. HTTP workers submit independently and stream from their - // per-request collectors; no server-wide engine mutex remains. - vllm::v1::AsyncLLM& engine = loaded->async_engine(); - const vllm::tok::Tokenizer& tokenizer = loaded->tokenizer(); - - if (args.cuda_profile_graph_replays > 0) { -#ifdef VT_BENCH_PROFILE_CONTROL - vt::cuda::ConfigureCudaGraphReplayProfiler( - static_cast(args.cuda_profile_graph_replays), - static_cast(args.cuda_profile_graph_batch)); - std::cerr << "[VT_CUDA_PROFILE] ready pid=" << getpid() - << " signal=SIGUSR2 target_replays=" - << args.cuda_profile_graph_replays << "\n"; -#else - throw std::invalid_argument( - "--cuda-profile-graph-replays requires " - "VLLM_CPP_BENCH_PROFILE_CONTROL=ON"); -#endif - } - - // ── OpenAI serving handlers. The chat handler is wired with the real chat - // template (Task 3) when tokenizer_config.json carries one; otherwise it - // keeps the default role-join fallback. ──────────────────────────────── - namespace oai = vllm::entrypoints::openai; - oai::OpenAIServingModels models(served_model_name); - oai::OpenAIServingCompletion completion( - engine, served_model_name, args.enable_force_include_usage); - - oai::ChatPromptFn chat_prompt_fn = oai::DefaultChatPromptFallback; - // Kept outside the try so the parser resolution below can sniff it when - // --tool-call-parser/--reasoning-parser are "auto"; empty when the model - // ships no template (auto then falls back to hermes / disabled). - std::string chat_template; - try { - chat_template = - vllm::entrypoints::LoadChatTemplateFromConfig(tokenizer_config_path); - const std::string bos = - tokenizer.BosId() >= 0 ? tokenizer.Decode({tokenizer.BosId()}) : ""; - const std::string eos = - tokenizer.EosId() >= 0 ? tokenizer.Decode({tokenizer.EosId()}) : ""; - chat_prompt_fn = - vllm::entrypoints::MakeChatTemplatePromptFn( - chat_template, bos, eos, args.enable_thinking); - std::cerr << "server: using chat template (" << chat_template.size() - << " chars) from " << tokenizer_config_path - << " or sibling chat_template.jinja" - << " enable_thinking=" - << (args.enable_thinking ? "true" : "false") << "\n"; - } catch (const std::exception& e) { - std::cerr << "server: no chat template (" << e.what() - << "); falling back to the simple role-join prompt\n"; - } - // Dialect selection. Defaults reproduce the previously hardcoded pair - // ("hermes", "") exactly; an unknown name throws std::invalid_argument - // listing every registered parser and aborts startup, rather than leaving - // tool/reasoning parsing silently off for the life of the process. - const std::string tool_parser_name = - oai::ResolveToolParserName(args.tool_call_parser, chat_template); - const std::string reasoning_parser_name = - oai::ResolveReasoningParserName(args.reasoning_parser, chat_template); - std::cerr << "server: tool-call parser " - << (tool_parser_name.empty() ? "disabled" : tool_parser_name) - << ", reasoning parser " - << (reasoning_parser_name.empty() ? "disabled" - : reasoning_parser_name) - << "\n"; - oai::OpenAIServingChat chat(engine, served_model_name, chat_prompt_fn, - tool_parser_name, reasoning_parser_name, - args.enable_force_include_usage); - - // SAMPLE-BEAM (C7): enable use_beam_search on the production AsyncLLM path. - // Both handlers need the tokenizer (prompt tok + per-beam detok) and the eos - // id (beam retirement); a use_beam_search request then routes through - // BeamSearchAsync (online.py) over the async engine. Without this, beam - // requests reject with "requires an engine and a tokenizer". - const std::optional beam_eos = - tokenizer.EosId() >= 0 - ? std::optional(tokenizer.EosId()) - : std::nullopt; - completion.set_beam_search_tokenizer(&tokenizer, beam_eos); - chat.set_beam_search_tokenizer(&tokenizer, beam_eos); - - // ── MM-SERVE-E2E: wire the multimodal chat seam for image-capable models. - // When the model dir carries a preprocessor_config.json the Qwen3-VL image - // processor loads, we construct the seam body (MakeQwen3VLImageChatFn) so an - // OpenAI image_url request renders the placeholder marker → tokenizes to the - // single image_pad id → EXPANDS to N image tokens + mm_features carried onto - // the engine request. A text-only model (no preprocessor_config.json) leaves - // the seam UNSET → the chat path is byte-identical. The container-format - // image codec (PNG/JPEG → RGB) is a NAMED residual: no codec is vendored, so - // the production codec rejects encoded images with a clear message (the M2c - // single-sequence gate consumes pre-decoded raw RGB). The mm FORWARD (vision - // tower + merge + MRoPE/DeepStack on the GPU worker consuming - // Request.mm_features) is the remaining MM-SERVE-E2E residual — the engine - // model runner has no mm-forward path yet. Kept alive for the server loop. - std::unique_ptr mm_image_proc; - const std::string preprocessor_config_path = - (dir / "preprocessor_config.json").string(); - if (fs::exists(preprocessor_config_path)) { - try { - vllm::multimodal::Qwen3VLProcessorConfig pcfg = - vllm::multimodal::LoadQwen3VLProcessorConfig( - preprocessor_config_path, config_path, served_model_name); - mm_image_proc = - std::make_unique(pcfg); - oai::ImageCodecFn codec = - [](const oai::DecodedMedia& media) -> oai::DecodedImageRgb { - // Raw-RGB passthrough (image/x-raw-rgb): the single-sequence e2e / - // gate fixture format. A square raw-RGB payload is decoded directly; - // any container format (PNG/JPEG) is the NAMED codec residual. - if (media.media_type == "image/x-raw-rgb") { - const std::size_t n = media.bytes.size(); - const std::size_t px = n / 3; - const auto side = - static_cast(std::llround(std::sqrt( - static_cast(px)))); - if (side <= 0 || static_cast(side * side * 3) != n) { - throw std::runtime_error( - "image/x-raw-rgb payload is not a square HxWx3 buffer"); - } - oai::DecodedImageRgb out; - out.rgb = media.bytes; - out.height = side; - out.width = side; - return out; - } - throw std::runtime_error( - "multimodal image: container-format decode (PNG/JPEG -> RGB) is a " - "named MM-SERVE residual; supply raw RGB (image/x-raw-rgb)"); - }; - chat.set_multimodal_chat_fn(oai::MakeQwen3VLImageChatFn( - *mm_image_proc, tokenizer, chat_prompt_fn, std::move(codec))); - std::cerr << "server: multimodal image seam wired (Qwen3-VL processor " - "from " - << preprocessor_config_path << ")\n"; - } catch (const std::exception& e) { - std::cerr << "server: no multimodal image seam (" << e.what() - << "); image requests fall back to the text path\n"; - } - } - - // Diagnostic opt-out exists only for same-binary attribution. Production - // defaults to the capacity-derived fixed pool. - const char* fixed_pool_env = std::getenv("VLLM_CPP_HTTP_FIXED_POOL"); - const auto worker_pool_mode = - fixed_pool_env != nullptr && std::string(fixed_pool_env) == "0" - ? oai::ApiServer::HttpWorkerPoolMode::kLegacyDynamic - : oai::ApiServer::HttpWorkerPoolMode::kCapacityFixed; - oai::ApiServer server(completion, chat, models, vllm::Version(), - static_cast(args.max_num_seqs), - worker_pool_mode); - - // ── C8 opt-in utility/admin endpoints (SERVE-UTILITY-ENDPOINTS / - // SERVE-ADMIN). Wire the setters from the LIVE engine + tokenizer through the - // single shared seam so the production server actually serves /tokenize, - // /detokenize, /tokenizer_info (flag) and /abort_requests (dev-mode flag), - // mirroring vLLM 0.26's per-endpoint default gating. /metrics and - // /reset_prefix_cache stay unwired (no live backing on the AsyncLLM path) — - // see ConfigureUtilityEndpoints + specs/{utility,admin}-endpoints.md. ──────── - // ── MiniMax-H3 video generation. OPT-IN: with no --video-dit the routes are - // never registered and the server is byte-identical to before. The whole - // pipeline lives in the LIBRARY seam (vllm::multimodal::MiniMaxH3VideoEngine, - // ARCH-ONE-SURFACE ROW 2) — the SAME entry point the C ABI's vllm_video_* - // and the minimax-h3-gen example drive, so HTTP and FFI cannot drift. This - // file keeps exactly what an example may own: flag plumbing, the job - // directory, and the ONE process spawn (ffmpeg, ratified 2026-08-03 — the - // library builds the argv and spawns nothing). ──────────────────────────── - std::shared_ptr video_engine; - if (!args.video_dit.empty()) { - std::cerr << "server: loading MiniMax-H3 video checkpoints...\n"; - vllm::multimodal::MiniMaxH3VideoModelParams vmp; - vmp.dit_path = args.video_dit; - vmp.encoder_path = args.video_encoder; - vmp.tokenizer_path = args.video_tokenizer; - vmp.video_vae_path = args.video_vae; - vmp.video_vae_config_path = args.video_vae_config; - vmp.audio_vae_path = args.audio_vae; - vmp.audio_vae_config_path = args.audio_vae_config; - vmp.prompt_embeds_path = args.video_prompt_embeds; - vmp.partition = args.video_partition; - vmp.device = args.video_device == "cuda" ? 1 : 0; - vmp.dequant_bf16 = args.video_dequant_bf16 ? 1 : 0; - vmp.encoder_max_layers = args.video_encoder_max_layers; - video_engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(vmp); - std::cerr << "server: /v1/videos on (device=" << args.video_device - << (args.video_dequant_bf16 ? ", dequant-bf16" : ", keep-quant") << ")\n"; - // HONEST LIMIT, stated at startup rather than buried: turning a PROMPT - // into conditioning needs the H3-Encoder; without one every request is - // conditioned on the SAME supplied embeddings. - if (video_engine->has_encoder()) { - std::cerr << "server: /v1/videos conditions on the request PROMPT\n"; - } else if (!video_engine->has_prompt_embeds()) { - std::cerr << "server: WARNING /v1/videos has neither --video-encoder nor " - "--video-prompt-embeds; requests will be REJECTED\n"; - } else { - std::cerr << "server: WARNING /v1/videos ignores the request PROMPT — pass " - "--video-encoder to condition on it\n"; - } - - auto counter = std::make_shared>(0); - const std::string workdir = args.video_workdir; - const std::string ffmpeg = args.video_ffmpeg; - server.set_video_runner([video_engine, counter, workdir, - ffmpeg](const vllm::openai::VideoRequest& req) -> std::string { - const int64_t id = counter->fetch_add(1); - const std::string dir = workdir + "/job" + std::to_string(id); - // The library-owned request mapping + generation: conditioning, task - // resolution, the #77 partition guard, reference encoding, artifacts. - const vllm::multimodal::MiniMaxH3VideoResult out = video_engine->Generate( - vllm::multimodal::MiniMaxH3VideoGenParamsFromRequest(req, dir)); - // The ONE process spawn: exec the argv the library composed. - std::vector argv_mux = out.mux_argv; - if (!argv_mux.empty()) argv_mux[0] = ffmpeg; - const int status = RunFfmpegArgv(argv_mux); - if (status != 0) { - throw std::runtime_error("ffmpeg exited " + std::to_string(status)); - } - return out.mux_output_path; - }); - } - - oai::UtilityEndpointOptions endpoint_opts; - endpoint_opts.enable_tokenizer_info_endpoint = - args.enable_tokenizer_info_endpoint; - endpoint_opts.enable_server_dev_mode = args.enable_server_dev_mode; - oai::ConfigureUtilityEndpoints(server, tokenizer, loaded->max_model_len(), - engine, endpoint_opts); - std::cerr << "server: utility endpoints: /tokenize /detokenize on" - << (args.enable_tokenizer_info_endpoint ? ", /tokenizer_info on" - : "") - << (args.enable_server_dev_mode ? ", /abort_requests on (dev-mode)" - : "") - << "\n"; - - // Prometheus /metrics (Python vLLM always-on family names). - std::unique_ptr prom_logger; - if (args.enable_metrics) { - prom_logger = std::make_unique( - served_model_name, loaded->max_model_len(), /*engine_index=*/0); - // Sync engine path records on step(); async may under-report until fully wired. - loaded->engine().set_stat_logger(prom_logger.get()); - server.set_metrics_logger(prom_logger.get()); - std::cerr << "server: GET /metrics enabled (PrometheusStatLogger)\n"; - } - - std::cerr << "server: listening on http://" << args.host << ":" << args.port - << " (model '" << served_model_name << "', HTTP worker pool "; - if (server.http_worker_count() == 0) { - std::cerr << "legacy-dynamic"; - } else { - std::cerr << server.http_worker_count() << " fixed"; - } - std::cerr << ")\n"; - -#ifdef VT_BENCH_PROFILE_CONTROL - std::atomic benchmark_shutdown_waiter_ready{false}; - std::atomic benchmark_shutdown_received{false}; - std::atomic benchmark_shutdown_failed{false}; - std::atomic benchmark_shutdown_cancelled{false}; - std::thread benchmark_shutdown_thread; - if (args.cuda_profile_graph_replays > 0) { - benchmark_shutdown_thread = std::thread([&]() { - const int shutdown_fd = - open(args.benchmark_shutdown_fifo.c_str(), - O_RDONLY | O_NONBLOCK | O_CLOEXEC | O_NOFOLLOW); - if (shutdown_fd < 0) { - const int status = errno; - std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=open status=" - << status << "\n"; - benchmark_shutdown_failed.store(true, std::memory_order_release); - return; - } - struct stat shutdown_stat {}; - const int stat_status = fstat(shutdown_fd, &shutdown_stat); - if (stat_status != 0 || !S_ISFIFO(shutdown_stat.st_mode)) { - const int status = stat_status != 0 ? errno : EINVAL; - std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=fstat status=" - << status << "\n"; - close(shutdown_fd); - benchmark_shutdown_failed.store(true, std::memory_order_release); - return; - } - benchmark_shutdown_waiter_ready.store(true, std::memory_order_release); - std::cerr << "[VT_BENCH_SHUTDOWN] ready pid=" << getpid() - << " control=fifo\n"; - while (!benchmark_shutdown_cancelled.load(std::memory_order_acquire)) { - char command = '\0'; - const ssize_t bytes = read(shutdown_fd, &command, 1); - if (bytes == 1) { - if (command == 'Q') { - benchmark_shutdown_received.store(true, - std::memory_order_release); - std::cerr - << "[VT_BENCH_SHUTDOWN] requested control=fifo\n"; - close(shutdown_fd); - server.stop(); - return; - } - std::cerr - << "[VT_BENCH_SHUTDOWN] failed operation=command status=" - << static_cast( - static_cast(command)) - << "\n"; - close(shutdown_fd); - benchmark_shutdown_failed.store(true, std::memory_order_release); - server.stop(); - return; - } - if (bytes < 0 && errno != EAGAIN && errno != EWOULDBLOCK && - errno != EINTR) { - const int status = errno; - std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=read status=" - << status << "\n"; - close(shutdown_fd); - benchmark_shutdown_failed.store(true, std::memory_order_release); - server.stop(); - return; - } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - close(shutdown_fd); - }); - while (!benchmark_shutdown_waiter_ready.load(std::memory_order_acquire) && - !benchmark_shutdown_failed.load(std::memory_order_acquire)) { - std::this_thread::yield(); - } - if (benchmark_shutdown_failed.load(std::memory_order_acquire)) { - benchmark_shutdown_cancelled.store(true, std::memory_order_release); - benchmark_shutdown_thread.join(); - return 1; - } - } -#endif - - const bool listen_ok = server.listen(args.host, args.port); - -#ifdef VT_BENCH_PROFILE_CONTROL - if (benchmark_shutdown_thread.joinable()) { - benchmark_shutdown_cancelled.store(true, std::memory_order_release); - benchmark_shutdown_thread.join(); - if (benchmark_shutdown_received.load(std::memory_order_acquire)) { - std::cerr << "[VT_BENCH_SHUTDOWN] completed control=fifo\n"; - } else { - if (!benchmark_shutdown_failed.load(std::memory_order_acquire)) { - std::cerr - << "[VT_BENCH_SHUTDOWN] failed operation=cancelled status=0\n"; - } - if (listen_ok) { - return 1; - } - } - } -#endif - - if (!listen_ok) { - std::cerr << "server: failed to bind " << args.host << ":" << args.port - << "\n"; - return 1; - } - return 0; - } catch (const std::exception& e) { - std::cerr << "server: fatal: " << e.what() << "\n"; - return 1; - } -} +int main(int argc, char** argv) { return vllm_server_main(argc, argv); } diff --git a/include/vllm.h b/include/vllm.h index 8028de31b..667477da8 100644 --- a/include/vllm.h +++ b/include/vllm.h @@ -129,8 +129,20 @@ extern "C" { * other's tasks LOUDLY: vllm_complete/vllm_chat on a pooling engine name * vllm_embed, and vllm_embed on a text engine names vllm_complete — the * SupportsTranscription-refusal precedent (v11) applied to the pooling task. - * Purely additive — no struct changed; zero values preserve behaviour. */ -#define VLLM_ABI_VERSION 16 + * Purely additive — no struct changed; zero values preserve behaviour. + * + * v17 — vllm_server_main: RUN THE OPENAI SERVER from the public surface. + * examples/server was the deepest ONE SURFACE breach (36 internal headers: the + * engine, the serving layers, metrics, the video and ASR seams), which is why it + * carried an example-abi-allowlist entry. That construction moved INTO the + * library (vllm/entrypoints/openai/server_main.h) and the example is now a thin + * client of this call. argv rather than a params struct is deliberate: the + * server takes ~57 flags and gains more with every serving feature, and a + * mirrored C struct would put that churn in the ABI where every field is + * permanent. The flag surface mirrors vLLM's cli_args.py, which is the real + * contract. Embedders wanting programmatic control keep the granular entry + * points. Purely additive. */ +#define VLLM_ABI_VERSION 17 /* ── Export macro ───────────────────────────────────────────────────────────── * Marks the symbols that make up the stable ABI. Default visibility now; Task 3 @@ -792,6 +804,23 @@ VLLM_API const char* vllm_last_error(void); * not free. */ VLLM_API const char* vllm_version(void); +/* ── OpenAI-compatible server ───────────────────────────────────────────────── + * Parse `argv` and RUN the OpenAI-compatible HTTP server until it exits, + * returning the process exit code (0 on clean shutdown). `--help` prints usage + * and returns 0; a bad argument or a startup failure prints the reason and + * returns non-zero. Never throws across this boundary. + * + * This is what `vllm-server` is: examples/server is a thin client of this call. + * It serves /v1/chat/completions, /v1/completions, /v1/models, /v1/embeddings, + * and — when the matching flags are supplied — /v1/videos (MiniMax-H3) and + * transcription, all through the SAME library seams the granular entry points + * below drive, so HTTP and FFI cannot drift. + * + * BLOCKS for the lifetime of the server. `argv` must hold `argc` NUL-terminated + * strings and stay valid for the duration; the library does not take ownership. + * The conventional argv[0] program name is expected at index 0. */ +VLLM_API int32_t vllm_server_main(int32_t argc, char** argv); + /* The ABI version the library was built with (compare against VLLM_ABI_VERSION). */ VLLM_API int32_t vllm_abi_version(void); diff --git a/scripts/example-abi-allowlist.txt b/scripts/example-abi-allowlist.txt index 2ce066a0a..9c4691e51 100644 --- a/scripts/example-abi-allowlist.txt +++ b/scripts/example-abi-allowlist.txt @@ -24,7 +24,6 @@ # --- Capability drivers whose FAST PATH is reachable ONLY here, not through the ABI --- examples/deepseek_v4_gen | fold=ARCH-ONE-SURFACE | DeepSeek-V4-Flash keep-quant GGUF greedy decode (DeepseekV4ForwardGguf(Cached) + DeepseekV4KvCache) is CLI-only; the registered DeepseekV4ForCausalLM forward is a W3-W8 stub (deepseek_v4_registry.cpp:22). Grow ABI keep-quant GGUF load+decode, rewrite as ABI client, delete the bespoke forward examples/laguna_gen | fold=ARCH-ONE-SURFACE | Laguna-S-2.1 keep-quant GGUF (multi-shard) + NVFP4 W4A4 device-resident decode (LagunaForwardGguf(Cached), Marlin residents, fp4-shared) is CLI-only; registered LagunaForCausalLM forward VT_CHECK(false)s on non-bf16 (laguna.cpp:156). Grow ABI, rewrite as client, delete bespoke forward -examples/server | fold=ARCH-ONE-SURFACE | The reference OpenAI server constructs the engine, metrics, video_runner and mm seam directly from internal C++ headers (LoadedEngine/AsyncLLM, minimax_h3.h, chat_mm). It should stand on the public surface (C ABI, or a curated public C++ API the ABI wraps). Grow the surface, rewrite the server against it # --- Dev / diagnostic tools: NO permanent exemption (developer-directed 2026-08-07) --- # Every one is a transition-tracker like the drivers above; the allowlist only shrinks. diff --git a/src/capi/vllm_c.cpp b/src/capi/vllm_c.cpp index 3d63ee56d..b42faa5f6 100644 --- a/src/capi/vllm_c.cpp +++ b/src/capi/vllm_c.cpp @@ -44,6 +44,7 @@ #include "vllm/model_executor/models/minimax_h3.h" // mux argv (v12) #include "vllm/multimodal/parakeet_transcription.h" // vllm_transcribe (v11) #include "vllm/multimodal/minimax_h3_video.h" // vllm_video_* (v12) +#include "vllm/entrypoints/openai/server_main.h" // vllm_server_main (v17) #include "vllm/outputs.h" #include "vllm/sampling_params.h" #include "vllm/transformers_utils/hf_config.h" // PeekHfArchitectures (v11) @@ -1615,6 +1616,22 @@ VLLM_API const char* vllm_version(void) { return kVersion.c_str(); } +VLLM_API int32_t vllm_server_main(int32_t argc, char** argv) { + // The server owns its own error reporting on stderr (it is a PROCESS entry + // point, not a request call), so this does not set vllm_last_error. What it + // must guarantee is that nothing throws across the C boundary. + try { + return static_cast( + vllm::entrypoints::openai::VllmServerMain(static_cast(argc), argv)); + } catch (const std::exception& e) { + std::fprintf(stderr, "vllm_server_main: %s\n", e.what()); + return 1; + } catch (...) { + std::fprintf(stderr, "vllm_server_main: unknown error\n"); + return 1; + } +} + VLLM_API int32_t vllm_abi_version(void) { return VLLM_ABI_VERSION; } } // extern "C" diff --git a/src/vllm/entrypoints/openai/server_main.cpp b/src/vllm/entrypoints/openai/server_main.cpp new file mode 100644 index 000000000..b72ba3ba5 --- /dev/null +++ b/src/vllm/entrypoints/openai/server_main.cpp @@ -0,0 +1,1055 @@ +// server: an OpenAI-compatible HTTP server over the vllm.cpp LLMEngine (M3.1 +// Task 4). Loads a supported model (safetensors or GGUF weights + tokenizer + a KV-cache config → +// LLMEngine), constructs the OpenAI serving handlers (chat wired with the real +// chat template via MakeChatTemplatePromptFn(LoadChatTemplateFromConfig(...))) +// and serves /v1/completions, /v1/chat/completions, /v1/models, /health, +// /version. +// +// server --model [--host 0.0.0.0] [--port 8000] +// [--tokenizer-config ] +// [--served-model-name ] +// [--block-size N] [--num-blocks N] [--max-model-len N] +// [--gpu-memory-utilization F] [--kv-cache-memory BYTES] +// [--max-num-seqs N] [--max-num-batched-tokens N] +// [--enable-force-include-usage] +// [--[no-]enable-prefix-caching] +// [--scheduling-policy fcfs|priority] +// [--tool-call-parser |auto|none] +// [--reasoning-parser |auto|none] +// [--kv-transfer-config ''] +// +// A directory holds config.json, tokenizer.json and supported safetensors +// shards. A supported GGUF file is also accepted and supplies model metadata +// plus embedded vocabulary. If --tokenizer-config is omitted for a directory it +// defaults to /tokenizer_config.json; when that file has no chat_template +// the chat endpoint falls back to the simple role-join prompt. +// +// NOTE: loading the real 35B checkpoint is a GPU/dgx concern; on the CPU CI box +// this binary is only built + smoke-tested against a synthetic engine (see +// tests/vllm/entrypoints/openai/test_api_server.cpp). The wiring below is the +// same either way. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef VT_BENCH_PROFILE_CONTROL +#include +#include +#include +#include +#include +#endif + +#include "vllm/config/device.h" +#include "vllm/config/kv_transfer.h" +#include "vllm/config/scheduler.h" +#include "vllm/entrypoints/chat_template.h" +#include "vllm/entrypoints/model_loader.h" +#include +#include "vllm/entrypoints/openai/server_main.h" +#include "vllm/entrypoints/openai/api_server.h" +#include "vllm/entrypoints/openai/chat_mm.h" +#include "vllm/entrypoints/openai/request_logger.h" +#include "vllm/entrypoints/openai/serving_chat.h" +#include "vllm/entrypoints/openai/serving_completion.h" +#include "vllm/entrypoints/openai/serving_models.h" +#include "vllm/v1/metrics/loggers.h" +#include "vllm/entrypoints/openai/reasoning_parsers/detect.h" +#include "vllm/entrypoints/openai/tool_parsers/detect.h" +#include "vllm/model_executor/model_loader/safetensors_reader.h" +#include "vllm/model_executor/models/qwen3_5_weights.h" +#include "vllm/transformers_utils/hf_config.h" +#include "vllm/model_executor/models/model_registry.h" +#include "vllm/multimodal/minimax_h3_video.h" +#include "vllm/multimodal/parakeet_transcription.h" +#include "vllm/tokenizer/tokenizer.h" +#include "vllm/version.h" +#include "vllm/v1/core/kv_cache_utils.h" +#include "vllm/v1/core/sched/scheduler.h" +#include "vllm/v1/engine/core.h" +#include "vllm/v1/engine/input_processor.h" +#include "vllm/v1/engine/llm_engine.h" +#include "vllm/v1/engine/output_processor.h" +#include "vllm/v1/executor/executor.h" +#include "vllm/v1/kv_cache_interface.h" +#include "vllm/v1/kv_offload/kv_connector.h" +#include "vllm/v1/worker/gpu/runner.h" +#include "vt/backend.h" +#ifdef VT_BENCH_PROFILE_CONTROL +#include "vt/cuda/cuda_profiler_control.h" +#endif +#include "vt/dtype.h" +#include "vt/tensor.h" + +namespace { + +namespace fs = std::filesystem; +using vllm::HfConfig; +using vllm::Qwen3_5MoeWeights; + +// Run an argv to completion and return its exit status — the ONE process +// spawn in the MiniMax-H3 path, and it lives HERE, in examples/, by the +// developer-ratified 2026-08-03 decision: the library (the +// MiniMaxH3VideoEngine seam behind /v1/videos) writes the artifacts and +// BUILDS this argv, and spawns nothing. +int RunFfmpegArgv(const std::vector& args) { + std::vector c_args; + c_args.reserve(args.size() + 1); + for (const std::string& arg : args) { + c_args.push_back(const_cast(arg.c_str())); + } + c_args.push_back(nullptr); + const pid_t pid = fork(); + if (pid < 0) throw std::runtime_error("fork failed"); + if (pid == 0) { + execvp(c_args[0], c_args.data()); + _exit(127); // exec failed; never run the parent's atexit handlers + } + int status = 0; + if (waitpid(pid, &status, 0) < 0) throw std::runtime_error("waitpid failed"); + if (WIFSIGNALED(status)) { + throw std::runtime_error("ffmpeg died on signal " + std::to_string(WTERMSIG(status))); + } + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; +} + +struct Args { + std::string model_dir; + std::string host = "0.0.0.0"; + int port = 8000; + std::string tokenizer_config; // default: /tokenizer_config.json + std::string served_model_name; // default: the model dir name + int block_size = 32; + // --num-blocks is the KV block-count OVERRIDE (0 => auto: sized by the knobs + // below, else the 256-block fallback). ROAD-V1-MEM M1. + int num_blocks = 0; + // --gpu-memory-utilization: fraction of free device memory for the whole + // engine (needs the M3 profile run; inert until then). --kv-cache-memory: an + // absolute KV-pool size in bytes that sizes the block count directly (0 => + // unset). + double gpu_memory_utilization = 0.92; + long long kv_cache_memory_bytes = 0; + int max_model_len = 0; // 0 => config.max_position_embeddings + int max_num_seqs = 8; + int max_num_batched_tokens = 0; // 0 => per-architecture default. + // --device: explicit device selection for the TEXT engine (ARCH-ONE-SURFACE + // ROW 8), the vLLM DeviceConfig.device names this build serves: "auto" + // (default — the accelerator-first probe, byte-identical to before the flag), + // "cpu" (force the CPU queue), "cuda" (require CUDA; an absent device fails + // startup LOUDLY, never a silent fallback). The video engine keeps its own + // --video-device below: the two engines are loaded from different + // checkpoints and may legitimately serve on different devices. + std::string device = "auto"; + // --- MiniMax-H3 video generation (opt-in; absent => /v1/videos is unregistered + // and the server behaves exactly as before). --- + std::string video_dit, video_vae, video_vae_config, audio_vae, audio_vae_config; + std::string video_prompt_embeds, video_workdir = "/tmp/vllm_h3_videos"; + std::string video_encoder, video_tokenizer; + int video_encoder_max_layers = 50; + std::string video_ffmpeg = "ffmpeg", video_device = "cuda"; + std::string video_partition; // served partition (fl2va|ref2va); see the #77 guard + // Keep-quant is the library seam's DEFAULT arm; --video-dequant-bf16 selects + // the bf16 dequant/stream arm (the throughput trade the gen example ships). + // --video-keep-quant is still accepted (it names the default). + bool video_dequant_bf16 = false; + int cuda_profile_graph_replays = 0; // trace-only diagnostic build seam. + int cuda_profile_graph_batch = 0; // 0 => accepted c16 trace contract. + std::string benchmark_shutdown_fifo; // paired trace-only control path. + std::optional enable_prefix_caching = std::nullopt; + bool enable_force_include_usage = false; + // GET /tokenizer_info gate. Mirrors vLLM's --enable-tokenizer-info-endpoint + // (entrypoints/openai/cli_args.py:140, default False; the route is registered + // only when serve/tokenize/api_router.py:95 sees the flag). Default off → the + // route 404s, byte-identical to before. + bool enable_tokenizer_info_endpoint = false; + // Dev/admin endpoint gate. Mirrors vLLM's VLLM_SERVER_DEV_MODE env + // (envs.py:157, default 0): build_app registers the dev/rlhf + dev/cache + // routers only under `if envs.VLLM_SERVER_DEV_MODE` (api_server.py:238). Off by + // default → /abort_requests 404s. Enables the /abort_requests production wiring. + bool enable_server_dev_mode = false; + bool verbose = false; + // Gemma4 HF/vLLM: --default-chat-template-kwargs enable_thinking (default OFF). + bool enable_thinking = false; + // Request logging (Python vLLM --enable-log-requests parity). Default ON. + bool enable_log_requests = true; + bool enable_log_outputs = false; + int max_log_len = 256; + // Attach Prometheus logger + GET /metrics (default ON for solid Hermes serve). + bool enable_metrics = true; + // Scheduling policy: "fcfs" (default), "priority" (mirrors vLLM's + // --scheduling-policy / SchedulerConfig.policy), or "lpm" (SGLang's + // cache-aware longest-prefix-match admission ordering, ENG-SGLANG-BEHAVIOR-FLAG; + // opt-in, output-neutral, resolves to fcfs when prefix caching is off). + // --schedule-policy is accepted as an SGLang-compatible alias. + std::string scheduling_policy = "fcfs"; + // Jump-forward decoding (ENG-SGLANG-BEHAVIOR-FLAG SW3): tri-state, mirrors the + // C-ABI vllm_model_params.enable_jump_forward. Unset (default) => OFF unless + // VT_ENABLE_JUMP_FORWARD is set; --enable-jump-forward forces on, + // --disable-jump-forward forces off (the env var still overrides). The + // token-unique forced-run subset only; see .agents/specs/sglang-enablement.md. + std::optional enable_jump_forward = std::nullopt; + // Tool-call / reasoning dialect selection (mirrors vLLM's --tool-call-parser + // and --reasoning-parser). THE DEFAULTS ARE TODAY'S HARDCODED BEHAVIOUR: + // "hermes" is exactly what OpenAIServingChat was constructed with before this + // flag existed, and "none" is the empty reasoning-parser name it passed. An + // invocation that names neither flag is therefore unchanged, byte for byte. + // "auto" opts into the chat-template detection the C ABI uses. + std::string tool_call_parser = "hermes"; + std::string reasoning_parser = "none"; + // vLLM's --kv-transfer-config: the external KV connector selection, as the + // same JSON object vLLM takes. Empty (default) == no connector == the inert + // production path. See docs/KV-OFFLOAD.md. + std::string kv_transfer_config; + // vLLM's --speculative-config: the speculative-decoding selection, as the same + // JSON object vLLM takes (e.g. '{"method":"mtp","num_speculative_tokens":1}'). + // Empty (default) == no speculation == the inert production path (SPEC-MTP I5d). + std::string speculative_config; +}; + +[[noreturn]] void Usage(const char* argv0, int code) { + std::cerr + << "usage: " << argv0 + << " --model [--host H] [--port P] [--tokenizer-config F]\n" + " [--served-model-name N] [--block-size N] " + "[--num-blocks N] [--max-model-len N]\n" + " [--gpu-memory-utilization F] " + "[--kv-cache-memory BYTES]\n" + " [--max-num-seqs N] " + "[--max-num-batched-tokens N]\n" + " [--device auto|cpu|cuda]\n" + " [--cuda-profile-graph-replays N]\n" + " [--cuda-profile-graph-batch N]\n" + " [--benchmark-shutdown-fifo F]\n" + " [--enable-force-include-usage]\n" + " [--enable-tokenizer-info-endpoint]\n" + " [--enable-server-dev-mode]\n" + " [--verbose]\n" + " [--enable-thinking|--no-enable-thinking]\n" + " [--enable-log-requests|--disable-log-requests]\n" + " [--enable-log-outputs] [--max-log-len N]\n" + " [--enable-metrics|--disable-metrics]\n" + " [--[no-]enable-prefix-caching]\n" + " [--[no-]enable-radix-attention]\n" + " [--scheduling-policy fcfs|priority|lpm]\n" + " [--[enable|disable]-jump-forward]\n" + " [--tool-call-parser |auto|none]\n" + " [--reasoning-parser |auto|none]\n" + " [--kv-transfer-config '']\n" + " [--speculative-config '']\n"; + std::exit(code); +} + +std::string NextArg(int argc, char** argv, int& i, const char* argv0) { + if (i + 1 >= argc) Usage(argv0, 2); + return argv[++i]; +} + +Args ParseArgs(int argc, char** argv) { + Args a; + for (int i = 1; i < argc; ++i) { + const std::string flag = argv[i]; + if (flag == "--model") { + a.model_dir = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--host") { + a.host = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--port") { + a.port = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--tokenizer-config") { + a.tokenizer_config = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--served-model-name") { + a.served_model_name = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--block-size") { + a.block_size = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--num-blocks") { + a.num_blocks = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--gpu-memory-utilization") { + a.gpu_memory_utilization = std::stod(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--kv-cache-memory") { + a.kv_cache_memory_bytes = std::stoll(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--max-model-len") { + a.max_model_len = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--max-num-seqs") { + a.max_num_seqs = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--max-num-batched-tokens") { + a.max_num_batched_tokens = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--device") { + // Text-engine device selection (mirrors vLLM's DeviceConfig.device + // names). Validated by vllm::DeviceFromString at engine construction; + // --video-device (below) stays the video engine's separate knob. + a.device = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--cuda-profile-graph-replays") { + a.cuda_profile_graph_replays = + std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--cuda-profile-graph-batch") { + a.cuda_profile_graph_batch = + std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--benchmark-shutdown-fifo") { + a.benchmark_shutdown_fifo = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--enable-force-include-usage") { + a.enable_force_include_usage = true; + } else if (flag == "--enable-tokenizer-info-endpoint") { + a.enable_tokenizer_info_endpoint = true; + } else if (flag == "--video-dit") { + a.video_dit = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-vae") { + a.video_vae = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-vae-config") { + a.video_vae_config = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--audio-vae") { + a.audio_vae = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--audio-vae-config") { + a.audio_vae_config = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-encoder") { + a.video_encoder = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-tokenizer") { + a.video_tokenizer = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-encoder-max-layers") { + a.video_encoder_max_layers = std::atoi(NextArg(argc, argv, i, argv[0]).c_str()); + } else if (flag == "--video-prompt-embeds") { + a.video_prompt_embeds = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-workdir") { + a.video_workdir = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-ffmpeg") { + a.video_ffmpeg = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-device") { + a.video_device = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-partition") { + a.video_partition = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--video-keep-quant") { + // the seam's default arm; accepted for pre-fold CLI compatibility + a.video_dequant_bf16 = false; + } else if (flag == "--video-dequant-bf16") { + a.video_dequant_bf16 = true; + } else if (flag == "--enable-server-dev-mode") { + a.enable_server_dev_mode = true; + } else if (flag == "--verbose" || flag == "-v") { + a.verbose = true; + } else if (flag == "--enable-thinking") { + a.enable_thinking = true; + } else if (flag == "--no-enable-thinking") { + a.enable_thinking = false; + } else if (flag == "--enable-log-requests") { + a.enable_log_requests = true; + } else if (flag == "--disable-log-requests") { + a.enable_log_requests = false; + } else if (flag == "--enable-log-outputs") { + a.enable_log_outputs = true; + } else if (flag == "--max-log-len") { + a.max_log_len = std::stoi(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--enable-metrics") { + a.enable_metrics = true; + } else if (flag == "--disable-metrics") { + a.enable_metrics = false; + } else if (flag == "--enable-prefix-caching" || + flag == "--no-enable-prefix-caching" || + flag == "--enable-radix-attention" || + flag == "--disable-radix-attention") { + // --[no-]enable-prefix-caching is vLLM's flag. --enable-radix-attention / + // --disable-radix-attention are SGLang-compatible ALIASES for the SAME + // toggle (RadixAttention is fused into our block-hash APC — there is no + // distinct radix code path; see .agents/specs/sglang-radixattention.md §1). + // They set the identical tri-state as the vLLM flag; last-wins is rejected + // (mirrors passing the vLLM flag twice) so a contradictory pair is caught. + if (a.enable_prefix_caching.has_value()) { + std::cerr << "server: prefix-caching flag (--[no-]enable-prefix-caching " + "/ --[disable|enable]-radix-attention) specified more than " + "once\n"; + Usage(argv[0], 2); + } + a.enable_prefix_caching = + flag == "--enable-prefix-caching" || flag == "--enable-radix-attention"; + } else if (flag == "--scheduling-policy" || flag == "--schedule-policy") { + // --scheduling-policy is vLLM's flag; --schedule-policy is SGLang's name, + // accepted as an alias. Both take fcfs|priority|lpm. + a.scheduling_policy = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--enable-jump-forward" || + flag == "--disable-jump-forward") { + // ENG-SGLANG-BEHAVIOR-FLAG SW3: opt into (or force off) jump-forward + // decoding — the token-unique grammar-speed subset (see + // .agents/specs/sglang-enablement.md). Absent => the default (OFF unless + // VT_ENABLE_JUMP_FORWARD is set). The env var, when set, still overrides. + if (a.enable_jump_forward.has_value()) { + std::cerr << "server: jump-forward flag " + "(--[enable|disable]-jump-forward) specified more than " + "once\n"; + Usage(argv[0], 2); + } + a.enable_jump_forward = flag == "--enable-jump-forward"; + } else if (flag == "--tool-call-parser") { + a.tool_call_parser = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--reasoning-parser") { + a.reasoning_parser = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--kv-transfer-config") { + a.kv_transfer_config = NextArg(argc, argv, i, argv[0]); + } else if (flag == "--speculative-config") { + a.speculative_config = NextArg(argc, argv, i, argv[0]); + } else if (flag == "-h" || flag == "--help") { + Usage(argv[0], 0); + } else { + std::cerr << "server: unknown argument '" << flag << "'\n"; + Usage(argv[0], 2); + } + } + if (a.model_dir.empty()) { + std::cerr << "server: --model is required\n"; + Usage(argv[0], 2); + } + if (a.max_num_seqs <= 0 || a.max_num_batched_tokens < 0 || + a.cuda_profile_graph_replays < 0 || a.cuda_profile_graph_batch < 0) { + std::cerr << "server: scheduler capacities must be positive " + "(--max-num-batched-tokens may be 0 for auto)\n"; + Usage(argv[0], 2); + } + if ((a.cuda_profile_graph_replays > 0) != + !a.benchmark_shutdown_fifo.empty()) { + std::cerr << "server: --cuda-profile-graph-replays and " + "--benchmark-shutdown-fifo must be specified together\n"; + Usage(argv[0], 2); + } + if (a.cuda_profile_graph_replays == 0 && a.cuda_profile_graph_batch != 0) { + std::cerr << "server: --cuda-profile-graph-batch requires " + "--cuda-profile-graph-replays\n"; + Usage(argv[0], 2); + } + if (a.cuda_profile_graph_replays > 0 && a.cuda_profile_graph_batch == 0) { + a.cuda_profile_graph_batch = 16; + } + if (a.cuda_profile_graph_batch > a.max_num_seqs) { + std::cerr << "server: --cuda-profile-graph-batch exceeds --max-num-seqs\n"; + Usage(argv[0], 2); + } + // Validate a NAMED parser dialect here, before the (multi-GB) model load, so a + // typo costs a second rather than a full load. "auto" cannot be checked yet — + // it resolves against the chat template — but detection only ever returns + // registered names, so it cannot fail later either. + namespace oai = vllm::entrypoints::openai; + if (a.tool_call_parser != "auto") { + (void)oai::ResolveToolParserName(a.tool_call_parser, ""); + } + if (a.reasoning_parser != "auto") { + (void)oai::ResolveReasoningParserName(a.reasoning_parser, ""); + } + return a; +} + +} // namespace + +namespace vllm { +namespace entrypoints { +namespace openai { + +int VllmServerMain(int argc, char** argv) { + try { + const Args args = ParseArgs(argc, argv); + if (args.verbose) { + setenv("VT_SERVER_VERBOSE", "1", /*overwrite=*/1); + std::cerr << "server: verbose stage logging enabled (debug_stages)\n"; + } + { + vllm::entrypoints::openai::RequestLogConfig log_cfg; + log_cfg.enable_log_requests = args.enable_log_requests; + log_cfg.enable_log_outputs = args.enable_log_outputs || args.verbose; + log_cfg.max_log_len = args.max_log_len; + log_cfg.debug_stages = args.verbose || + (std::getenv("VT_SERVER_VERBOSE") && + std::getenv("VT_SERVER_VERBOSE")[0] == '1'); + vllm::entrypoints::openai::ConfigureRequestLogger(log_cfg); + std::cerr << "server: request logging " + << (log_cfg.enable_log_requests ? "ON" : "OFF") + << " outputs=" << (log_cfg.enable_log_outputs ? "ON" : "OFF") + << " max_log_len=" << log_cfg.max_log_len + << " debug_stages=" << (log_cfg.debug_stages ? "ON" : "OFF") + << "\n"; + } + + const fs::path dir(args.model_dir); + const std::string config_path = (dir / "config.json").string(); + const std::string tokenizer_path = (dir / "tokenizer.json").string(); + const std::string tokenizer_config_path = + args.tokenizer_config.empty() + ? (dir / "tokenizer_config.json").string() + : args.tokenizer_config; + const std::string served_model_name = + args.served_model_name.empty() + ? (dir.has_filename() ? dir.filename().string() + : dir.parent_path().filename().string()) + : args.served_model_name; + + // ── TASK DISPATCH (ARCH-ONE-SURFACE ROW 1): a model dir whose + // architectures resolve to a SupportsTranscription-ONLY registration + // (Parakeet CTC/RNNT/TDT) serves /v1/audio/transcriptions through the ONE + // library seam — the same ParakeetTranscriber vllm_transcribe drives — and + // registers NO generate routes (vLLM's task-conditional registration, + // api_server.py:255-265). Every other model takes the text path below, + // byte-identical to before. ──────────────────────────────────────────────── + { + bool transcription_only = false; + const std::vector archs = + vllm::PeekHfArchitectures(config_path); + if (!archs.empty()) { + try { + transcription_only = + vllm::ModelRegistry::Resolve(std::span(archs)) + .info.supports_transcription_only; + } catch (const std::exception&) { + transcription_only = false; // unknown arch: the text path diagnoses + } + } + // ── POOLING TASK DISPATCH (ARCH-ONE-SURFACE ROW 6): a model dir whose + // architectures resolve to a POOLING registration (is_pooling_model, + // e.g. "LlamaModel" — vLLM _EMBEDDING_MODELS registry.py:230) serves + // /v1/embeddings through the ONE engine path (LoadedEngine -> + // LLMEngine::embed -> registry forward -> PoolingRunner) — the same + // path vllm_embed drives — and registers NO generate routes (vLLM's + // task-conditional registration, api_server.py:255-265). ────────────── + bool pooling_model = false; + if (!archs.empty()) { + try { + pooling_model = + vllm::ModelRegistry::Resolve(std::span(archs)) + .info.is_pooling_model; + } catch (const std::exception&) { + pooling_model = false; // unknown arch: the text path diagnoses + } + } + if (pooling_model) { + std::cerr << "server: pooling (embedding) model (" << archs[0] + << "); serving /v1/embeddings\n"; + vllm::entrypoints::EngineParams embed_params; + embed_params.block_size = args.block_size; + embed_params.num_blocks = args.num_blocks; + embed_params.gpu_memory_utilization = args.gpu_memory_utilization; + embed_params.kv_cache_memory_bytes = args.kv_cache_memory_bytes; + embed_params.max_model_len = args.max_model_len; + embed_params.max_num_seqs = args.max_num_seqs; + embed_params.max_num_batched_tokens = args.max_num_batched_tokens; + embed_params.enable_prefix_caching = args.enable_prefix_caching; + auto loaded_embed = std::shared_ptr( + vllm::entrypoints::LoadedEngine::FromModelDir(args.model_dir, + embed_params)); + namespace oai = vllm::entrypoints::openai; + oai::OpenAIServingModels embed_models(served_model_name); + oai::ApiServer embed_server(embed_models, vllm::Version()); + auto embed_mutex = std::make_shared(); + auto embed_counter = std::make_shared>(0); + embed_server.set_embedder( + [loaded_embed, embed_mutex, embed_counter]( + const std::vector& inputs) { + // Serialize batches: the pooling path drives the SYNCHRONOUS + // LLMEngine (async scheduling resolves OFF for pooling models). + std::lock_guard lock(*embed_mutex); + oai::ApiServer::EmbeddingBatch batch; + for (const std::string& text : inputs) { + std::vector ids = + loaded_embed->tokenizer().EncodeWithSpecialTokens(text); + if (ids.empty()) { + throw std::runtime_error( + "input tokenized to an empty prompt"); + } + batch.prompt_tokens += static_cast(ids.size()); + vllm::RequestOutput ro = loaded_embed->engine().embed( + std::move(ids), vllm::PoolingParams{}, + "embd-" + std::to_string(embed_counter->fetch_add(1))); + if (!ro.finished || !ro.pooling_output.has_value()) { + throw std::runtime_error( + "engine produced no pooled output"); + } + batch.embeddings.push_back(std::move(*ro.pooling_output)); + } + return batch; + }); + std::cerr << "server: listening on http://" << args.host << ":" + << args.port << "\n"; + if (!embed_server.listen(args.host, args.port)) { + std::cerr << "server: failed to bind " << args.host << ":" + << args.port << "\n"; + return 1; + } + return 0; + } + + if (transcription_only) { + std::cerr << "server: transcription-only model (" << archs[0] + << "); serving /v1/audio/transcriptions\n"; + auto transcriber = + std::make_shared( + vllm::multimodal::ParakeetTranscriber::FromDir(args.model_dir)); + namespace oai = vllm::entrypoints::openai; + oai::OpenAIServingModels asr_models(served_model_name); + oai::ApiServer asr_server(asr_models, vllm::Version()); + asr_server.set_transcriber( + [transcriber](const uint8_t* wav, size_t n) { + return transcriber->TranscribeWavBytes(wav, n); + }); + std::cerr << "server: listening on http://" << args.host << ":" + << args.port << "\n"; + if (!asr_server.listen(args.host, args.port)) { + std::cerr << "server: failed to bind " << args.host << ":" + << args.port << "\n"; + return 1; + } + return 0; + } + } + + // ── Load the model + build the full engine stack via the shared loader + // (src/vllm/entrypoints/model_loader.cpp) — the same path the C ABI drives. + // It loads config.json + tokenizer.json + *.safetensors and wires the M1.8 + // LLMEngine over Scheduler + runner + KV + processors. ───────────────────── + std::cerr << "server: loading model from " << args.model_dir << " (config " + << config_path << ", tokenizer " << tokenizer_path << ")\n"; + vllm::entrypoints::EngineParams engine_params; + engine_params.block_size = args.block_size; + engine_params.num_blocks = args.num_blocks; + engine_params.gpu_memory_utilization = args.gpu_memory_utilization; + engine_params.kv_cache_memory_bytes = args.kv_cache_memory_bytes; + engine_params.max_model_len = args.max_model_len; // 0 => from config. + engine_params.max_num_seqs = args.max_num_seqs; + engine_params.max_num_batched_tokens = args.max_num_batched_tokens; + engine_params.enable_prefix_caching = args.enable_prefix_caching; + // --device: explicit device selection (ARCH-ONE-SURFACE ROW 8). "auto" + // (default) keeps the accelerator-first probe byte-identical; an unknown + // name throws HERE (a startup error), and an explicitly named ABSENT + // device fails FromModelDir loudly — never a silent fallback + // (vllm/config/device.py:61-66). + engine_params.device = vllm::DeviceFromString(args.device); + // Reject an unknown policy string (mirrors upstream SchedulingPolicy(value)). + engine_params.policy = vllm::SchedulerPolicyFromString(args.scheduling_policy); + // ENG-SGLANG-BEHAVIOR-FLAG (SW1): `lpm` needs prefix caching to have any + // cache to match against; with APC explicitly off it degrades to fcfs + // (the scheduler leaves arrival order intact). Warn once at load so the + // no-op is visible (mirrors the spec's lpm+cache-off resolution). + if (engine_params.policy == vllm::SchedulerPolicy::kLPM && + args.enable_prefix_caching.has_value() && + !args.enable_prefix_caching.value()) { + std::cerr << "server: --scheduling-policy lpm has no effect with prefix " + "caching disabled; falling back to fcfs admission order\n"; + } + // ENG-SGLANG-BEHAVIOR-FLAG SW3: jump-forward decoding. Unset => the default + // (env-resolved, OFF); --[enable|disable]-jump-forward forces it, and + // VT_ENABLE_JUMP_FORWARD still overrides at resolution time. + engine_params.enable_jump_forward = args.enable_jump_forward; + // --kv-transfer-config: the external KV connector, mirroring vLLM's own + // flag and JSON shape. Absent (default) leaves the optional unset, which is + // the inert no-connector path the server has always run. A malformed + // document, an unknown key/role, or a connector whose worker half cannot + // move bytes on this device (the D1 guard, inside LoadedEngine) all throw + // out of here and are reported at startup by the catch in main. + if (!args.kv_transfer_config.empty()) { + vllm::KVTransferConfig kv_cfg = + vllm::ParseKVTransferConfigJson(args.kv_transfer_config); + if (kv_cfg.kv_connector.has_value() && + !vllm::v1::kv_offload::KVConnectorFactory::IsRegistered( + *kv_cfg.kv_connector)) { + std::string msg = "unknown kv_connector \"" + *kv_cfg.kv_connector + + "\" (registered connectors: "; + const std::vector names = + vllm::v1::kv_offload::KVConnectorFactory::RegisteredNames(); + for (size_t n = 0; n < names.size(); ++n) { + if (n != 0) msg += ", "; + msg += names[n]; + } + msg += ")"; + throw std::invalid_argument(msg); + } + engine_params.kv_transfer_config = std::move(kv_cfg); + } + // --speculative-config: speculative decoding (SPEC-MTP I5d). Absent (default) + // leaves the optional unset — the byte-identical no-speculation path. The + // parse validates method/k here; n_predict + the resolved k are finalized in + // LoadedEngine once the checkpoint's mtp_num_hidden_layers is known. A + // malformed document or unsupported method throws and is reported at startup. + if (!args.speculative_config.empty()) { + engine_params.speculative_config = + vllm::ParseSpeculativeConfigJson(args.speculative_config); + } + std::unique_ptr loaded = + vllm::entrypoints::LoadedEngine::FromModelDir(args.model_dir, + engine_params); + std::cerr << "server: prefix caching " + << (loaded->prefix_caching_enabled() ? "enabled" : "disabled") + << "\n"; + // W2: the production server uses AsyncLLM over EngineCoreProc's dedicated + // engine thread. HTTP workers submit independently and stream from their + // per-request collectors; no server-wide engine mutex remains. + vllm::v1::AsyncLLM& engine = loaded->async_engine(); + const vllm::tok::Tokenizer& tokenizer = loaded->tokenizer(); + + if (args.cuda_profile_graph_replays > 0) { +#ifdef VT_BENCH_PROFILE_CONTROL + vt::cuda::ConfigureCudaGraphReplayProfiler( + static_cast(args.cuda_profile_graph_replays), + static_cast(args.cuda_profile_graph_batch)); + std::cerr << "[VT_CUDA_PROFILE] ready pid=" << getpid() + << " signal=SIGUSR2 target_replays=" + << args.cuda_profile_graph_replays << "\n"; +#else + throw std::invalid_argument( + "--cuda-profile-graph-replays requires " + "VLLM_CPP_BENCH_PROFILE_CONTROL=ON"); +#endif + } + + // ── OpenAI serving handlers. The chat handler is wired with the real chat + // template (Task 3) when tokenizer_config.json carries one; otherwise it + // keeps the default role-join fallback. ──────────────────────────────── + namespace oai = vllm::entrypoints::openai; + oai::OpenAIServingModels models(served_model_name); + oai::OpenAIServingCompletion completion( + engine, served_model_name, args.enable_force_include_usage); + + oai::ChatPromptFn chat_prompt_fn = oai::DefaultChatPromptFallback; + // Kept outside the try so the parser resolution below can sniff it when + // --tool-call-parser/--reasoning-parser are "auto"; empty when the model + // ships no template (auto then falls back to hermes / disabled). + std::string chat_template; + try { + chat_template = + vllm::entrypoints::LoadChatTemplateFromConfig(tokenizer_config_path); + const std::string bos = + tokenizer.BosId() >= 0 ? tokenizer.Decode({tokenizer.BosId()}) : ""; + const std::string eos = + tokenizer.EosId() >= 0 ? tokenizer.Decode({tokenizer.EosId()}) : ""; + chat_prompt_fn = + vllm::entrypoints::MakeChatTemplatePromptFn( + chat_template, bos, eos, args.enable_thinking); + std::cerr << "server: using chat template (" << chat_template.size() + << " chars) from " << tokenizer_config_path + << " or sibling chat_template.jinja" + << " enable_thinking=" + << (args.enable_thinking ? "true" : "false") << "\n"; + } catch (const std::exception& e) { + std::cerr << "server: no chat template (" << e.what() + << "); falling back to the simple role-join prompt\n"; + } + // Dialect selection. Defaults reproduce the previously hardcoded pair + // ("hermes", "") exactly; an unknown name throws std::invalid_argument + // listing every registered parser and aborts startup, rather than leaving + // tool/reasoning parsing silently off for the life of the process. + const std::string tool_parser_name = + oai::ResolveToolParserName(args.tool_call_parser, chat_template); + const std::string reasoning_parser_name = + oai::ResolveReasoningParserName(args.reasoning_parser, chat_template); + std::cerr << "server: tool-call parser " + << (tool_parser_name.empty() ? "disabled" : tool_parser_name) + << ", reasoning parser " + << (reasoning_parser_name.empty() ? "disabled" + : reasoning_parser_name) + << "\n"; + oai::OpenAIServingChat chat(engine, served_model_name, chat_prompt_fn, + tool_parser_name, reasoning_parser_name, + args.enable_force_include_usage); + + // SAMPLE-BEAM (C7): enable use_beam_search on the production AsyncLLM path. + // Both handlers need the tokenizer (prompt tok + per-beam detok) and the eos + // id (beam retirement); a use_beam_search request then routes through + // BeamSearchAsync (online.py) over the async engine. Without this, beam + // requests reject with "requires an engine and a tokenizer". + const std::optional beam_eos = + tokenizer.EosId() >= 0 + ? std::optional(tokenizer.EosId()) + : std::nullopt; + completion.set_beam_search_tokenizer(&tokenizer, beam_eos); + chat.set_beam_search_tokenizer(&tokenizer, beam_eos); + + // ── MM-SERVE-E2E: wire the multimodal chat seam for image-capable models. + // When the model dir carries a preprocessor_config.json the Qwen3-VL image + // processor loads, we construct the seam body (MakeQwen3VLImageChatFn) so an + // OpenAI image_url request renders the placeholder marker → tokenizes to the + // single image_pad id → EXPANDS to N image tokens + mm_features carried onto + // the engine request. A text-only model (no preprocessor_config.json) leaves + // the seam UNSET → the chat path is byte-identical. The container-format + // image codec (PNG/JPEG → RGB) is a NAMED residual: no codec is vendored, so + // the production codec rejects encoded images with a clear message (the M2c + // single-sequence gate consumes pre-decoded raw RGB). The mm FORWARD (vision + // tower + merge + MRoPE/DeepStack on the GPU worker consuming + // Request.mm_features) is the remaining MM-SERVE-E2E residual — the engine + // model runner has no mm-forward path yet. Kept alive for the server loop. + std::unique_ptr mm_image_proc; + const std::string preprocessor_config_path = + (dir / "preprocessor_config.json").string(); + if (fs::exists(preprocessor_config_path)) { + try { + vllm::multimodal::Qwen3VLProcessorConfig pcfg = + vllm::multimodal::LoadQwen3VLProcessorConfig( + preprocessor_config_path, config_path, served_model_name); + mm_image_proc = + std::make_unique(pcfg); + oai::ImageCodecFn codec = + [](const oai::DecodedMedia& media) -> oai::DecodedImageRgb { + // Raw-RGB passthrough (image/x-raw-rgb): the single-sequence e2e / + // gate fixture format. A square raw-RGB payload is decoded directly; + // any container format (PNG/JPEG) is the NAMED codec residual. + if (media.media_type == "image/x-raw-rgb") { + const std::size_t n = media.bytes.size(); + const std::size_t px = n / 3; + const auto side = + static_cast(std::llround(std::sqrt( + static_cast(px)))); + if (side <= 0 || static_cast(side * side * 3) != n) { + throw std::runtime_error( + "image/x-raw-rgb payload is not a square HxWx3 buffer"); + } + oai::DecodedImageRgb out; + out.rgb = media.bytes; + out.height = side; + out.width = side; + return out; + } + throw std::runtime_error( + "multimodal image: container-format decode (PNG/JPEG -> RGB) is a " + "named MM-SERVE residual; supply raw RGB (image/x-raw-rgb)"); + }; + chat.set_multimodal_chat_fn(oai::MakeQwen3VLImageChatFn( + *mm_image_proc, tokenizer, chat_prompt_fn, std::move(codec))); + std::cerr << "server: multimodal image seam wired (Qwen3-VL processor " + "from " + << preprocessor_config_path << ")\n"; + } catch (const std::exception& e) { + std::cerr << "server: no multimodal image seam (" << e.what() + << "); image requests fall back to the text path\n"; + } + } + + // Diagnostic opt-out exists only for same-binary attribution. Production + // defaults to the capacity-derived fixed pool. + const char* fixed_pool_env = std::getenv("VLLM_CPP_HTTP_FIXED_POOL"); + const auto worker_pool_mode = + fixed_pool_env != nullptr && std::string(fixed_pool_env) == "0" + ? oai::ApiServer::HttpWorkerPoolMode::kLegacyDynamic + : oai::ApiServer::HttpWorkerPoolMode::kCapacityFixed; + oai::ApiServer server(completion, chat, models, vllm::Version(), + static_cast(args.max_num_seqs), + worker_pool_mode); + + // ── C8 opt-in utility/admin endpoints (SERVE-UTILITY-ENDPOINTS / + // SERVE-ADMIN). Wire the setters from the LIVE engine + tokenizer through the + // single shared seam so the production server actually serves /tokenize, + // /detokenize, /tokenizer_info (flag) and /abort_requests (dev-mode flag), + // mirroring vLLM 0.26's per-endpoint default gating. /metrics and + // /reset_prefix_cache stay unwired (no live backing on the AsyncLLM path) — + // see ConfigureUtilityEndpoints + specs/{utility,admin}-endpoints.md. ──────── + // ── MiniMax-H3 video generation. OPT-IN: with no --video-dit the routes are + // never registered and the server is byte-identical to before. The whole + // pipeline lives in the LIBRARY seam (vllm::multimodal::MiniMaxH3VideoEngine, + // ARCH-ONE-SURFACE ROW 2) — the SAME entry point the C ABI's vllm_video_* + // and the minimax-h3-gen example drive, so HTTP and FFI cannot drift. This + // file keeps exactly what an example may own: flag plumbing, the job + // directory, and the ONE process spawn (ffmpeg, ratified 2026-08-03 — the + // library builds the argv and spawns nothing). ──────────────────────────── + std::shared_ptr video_engine; + if (!args.video_dit.empty()) { + std::cerr << "server: loading MiniMax-H3 video checkpoints...\n"; + vllm::multimodal::MiniMaxH3VideoModelParams vmp; + vmp.dit_path = args.video_dit; + vmp.encoder_path = args.video_encoder; + vmp.tokenizer_path = args.video_tokenizer; + vmp.video_vae_path = args.video_vae; + vmp.video_vae_config_path = args.video_vae_config; + vmp.audio_vae_path = args.audio_vae; + vmp.audio_vae_config_path = args.audio_vae_config; + vmp.prompt_embeds_path = args.video_prompt_embeds; + vmp.partition = args.video_partition; + vmp.device = args.video_device == "cuda" ? 1 : 0; + vmp.dequant_bf16 = args.video_dequant_bf16 ? 1 : 0; + vmp.encoder_max_layers = args.video_encoder_max_layers; + video_engine = vllm::multimodal::MiniMaxH3VideoEngine::Load(vmp); + std::cerr << "server: /v1/videos on (device=" << args.video_device + << (args.video_dequant_bf16 ? ", dequant-bf16" : ", keep-quant") << ")\n"; + // HONEST LIMIT, stated at startup rather than buried: turning a PROMPT + // into conditioning needs the H3-Encoder; without one every request is + // conditioned on the SAME supplied embeddings. + if (video_engine->has_encoder()) { + std::cerr << "server: /v1/videos conditions on the request PROMPT\n"; + } else if (!video_engine->has_prompt_embeds()) { + std::cerr << "server: WARNING /v1/videos has neither --video-encoder nor " + "--video-prompt-embeds; requests will be REJECTED\n"; + } else { + std::cerr << "server: WARNING /v1/videos ignores the request PROMPT — pass " + "--video-encoder to condition on it\n"; + } + + auto counter = std::make_shared>(0); + const std::string workdir = args.video_workdir; + const std::string ffmpeg = args.video_ffmpeg; + server.set_video_runner([video_engine, counter, workdir, + ffmpeg](const vllm::openai::VideoRequest& req) -> std::string { + const int64_t id = counter->fetch_add(1); + const std::string dir = workdir + "/job" + std::to_string(id); + // The library-owned request mapping + generation: conditioning, task + // resolution, the #77 partition guard, reference encoding, artifacts. + const vllm::multimodal::MiniMaxH3VideoResult out = video_engine->Generate( + vllm::multimodal::MiniMaxH3VideoGenParamsFromRequest(req, dir)); + // The ONE process spawn: exec the argv the library composed. + std::vector argv_mux = out.mux_argv; + if (!argv_mux.empty()) argv_mux[0] = ffmpeg; + const int status = RunFfmpegArgv(argv_mux); + if (status != 0) { + throw std::runtime_error("ffmpeg exited " + std::to_string(status)); + } + return out.mux_output_path; + }); + } + + oai::UtilityEndpointOptions endpoint_opts; + endpoint_opts.enable_tokenizer_info_endpoint = + args.enable_tokenizer_info_endpoint; + endpoint_opts.enable_server_dev_mode = args.enable_server_dev_mode; + oai::ConfigureUtilityEndpoints(server, tokenizer, loaded->max_model_len(), + engine, endpoint_opts); + std::cerr << "server: utility endpoints: /tokenize /detokenize on" + << (args.enable_tokenizer_info_endpoint ? ", /tokenizer_info on" + : "") + << (args.enable_server_dev_mode ? ", /abort_requests on (dev-mode)" + : "") + << "\n"; + + // Prometheus /metrics (Python vLLM always-on family names). + std::unique_ptr prom_logger; + if (args.enable_metrics) { + prom_logger = std::make_unique( + served_model_name, loaded->max_model_len(), /*engine_index=*/0); + // Sync engine path records on step(); async may under-report until fully wired. + loaded->engine().set_stat_logger(prom_logger.get()); + server.set_metrics_logger(prom_logger.get()); + std::cerr << "server: GET /metrics enabled (PrometheusStatLogger)\n"; + } + + std::cerr << "server: listening on http://" << args.host << ":" << args.port + << " (model '" << served_model_name << "', HTTP worker pool "; + if (server.http_worker_count() == 0) { + std::cerr << "legacy-dynamic"; + } else { + std::cerr << server.http_worker_count() << " fixed"; + } + std::cerr << ")\n"; + +#ifdef VT_BENCH_PROFILE_CONTROL + std::atomic benchmark_shutdown_waiter_ready{false}; + std::atomic benchmark_shutdown_received{false}; + std::atomic benchmark_shutdown_failed{false}; + std::atomic benchmark_shutdown_cancelled{false}; + std::thread benchmark_shutdown_thread; + if (args.cuda_profile_graph_replays > 0) { + benchmark_shutdown_thread = std::thread([&]() { + const int shutdown_fd = + open(args.benchmark_shutdown_fifo.c_str(), + O_RDONLY | O_NONBLOCK | O_CLOEXEC | O_NOFOLLOW); + if (shutdown_fd < 0) { + const int status = errno; + std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=open status=" + << status << "\n"; + benchmark_shutdown_failed.store(true, std::memory_order_release); + return; + } + struct stat shutdown_stat {}; + const int stat_status = fstat(shutdown_fd, &shutdown_stat); + if (stat_status != 0 || !S_ISFIFO(shutdown_stat.st_mode)) { + const int status = stat_status != 0 ? errno : EINVAL; + std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=fstat status=" + << status << "\n"; + close(shutdown_fd); + benchmark_shutdown_failed.store(true, std::memory_order_release); + return; + } + benchmark_shutdown_waiter_ready.store(true, std::memory_order_release); + std::cerr << "[VT_BENCH_SHUTDOWN] ready pid=" << getpid() + << " control=fifo\n"; + while (!benchmark_shutdown_cancelled.load(std::memory_order_acquire)) { + char command = '\0'; + const ssize_t bytes = read(shutdown_fd, &command, 1); + if (bytes == 1) { + if (command == 'Q') { + benchmark_shutdown_received.store(true, + std::memory_order_release); + std::cerr + << "[VT_BENCH_SHUTDOWN] requested control=fifo\n"; + close(shutdown_fd); + server.stop(); + return; + } + std::cerr + << "[VT_BENCH_SHUTDOWN] failed operation=command status=" + << static_cast( + static_cast(command)) + << "\n"; + close(shutdown_fd); + benchmark_shutdown_failed.store(true, std::memory_order_release); + server.stop(); + return; + } + if (bytes < 0 && errno != EAGAIN && errno != EWOULDBLOCK && + errno != EINTR) { + const int status = errno; + std::cerr << "[VT_BENCH_SHUTDOWN] failed operation=read status=" + << status << "\n"; + close(shutdown_fd); + benchmark_shutdown_failed.store(true, std::memory_order_release); + server.stop(); + return; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + close(shutdown_fd); + }); + while (!benchmark_shutdown_waiter_ready.load(std::memory_order_acquire) && + !benchmark_shutdown_failed.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + if (benchmark_shutdown_failed.load(std::memory_order_acquire)) { + benchmark_shutdown_cancelled.store(true, std::memory_order_release); + benchmark_shutdown_thread.join(); + return 1; + } + } +#endif + + const bool listen_ok = server.listen(args.host, args.port); + +#ifdef VT_BENCH_PROFILE_CONTROL + if (benchmark_shutdown_thread.joinable()) { + benchmark_shutdown_cancelled.store(true, std::memory_order_release); + benchmark_shutdown_thread.join(); + if (benchmark_shutdown_received.load(std::memory_order_acquire)) { + std::cerr << "[VT_BENCH_SHUTDOWN] completed control=fifo\n"; + } else { + if (!benchmark_shutdown_failed.load(std::memory_order_acquire)) { + std::cerr + << "[VT_BENCH_SHUTDOWN] failed operation=cancelled status=0\n"; + } + if (listen_ok) { + return 1; + } + } + } +#endif + + if (!listen_ok) { + std::cerr << "server: failed to bind " << args.host << ":" << args.port + << "\n"; + return 1; + } + return 0; + } catch (const std::exception& e) { + std::cerr << "server: fatal: " << e.what() << "\n"; + return 1; + } +} + +} // namespace openai +} // namespace entrypoints +} // namespace vllm diff --git a/src/vllm/entrypoints/openai/server_main.h b/src/vllm/entrypoints/openai/server_main.h new file mode 100644 index 000000000..7cfe00bf6 --- /dev/null +++ b/src/vllm/entrypoints/openai/server_main.h @@ -0,0 +1,35 @@ +// vllm.cpp original. The OpenAI server's ENTRY POINT, owned by the library. +// +// ARCH-ONE-SURFACE: `examples/server` used to construct the engine, the serving +// layers, metrics, the video seam and the ASR seam directly from 36 internal +// headers -- the deepest breach of the ONE SURFACE directive +// (.agents/specs/one-surface-abi.md), and the reason the example carried an +// entry in scripts/example-abi-allowlist.txt. The construction moved HERE +// verbatim; the example is now a thin client of the C ABI's `vllm_server_main`, +// which wraps this. +// +// argv rather than a params STRUCT is deliberate. The server takes ~57 flags and +// grows more with every serving feature; a mirrored C struct would put that +// churn in the ABI, where every field is permanent. The flag surface is already +// specified by vLLM's cli_args.py, which is the contract this mirrors, so argv +// IS the stable interface. Embedders that want programmatic control keep the +// granular entry points (vllm_engine_load / vllm_chat / vllm_video_generate / +// vllm_transcribe) -- this one exists to RUN THE SERVER. +#ifndef VLLM_ENTRYPOINTS_OPENAI_SERVER_MAIN_H_ +#define VLLM_ENTRYPOINTS_OPENAI_SERVER_MAIN_H_ + +namespace vllm { +namespace entrypoints { +namespace openai { + +// Parse `argv` and run the OpenAI-compatible server until it exits. Returns the +// process exit code (0 on clean shutdown). Prints usage and returns 0 for +// `--help`, and returns non-zero after printing the reason on a bad argument or +// a startup failure -- it does NOT throw across the C ABI boundary. +int VllmServerMain(int argc, char** argv); + +} // namespace openai +} // namespace entrypoints +} // namespace vllm + +#endif // VLLM_ENTRYPOINTS_OPENAI_SERVER_MAIN_H_