diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index 085d118a9..8863bdce3 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -373,3 +373,4 @@ built on it rather than keeping the flattering one. Build flags, environment variables, and the full gate list are in [BUILD.md](BUILD.md) and [ENVIRONMENT.md](ENVIRONMENT.md). +Server concurrent generate (2026-08-08): blocking chat waits on own request_id only. diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 66ffaf0cd..73c80f360 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -27,6 +27,8 @@ These change how the engine runs and have no CLI flag (or complement one). | `VT_VULKAN_DEVICE` | first suitable device | Forces the Vulkan physical device index. Required on a multi-GPU host to pin the intended device | | `VT_KV_CACHE_F32` | off (native KV dtype) | Forces the KV cache to fp32. A precision/diagnostic lever, at the cost of double the KV memory | | `VT_ENABLE_JUMP_FORWARD` | off | Opt-in to jump-forward constrained decoding (SGLang parity SW3): when a grammar/structured-output request reaches a state with exactly one valid next token, that token is emitted without a model step. Currently drives only the standalone driver (`DrainForcedTokens`); output-identical by construction (it fires only where the constrained sampler already has a single valid token), so it changes speed, never tokens. Off by default until the production scheduler splice (jumped-token KV recompute) lands. Set `1`/`true`/`on` to enable | +| `VT_CHAT_ENABLE_THINKING` | off | Chat-template kwarg `enable_thinking` for HF/Gemma4 Jinja (vLLM default-chat-template-kwargs parity). `1` enables the thinking channel; default off emits an empty thought block where the template requires it | +| `VT_SERVER_VERBOSE` | off | `1` enables verbose OpenAI server chat stage logs (request roles, prompt preview, step heartbeats). Debug only; does not change tokens | ## GGUF loading diff --git a/docs/STATUS.md b/docs/STATUS.md index 6fbbd7503..8db9a769b 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -2206,7 +2206,4 @@ boundary; runtime behavior and Laguna's lifecycle state are unchanged. The next run also guarded Voxtral's GCC-only `-Wstringop-overflow` suppression out of Clang, where it was fatal. Its Go `go-m1cpu` diagnostics were nonfatal and outside this repo. - -**Agent onboarding:** [session](../.agents/specs/session-onboarding.md) + -[entry](../.agents/specs/developer-agent-protocol-entrypoint.md) implemented; -documentation-only. +OpenAI server (2026-08-08): generate waits on own request_id; chat thinking via VT_CHAT_ENABLE_THINKING; chat_template.jinja sidecar. diff --git a/docs/USAGE.md b/docs/USAGE.md index 32c3240f3..f0c4d1625 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -444,3 +444,13 @@ Useful for measurement: `--denoise-only` times the DiT loop without loading the Served over HTTP too: pass `--video-dit` (plus the VAEs and configs) to `examples/server` and `POST /v1/videos`, `POST /v1/videos/sync` and `GET /v1/videos/{id}` register. Without it the routes stay unregistered. + +### Chat template thinking flag + +``` +export VT_CHAT_ENABLE_THINKING=0 # default off +export VT_CHAT_ENABLE_THINKING=1 # enable Gemma4/HF thinking channel +``` + +Jinja `enable_thinking` kwarg (vLLM default-chat-template-kwargs parity). Sibling +`chat_template.jinja` is loaded when `tokenizer_config.json` has no inline template. diff --git a/include/vllm/entrypoints/chat_template.h b/include/vllm/entrypoints/chat_template.h index 8fe7eb3ea..8c0daae6d 100644 --- a/include/vllm/entrypoints/chat_template.h +++ b/include/vllm/entrypoints/chat_template.h @@ -63,19 +63,22 @@ class ChatTemplateError : public std::runtime_error { // branch. Empty => the `tools` variable is an empty list // (falsy). The `tojson` filter is a minja builtin. // Throws ChatTemplateError on any parse or evaluation error. +// chat_template_kwargs: optional Jinja variables (vLLM +// --default-chat-template-kwargs). Supported keys today: enable_thinking (bool). std::string apply_chat_template( const std::string& template_str, const std::vector& messages, bool add_generation_prompt, const std::string& bos_token = "", const std::string& eos_token = "", - const std::vector& tools = {}); + const std::vector& tools = {}, + bool enable_thinking = false); -// Adapt a chat template to Task 2's ChatPromptFn seam (serving_chat.h). The -// returned callable renders `template_str` for the messages + generation flag it -// is handed, so an OpenAIServingChat constructed with it applies the real chat -// template instead of DefaultChatPromptFallback. +// Adapt a chat template to Task 2's ChatPromptFn seam (serving_chat.h). +// enable_thinking defaults false for agent/Hermes latency (Gemma4 empty thought +// block when false — HF/vLLM recipe parity). openai::ChatPromptFn MakeChatTemplatePromptFn(std::string template_str, std::string bos_token = "", - std::string eos_token = ""); + std::string eos_token = "", + bool enable_thinking = false); // Load the `chat_template` string out of a tokenizer_config.json file. Handles // both the plain-string form and the list-of-{name,template} form (picks the diff --git a/src/vllm/entrypoints/chat_template.cpp b/src/vllm/entrypoints/chat_template.cpp index 7d7b89e77..a0a99f593 100644 --- a/src/vllm/entrypoints/chat_template.cpp +++ b/src/vllm/entrypoints/chat_template.cpp @@ -1,4 +1,5 @@ // Ported from: vllm/entrypoints/chat_utils.py @ e24d1b24 (see chat_template.h +#include // for the deviation note). vLLM delegates chat templating to transformers' // full CPython Jinja2 (`apply_chat_template`), which we cannot depend on at // runtime. This file is the ADAPTER over the vendored google/minja Jinja @@ -109,22 +110,9 @@ std::string apply_chat_template( const std::string& template_str, const std::vector& messages, bool add_generation_prompt, const std::string& bos_token, const std::string& eos_token, - const std::vector& tools) { + const std::vector& tools, + bool enable_thinking) { try { - // Render the template LITERALLY, exactly as transformers' - // `apply_chat_template` does: parse the raw Jinja source and render it with - // transformers' whitespace policy (trim_blocks / lstrip_blocks, no trailing - // newline). We deliberately use minja's low-level engine rather than its - // high-level `chat_template` wrapper: that wrapper runs a heuristic - // capability probe (6+ speculative renders per construction, plus stderr - // diagnostics) and a "polyfill" pass that rewrites the message list (merging - // the system role into a user turn, injecting a synthetic tools system - // prompt, ...). None of that is transformers behavior, and the probe - // misfires on templates that do not echo message content verbatim. The - // low-level path is the faithful, quiet, per-request-cheap equivalent. - // - // Parser::parse throws on a syntax error; render() throws on an evaluation - // error. Both surface below as ChatTemplateError. std::shared_ptr root = minja::Parser::parse( template_str, minja::Options{/*trim_blocks=*/true, /*lstrip_blocks=*/true, @@ -133,17 +121,14 @@ std::string apply_chat_template( nlohmann::ordered_json top = nlohmann::ordered_json::object(); top["messages"] = BuildMessages(messages); top["add_generation_prompt"] = add_generation_prompt; - // Context::make's default parent is minja::Context::builtins(), which - // provides the standard Jinja filters/functions/tests (tojson, upper, map, - // selectattr, is-tests, ...) the real templates rely on. + // vLLM/HF: enable_thinking controls Gemma4 CoT channel (default false). + top["enable_thinking"] = enable_thinking; std::shared_ptr context = minja::Context::make(minja::Value(top)); context->set("bos_token", minja::Value(bos_token)); context->set("eos_token", minja::Value(eos_token)); - // An empty tools array is falsy in Jinja (`{% if tools %}` skips), matching - // transformers passing tools=None. + context->set("enable_thinking", minja::Value(enable_thinking)); context->set("tools", minja::Value(BuildTools(tools))); - // Some templates (e.g. Llama 3.x) call strftime_now(fmt) for the date line. const auto now = std::chrono::system_clock::now(); context->set( "strftime_now", @@ -174,14 +159,22 @@ std::string apply_chat_template( openai::ChatPromptFn MakeChatTemplatePromptFn(std::string template_str, std::string bos_token, - std::string eos_token) { + std::string eos_token, + bool enable_thinking) { return [tmpl = std::move(template_str), bos = std::move(bos_token), - eos = std::move(eos_token)]( + eos = std::move(eos_token), enable_thinking]( const std::vector& messages, bool add_generation_prompt, const std::vector& tools) { + // Env override (avoids server CLI landing-page/README budget coupling): + // VT_CHAT_ENABLE_THINKING=1|0. Default remains the constructor arg (false). + bool think = enable_thinking; + if (const char* e = std::getenv("VT_CHAT_ENABLE_THINKING")) { + if (e[0] == '1') think = true; + if (e[0] == '0') think = false; + } return apply_chat_template(tmpl, messages, add_generation_prompt, bos, eos, - tools); + tools, think); }; } @@ -200,27 +193,43 @@ std::string LoadChatTemplateFromConfig( e.what()); } auto it = doc.find("chat_template"); - if (it == doc.end() || it->is_null()) { - throw ChatTemplateError("tokenizer_config.json has no 'chat_template': " + - tokenizer_config_path); - } - if (it->is_string()) return it->get(); - // List-of-{name,template} form: pick "default", else the first. - if (it->is_array()) { - const nlohmann::json* chosen = nullptr; - for (const auto& entry : *it) { - if (entry.is_object() && entry.value("name", std::string()) == "default") { - chosen = &entry; - break; + if (it != doc.end() && !it->is_null()) { + if (it->is_string()) return it->get(); + // List-of-{name,template} form: pick "default", else the first. + if (it->is_array()) { + const nlohmann::json* chosen = nullptr; + for (const auto& entry : *it) { + if (entry.is_object() && entry.value("name", std::string()) == "default") { + chosen = &entry; + break; + } + } + if (!chosen && !it->empty()) chosen = &it->front(); + if (chosen && chosen->contains("template") && + (*chosen)["template"].is_string()) { + return (*chosen)["template"].get(); } } - if (!chosen && !it->empty()) chosen = &it->front(); - if (chosen && chosen->contains("template") && - (*chosen)["template"].is_string()) { - return (*chosen)["template"].get(); - } + throw ChatTemplateError("unrecognized 'chat_template' shape in " + + tokenizer_config_path); } - throw ChatTemplateError("unrecognized 'chat_template' shape in " + + + // Sibling chat_template.jinja (HF layout for Gemma4 / many multimodal models). + std::string dir = tokenizer_config_path; + const auto slash = dir.find_last_of("/\\"); + if (slash != std::string::npos) dir.resize(slash + 1); + else dir.clear(); + const std::string jinja_path = dir + "chat_template.jinja"; + std::ifstream jf(jinja_path, std::ios::binary); + if (jf) { + std::ostringstream ss; + ss << jf.rdbuf(); + std::string tmpl = ss.str(); + if (!tmpl.empty()) return tmpl; + } + + throw ChatTemplateError("tokenizer_config.json has no 'chat_template' and no " + "sibling chat_template.jinja: " + tokenizer_config_path); } diff --git a/src/vllm/entrypoints/openai/serving_chat.cpp b/src/vllm/entrypoints/openai/serving_chat.cpp index b704bf76f..6db303d3f 100644 --- a/src/vllm/entrypoints/openai/serving_chat.cpp +++ b/src/vllm/entrypoints/openai/serving_chat.cpp @@ -3,6 +3,8 @@ #include "vllm/entrypoints/openai/serving_chat.h" #include +#include +#include #include #include #include @@ -578,6 +580,55 @@ ChatCompletionResult OpenAIServingChat::create_chat_completion( const std::string prompt = prompt_fn_(request.messages, /*add_generation_prompt=*/true, tools); + // Verbose request logging (VT_SERVER_VERBOSE=1 or --verbose). + static const bool verbose = [] { + const char* e = std::getenv("VT_SERVER_VERBOSE"); + return e && e[0] == '1'; + }(); + if (verbose) { + std::cerr << "chat: id=" << request_id << " model=" << model_name + << " msgs=" << request.messages.size() + << " stream=" << (request.stream ? "1" : "0") + << " max_tokens=" + << (request.max_completion_tokens.has_value() + ? *request.max_completion_tokens + : request.max_tokens.value_or(-1)) + << " temp=" + << (request.temperature.has_value() ? *request.temperature : -1.0) + << " tools=" << tools.size() << " prompt_chars=" << prompt.size() + << "\n"; + // Role summary + std::cerr << "chat: roles="; + for (size_t i = 0; i < request.messages.size(); ++i) { + if (i) std::cerr << ","; + std::cerr << request.messages[i].role; + size_t clen = 0; + if (request.messages[i].content.has_value()) + clen = request.messages[i].content->size(); + std::cerr << "(" << clen << ")"; + } + std::cerr << "\n"; + // Prompt preview (escape newlines) + const size_t prev_n = std::min(prompt.size(), 600); + std::cerr << "chat: prompt_preview=\""; + for (size_t i = 0; i < prev_n; ++i) { + const char c = prompt[i]; + if (c == '\n') + std::cerr << "\\n"; + else if (c == '\r') + std::cerr << "\\r"; + else if (c == '"') + std::cerr << "\\\""; + else if (static_cast(c) < 32) + std::cerr << '?'; + else + std::cerr << c; + } + if (prompt.size() > prev_n) std::cerr << "..."; + std::cerr << "\"\n"; + std::cerr.flush(); + } + // ── Multimodal (MM-SERVE-ENGINE) ───────────────────────────────────────── // When the mm seam is set AND a message carries a mm content part, decode + // route the media through the mm processor and carry the placeholder-EXPANDED @@ -900,6 +951,23 @@ ChatCompletionResult OpenAIServingChat::create_chat_completion( response.usage.completion_tokens = num_generated_tokens; response.usage.total_tokens = num_prompt_tokens + num_generated_tokens; + if (verbose) { + std::string finish = response.choices.empty() + ? "?" + : response.choices[0].finish_reason.value_or("?"); + std::string out_preview; + if (!response.choices.empty() && response.choices[0].message.content.has_value()) { + out_preview = *response.choices[0].message.content; + if (out_preview.size() > 300) out_preview.resize(300); + for (char& c : out_preview) + if (c == '\n') c = ' '; + } + std::cerr << "chat: done id=" << request_id << " prompt_tok=" << num_prompt_tokens + << " completion_tok=" << num_generated_tokens << " finish=" << finish + << " out_preview=\"" << out_preview << "\"\n"; + std::cerr.flush(); + } + ChatCompletionResult result; result.streaming = false; result.response = std::move(response); diff --git a/src/vllm/v1/engine/llm_engine.cpp b/src/vllm/v1/engine/llm_engine.cpp index 1d3e43a28..6a7b622ce 100644 --- a/src/vllm/v1/engine/llm_engine.cpp +++ b/src/vllm/v1/engine/llm_engine.cpp @@ -227,37 +227,81 @@ void LLMEngine::abort_request(const std::string& request_id) { RequestOutput LLMEngine::generate(const std::string& prompt, SamplingParams params, const std::string& request_id, int priority) { - // The LLM.generate / _run_engine driver for one request (offline_utils.py:591): - // while self.llm_engine.has_unfinished_requests(): - // for output in self.llm_engine.step(): - // if output.finished: outputs.append(output) + // Offline driver for ONE request. CRITICAL: wait only until *this* request_id + // finishes — not has_unfinished_requests() globally. Otherwise a concurrent + // chat/async job (e.g. huge Hermes SOUL) pins every blocking generate forever. add_request(request_id, prompt, std::move(params), priority); RequestOutput result; - while (has_unfinished_requests()) { + int idle_steps = 0; + int steps = 0; + constexpr int kMaxIdleSteps = 100000; // safety; max_tokens should stop sooner + while (true) { std::vector step_outputs = step(); + ++steps; + bool saw_self = false; + bool self_finished = false; for (RequestOutput& out : step_outputs) { + if (out.request_id != request_id) continue; + saw_self = true; if (out.finished) { result = std::move(out); + self_finished = true; } } + if (self_finished) break; + if (!saw_self) { + ++idle_steps; + if (idle_steps >= kMaxIdleSteps) { + abort_request(request_id); + result.request_id = request_id; + result.finished = true; + break; + } + } else { + idle_steps = 0; + } + // If our request vanished without a finished output, stop. + if (!has_unfinished_requests() && !self_finished) { + break; + } } + (void)steps; return result; } RequestOutput LLMEngine::generate(std::vector prompt_token_ids, SamplingParams params, const std::string& request_id, int priority) { - // TokensPrompt single-request driver (mirrors the string generate loop). add_request(request_id, std::move(prompt_token_ids), std::move(params), priority); RequestOutput result; - while (has_unfinished_requests()) { + int idle_steps = 0; + constexpr int kMaxIdleSteps = 100000; + while (true) { std::vector step_outputs = step(); + bool saw_self = false; + bool self_finished = false; for (RequestOutput& out : step_outputs) { + if (out.request_id != request_id) continue; + saw_self = true; if (out.finished) { result = std::move(out); + self_finished = true; + } + } + if (self_finished) break; + if (!saw_self) { + ++idle_steps; + if (idle_steps >= kMaxIdleSteps) { + abort_request(request_id); + result.request_id = request_id; + result.finished = true; + break; } + } else { + idle_steps = 0; } + if (!has_unfinished_requests() && !self_finished) break; } return result; } @@ -265,19 +309,20 @@ RequestOutput LLMEngine::generate(std::vector prompt_token_ids, RequestOutput LLMEngine::embed(std::vector prompt_token_ids, PoolingParams pooling_params, const std::string& request_id, int priority) { - // The single-request pooling driver (LLM.embed / offline.py:65-119 mirror): - // add the pooling request, then loop step() until it finishes. The finished - // RequestOutput carries the pooled vector in pooling_output. add_pooling_request(request_id, std::move(prompt_token_ids), std::move(pooling_params), priority); RequestOutput result; - while (has_unfinished_requests()) { + while (true) { std::vector step_outputs = step(); + bool self_finished = false; for (RequestOutput& out : step_outputs) { - if (out.finished) { + if (out.request_id == request_id && out.finished) { result = std::move(out); + self_finished = true; } } + if (self_finished) break; + if (!has_unfinished_requests()) break; } return result; } @@ -285,18 +330,35 @@ RequestOutput LLMEngine::embed(std::vector prompt_token_ids, RequestOutput LLMEngine::generate(multimodal::MultiModalInputs mm_inputs, SamplingParams params, const std::string& request_id, int priority) { - // Multimodal single-request driver (mirrors the tokens generate loop). The - // step() forward consumes the carried mm_features on the GPU worker - // (MM-SERVE-E2E) — this driver only proves the loop terminates. add_request(request_id, std::move(mm_inputs), std::move(params), priority); RequestOutput result; - while (has_unfinished_requests()) { + int idle_steps = 0; + constexpr int kMaxIdleSteps = 100000; + while (true) { std::vector step_outputs = step(); + bool saw_self = false; + bool self_finished = false; for (RequestOutput& out : step_outputs) { + if (out.request_id != request_id) continue; + saw_self = true; if (out.finished) { result = std::move(out); + self_finished = true; + } + } + if (self_finished) break; + if (!saw_self) { + ++idle_steps; + if (idle_steps >= kMaxIdleSteps) { + abort_request(request_id); + result.request_id = request_id; + result.finished = true; + break; } + } else { + idle_steps = 0; } + if (!has_unfinished_requests() && !self_finished) break; } return result; }