diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index fe9b8254a19b..772334e6cf14 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -821,7 +821,7 @@ def serve( f"Cannot auto-detect tool parser for model '{model}'. " f"Supported model types for auto-detection: qwen2, qwen3, " f"qwen3_moe, qwen3_5, qwen3_5_moe, qwen3_next, deepseek_v3, " - f"deepseek_v32, kimi_k2, kimi_k25, glm4. " + f"deepseek_v32, deepseek_v4, kimi_k2, kimi_k25, glm4. " f"Please specify a parser explicitly: " f"{list(ToolParserFactory.parsers.keys())}", param_hint="--tool_parser") @@ -835,7 +835,7 @@ def serve( f"Cannot auto-detect reasoning parser for model '{model}'. " f"Supported model types for auto-detection: qwen3, qwen3_moe, " f"qwen3_5, qwen3_5_moe, qwen3_next, deepseek_v3 (R1 only), " - f"deepseek_v32 (R1 only), nemotron_h. " + f"deepseek_v32 (R1 only), deepseek_v4, nemotron_h. " f"Please specify a parser explicitly: " f"{list(ReasoningParserFactory.keys())}", param_hint="--reasoning_parser") diff --git a/tensorrt_llm/llmapi/reasoning_parser.py b/tensorrt_llm/llmapi/reasoning_parser.py index ded97ba00992..b2c9c1fa4242 100644 --- a/tensorrt_llm/llmapi/reasoning_parser.py +++ b/tensorrt_llm/llmapi/reasoning_parser.py @@ -80,6 +80,19 @@ def finish(self) -> ReasoningParserResult: return ReasoningParserResult() +class IdentityReasoningParser(BaseReasoningParser): + """Reasoning parser that treats all model output as visible content.""" + + reasoning_start = "" + reasoning_end = "" + + def parse(self, text: str) -> ReasoningParserResult: + return ReasoningParserResult(content=text) + + def parse_delta(self, delta_text: str) -> ReasoningParserResult: + return ReasoningParserResult(content=delta_text) + + @register_reasoning_parser("deepseek-r1", reasoning_at_start=True) @register_reasoning_parser("qwen3") @register_reasoning_parser("minimax_m2", reasoning_at_start=True) @@ -173,6 +186,42 @@ def parse_delta(self, delta_text: str) -> ReasoningParserResult: "Unreachable code reached in `DeepSeekR1Parser.parse_delta`") +@register_reasoning_parser("deepseek_v4") +class DeepSeekV4ReasoningParser(BaseReasoningParser): + """DeepSeek-V4 parser selected by thinking-mode chat template kwargs.""" + + reasoning_start = "" + reasoning_end = "" + + def __init__( + self, + *, + chat_template_kwargs: Optional[dict[str, Any]] = None, + ) -> None: + super().__init__(chat_template_kwargs=chat_template_kwargs) + chat_template_kwargs = chat_template_kwargs or {} + thinking = bool( + chat_template_kwargs.get("thinking", False) + or chat_template_kwargs.get("enable_thinking", False)) + if thinking: + self._parser = DeepSeekR1Parser( + reasoning_at_start=True, + chat_template_kwargs=chat_template_kwargs, + ) + else: + self._parser = IdentityReasoningParser( + chat_template_kwargs=chat_template_kwargs) + + def parse(self, text: str) -> ReasoningParserResult: + return self._parser.parse(text) + + def parse_delta(self, delta_text: str) -> ReasoningParserResult: + return self._parser.parse_delta(delta_text) + + def finish(self) -> ReasoningParserResult: + return self._parser.finish() + + MODEL_TYPE_TO_REASONING_PARSER: dict[str, str] = { "qwen3": "qwen3", "qwen3_moe": "qwen3", @@ -181,6 +230,7 @@ def parse_delta(self, delta_text: str) -> ReasoningParserResult: "qwen3_next": "qwen3", "deepseek_v3": "deepseek-r1", "deepseek_v32": "deepseek-r1", + "deepseek_v4": "deepseek_v4", "nemotron_h": "nano-v3", } diff --git a/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py b/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py index 25c49ae2a2cb..3362ab7cb68a 100644 --- a/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py +++ b/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py @@ -133,7 +133,7 @@ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult try: # Extract content between function_calls tags function_calls_match = re.search( - r"<｜DSML｜function_calls>(.*?)", + re.escape(self.bot_token) + r"(.*?)" + re.escape(self.eot_token), text, re.DOTALL, ) diff --git a/tensorrt_llm/serve/tool_parser/deepseekv4_parser.py b/tensorrt_llm/serve/tool_parser/deepseekv4_parser.py new file mode 100644 index 000000000000..05896561f839 --- /dev/null +++ b/tensorrt_llm/serve/tool_parser/deepseekv4_parser.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .deepseekv32_parser import DeepSeekV32Parser + + +class DeepSeekV4Parser(DeepSeekV32Parser): + """Tool parser for the DeepSeek V4 DSML tool call format.""" + + def __init__(self) -> None: + super().__init__() + self.bot_token = "<｜DSML｜tool_calls>" # nosec B105 + self.eot_token = "" # nosec B105 diff --git a/tensorrt_llm/serve/tool_parser/tool_parser_factory.py b/tensorrt_llm/serve/tool_parser/tool_parser_factory.py index 1d6495d1424f..0886caa2385f 100644 --- a/tensorrt_llm/serve/tool_parser/tool_parser_factory.py +++ b/tensorrt_llm/serve/tool_parser/tool_parser_factory.py @@ -4,6 +4,7 @@ from .base_tool_parser import BaseToolParser from .deepseekv3_parser import DeepSeekV3Parser +from .deepseekv4_parser import DeepSeekV4Parser from .deepseekv31_parser import DeepSeekV31Parser from .deepseekv32_parser import DeepSeekV32Parser from .glm4_parser import Glm4ToolParser @@ -21,6 +22,7 @@ "qwen3_next": "qwen3", "deepseek_v3": "deepseek_v3", "deepseek_v32": "deepseek_v32", + "deepseek_v4": "deepseek_v4", "kimi_k2": "kimi_k2", "kimi_k25": "kimi_k2", "glm4": "glm4", @@ -48,6 +50,7 @@ class ToolParserFactory: "deepseek_v3": DeepSeekV3Parser, "deepseek_v31": DeepSeekV31Parser, "deepseek_v32": DeepSeekV32Parser, + "deepseek_v4": DeepSeekV4Parser, "glm4": Glm4ToolParser, "minimax_m2": MiniMaxM2ToolParser, } diff --git a/tensorrt_llm/tokenizer/deepseek_v4/tokenizer.py b/tensorrt_llm/tokenizer/deepseek_v4/tokenizer.py index 9149d229dd55..a5600816b072 100644 --- a/tensorrt_llm/tokenizer/deepseek_v4/tokenizer.py +++ b/tensorrt_llm/tokenizer/deepseek_v4/tokenizer.py @@ -12,7 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# ruff: noqa: E501 +import copy +import json from pathlib import Path from typing import Any @@ -24,7 +27,64 @@ EOS_TOKEN = "<｜end▁of▁sentence｜>" # nosec B105 USER_TOKEN = "<｜User｜>" # nosec B105 ASSISTANT_TOKEN = "<｜Assistant｜>" # nosec B105 +LATEST_REMINDER_TOKEN = "<｜latest_reminder｜>" # nosec B105 +THINKING_START_TOKEN = "" # nosec B105 THINKING_END_TOKEN = "" # nosec B105 +DSML_TOKEN = "｜DSML｜" # nosec B105 + +TOOL_CALLS_BLOCK_NAME = "tool_calls" +VALID_TASKS = { + "action": "<｜action｜>", + "query": "<｜query｜>", + "authority": "<｜authority｜>", + "domain": "<｜domain｜>", + "title": "<｜title｜>", + "read_url": "<｜read_url｜>", +} + +REASONING_EFFORT_MAX = ( + "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" + "You MUST be very thorough in your thinking and comprehensively decompose " + "the problem to resolve the root cause, rigorously stress-testing your " + "logic against all potential paths, edge cases, and adversarial scenarios.\n" + "Explicitly write out your entire deliberation process, documenting every " + "intermediate step, considered alternative, and rejected hypothesis to " + "ensure absolutely no assumption is left unchecked.\n\n" +) + +TOOLS_TEMPLATE = """## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following: + +<{dsml_token}tool_calls> +<{dsml_token}invoke name="$TOOL_NAME"> +<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<{dsml_token}invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response. + +Otherwise, output directly after {thinking_end_token} with tool calls or final response. + +### Available Tool Schemas + +{tool_schemas} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +""" + +RESPONSE_FORMAT_TEMPLATE = ( + "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}" +) +TOOL_CALL_TEMPLATE = '<{dsml_token}invoke name="{name}">\n{arguments}\n' +TOOL_CALLS_TEMPLATE = "<{dsml_token}{block_name}>\n{tool_calls}\n" +TOOL_OUTPUT_TEMPLATE = "{content}" def _message_content_to_text(content: Any) -> str: @@ -43,6 +103,310 @@ def _message_content_to_text(content: Any) -> str: return str(content) +def _to_json(value: Any) -> str: + try: + return json.dumps(value, ensure_ascii=False) + except TypeError: + return json.dumps(value, ensure_ascii=True) + + +def _tools_from_openai_format(tools: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [tool["function"] for tool in tools] + + +def _tool_calls_from_openai_format(tool_calls: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + { + "name": tool_call["function"]["name"], + "arguments": tool_call["function"]["arguments"], + } + for tool_call in tool_calls + ] + + +def _encode_arguments_to_dsml(tool_call: dict[str, Any]) -> str: + raw_args = tool_call["arguments"] + arguments = json.loads(raw_args) if isinstance(raw_args, str) else raw_args + if not isinstance(arguments, dict): + raise ValueError("DeepSeek-V4 tool call arguments must be a JSON object.") + + parameters = [] + for key, value in arguments.items(): + parameters.append( + f'<{DSML_TOKEN}parameter name="{key}" ' + f'string="{"true" if isinstance(value, str) else "false"}">' + f"{value if isinstance(value, str) else _to_json(value)}" + ) + return "\n".join(parameters) + + +def _render_tools(tools: list[dict[str, Any]]) -> str: + return TOOLS_TEMPLATE.format( + tool_schemas="\n".join(_to_json(tool) for tool in tools), + dsml_token=DSML_TOKEN, + thinking_start_token=THINKING_START_TOKEN, + thinking_end_token=THINKING_END_TOKEN, + ) + + +def _find_last_user_index(messages: list[dict[str, Any]]) -> int: + for index in range(len(messages) - 1, -1, -1): + if messages[index].get("role") in ("user", "developer"): + return index + return -1 + + +def _merge_tool_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + merged: list[dict[str, Any]] = [] + for message in messages: + message = copy.deepcopy(message) + role = message.get("role") + + if role == "tool": + tool_block = { + "type": "tool_result", + "tool_use_id": message.get("tool_call_id", ""), + "content": message.get("content", ""), + } + if merged and merged[-1].get("role") == "user" and "content_blocks" in merged[-1]: + merged[-1]["content_blocks"].append(tool_block) + else: + merged.append({"role": "user", "content_blocks": [tool_block]}) + elif role == "user": + text_block = {"type": "text", "text": _message_content_to_text(message.get("content"))} + if ( + merged + and merged[-1].get("role") == "user" + and "content_blocks" in merged[-1] + and merged[-1].get("task") is None + ): + merged[-1]["content_blocks"].append(text_block) + else: + new_message = { + "role": "user", + "content": message.get("content", ""), + "content_blocks": [text_block], + } + for key in ("task", "wo_eos", "mask"): + if key in message: + new_message[key] = message[key] + merged.append(new_message) + else: + merged.append(message) + + return merged + + +def _sort_tool_results_by_call_order(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + last_tool_call_order: dict[str, int] = {} + + for message in messages: + if message.get("role") == "assistant" and message.get("tool_calls"): + last_tool_call_order = {} + for index, tool_call in enumerate(message["tool_calls"]): + tool_call_id = tool_call.get("id") or tool_call.get("function", {}).get("id", "") + if tool_call_id: + last_tool_call_order[tool_call_id] = index + elif message.get("role") == "user" and message.get("content_blocks"): + tool_blocks = [ + block for block in message["content_blocks"] if block.get("type") == "tool_result" + ] + if len(tool_blocks) > 1 and last_tool_call_order: + sorted_blocks = sorted( + tool_blocks, + key=lambda block: last_tool_call_order.get(block.get("tool_use_id", ""), 0), + ) + sorted_index = 0 + new_blocks = [] + for block in message["content_blocks"]: + if block.get("type") == "tool_result": + new_blocks.append(sorted_blocks[sorted_index]) + sorted_index += 1 + else: + new_blocks.append(block) + message["content_blocks"] = new_blocks + + return messages + + +def _drop_thinking_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + last_user_index = _find_last_user_index(messages) + result = [] + keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"} + + for index, message in enumerate(messages): + role = message.get("role") + if role in keep_roles or index >= last_user_index: + result.append(message) + elif role == "assistant": + message_without_reasoning = copy.copy(message) + message_without_reasoning.pop("reasoning", None) + message_without_reasoning.pop("reasoning_content", None) + result.append(message_without_reasoning) + + return result + + +def _render_user_content(message: dict[str, Any]) -> str: + content_blocks = message.get("content_blocks") + if not content_blocks: + return _message_content_to_text(message.get("content")) + + parts = [] + for block in content_blocks: + block_type = block.get("type") + if block_type == "text": + parts.append(str(block.get("text", ""))) + elif block_type == "tool_result": + tool_content = block.get("content", "") + if isinstance(tool_content, list): + text_parts = [] + for item in tool_content: + if isinstance(item, dict) and item.get("type") == "text": + text_parts.append(str(item.get("text", ""))) + elif isinstance(item, dict): + text_parts.append(f"[Unsupported {item.get('type')}]") + else: + text_parts.append(str(item)) + tool_content = "\n\n".join(text_parts) + parts.append(TOOL_OUTPUT_TEMPLATE.format(content=tool_content)) + else: + parts.append(f"[Unsupported {block_type}]") + return "\n\n".join(parts) + + +def _render_message( + index: int, + messages: list[dict[str, Any]], + thinking_mode: str, + drop_thinking: bool, + add_generation_prompt: bool, + reasoning_effort: str | None, +) -> str: + if thinking_mode not in ("chat", "thinking"): + raise ValueError(f"Invalid thinking_mode: {thinking_mode}") + + message = messages[index] + last_user_index = _find_last_user_index(messages) + role = message.get("role") + content = _message_content_to_text(message.get("content")) + tools = message.get("tools") + response_format = message.get("response_format") + tool_calls = message.get("tool_calls") + reasoning = message.get("reasoning") or message.get("reasoning_content") or "" + prompt = "" + + if tools: + tools = _tools_from_openai_format(tools) + if tool_calls: + tool_calls = _tool_calls_from_openai_format(tool_calls) + + if index == 0 and thinking_mode == "thinking" and reasoning_effort == "max": + prompt += REASONING_EFFORT_MAX + + if role == "system": + prompt += content + if tools: + prompt += "\n\n" + _render_tools(tools) + if response_format: + prompt += "\n\n" + RESPONSE_FORMAT_TEMPLATE.format(schema=_to_json(response_format)) + elif role == "developer": + prompt += USER_TOKEN + content + if tools: + prompt += "\n\n" + _render_tools(tools) + if response_format: + prompt += "\n\n" + RESPONSE_FORMAT_TEMPLATE.format(schema=_to_json(response_format)) + elif role == "user": + prompt += USER_TOKEN + _render_user_content(message) + elif role == "latest_reminder": + prompt += LATEST_REMINDER_TOKEN + content + elif role == "tool": + raise NotImplementedError( + "DeepSeek-V4 merges tool messages into user messages; " + "preprocess with _merge_tool_messages()." + ) + elif role == "assistant": + tool_calls_content = "" + if tool_calls: + rendered_tool_calls = [ + TOOL_CALL_TEMPLATE.format( + dsml_token=DSML_TOKEN, + name=tool_call.get("name"), + arguments=_encode_arguments_to_dsml(tool_call), + ) + for tool_call in tool_calls + ] + tool_calls_content += "\n\n" + TOOL_CALLS_TEMPLATE.format( + dsml_token=DSML_TOKEN, + block_name=TOOL_CALLS_BLOCK_NAME, + tool_calls="\n".join(rendered_tool_calls), + ) + + thinking_part = "" + prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None + if thinking_mode == "thinking" and not prev_has_task: + if not drop_thinking or index > last_user_index: + thinking_part = reasoning + THINKING_END_TOKEN + + if message.get("wo_eos", False): + prompt += thinking_part + content + tool_calls_content + else: + prompt += thinking_part + content + tool_calls_content + EOS_TOKEN + else: + raise NotImplementedError(f"Unsupported DeepSeek-V4 message role: {role}") + + next_role = messages[index + 1].get("role") if index + 1 < len(messages) else None + if next_role is not None and next_role not in ("assistant", "latest_reminder"): + return prompt + + task = message.get("task") + if task is not None: + if task not in VALID_TASKS: + raise ValueError(f"Invalid DeepSeek-V4 task: {task}") + if task == "action": + prompt += ASSISTANT_TOKEN + prompt += THINKING_START_TOKEN if thinking_mode == "thinking" else THINKING_END_TOKEN + prompt += VALID_TASKS[task] + elif role in ("user", "developer") and (next_role == "assistant" or add_generation_prompt): + prompt += ASSISTANT_TOKEN + if thinking_mode == "thinking" and (not drop_thinking or index >= last_user_index): + prompt += THINKING_START_TOKEN + else: + prompt += THINKING_END_TOKEN + + return prompt + + +def _encode_messages( + messages: list[dict[str, Any]], + thinking_mode: str, + drop_thinking: bool, + add_generation_prompt: bool, + reasoning_effort: str | None, +) -> str: + messages = _merge_tool_messages(messages) + messages = _sort_tool_results_by_call_order(messages) + + effective_drop_thinking = drop_thinking + if any(message.get("tools") for message in messages): + effective_drop_thinking = False + + if thinking_mode == "thinking" and effective_drop_thinking: + messages = _drop_thinking_messages(messages) + + prompt = BOS_TOKEN + for index in range(len(messages)): + prompt += _render_message( + index, + messages, + thinking_mode=thinking_mode, + drop_thinking=effective_drop_thinking, + add_generation_prompt=add_generation_prompt, + reasoning_effort=reasoning_effort, + ) + return prompt + + class DeepseekV4Tokenizer(TransformersTokenizer): """DeepSeek-V4 tokenizer with the checkpoint reference chat format.""" @@ -65,29 +429,29 @@ def from_pretrained( return cls(tokenizer) def apply_chat_template(self, messages, tools=None, **kwargs): - if tools: - raise NotImplementedError("DeepSeek-V4 tool-call chat formatting is not supported yet.") - - add_generation_prompt = kwargs.get("add_generation_prompt", True) tokenize = kwargs.get("tokenize", False) + thinking = kwargs.get("thinking", False) or kwargs.get("enable_thinking", False) + thinking_mode = "thinking" if thinking else "chat" + reasoning_effort = kwargs.get("reasoning_effort") + if reasoning_effort not in ("max", "high"): + reasoning_effort = None - rendered = BOS_TOKEN - for idx, message in enumerate(messages): - role = message.get("role") - content = _message_content_to_text(message.get("content")) - next_role = messages[idx + 1].get("role") if idx + 1 < len(messages) else None - - if role == "system": - rendered += content - elif role in ("user", "developer"): - rendered += USER_TOKEN + content - if next_role == "assistant" or (next_role is None and add_generation_prompt): - rendered += ASSISTANT_TOKEN + THINKING_END_TOKEN - elif role == "assistant": - rendered += content + EOS_TOKEN - else: - raise NotImplementedError(f"Unsupported DeepSeek-V4 message role: {role}") + conversation = kwargs.get("conversation", messages) + messages = list(conversation) + if tools: + messages.insert(0, {"role": "system", "tools": tools}) + + rendered = _encode_messages( + messages=messages, + thinking_mode=thinking_mode, + drop_thinking=kwargs.get("drop_thinking", True), + add_generation_prompt=True, + reasoning_effort=reasoning_effort, + ) if tokenize: - return self.encode(rendered, add_special_tokens=False) + tokenizer_kwargs = { + key: kwargs[key] for key in ("truncation", "max_length") if key in kwargs + } + return self.encode(rendered, add_special_tokens=False, **tokenizer_kwargs) return rendered diff --git a/tests/unittest/llmapi/apps/test_tool_parsers.py b/tests/unittest/llmapi/apps/test_tool_parsers.py index ff73c26a70f7..c4ccd496bae3 100644 --- a/tests/unittest/llmapi/apps/test_tool_parsers.py +++ b/tests/unittest/llmapi/apps/test_tool_parsers.py @@ -24,6 +24,7 @@ from tensorrt_llm.serve.tool_parser.base_tool_parser import BaseToolParser from tensorrt_llm.serve.tool_parser.core_types import StructureInfo from tensorrt_llm.serve.tool_parser.deepseekv3_parser import DeepSeekV3Parser +from tensorrt_llm.serve.tool_parser.deepseekv4_parser import DeepSeekV4Parser from tensorrt_llm.serve.tool_parser.deepseekv31_parser import DeepSeekV31Parser from tensorrt_llm.serve.tool_parser.deepseekv32_parser import DeepSeekV32Parser from tensorrt_llm.serve.tool_parser.glm4_parser import Glm4ToolParser @@ -1493,6 +1494,60 @@ def test_encode_messages_multi_turn_with_tool_calls(self): assert ">ls<" in result +# ============================================================================ +# DeepSeekV4Parser Tests +# ============================================================================ + + +class TestDeepSeekV4Parser(BaseToolParserTestClass): + """Test suite for DeepSeekV4Parser class.""" + + def make_parser(self): + return DeepSeekV4Parser() + + def make_tool_parser_test_cases(self): + return ToolParserTestCases( + has_tool_call_true= + ('Some text <｜DSML｜tool_calls> <｜DSML｜invoke name="get_weather"> ' + '<｜DSML｜parameter name="location" string="true">NYC ' + " "), + detect_and_parse_single_tool=( + ('Normal text<｜DSML｜tool_calls> <｜DSML｜invoke name="get_weather"> ' + '<｜DSML｜parameter name="location" string="true">NYC ' + " "), + "Normal text", + "get_weather", + { + "location": "NYC" + }, + ), + detect_and_parse_multiple_tools=( + ('<｜DSML｜tool_calls> <｜DSML｜invoke name="get_weather"> ' + '<｜DSML｜parameter name="location" string="true">NYC ' + ' <｜DSML｜invoke name="search_web"> ' + '{ "query": "AI" } '), + ("get_weather", "search_web"), + ), + detect_and_parse_malformed_tool= + ('<|DSML|tool_calls> <|DSML|invoke name="get_weather"> ' + '<|DSML|parameter name="location" string="true">NYC ' + " "), + detect_and_parse_with_parameters_key=( + ('<｜DSML｜tool_calls> <｜DSML｜invoke name="search_web"> ' + '{ "query": "test" } '), + "search_web", + { + "query": "test" + }, + ), + parse_streaming_increment_partial_bot_token="<｜DSML｜tool", + undefined_tool= + ('<｜DSML｜tool_calls> <｜DSML｜invoke name="undefined_func"> ' + '<｜DSML｜parameter name="arg" string="true">value ' + " "), + ) + + # ============================================================================ # Glm4ToolParser Tests # ============================================================================ diff --git a/tests/unittest/llmapi/test_deepseek_v4_tokenizer.py b/tests/unittest/llmapi/test_deepseek_v4_tokenizer.py index a95525fe64fd..dec508d49cbc 100644 --- a/tests/unittest/llmapi/test_deepseek_v4_tokenizer.py +++ b/tests/unittest/llmapi/test_deepseek_v4_tokenizer.py @@ -69,6 +69,326 @@ def test_deepseek_v4_chat_template_tokenize_uses_rendered_prompt(): ) +def test_deepseek_v4_chat_template_supports_thinking_mode(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + + prompt = tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": "hello", + } + ], + tokenize=False, + enable_thinking=True, + ) + + assert prompt == ("<｜begin▁of▁sentence｜><｜User｜>hello<｜Assistant｜>") + + +def test_deepseek_v4_chat_template_supports_thinking_alias(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + + prompt = tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": "hello", + } + ], + tokenize=False, + thinking=True, + ) + + assert prompt == ("<｜begin▁of▁sentence｜><｜User｜>hello<｜Assistant｜>") + + +def test_deepseek_v4_chat_template_matches_vllm_add_generation_prompt_behavior(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + + prompt = tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": "hello", + } + ], + tokenize=False, + add_generation_prompt=False, + ) + + assert prompt == ("<｜begin▁of▁sentence｜><｜User｜>hello<｜Assistant｜>") + + +def test_deepseek_v4_chat_template_accepts_openai_reasoning_effort_values(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + + for reasoning_effort in ("none", "low", "medium", "high"): + prompt = tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": "hello", + } + ], + tokenize=False, + enable_thinking=True, + reasoning_effort=reasoning_effort, + ) + + assert prompt.endswith("<｜Assistant｜>") + assert "Reasoning Effort: Absolute maximum" not in prompt + + +def test_deepseek_v4_chat_template_preserves_reference_max_reasoning_effort(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + + prompt = tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": "hello", + } + ], + tokenize=False, + enable_thinking=True, + reasoning_effort="max", + ) + + assert prompt.startswith("<｜begin▁of▁sentence｜>Reasoning Effort: Absolute maximum") + assert prompt.endswith("<｜User｜>hello<｜Assistant｜>") + + +def test_deepseek_v4_chat_template_drops_historical_thinking_without_tools(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + + prompt = tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": "first", + }, + { + "role": "assistant", + "reasoning": "hidden chain", + "content": "answer", + }, + { + "role": "user", + "content": "second", + }, + ], + tokenize=False, + enable_thinking=True, + ) + + assert "hidden chain" not in prompt + assert "answer<｜end▁of▁sentence｜>" in prompt + assert prompt.endswith("<｜User｜>second<｜Assistant｜>") + + +def test_deepseek_v4_chat_template_keeps_historical_thinking_with_tools(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + tools = [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search", + "parameters": {"type": "object"}, + }, + } + ] + + prompt = tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": "first", + }, + { + "role": "assistant", + "reasoning": "kept chain", + "content": "answer", + }, + { + "role": "user", + "content": "second", + }, + ], + tools=tools, + tokenize=False, + enable_thinking=True, + ) + + assert "kept chainanswer<｜end▁of▁sentence｜>" in prompt + assert prompt.endswith("<｜User｜>second<｜Assistant｜>") + + +def test_deepseek_v4_chat_template_renders_developer_tools_and_latest_reminder(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + tools = [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search", + "parameters": {"type": "object"}, + }, + } + ] + messages = [ + { + "role": "system", + "content": "sys", + }, + { + "role": "latest_reminder", + "content": "today", + }, + { + "role": "developer", + "content": "dev", + "tools": tools, + }, + { + "role": "assistant", + "reasoning": "need search", + "content": "", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "search", + "arguments": '{"query": "x"}', + }, + } + ], + }, + { + "role": "tool", + "content": "[0]", + }, + ] + + prompt = tokenizer.apply_chat_template( + messages, + tokenize=False, + enable_thinking=True, + ) + + assert prompt.startswith("<｜begin▁of▁sentence｜>sys<｜latest_reminder｜>today<｜User｜>dev") + assert "## Tools" in prompt + assert '<｜DSML｜invoke name="search">' in prompt + assert "need search" in prompt + assert "<｜User｜>[0]<｜Assistant｜>" in prompt + + +def test_deepseek_v4_chat_template_renders_action_task_token(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + + prompt = tokenizer.apply_chat_template( + [ + { + "role": "system", + "content": "sys", + }, + { + "role": "latest_reminder", + "content": "today", + }, + { + "role": "user", + "content": "search this", + "task": "action", + }, + { + "role": "assistant", + "content": "Search", + }, + ], + tokenize=False, + ) + + assert prompt == ( + "<｜begin▁of▁sentence｜>sys<｜latest_reminder｜>today" + "<｜User｜>search this<｜Assistant｜><｜action｜>" + "Search<｜end▁of▁sentence｜>" + ) + + +def test_deepseek_v4_chat_template_uses_v4_tool_prompt_from_request_tools(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a city", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + + prompt = tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": "Weather?", + } + ], + tools=tools, + tokenize=False, + ) + + assert "## Tools" in prompt + assert "<｜DSML｜tool_calls>" in prompt + assert "" in prompt + assert "function_calls" not in prompt + assert '"name": "get_weather"' in prompt + assert prompt.endswith("<｜User｜>Weather?<｜Assistant｜>") + + +def test_deepseek_v4_chat_template_renders_tool_call_history(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + messages = [ + { + "role": "user", + "content": "List the repo", + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "str_replace_editor", + "arguments": '{"command": "view", "path": "/testbed"}', + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "file list", + }, + ] + + prompt = tokenizer.apply_chat_template(messages, tokenize=False) + + assert '<｜DSML｜invoke name="str_replace_editor">' in prompt + assert '<｜DSML｜parameter name="command" string="true">view' in prompt + assert '<｜DSML｜parameter name="path" string="true">/testbed' in prompt + assert "<｜User｜>file list<｜Assistant｜>" in prompt + assert 'parameter name="arguments"' not in prompt + + def test_deepseek_v4_custom_tokenizer_reuses_loaded_wrapper(): tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) @@ -95,3 +415,35 @@ def test_deepseek_v4_server_chat_template_path_uses_custom_tokenizer(): ) assert prompt == ("<｜begin▁of▁sentence｜><｜User｜>hello<｜Assistant｜>") + + +def test_deepseek_v4_server_chat_template_path_forwards_tools(): + tokenizer = DeepseekV4Tokenizer(_DummyTokenizer()) + tools = [ + { + "type": "function", + "function": { + "name": "search", + "description": "Search", + "parameters": {"type": "object"}, + }, + } + ] + + prompt = apply_chat_template( + model_type="deepseek_v4", + tokenizer=tokenizer, + processor=None, + conversation=[ + { + "role": "user", + "content": "hello", + } + ], + add_generation_prompt=True, + mm_placeholder_counts=[{}], + tools=tools, + ) + + assert "<｜DSML｜tool_calls>" in prompt + assert '"name": "search"' in prompt diff --git a/tests/unittest/llmapi/test_reasoning_parser.py b/tests/unittest/llmapi/test_reasoning_parser.py index 2ce11aecc830..5a18f0be0d6f 100644 --- a/tests/unittest/llmapi/test_reasoning_parser.py +++ b/tests/unittest/llmapi/test_reasoning_parser.py @@ -44,6 +44,34 @@ def test_deepseek_r1_reasoning_parser_stream(delta_texts: list, content: list, assert result.reasoning_content == reasoning_context[i] +@pytest.mark.parametrize("chat_template_kwargs", [{ + "thinking": True +}, { + "enable_thinking": True +}]) +def test_deepseek_v4_reasoning_parser_extracts_when_thinking( + chat_template_kwargs: dict): + reasoning_parser = ReasoningParserFactory.create_reasoning_parser( + "deepseek_v4", chat_template_kwargs) + + result = reasoning_parser.parse(f"hidden{R1_END}visible") + + assert result.content == "visible" + assert result.reasoning_content == "hidden" + + +def test_deepseek_v4_reasoning_parser_streams_when_thinking(): + reasoning_parser = ReasoningParserFactory.create_reasoning_parser( + "deepseek_v4", {"enable_thinking": True}) + + deltas = ["hid", f"den{R1_END}visible", " tail"] + results = [reasoning_parser.parse_delta(delta) for delta in deltas] + + assert [result.content for result in results] == ["", "visible", " tail"] + assert [result.reasoning_content + for result in results] == ["hid", "den", ""] + + TOOL_START = "<|tool_calls_section_begin|>"