From fed1b4a6073b728e0250be9c5c4c00b9047426b3 Mon Sep 17 00:00:00 2001 From: Dom Brown <3886319+DomBrown@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:44:58 +0000 Subject: [PATCH 1/4] [TRTLLM-14953][feat] Add generation config sampling defaults Allow PyTorch requests to opt into model-provided sampling defaults while preserving existing TRT-LLM behavior by default. Signed-off-by: Dom Brown <3886319+DomBrown@users.noreply.github.com> --- docs/source/features/sampling.md | 48 ++++++- tensorrt_llm/commands/serve.py | 20 ++- tensorrt_llm/llmapi/llm.py | 18 +++ tensorrt_llm/llmapi/llm_args.py | 11 ++ tensorrt_llm/llmapi/llm_utils.py | 23 ++++ tensorrt_llm/sampling_params.py | 57 +++++++- tensorrt_llm/serve/openai_protocol.py | 14 +- .../usage/llm_args_golden_manifest.json | 10 ++ .../integration/test_lists/test-db/l0_a10.yml | 1 + .../api_stability/references/llm.yaml | 4 + .../references/trtllm_serve_cli.yaml | 9 ++ tests/unittest/llmapi/test_llm_args.py | 36 +++++ tests/unittest/llmapi/test_llm_utils.py | 33 +++++ tests/unittest/llmapi/test_sampling_params.py | 125 ++++++++++++++++++ 14 files changed, 402 insertions(+), 7 deletions(-) diff --git a/docs/source/features/sampling.md b/docs/source/features/sampling.md index 3bee98bb87e7..c0b3e1fbd260 100644 --- a/docs/source/features/sampling.md +++ b/docs/source/features/sampling.md @@ -72,6 +72,52 @@ llm.generate(["Hello, my name is", sampling_params_1]) ``` +### Model generation config defaults + +The PyTorch backend can use compatible sampling defaults explicitly specified +in a model's `generation_config.json`. This behavior is opt-in: + +```python +from tensorrt_llm import LLM + +llm = LLM(model='nvidia/Llama-3.1-8B-Instruct-FP8', + generation_config='auto') +``` + +For `trtllm-serve`, enable it on the command line: + +```bash +trtllm-serve nvidia/Llama-3.1-8B-Instruct-FP8 --generation-config auto +``` + +or in the server YAML configuration: + +```yaml +generation_config: auto +``` + +The `generation_config` option has two modes: + +* `trtllm` (default) keeps the TRT-LLM sampling behavior and defaults. +* `auto` loads supported sampling values from the model's + `generation_config.json`. + +In `auto` mode, values are resolved in this order: + +1. A value explicitly specified by the request. +2. A value explicitly present in `generation_config.json`. +3. The existing default for the LLM API or serving protocol. + +The supported fields are `temperature`, `top_p`, `top_k`, `min_p`, +`repetition_penalty`, `no_repeat_ngram_size`, `length_penalty`, and +`early_stopping` when its value is a boolean or integer. Defaults synthesized +by Hugging Face Transformers for fields absent from the JSON file are not +applied. + +TRT-LLM's existing model-specific handling of `eos_token_id`, BART +`forced_bos_token_id`, and Whisper suppression tokens remains active in both +modes. + ### LLM API sampling behavior when using Torch Sampler * The sampling is controlled via `SamplingParams`. @@ -120,7 +166,7 @@ llm.generate(["Hello, my name is", * Top-P decay is not supported in combination with beam search or with speculative decoding modes that route draft tokens through the Torch Sampler; such requests are rejected. -* Positive Min-P is not supported in combination with one-model speculative decoding. Such +* Positive Min-P is not supported in combination with one-model speculative decoding. Such requests are rejected at admission. * Occurrence penalties are supported: `repetition_penalty`, `presence_penalty` and diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index 65691acbf91c..bdb541cb9aee 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -197,6 +197,7 @@ def get_llm_args( custom_tokenizer: Optional[str] = None, post_processor_hook: Optional[str] = None, backend: str = "pytorch", + generation_config: str = _LLM_ARGS_FIELDS["generation_config"].default, max_beam_width: int = _LLM_ARGS_FIELDS["max_beam_width"].default, max_batch_size: int = _LLM_ARGS_FIELDS["max_batch_size"].default, max_num_tokens: int = _LLM_ARGS_FIELDS["max_num_tokens"].default, @@ -247,6 +248,8 @@ def get_llm_args( model, "backend": backend, + "generation_config": + generation_config, "tokenizer": tokenizer, "custom_tokenizer": @@ -989,6 +992,13 @@ def launch_visual_gen_server( default="pytorch", help="The backend to use to serve the model. Default is pytorch backend.", status="beta") +@stability_option( + "--generation-config", + type=click.Choice(["auto", "trtllm"]), + default=_LLM_ARGS_FIELDS["generation_config"].default, + help="Sampling defaults source. 'auto' loads supported values from the " + "model's generation_config.json; 'trtllm' uses TRT-LLM defaults.", + status="prototype") @stability_option("--custom_module_dirs", type=click.Path(exists=True, readable=True, @@ -1277,10 +1287,11 @@ def launch_visual_gen_server( status="prototype") def serve(model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str], post_processor_hook: Optional[str], host: str, port: int, - log_level: str, backend: str, max_beam_width: int, - max_batch_size: int, max_num_tokens: int, max_seq_len: int, - tensor_parallel_size: int, pipeline_parallel_size: int, - context_parallel_size: int, moe_expert_parallel_size: Optional[int], + log_level: str, backend: str, generation_config: str, + max_beam_width: int, max_batch_size: int, max_num_tokens: int, + max_seq_len: int, tensor_parallel_size: int, + pipeline_parallel_size: int, context_parallel_size: int, + moe_expert_parallel_size: Optional[int], moe_cluster_parallel_size: Optional[int], gpus_per_node: Optional[int], free_gpu_memory_fraction: float, kv_cache_dtype: str, num_postprocess_workers: int, @@ -1367,6 +1378,7 @@ def _serve_llm(): custom_tokenizer=custom_tokenizer, post_processor_hook=post_processor_hook, backend=backend, + generation_config=generation_config, max_beam_width=max_beam_width, max_batch_size=max_batch_size, max_num_tokens=max_num_tokens, diff --git a/tensorrt_llm/llmapi/llm.py b/tensorrt_llm/llmapi/llm.py index 2b34297c8436..825e26fb3455 100644 --- a/tensorrt_llm/llmapi/llm.py +++ b/tensorrt_llm/llmapi/llm.py @@ -402,6 +402,8 @@ def __init__(self, self._hf_model_dir: Optional[Path] = None self._hf_model_config = None self._generation_config = None + # Raw JSON preserves explicit keys; GenerationConfig fills defaults. + self._generation_config_explicit_values: dict[str, Any] = {} self.llm_build_stats = LlmBuildStats() self._build_model() @@ -1365,12 +1367,20 @@ def _process_env_overrides(self, os.environ[key] = str_value logger.info(f"Setting {key}='{str_value}'") + def _apply_generation_config_sampling_defaults( + self, sampling_params: SamplingParams) -> None: + if (self.args.backend == "pytorch" + and self.args.generation_config == "auto"): + sampling_params._apply_generation_config_defaults( + self._generation_config_explicit_values) + def _prepare_sampling_params( self, sampling_params: Optional[SamplingParams] = None) -> SamplingParams: if sampling_params is None: sampling_params = SamplingParams() if isinstance(sampling_params, SamplingParams): + self._apply_generation_config_sampling_defaults(sampling_params) if sampling_params.end_id is None: if self.tokenizer is None: raise ValueError( @@ -1588,6 +1598,12 @@ def _try_load_generation_config( self) -> Optional[transformers.GenerationConfig]: return ModelLoader.load_hf_generation_config(self.args.model) + def _try_load_generation_config_explicit_values(self) -> dict[str, Any]: + if self.args.backend != "pytorch" or self.args.generation_config != "auto": + return {} + model_dir = self._hf_model_dir or self.args.model + return ModelLoader.load_hf_generation_config_dict(model_dir) + def _try_load_hf_model_config( self) -> Optional[transformers.PretrainedConfig]: return ModelLoader.load_hf_model_config( @@ -1730,6 +1746,8 @@ def _build_model(self): self._tokenizer = self._try_load_tokenizer() self._hf_model_config = self._try_load_hf_model_config() self._generation_config = self._try_load_generation_config() + self._generation_config_explicit_values = self._try_load_generation_config_explicit_values( + ) # Multimodal special handling: # 1. Default load_tokenizer may fail because MM has different tokenizer configuration. Hence we initialize it inside input processor diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 1f7e855280c3..55dbfa0f2ab8 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -5047,6 +5047,17 @@ def set_default_capture_num_tokens(self) -> 'TorchCompileConfig': class TorchLlmArgs(BaseLlmArgs): # PyTorch backend specific configurations + generation_config: Literal["auto", "trtllm"] = Field( + default="trtllm", + description= + "Controls whether sampling defaults are loaded from the model's " + "generation_config.json. 'auto' applies supported values when the " + "request does not specify them; 'trtllm' preserves TRT-LLM defaults. " + "Precedence is request values, generation_config.json values, then " + "TRT-LLM defaults.", + status="prototype", + json_schema_extra={"type": "Literal['auto', 'trtllm']"}) + garbage_collection_gen0_threshold: int = Field( default=20000, description= diff --git a/tensorrt_llm/llmapi/llm_utils.py b/tensorrt_llm/llmapi/llm_utils.py index 63b360f5584b..b15798419ca1 100644 --- a/tensorrt_llm/llmapi/llm_utils.py +++ b/tensorrt_llm/llmapi/llm_utils.py @@ -310,6 +310,29 @@ def load_hf_generation_config( ) return None + @staticmethod + def load_hf_generation_config_dict( + model_dir: Union[str, Path]) -> Dict[str, Any]: + """Load only values explicitly present in generation_config.json.""" + generation_config_path = Path(model_dir) / "generation_config.json" + try: + with open(generation_config_path, "r") as config_file: + generation_config = json.load(config_file) + except FileNotFoundError: + return {} + except (OSError, json.JSONDecodeError) as e: + logger.warning( + f"Failed to load generation config values from {generation_config_path}, encountered error: {e}" + ) + return {} + + if not isinstance(generation_config, dict): + logger.warning( + f"Ignoring generation config from {generation_config_path}: expected a JSON object." + ) + return {} + return generation_config + @staticmethod def load_hf_model_config( model_dir, diff --git a/tensorrt_llm/sampling_params.py b/tensorrt_llm/sampling_params.py index 34c92979fe77..f32606592366 100644 --- a/tensorrt_llm/sampling_params.py +++ b/tensorrt_llm/sampling_params.py @@ -15,8 +15,9 @@ import json import os from abc import ABC, abstractmethod +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field, fields -from typing import List, NamedTuple, Optional, Tuple, Union +from typing import Any, List, NamedTuple, Optional, Tuple, Union import torch from pydantic import BaseModel @@ -27,6 +28,19 @@ MAX_TOP_LOGPROBS = 100 +_GENERATION_CONFIG_SAMPLING_FIELDS = frozenset( + { + "early_stopping", + "length_penalty", + "min_p", + "no_repeat_ngram_size", + "repetition_penalty", + "temperature", + "top_k", + "top_p", + } +) + def validate_thinking_token_budget(value: Optional[Union[int, float, bool]]) -> Optional[int]: """Validate ``thinking_token_budget``; return ``None`` if unset.""" @@ -349,6 +363,10 @@ class SamplingParams: # Currently, _stream_interval is only used to pass llm.args.stream_interval to tokenizer. # TODO: make this a per-request parameter. _stream_interval: Optional[int] = field(default=None, init=False, repr=False) + # None identifies direct LLM API SamplingParams, where non-None values are + # request-provided. Serving adapters set this to preserve which fields were + # explicitly present before they materialize their protocol defaults. + _request_provided_fields: Optional[frozenset[str]] = field(default=None, init=False, repr=False) def __post_init__(self): if self.pad_id is None: @@ -448,6 +466,43 @@ def _validate(self): if self.logprobs_simple_format and self.use_beam_search: raise ValueError("logprobs_simple_format is not supported with beam search") + def _set_request_provided_fields(self, field_names: Iterable[str]) -> None: + """Record sampler fields explicitly supplied by a serving request.""" + self._request_provided_fields = frozenset(field_names) & _GENERATION_CONFIG_SAMPLING_FIELDS + + def _apply_generation_config_defaults(self, generation_config: Mapping[str, Any]) -> None: + """Apply compatible model defaults without overriding request values.""" + for field_name in _GENERATION_CONFIG_SAMPLING_FIELDS: + if field_name not in generation_config: + continue + + # Direct LLM API calls preserve None as the unset sentinel. Serving + # adapters materialize protocol defaults, so they instead record + # which fields the request explicitly supplied. + if self._request_provided_fields is None: + request_provided = getattr(self, field_name) is not None + else: + request_provided = field_name in self._request_provided_fields + if request_provided: + continue + + # A JSON null is also unset and must fall through to the existing + # TRT-LLM or serving default. + value = generation_config[field_name] + if value is None: + continue + # Hugging Face also permits "never", which the TRT-LLM integer + # early-stopping setting cannot represent. + if field_name == "early_stopping" and not isinstance(value, (bool, int)): + logger.warning( + "Ignoring unsupported generation_config.json early_stopping value " + f"{value!r}; TRT-LLM supports only boolean or integer values." + ) + continue + setattr(self, field_name, value) + + self._validate() + # NB: The predicates below are static because downstream code (e.g. # sampler_strategy.resolve_sampling_strategy) only holds instances of # bindings.SamplingConfig (not SamplingParams). They are the single diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 7ec9706e99ee..0cce73d0b5f8 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -492,6 +492,14 @@ def _response_format_text_config_to_guided_decoding_params( resp_format, reasoning_parser=reasoning_parser) +def _record_sampling_params_request_fields( + request: OpenAIBaseModel, sampling_params: SamplingParams) -> None: + """Preserve explicitly supplied fields across protocol defaulting.""" + sampling_params._set_request_provided_fields( + field_name for field_name in request.model_fields_set + if getattr(request, field_name, None) is not None) + + class CompletionRequest(OpenAIBaseModel): # Ordered by official OpenAI API documentation # https://platform.openai.com/docs/api-reference/completions/create @@ -631,6 +639,7 @@ def to_sampling_params(self, ) if return_log_probs: sampling_params._return_log_probs = True + _record_sampling_params_request_fields(self, sampling_params) return sampling_params @model_validator(mode="before") @@ -1044,6 +1053,7 @@ def to_sampling_params(self, ) if return_log_probs: sampling_params._return_log_probs = True + _record_sampling_params_request_fields(self, sampling_params) return sampling_params @model_validator(mode='before') @@ -1221,7 +1231,7 @@ def to_sampling_params( guided_decoding = _response_format_text_config_to_guided_decoding_params( self.text.format, reasoning_parser=reasoning_parser) - return SamplingParams( + sampling_params = SamplingParams( temperature=temperature, top_p=top_p, max_tokens=max_tokens, @@ -1230,6 +1240,8 @@ def to_sampling_params( guided_decoding=guided_decoding, thinking_token_budget=self.thinking_token_budget, ) + _record_sampling_params_request_fields(self, sampling_params) + return sampling_params @model_validator(mode="before") @classmethod diff --git a/tensorrt_llm/usage/llm_args_golden_manifest.json b/tensorrt_llm/usage/llm_args_golden_manifest.json index f7b26ee0a1ff..4cc3d26b1894 100644 --- a/tensorrt_llm/usage/llm_args_golden_manifest.json +++ b/tensorrt_llm/usage/llm_args_golden_manifest.json @@ -519,6 +519,16 @@ "kind": "value", "path": "gather_generation_logits" }, + { + "allowed_values": [ + "auto", + "trtllm" + ], + "annotation": "Literal['auto', 'trtllm']", + "converter": "", + "kind": "categorical", + "path": "generation_config" + }, { "allowed_values": [ "auto", diff --git a/tests/integration/test_lists/test-db/l0_a10.yml b/tests/integration/test_lists/test-db/l0_a10.yml index b3b8bdb0e9fa..a401039e68af 100644 --- a/tests/integration/test_lists/test-db/l0_a10.yml +++ b/tests/integration/test_lists/test-db/l0_a10.yml @@ -115,6 +115,7 @@ l0_a10: # llmapi - unittest/llmapi/test_rlhf_utils.py - unittest/llmapi/test_llm_args.py + - unittest/llmapi/test_sampling_params.py - unittest/llmapi/test_additional_model_outputs.py -m "gpu1" # executor - unittest/executor/test_postprocessor_hook.py diff --git a/tests/unittest/api_stability/references/llm.yaml b/tests/unittest/api_stability/references/llm.yaml index 5cac19e246df..ac5781a0b03e 100644 --- a/tests/unittest/api_stability/references/llm.yaml +++ b/tests/unittest/api_stability/references/llm.yaml @@ -82,6 +82,10 @@ methods: default: null status: prototype # Runtime behavior + generation_config: + annotation: Literal['auto', 'trtllm'] + default: trtllm + status: prototype garbage_collection_gen0_threshold: annotation: int default: 20000 diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml index c3430712d780..8b2a8774c292 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml @@ -166,6 +166,15 @@ commands: flags: - "--free_gpu_memory_fraction" - "--kv_cache_free_gpu_memory_fraction" + generation_config: + type: Choice(['auto', 'trtllm']) + default: trtllm + status: prototype + required: false + multiple: false + is_flag: false + flags: + - "--generation-config" gpus_per_node: type: int default: null diff --git a/tests/unittest/llmapi/test_llm_args.py b/tests/unittest/llmapi/test_llm_args.py index 61a1c94b083b..5f588febd6be 100644 --- a/tests/unittest/llmapi/test_llm_args.py +++ b/tests/unittest/llmapi/test_llm_args.py @@ -69,6 +69,16 @@ from .test_llm import llama_model_path +@pytest.mark.cpu_only +def test_generation_config_mode_defaults_and_validation(): + assert TorchLlmArgs(model=llama_model_path).generation_config == "trtllm" + assert (TorchLlmArgs(model=llama_model_path, + generation_config="auto").generation_config == "auto") + + with pytest.raises(ValidationError, match="generation_config"): + TorchLlmArgs(model=llama_model_path, generation_config="invalid") + + @pytest.mark.cpu_only def test_LookaheadDecodingConfig(): # from constructor @@ -2523,6 +2533,32 @@ def test_serve_explicit_cli_default_value_wins_over_yaml(self): ) assert merged["tensor_parallel_size"] == 1 + def test_serve_generation_config_cli_over_yaml_precedence(self): + """YAML wins when CLI omits the mode; an explicit CLI mode wins otherwise.""" + default_args, _ = get_llm_args( + model=llama_model_path, + backend="pytorch", + generation_config="trtllm", + ) + assert "generation_config" not in default_args + + yaml_merged = update_llm_args_with_extra_dict( + default_args, {"generation_config": "auto"}) + assert yaml_merged["generation_config"] == "auto" + + explicit_args, _ = get_llm_args( + model=llama_model_path, + backend="pytorch", + generation_config="trtllm", + explicit_cli_keys={"generation_config"}, + ) + cli_merged = update_llm_args_with_extra_dict( + explicit_args, + {"generation_config": "auto"}, + explicit_cli_keys={"generation_config"}, + ) + assert cli_merged["generation_config"] == "trtllm" + def test_serve_is_non_default_or_required_helper(self): # Test always_include parameters assert is_non_default_or_required("model", "test-model", "pytorch", diff --git a/tests/unittest/llmapi/test_llm_utils.py b/tests/unittest/llmapi/test_llm_utils.py index 1e3321eac7f8..00ea3260e1fb 100644 --- a/tests/unittest/llmapi/test_llm_utils.py +++ b/tests/unittest/llmapi/test_llm_utils.py @@ -1,4 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import asyncio +import json import threading import time @@ -6,6 +21,7 @@ import torch from tensorrt_llm.llmapi.llm_args import TorchLlmArgs +from tensorrt_llm.llmapi.llm_utils import ModelLoader from tensorrt_llm.llmapi.utils import AsyncQueue # isort: off @@ -13,6 +29,23 @@ # isort: on +@pytest.mark.cpu_only +def test_load_hf_generation_config_dict_preserves_explicit_values(tmp_path): + expected = { + "temperature": 1.0, + "top_p": 0.9, + } + (tmp_path / "generation_config.json").write_text(json.dumps(expected), + encoding="utf-8") + + assert ModelLoader.load_hf_generation_config_dict(tmp_path) == expected + + +@pytest.mark.cpu_only +def test_load_hf_generation_config_dict_returns_empty_without_file(tmp_path): + assert ModelLoader.load_hf_generation_config_dict(tmp_path) == {} + + @pytest.mark.cpu_only def test_LlmArgs_default_gpus_per_node(): # default diff --git a/tests/unittest/llmapi/test_sampling_params.py b/tests/unittest/llmapi/test_sampling_params.py index 75d2a6c45e4d..ca6a73caa6b2 100644 --- a/tests/unittest/llmapi/test_sampling_params.py +++ b/tests/unittest/llmapi/test_sampling_params.py @@ -14,10 +14,12 @@ # limitations under the License. import asyncio import json +from types import SimpleNamespace import pytest import torch +from tensorrt_llm.llmapi.llm import BaseLLM from tensorrt_llm.llmapi.thinking_budget import ( ThinkingBudgetLogitsProcessor, add_thinking_budget_logits_processor, @@ -27,6 +29,7 @@ ChatCompletionRequest, CompletionRequest, KVCacheTruncateRequest, + ResponsesRequest, ensure_request_chat_template_allowed, ) from tensorrt_llm.serve.resource_governor import ResourceGovernor @@ -34,6 +37,128 @@ pytestmark = pytest.mark.cpu_only +def _apply_generation_config_sampling_defaults( + mode: str, + sampling_params: SamplingParams, + generation_config_explicit_values: dict, +) -> SamplingParams: + llm = SimpleNamespace( + args=SimpleNamespace(backend="pytorch", generation_config=mode), + _generation_config_explicit_values=generation_config_explicit_values, + ) + BaseLLM._apply_generation_config_sampling_defaults(llm, sampling_params) + return sampling_params + + +def test_generation_config_mode_controls_sampling_defaults(): + values = {"temperature": 0.7} + + trtllm_params = _apply_generation_config_sampling_defaults( + "trtllm", SamplingParams(end_id=1), values + ) + auto_params = _apply_generation_config_sampling_defaults( + "auto", SamplingParams(end_id=1), values + ) + + assert trtllm_params.temperature is None + assert auto_params.temperature == 0.7 + + +def test_generation_config_sampling_precedence(): + params = SamplingParams(end_id=1, temperature=0.2) + prepared = _apply_generation_config_sampling_defaults( + "auto", + params, + { + "temperature": 0.7, + "top_p": 0.9, + }, + ) + + # Check each precedence level: + # 1. Request-provided values + # 2. Generation config values + # 3. TRT-LLM defaults + assert prepared.temperature == 0.2 + assert prepared.top_p == 0.9 + assert prepared.top_k is None + + +def test_generation_config_applies_all_supported_sampling_fields(): + # Use typical_p because it is not supported by the generation config. + values = { + "early_stopping": True, + "length_penalty": 0.8, + "min_p": 0.05, + "no_repeat_ngram_size": 3, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.9, + "typical_p": 0.95, + } + + prepared = _apply_generation_config_sampling_defaults("auto", SamplingParams(end_id=1), values) + + for field_name, expected in values.items(): + if field_name != "typical_p": + assert getattr(prepared, field_name) == expected + + +@pytest.mark.parametrize( + "request_obj", + [ + CompletionRequest(model="test", prompt="hi"), + ChatCompletionRequest(model="test", messages=[{"role": "user", "content": "hi"}]), + ResponsesRequest(model="test", input="hi"), + ], +) +def test_generation_config_overrides_omitted_serve_defaults(request_obj): + params = request_obj.to_sampling_params() + + assert params.temperature == 1.0 + prepared = _apply_generation_config_sampling_defaults( + "auto", params, {"temperature": 0.7, "top_p": 0.9} + ) + + assert prepared.temperature == 0.7 + assert prepared.top_p == 0.9 + + +@pytest.mark.parametrize( + "request_obj", + [ + CompletionRequest(model="test", prompt="hi", temperature=0.2), + ChatCompletionRequest( + model="test", + messages=[{"role": "user", "content": "hi"}], + temperature=0.2, + ), + ResponsesRequest(model="test", input="hi", temperature=0.2), + ], +) +def test_explicit_serve_request_overrides_generation_config(request_obj): + prepared = _apply_generation_config_sampling_defaults( + "auto", + request_obj.to_sampling_params(), + {"temperature": 0.7}, + ) + + assert prepared.temperature == 0.2 + + +def test_absent_generation_config_value_preserves_existing_defaults(): + direct_params = _apply_generation_config_sampling_defaults("auto", SamplingParams(end_id=1), {}) + serve_params = _apply_generation_config_sampling_defaults( + "auto", + CompletionRequest(model="test", prompt="hi").to_sampling_params(), + {}, + ) + + assert direct_params.temperature is None + assert serve_params.temperature == 1.0 + + @pytest.mark.parametrize("field", ["logprobs", "prompt_logprobs", "top_logprobs"]) def test_check_logprobs_limit(field): check_logprobs_limit(field, None) From 6ae1d78193c6e661009ef7b194a1a6d917d25e87 Mon Sep 17 00:00:00 2001 From: Dom Brown <3886319+DomBrown@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:32:19 +0000 Subject: [PATCH 2/4] [TRTLLM-14953][fix] Validate generation config edge cases Reject unsupported AutoDeploy defaults and cover CLI precedence and invalid generation configuration values. Signed-off-by: Dom Brown <3886319+DomBrown@users.noreply.github.com> --- tensorrt_llm/_torch/auto_deploy/llm_args.py | 7 ++ tests/unittest/llmapi/test_llm_args.py | 71 +++++++++++++------ tests/unittest/llmapi/test_llm_utils.py | 15 ++++ tests/unittest/llmapi/test_sampling_params.py | 11 +++ 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/tensorrt_llm/_torch/auto_deploy/llm_args.py b/tensorrt_llm/_torch/auto_deploy/llm_args.py index f0c2af09cf6b..02f473b2b77d 100644 --- a/tensorrt_llm/_torch/auto_deploy/llm_args.py +++ b/tensorrt_llm/_torch/auto_deploy/llm_args.py @@ -93,6 +93,13 @@ def ensure_no_beam_search(cls, value: Any) -> Any: raise ValueError("AutoDeploy does not support beam search (max_beam_width > 1).") return value + @field_validator("generation_config", mode="after") + @classmethod + def ensure_no_generation_config_defaults(cls, value: str) -> str: + if value != "trtllm": + raise ValueError("AutoDeploy does not support generation_config='auto'; use 'trtllm'.") + return value + @field_validator( "tensor_parallel_size", "pipeline_parallel_size", diff --git a/tests/unittest/llmapi/test_llm_args.py b/tests/unittest/llmapi/test_llm_args.py index 5f588febd6be..546998492a3a 100644 --- a/tests/unittest/llmapi/test_llm_args.py +++ b/tests/unittest/llmapi/test_llm_args.py @@ -70,7 +70,7 @@ @pytest.mark.cpu_only -def test_generation_config_mode_defaults_and_validation(): +def test_generation_config_mode_defaults_and_validation() -> None: assert TorchLlmArgs(model=llama_model_path).generation_config == "trtllm" assert (TorchLlmArgs(model=llama_model_path, generation_config="auto").generation_config == "auto") @@ -79,6 +79,13 @@ def test_generation_config_mode_defaults_and_validation(): TorchLlmArgs(model=llama_model_path, generation_config="invalid") +@pytest.mark.cpu_only +def test_generation_config_auto_rejects_autodeploy() -> None: + with pytest.raises(ValidationError, + match="AutoDeploy does not support generation_config"): + AutoDeployLlmArgs(model=llama_model_path, generation_config="auto") + + @pytest.mark.cpu_only def test_LookaheadDecodingConfig(): # from constructor @@ -2533,31 +2540,49 @@ def test_serve_explicit_cli_default_value_wins_over_yaml(self): ) assert merged["tensor_parallel_size"] == 1 - def test_serve_generation_config_cli_over_yaml_precedence(self): + def test_serve_generation_config_cli_over_yaml_precedence(self, + tmp_path) -> None: """YAML wins when CLI omits the mode; an explicit CLI mode wins otherwise.""" - default_args, _ = get_llm_args( - model=llama_model_path, - backend="pytorch", - generation_config="trtllm", - ) - assert "generation_config" not in default_args + from unittest import mock - yaml_merged = update_llm_args_with_extra_dict( - default_args, {"generation_config": "auto"}) - assert yaml_merged["generation_config"] == "auto" + from tensorrt_llm.commands.serve import main as serve_main - explicit_args, _ = get_llm_args( - model=llama_model_path, - backend="pytorch", - generation_config="trtllm", - explicit_cli_keys={"generation_config"}, - ) - cli_merged = update_llm_args_with_extra_dict( - explicit_args, - {"generation_config": "auto"}, - explicit_cli_keys={"generation_config"}, - ) - assert cli_merged["generation_config"] == "trtllm" + config_path = tmp_path / "config.yaml" + config_path.write_text("generation_config: auto\n", encoding="utf-8") + + with ( + mock.patch( + "tensorrt_llm.commands.serve.get_is_diffusion_only_model", + return_value=False), + mock.patch("tensorrt_llm.commands.serve.device_count", + return_value=1), + mock.patch("tensorrt_llm.commands.serve.launch_server") as + mock_launch_server, + ): + serve_main( + args=["dummy/model", "--config", + str(config_path)], + standalone_mode=False, + ) + assert mock_launch_server.call_args.args[2][ + "generation_config"] == "auto" + + mock_launch_server.reset_mock() + config_path.write_text("generation_config: trtllm\n", + encoding="utf-8") + serve_main( + args=[ + "dummy/model", + "--config", + str(config_path), + "--generation-config", + "auto", + ], + standalone_mode=False, + ) + + assert mock_launch_server.call_args.args[2][ + "generation_config"] == "auto" def test_serve_is_non_default_or_required_helper(self): # Test always_include parameters diff --git a/tests/unittest/llmapi/test_llm_utils.py b/tests/unittest/llmapi/test_llm_utils.py index 00ea3260e1fb..40539d67f3b5 100644 --- a/tests/unittest/llmapi/test_llm_utils.py +++ b/tests/unittest/llmapi/test_llm_utils.py @@ -46,6 +46,21 @@ def test_load_hf_generation_config_dict_returns_empty_without_file(tmp_path): assert ModelLoader.load_hf_generation_config_dict(tmp_path) == {} +@pytest.mark.cpu_only +def test_load_hf_generation_config_dict_returns_empty_for_malformed_json( + tmp_path): + (tmp_path / "generation_config.json").write_text("{", encoding="utf-8") + + assert ModelLoader.load_hf_generation_config_dict(tmp_path) == {} + + +@pytest.mark.cpu_only +def test_load_hf_generation_config_dict_returns_empty_for_json_array(tmp_path): + (tmp_path / "generation_config.json").write_text("[1, 2]", encoding="utf-8") + + assert ModelLoader.load_hf_generation_config_dict(tmp_path) == {} + + @pytest.mark.cpu_only def test_LlmArgs_default_gpus_per_node(): # default diff --git a/tests/unittest/llmapi/test_sampling_params.py b/tests/unittest/llmapi/test_sampling_params.py index ca6a73caa6b2..889a8abcee9a 100644 --- a/tests/unittest/llmapi/test_sampling_params.py +++ b/tests/unittest/llmapi/test_sampling_params.py @@ -104,6 +104,17 @@ def test_generation_config_applies_all_supported_sampling_fields(): if field_name != "typical_p": assert getattr(prepared, field_name) == expected + unsupported_values = _apply_generation_config_sampling_defaults( + "auto", + SamplingParams(end_id=1), + { + "top_p": None, + "early_stopping": "never", + }, + ) + assert unsupported_values.top_p is None + assert unsupported_values.early_stopping is None + @pytest.mark.parametrize( "request_obj", From 04af9666ab4c45579d418f24e6dc8f8446b09aff Mon Sep 17 00:00:00 2001 From: Dom Brown <3886319+DomBrown@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:24:05 +0000 Subject: [PATCH 3/4] fix: Address review comment regarding documentation Signed-off-by: Dom Brown <3886319+DomBrown@users.noreply.github.com> --- docs/source/features/sampling.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/features/sampling.md b/docs/source/features/sampling.md index c0b3e1fbd260..4f8eb101f450 100644 --- a/docs/source/features/sampling.md +++ b/docs/source/features/sampling.md @@ -123,7 +123,10 @@ modes. * The sampling is controlled via `SamplingParams`. * By default (`temperature = top_p = top_k = None`), greedy sampling is used - (unless min-p or top-p decay is active, see below). + (unless min-p or top-p decay is active, see below). With + `generation_config='auto'`, values explicitly specified in the model's + `generation_config.json` take the place of these defaults; see + [Model generation config defaults](#model-generation-config-defaults). * If either `temperature = 0`, `top_p = 0`, `top_k = 1`, and/or `min_p = 1`, is specified, sampling is greedy, irrespective of the values of the remaining parameters. From dd072d2465829215748e8fb2723c1b1741029de5 Mon Sep 17 00:00:00 2001 From: Dom Brown <3886319+DomBrown@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:42:28 +0000 Subject: [PATCH 4/4] fix: remove erroneous addition of CPU only tests to A10 GPU CI stage Signed-off-by: Dom Brown <3886319+DomBrown@users.noreply.github.com> --- tests/integration/test_lists/test-db/l0_a10.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/test-db/l0_a10.yml b/tests/integration/test_lists/test-db/l0_a10.yml index a401039e68af..b3b8bdb0e9fa 100644 --- a/tests/integration/test_lists/test-db/l0_a10.yml +++ b/tests/integration/test_lists/test-db/l0_a10.yml @@ -115,7 +115,6 @@ l0_a10: # llmapi - unittest/llmapi/test_rlhf_utils.py - unittest/llmapi/test_llm_args.py - - unittest/llmapi/test_sampling_params.py - unittest/llmapi/test_additional_model_outputs.py -m "gpu1" # executor - unittest/executor/test_postprocessor_hook.py