[TRTLLM-14953][feat] Add ability to honour generation_config.json sampling defaults

docs/source/features/sampling.md

@@ -72,12 +72,61 @@ llm.generate(["Hello, my name is",
             sampling_params_1])
 ```
 
+### Model generation config defaults
+
+The PyTorch backend can use compatible sampling defaults explicitly specified
+in a model's `generation_config.json`. This behavior is opt-in:
+
+```python
+from tensorrt_llm import LLM
+
+llm = LLM(model='nvidia/Llama-3.1-8B-Instruct-FP8',
+          generation_config='auto')
+```
+
+For `trtllm-serve`, enable it on the command line:
+
+```bash
+trtllm-serve nvidia/Llama-3.1-8B-Instruct-FP8 --generation-config auto
+```
+
+or in the server YAML configuration:
+
+```yaml
+generation_config: auto
+```
+
+The `generation_config` option has two modes:
+
+* `trtllm` (default) keeps the TRT-LLM sampling behavior and defaults.
+* `auto` loads supported sampling values from the model's
+  `generation_config.json`.
+
+In `auto` mode, values are resolved in this order:
+
+1. A value explicitly specified by the request.
+2. A value explicitly present in `generation_config.json`.
+3. The existing default for the LLM API or serving protocol.
+
+The supported fields are `temperature`, `top_p`, `top_k`, `min_p`,
+`repetition_penalty`, `no_repeat_ngram_size`, `length_penalty`, and
+`early_stopping` when its value is a boolean or integer. Defaults synthesized
+by Hugging Face Transformers for fields absent from the JSON file are not
+applied.
+
+TRT-LLM's existing model-specific handling of `eos_token_id`, BART
+`forced_bos_token_id`, and Whisper suppression tokens remains active in both
+modes.
+
 ### LLM API sampling behavior when using Torch Sampler
 
 * The sampling is controlled via `SamplingParams`.
 
 * By default (`temperature = top_p = top_k = None`), greedy sampling is used
-  (unless min-p or top-p decay is active, see below).
+  (unless min-p or top-p decay is active, see below). With
+  `generation_config='auto'`, values explicitly specified in the model's
+  `generation_config.json` take the place of these defaults; see
+  [Model generation config defaults](#model-generation-config-defaults).
 
 * If either `temperature = 0`, `top_p = 0`, `top_k = 1`, and/or `min_p = 1`, is specified,
   sampling is greedy, irrespective of the values of the remaining parameters.
@@ -120,7 +169,7 @@ llm.generate(["Hello, my name is",
   * Top-P decay is not supported in combination with beam search or with speculative decoding
     modes that route draft tokens through the Torch Sampler; such requests are rejected.
 
-* Positive Min-P is not supported in combination with one-model speculative decoding. Such 
+* Positive Min-P is not supported in combination with one-model speculative decoding. Such
   requests are rejected at admission.
 
 * Occurrence penalties are supported: `repetition_penalty`, `presence_penalty` and

tensorrt_llm/_torch/auto_deploy/llm_args.py

@@ -93,6 +93,13 @@ def ensure_no_beam_search(cls, value: Any) -> Any:
             raise ValueError("AutoDeploy does not support beam search (max_beam_width > 1).")
         return value
 
+    @field_validator("generation_config", mode="after")
+    @classmethod
+    def ensure_no_generation_config_defaults(cls, value: str) -> str:
+        if value != "trtllm":
+            raise ValueError("AutoDeploy does not support generation_config='auto'; use 'trtllm'.")
+        return value
+
     @field_validator(
         "tensor_parallel_size",
         "pipeline_parallel_size",

tensorrt_llm/commands/serve.py

@@ -197,6 +197,7 @@ def get_llm_args(
         custom_tokenizer: Optional[str] = None,
         post_processor_hook: Optional[str] = None,
         backend: str = "pytorch",
+        generation_config: str = _LLM_ARGS_FIELDS["generation_config"].default,
         max_beam_width: int = _LLM_ARGS_FIELDS["max_beam_width"].default,
         max_batch_size: int = _LLM_ARGS_FIELDS["max_batch_size"].default,
         max_num_tokens: int = _LLM_ARGS_FIELDS["max_num_tokens"].default,
@@ -247,6 +248,8 @@ def get_llm_args(
         model,
         "backend":
         backend,
+        "generation_config":
+        generation_config,
         "tokenizer":
         tokenizer,
         "custom_tokenizer":
@@ -989,6 +992,13 @@ def launch_visual_gen_server(
     default="pytorch",
     help="The backend to use to serve the model. Default is pytorch backend.",
     status="beta")
+@stability_option(
+    "--generation-config",
+    type=click.Choice(["auto", "trtllm"]),
+    default=_LLM_ARGS_FIELDS["generation_config"].default,
+    help="Sampling defaults source. 'auto' loads supported values from the "
+    "model's generation_config.json; 'trtllm' uses TRT-LLM defaults.",
+    status="prototype")
 @stability_option("--custom_module_dirs",
                   type=click.Path(exists=True,
                                   readable=True,
@@ -1277,10 +1287,11 @@ def launch_visual_gen_server(
     status="prototype")
 def serve(model: str, tokenizer: Optional[str], custom_tokenizer: Optional[str],
           post_processor_hook: Optional[str], host: str, port: int,
-          log_level: str, backend: str, max_beam_width: int,
-          max_batch_size: int, max_num_tokens: int, max_seq_len: int,
-          tensor_parallel_size: int, pipeline_parallel_size: int,
-          context_parallel_size: int, moe_expert_parallel_size: Optional[int],
+          log_level: str, backend: str, generation_config: str,
+          max_beam_width: int, max_batch_size: int, max_num_tokens: int,
+          max_seq_len: int, tensor_parallel_size: int,
+          pipeline_parallel_size: int, context_parallel_size: int,
+          moe_expert_parallel_size: Optional[int],
           moe_cluster_parallel_size: Optional[int],
           gpus_per_node: Optional[int], free_gpu_memory_fraction: float,
           kv_cache_dtype: str, num_postprocess_workers: int,
@@ -1367,6 +1378,7 @@ def _serve_llm():
             custom_tokenizer=custom_tokenizer,
             post_processor_hook=post_processor_hook,
             backend=backend,
+            generation_config=generation_config,
             max_beam_width=max_beam_width,
             max_batch_size=max_batch_size,
             max_num_tokens=max_num_tokens,

tensorrt_llm/llmapi/llm.py

@@ -402,6 +402,8 @@ def __init__(self,
             self._hf_model_dir: Optional[Path] = None
             self._hf_model_config = None
             self._generation_config = None
+            # Raw JSON preserves explicit keys; GenerationConfig fills defaults.
+            self._generation_config_explicit_values: dict[str, Any] = {}
 
             self.llm_build_stats = LlmBuildStats()
             self._build_model()
@@ -1365,12 +1367,20 @@ def _process_env_overrides(self,
                 os.environ[key] = str_value
                 logger.info(f"Setting {key}='{str_value}'")
 
+    def _apply_generation_config_sampling_defaults(
+            self, sampling_params: SamplingParams) -> None:
+        if (self.args.backend == "pytorch"
+                and self.args.generation_config == "auto"):
+            sampling_params._apply_generation_config_defaults(
+                self._generation_config_explicit_values)
+
     def _prepare_sampling_params(
             self,
             sampling_params: Optional[SamplingParams] = None) -> SamplingParams:
         if sampling_params is None:
             sampling_params = SamplingParams()
         if isinstance(sampling_params, SamplingParams):
+            self._apply_generation_config_sampling_defaults(sampling_params)
             if sampling_params.end_id is None:
                 if self.tokenizer is None:
                     raise ValueError(
@@ -1588,6 +1598,12 @@ def _try_load_generation_config(
             self) -> Optional[transformers.GenerationConfig]:
         return ModelLoader.load_hf_generation_config(self.args.model)
 
+    def _try_load_generation_config_explicit_values(self) -> dict[str, Any]:
+        if self.args.backend != "pytorch" or self.args.generation_config != "auto":
+            return {}
+        model_dir = self._hf_model_dir or self.args.model
+        return ModelLoader.load_hf_generation_config_dict(model_dir)
+
     def _try_load_hf_model_config(
             self) -> Optional[transformers.PretrainedConfig]:
         return ModelLoader.load_hf_model_config(
@@ -1730,6 +1746,8 @@ def _build_model(self):
         self._tokenizer = self._try_load_tokenizer()
         self._hf_model_config = self._try_load_hf_model_config()
         self._generation_config = self._try_load_generation_config()
+        self._generation_config_explicit_values = self._try_load_generation_config_explicit_values(
+        )
 
         # Multimodal special handling:
         # 1. Default load_tokenizer may fail because MM has different tokenizer configuration. Hence we initialize it inside input processor

tensorrt_llm/llmapi/llm_args.py

@@ -5047,6 +5047,17 @@ def set_default_capture_num_tokens(self) -> 'TorchCompileConfig':
 
 class TorchLlmArgs(BaseLlmArgs):
     # PyTorch backend specific configurations
+    generation_config: Literal["auto", "trtllm"] = Field(
+        default="trtllm",
+        description=
+        "Controls whether sampling defaults are loaded from the model's "
+        "generation_config.json. 'auto' applies supported values when the "
+        "request does not specify them; 'trtllm' preserves TRT-LLM defaults. "
+        "Precedence is request values, generation_config.json values, then "
+        "TRT-LLM defaults.",
+        status="prototype",
+        json_schema_extra={"type": "Literal['auto', 'trtllm']"})
+
     garbage_collection_gen0_threshold: int = Field(
         default=20000,
         description=

tensorrt_llm/llmapi/llm_utils.py

@@ -310,6 +310,29 @@ def load_hf_generation_config(
             )
             return None
 
+    @staticmethod
+    def load_hf_generation_config_dict(
+            model_dir: Union[str, Path]) -> Dict[str, Any]:
+        """Load only values explicitly present in generation_config.json."""
+        generation_config_path = Path(model_dir) / "generation_config.json"
+        try:
+            with open(generation_config_path, "r") as config_file:
+                generation_config = json.load(config_file)
+        except FileNotFoundError:
+            return {}
+        except (OSError, json.JSONDecodeError) as e:
+            logger.warning(
+                f"Failed to load generation config values from {generation_config_path}, encountered error: {e}"
+            )
+            return {}
+
+        if not isinstance(generation_config, dict):
+            logger.warning(
+                f"Ignoring generation config from {generation_config_path}: expected a JSON object."
+            )
+            return {}
+        return generation_config
+
     @staticmethod
     def load_hf_model_config(
             model_dir,

tensorrt_llm/sampling_params.py

@@ -15,8 +15,9 @@
 import json
 import os
 from abc import ABC, abstractmethod
+from collections.abc import Iterable, Mapping
 from dataclasses import dataclass, field, fields
-from typing import List, NamedTuple, Optional, Tuple, Union
+from typing import Any, List, NamedTuple, Optional, Tuple, Union
 
 import torch
 from pydantic import BaseModel
@@ -27,6 +28,19 @@
 
 MAX_TOP_LOGPROBS = 100
 
+_GENERATION_CONFIG_SAMPLING_FIELDS = frozenset(
+    {
+        "early_stopping",
+        "length_penalty",
+        "min_p",
+        "no_repeat_ngram_size",
+        "repetition_penalty",
+        "temperature",
+        "top_k",
+        "top_p",
+    }
+)
+
 
 def validate_thinking_token_budget(value: Optional[Union[int, float, bool]]) -> Optional[int]:
     """Validate ``thinking_token_budget``; return ``None`` if unset."""
@@ -349,6 +363,10 @@ class SamplingParams:
     # Currently, _stream_interval is only used to pass llm.args.stream_interval to tokenizer.
     # TODO: make this a per-request parameter.
     _stream_interval: Optional[int] = field(default=None, init=False, repr=False)
+    # None identifies direct LLM API SamplingParams, where non-None values are
+    # request-provided. Serving adapters set this to preserve which fields were
+    # explicitly present before they materialize their protocol defaults.
+    _request_provided_fields: Optional[frozenset[str]] = field(default=None, init=False, repr=False)
 
     def __post_init__(self):
         if self.pad_id is None:
@@ -448,6 +466,43 @@ def _validate(self):
         if self.logprobs_simple_format and self.use_beam_search:
             raise ValueError("logprobs_simple_format is not supported with beam search")
 
+    def _set_request_provided_fields(self, field_names: Iterable[str]) -> None:
+        """Record sampler fields explicitly supplied by a serving request."""
+        self._request_provided_fields = frozenset(field_names) & _GENERATION_CONFIG_SAMPLING_FIELDS
+
+    def _apply_generation_config_defaults(self, generation_config: Mapping[str, Any]) -> None:
+        """Apply compatible model defaults without overriding request values."""
+        for field_name in _GENERATION_CONFIG_SAMPLING_FIELDS:
+            if field_name not in generation_config:
+                continue
+
+            # Direct LLM API calls preserve None as the unset sentinel. Serving
+            # adapters materialize protocol defaults, so they instead record
+            # which fields the request explicitly supplied.
+            if self._request_provided_fields is None:
+                request_provided = getattr(self, field_name) is not None
+            else:
+                request_provided = field_name in self._request_provided_fields
+            if request_provided:
+                continue
+
+            # A JSON null is also unset and must fall through to the existing
+            # TRT-LLM or serving default.
+            value = generation_config[field_name]
+            if value is None:
+                continue
+            # Hugging Face also permits "never", which the TRT-LLM integer
+            # early-stopping setting cannot represent.
+            if field_name == "early_stopping" and not isinstance(value, (bool, int)):
+                logger.warning(
+                    "Ignoring unsupported generation_config.json early_stopping value "
+                    f"{value!r}; TRT-LLM supports only boolean or integer values."
+                )
+                continue
+            setattr(self, field_name, value)
+
+        self._validate()
+
     # NB: The predicates below are static because downstream code (e.g.
     #     sampler_strategy.resolve_sampling_strategy) only holds instances of
     #     bindings.SamplingConfig (not SamplingParams). They are the single

tensorrt_llm/serve/openai_protocol.py

@@ -492,6 +492,14 @@ def _response_format_text_config_to_guided_decoding_params(
         resp_format, reasoning_parser=reasoning_parser)
 
 
+def _record_sampling_params_request_fields(
+        request: OpenAIBaseModel, sampling_params: SamplingParams) -> None:
+    """Preserve explicitly supplied fields across protocol defaulting."""
+    sampling_params._set_request_provided_fields(
+        field_name for field_name in request.model_fields_set
+        if getattr(request, field_name, None) is not None)
+
+
 class CompletionRequest(OpenAIBaseModel):
     # Ordered by official OpenAI API documentation
     # https://platform.openai.com/docs/api-reference/completions/create
@@ -631,6 +639,7 @@ def to_sampling_params(self,
         )
         if return_log_probs:
             sampling_params._return_log_probs = True
+        _record_sampling_params_request_fields(self, sampling_params)
         return sampling_params
 
     @model_validator(mode="before")
@@ -1044,6 +1053,7 @@ def to_sampling_params(self,
         )
         if return_log_probs:
             sampling_params._return_log_probs = True
+        _record_sampling_params_request_fields(self, sampling_params)
         return sampling_params
 
     @model_validator(mode='before')
@@ -1221,7 +1231,7 @@ def to_sampling_params(
             guided_decoding = _response_format_text_config_to_guided_decoding_params(
                 self.text.format, reasoning_parser=reasoning_parser)
 
-        return SamplingParams(
+        sampling_params = SamplingParams(
             temperature=temperature,
             top_p=top_p,
             max_tokens=max_tokens,
@@ -1230,6 +1240,8 @@ def to_sampling_params(
             guided_decoding=guided_decoding,
             thinking_token_budget=self.thinking_token_budget,
         )
+        _record_sampling_params_request_fields(self, sampling_params)
+        return sampling_params
 
     @model_validator(mode="before")
     @classmethod

tensorrt_llm/usage/llm_args_golden_manifest.json

@@ -519,6 +519,16 @@
       "kind": "value",
       "path": "gather_generation_logits"
     },
+    {
+      "allowed_values": [
+        "auto",
+        "trtllm"
+      ],
+      "annotation": "Literal['auto', 'trtllm']",
+      "converter": "",
+      "kind": "categorical",
+      "path": "generation_config"
+    },
     {
       "allowed_values": [
         "auto",