Skip to content

[TRTLLM-8685][chore] Unify CLI options for LLM API across bench, serve, and eval commands #8328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
anish-shanbhag wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

[TRTLLM-8685][chore] Unify CLI options for LLM API across bench, serve, and eval commands #8328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1f71907
[None][chore] Unify CLI options for LLM API configuration across benc…
anish-shanbhag Oct 13, 2025
9428ee5
Simplify option generation
anish-shanbhag Oct 14, 2025
24969b8
Add common options to encoder serve command
anish-shanbhag Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 3 additions & 42 deletions tensorrt_llm/bench/benchmark/low_latency.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
from tensorrt_llm.bench.utils.data import (create_dataset_from_stream,
initialize_tokenizer,
update_metadata_for_multimodal)
from tensorrt_llm.commands.common_llm_options import common_llm_options
from tensorrt_llm.logger import logger
from tensorrt_llm.sampling_params import SamplingParams

Expand All @@ -45,30 +46,6 @@
default=None,
help="Path to a serialized TRT-LLM engine.",
)
@optgroup.option(
"--extra_llm_api_options",
type=str,
default=None,
help=
"Path to a YAML file that overwrites the parameters specified by trtllm-bench."
)
@optgroup.option(
"--backend",
type=click.Choice(ALL_SUPPORTED_BACKENDS),
default="pytorch",
help="The backend to use for benchmark. Default is pytorch backend.")
@optgroup.option(
"--kv_cache_free_gpu_mem_fraction",
type=float,
default=.90,
help="The percentage of memory to use for KV Cache after model load.",
)
@optgroup.option(
"--max_seq_len",
type=int,
default=None,
help="Maximum sequence length.",
)
@optgroup.group(
"Engine Input Configuration",
help="Input configuration for driving the engine.",
Expand Down Expand Up @@ -109,24 +86,6 @@
default=2,
help="Number of requests warm up benchmark.",
)
@optgroup.option(
"--tp",
type=int,
default=1,
help="tensor parallelism size",
)
@optgroup.option(
"--pp",
type=int,
default=1,
help="pipeline parallelism size",
)
@optgroup.option(
"--ep",
type=int,
default=None,
help="expert parallelism size",
)
@optgroup.group("Request Load Control Options",
cls=MutuallyExclusiveOptionGroup,
help="Limits how requests are loaded.")
Expand Down Expand Up @@ -185,6 +144,7 @@
required=False,
help="Path where iteration logging is written to.",
)
@common_llm_options
@click.pass_obj
def latency_command(
bench_env: BenchmarkEnvironment,
Expand Down Expand Up @@ -285,6 +245,7 @@ def latency_command(
# Construct the runtime configuration dataclass.
runtime_config = RuntimeConfig(**exec_settings)

# TODO: unify LlmArgs parsing via Pydantic
llm = None
kwargs = kwargs | runtime_config.get_llm_args()
kwargs['backend'] = options.backend
Expand Down
91 changes: 9 additions & 82 deletions tensorrt_llm/bench/benchmark/throughput.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
from tensorrt_llm.bench.utils.data import (create_dataset_from_stream,
initialize_tokenizer,
update_metadata_for_multimodal)
from tensorrt_llm.commands.common_llm_options import common_llm_options
from tensorrt_llm.llmapi import CapacitySchedulerPolicy
from tensorrt_llm.logger import logger
from tensorrt_llm.sampling_params import SamplingParams
Expand All @@ -45,11 +46,6 @@
default=None,
help="Path to a serialized TRT-LLM engine.",
)
@optgroup.option(
"--backend",
type=click.Choice(ALL_SUPPORTED_BACKENDS),
default="pytorch",
help="The backend to use for benchmark. Default is pytorch backend.")
@optgroup.option(
"--custom_module_dirs",
type=click.Path(exists=True,
Expand All @@ -60,48 +56,19 @@
multiple=True,
help="Paths to custom module directories to import.",
)
@optgroup.option(
"--extra_llm_api_options",
type=str,
default=None,
help=
"Path to a YAML file that overwrites the parameters specified by trtllm-bench."
)
@optgroup.option("--sampler_options",
type=click.Path(exists=True,
readable=True,
path_type=Path,
resolve_path=True),
default=None,
help="Path to a YAML file that sets sampler options.")
@optgroup.option(
"--max_batch_size",
type=int,
help="Maximum runtime batch size to run the engine with.",
)
@optgroup.option(
"--max_num_tokens",
type=int,
help="Maximum runtime tokens that an engine can accept.",
)
@optgroup.option(
"--max_seq_len",
type=int,
default=None,
help="Maximum sequence length.",
)
@optgroup.option(
"--beam_width",
type=int,
default=1,
help="Number of search beams.",
)
@optgroup.option(
"--kv_cache_free_gpu_mem_fraction",
type=float,
default=.90,
help="The percentage of memory to use for KV Cache after model load.",
)
@optgroup.group(
"Engine Input Configuration",
help="Input configuration for driving the engine.",
Expand All @@ -116,16 +83,6 @@
required=False,
help="Pass in a dataset file for parsing instead of stdin.",
)
# For text models, tokenizer initialization is not needed when loading the model since the dataset is already tokenized.
# For this reason, we skip tokenizer initialization by default.
# However, for VLM models, tokenizer initialization is needed inside the model since the dataset contains texts and
# raw media data. We cannot skip tokenizer initialization in this case.
@optgroup.option(
"--no_skip_tokenizer_init",
is_flag=True,
default=False,
help="Do not skip tokenizer initialization when loading the model.",
)
@optgroup.option(
"--eos_id",
type=int,
Expand All @@ -152,14 +109,6 @@
default="cuda",
help="Device to load the multimodal data on.",
)
@optgroup.option(
"--max_input_len",
type=int,
default=4096,
help=
"Maximum input sequence length to use for multimodal models. This is used only when --modality "
"is specified since the actual number of vision tokens is unknown before the model is run.",
)
@optgroup.option(
"--num_requests",
type=int,
Expand All @@ -186,34 +135,6 @@
type=click.IntRange(min=1),
help="Target (average) sequence length for tuning heuristics.",
)
@optgroup.group(
"World Configuration",
help="Options for configuring the backend multi-GPU world.",
)
@optgroup.option(
"--tp",
type=int,
default=1,
help="tensor parallelism size",
)
@optgroup.option(
"--pp",
type=int,
default=1,
help="pipeline parallelism size",
)
@optgroup.option(
"--ep",
type=int,
default=None,
help="expert parallelism size",
)
@optgroup.option(
"--cluster_size",
type=int,
default=None,
help="expert cluster parallelism size",
)
@optgroup.group("Request Load Control Options",
cls=MutuallyExclusiveOptionGroup,
help="Limits how requests are loaded.")
Expand Down Expand Up @@ -286,6 +207,7 @@
help=
"KV cache scheduler policy: guaranteed_no_evict prevents request eviction, max_utilization optimizes for throughput.",
)
@common_llm_options
@click.pass_obj
def throughput_command(
bench_env: BenchmarkEnvironment,
Expand All @@ -296,7 +218,11 @@ def throughput_command(
# Parameters from CLI
image_data_format: str = params.get("image_data_format", "pt")
data_device: str = params.get("data_device", "cpu")
no_skip_tokenizer_init: bool = params.get("no_skip_tokenizer_init", False)
# For text models, tokenizer initialization is not needed when loading the model since the dataset is already
# tokenized. For this reason, we skip tokenizer initialization by default.
# However, for VLM models, tokenizer initialization is needed inside the model since the dataset contains texts and
# raw media data. We cannot skip tokenizer initialization in this case, hence we expose the option to the user.
skip_tokenizer_init: bool = params.get("skip_tokenizer_init", True)

# Get general CLI options using the centralized function
options: GeneralExecSettings = get_general_cli_options(params, bench_env)
Expand Down Expand Up @@ -396,6 +322,7 @@ def throughput_command(
# Dynamic runtime features.
exec_settings["settings_config"]["dynamic_max_batch_size"] = True

# TODO: unify LlmArgs parsing via Pydantic
# LlmArgs
exec_settings["extra_llm_api_options"] = params.pop("extra_llm_api_options")
exec_settings["iteration_log"] = options.iteration_log
Expand All @@ -407,7 +334,7 @@ def throughput_command(
try:
logger.info("Setting up throughput benchmark.")
kwargs = kwargs | runtime_config.get_llm_args()
kwargs['skip_tokenizer_init'] = not no_skip_tokenizer_init
kwargs['skip_tokenizer_init'] = skip_tokenizer_init
kwargs['backend'] = options.backend

llm = get_llm(runtime_config, kwargs)
Expand Down
Loading