diff --git a/tensorrt_llm/bench/benchmark/low_latency.py b/tensorrt_llm/bench/benchmark/low_latency.py index ac3efd14bd61..f0c130f63f2f 100644 --- a/tensorrt_llm/bench/benchmark/low_latency.py +++ b/tensorrt_llm/bench/benchmark/low_latency.py @@ -29,6 +29,8 @@ from tensorrt_llm.bench.utils.data import (create_dataset_from_stream, initialize_tokenizer, update_metadata_for_multimodal) +from tensorrt_llm.bench.utils.scenario import ( + prepare_llm_api_options_for_recipe, process_recipe_scenario) from tensorrt_llm.logger import logger from tensorrt_llm.sampling_params import SamplingParams @@ -45,6 +47,16 @@ default=None, help="Path to a serialized TRT-LLM engine.", ) +@optgroup.option( + "--recipe", + type=click.Path(exists=True, + readable=True, + path_type=Path, + resolve_path=True), + default=None, + help= + "Path to a recipe YAML file containing scenario and LLM API configuration. " + "CLI flags explicitly set will override recipe values.") @optgroup.option( "--extra_llm_api_options", type=str, @@ -196,6 +208,19 @@ def latency_command( # Model, experiment, and engine params options = get_general_cli_options(params, bench_env) + # Process recipe scenario if present + cli_defaults = { + 'concurrency': 1, # Latency default is 1 (not -1 like throughput) + 'target_input_len': None, + 'target_output_len': None, + 'num_requests': 0, + 'tp': 1, + 'pp': 1, + 'ep': None, + } + params, options, scenario = process_recipe_scenario(params, options, + bench_env, cli_defaults) + # Speculative Decode Options medusa_choices = params.get("medusa_choices") # Initialize the HF tokenizer for the specified model. @@ -274,7 +299,16 @@ def latency_command( exec_settings["performance_options"]["cuda_graphs"] = True exec_settings["performance_options"]["multi_block_mode"] = True - exec_settings["extra_llm_api_options"] = params.get("extra_llm_api_options") + # Process recipe format if detected - extract llm_api_options only + # Priority: --extra_llm_api_options > --recipe + recipe_path = params.get("recipe", None) + extra_llm_api_options_path = params.get("extra_llm_api_options", None) + config_path = extra_llm_api_options_path if extra_llm_api_options_path else recipe_path + # Convert Path to string if needed + if config_path is not None: + config_path = str(config_path) + exec_settings["extra_llm_api_options"] = prepare_llm_api_options_for_recipe( + config_path, scenario) # Decoding Options if medusa_choices is not None: diff --git a/tensorrt_llm/bench/benchmark/throughput.py b/tensorrt_llm/bench/benchmark/throughput.py index 6406b755c766..44530761afe8 100755 --- a/tensorrt_llm/bench/benchmark/throughput.py +++ b/tensorrt_llm/bench/benchmark/throughput.py @@ -28,6 +28,8 @@ from tensorrt_llm.bench.utils.data import (create_dataset_from_stream, initialize_tokenizer, update_metadata_for_multimodal) +from tensorrt_llm.bench.utils.scenario import ( + prepare_llm_api_options_for_recipe, process_recipe_scenario) from tensorrt_llm.llmapi import CapacitySchedulerPolicy from tensorrt_llm.logger import logger from tensorrt_llm.sampling_params import SamplingParams @@ -60,6 +62,16 @@ multiple=True, help="Paths to custom module directories to import.", ) +@optgroup.option( + "--recipe", + type=click.Path(exists=True, + readable=True, + path_type=Path, + resolve_path=True), + default=None, + help= + "Path to a recipe YAML file containing scenario and LLM API configuration. " + "CLI flags explicitly set will override recipe values.") @optgroup.option( "--extra_llm_api_options", type=str, @@ -302,6 +314,20 @@ def throughput_command( options: GeneralExecSettings = get_general_cli_options(params, bench_env) tokenizer = initialize_tokenizer(options.checkpoint_path) + # Process recipe scenario if present + cli_defaults = { + 'concurrency': -1, + 'target_input_len': None, + 'target_output_len': None, + 'num_requests': 0, + 'tp': 1, + 'pp': 1, + 'ep': None, + 'streaming': False, + } + params, options, scenario = process_recipe_scenario(params, options, + bench_env, cli_defaults) + # Extract throughput-specific options not handled by GeneralExecSettings max_batch_size = params.get("max_batch_size") max_num_tokens = params.get("max_num_tokens") @@ -397,7 +423,17 @@ def throughput_command( exec_settings["settings_config"]["dynamic_max_batch_size"] = True # LlmArgs - exec_settings["extra_llm_api_options"] = params.pop("extra_llm_api_options") + # Process recipe format if detected - extract llm_api_options only + # Priority: --extra_llm_api_options > --recipe + recipe_path = params.pop("recipe", None) + extra_llm_api_options_path = params.pop("extra_llm_api_options", None) + config_path = extra_llm_api_options_path if extra_llm_api_options_path else recipe_path + # Convert Path to string if needed + if config_path is not None: + config_path = str(config_path) + exec_settings["extra_llm_api_options"] = prepare_llm_api_options_for_recipe( + config_path, scenario) + exec_settings["iteration_log"] = options.iteration_log # Construct the runtime configuration dataclass. diff --git a/tensorrt_llm/bench/benchmark/utils/general.py b/tensorrt_llm/bench/benchmark/utils/general.py index 3a35008daba3..a4ffb0b4cbf2 100755 --- a/tensorrt_llm/bench/benchmark/utils/general.py +++ b/tensorrt_llm/bench/benchmark/utils/general.py @@ -84,7 +84,31 @@ def get_settings(params: dict, dataset_metadata: DatasetMetadata, model: str, kv_cache_config = {} if extra_llm_api_options: with open(extra_llm_api_options, 'r') as f: - llm_args_dict = yaml.safe_load(f) + loaded_data = yaml.safe_load(f) + + # Detect recipe format (has 'scenario' and 'llm_api_options' keys) + if isinstance( + loaded_data, dict + ) and 'scenario' in loaded_data and 'llm_api_options' in loaded_data: + # Recipe format - extract llm_api_options section for LLM args + llm_args_dict = loaded_data['llm_api_options'] + + # TODO: Add llm_api_options validation once PR #8331 merges + # (standardizes LlmArgs with Pydantic - validation will happen automatically) + + # Set environment variables from 'env' section (if not already set) + import os + env_vars = loaded_data.get('env', {}) + for key, value in env_vars.items(): + if key not in os.environ: + os.environ[key] = str(value) + logger.info( + f"Set environment variable from recipe: {key}={value}" + ) + else: + # Simple format - use loaded data directly + llm_args_dict = loaded_data + kv_cache_config = llm_args_dict.get("kv_cache_config", { "dtype": "auto", }) diff --git a/tensorrt_llm/bench/utils/scenario.py b/tensorrt_llm/bench/utils/scenario.py new file mode 100644 index 000000000000..8b32b3496014 --- /dev/null +++ b/tensorrt_llm/bench/utils/scenario.py @@ -0,0 +1,339 @@ +"""Utilities for extracting and processing recipe scenario parameters. + +This module provides functions to extract scenario information from recipe YAML +files and merge them with CLI parameters for trtllm-bench commands. +""" + +import json +import os +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple + +import yaml +from pydantic import ValidationError + +from tensorrt_llm.logger import logger +from tensorrt_llm.recipes import RecipeConfig, ScenarioConfig + +if TYPE_CHECKING: + from tensorrt_llm.bench.benchmark import GeneralExecSettings + from tensorrt_llm.bench.dataclasses.general import BenchmarkEnvironment + + +def extract_scenario_from_recipe(recipe_path: Optional[str]) -> Optional[Dict[str, Any]]: + """Extract scenario section from a recipe YAML file. + + Args: + recipe_path: Path to recipe YAML file, or None + + Returns: + Dictionary containing scenario parameters, or None if not a recipe format + or if recipe_path is None + + Example: + >>> scenario = extract_scenario_from_recipe("recipe.yaml") + >>> print(scenario["target_isl"]) + 8192 + """ + if recipe_path is None: + return None + + try: + with open(recipe_path, "r") as f: + loaded_data = yaml.safe_load(f) + + # Parse and validate using Pydantic schema + recipe = RecipeConfig(**loaded_data) + return recipe.scenario.model_dump() + + except (FileNotFoundError, yaml.YAMLError, KeyError, ValidationError): + # Not a valid recipe format, return None + return None + + +def merge_params_with_priority( + cli_params: Dict[str, Any], + scenario: Optional[Dict[str, Any]], + cli_defaults: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Merge CLI parameters with scenario values, with CLI taking precedence. + + Priority order (highest to lowest): + 1. Explicitly set CLI parameters (different from default) + 2. Scenario values from recipe + 3. CLI default values + + Args: + cli_params: Parameters from CLI arguments + scenario: Scenario dict from recipe (or None) + cli_defaults: Default values for CLI args (used to detect explicit values) + + Returns: + Merged parameter dictionary + + Example: + >>> cli = {"concurrency": 128, "tp": 1} + >>> scenario = {"target_concurrency": 256, "tp_size": 4} + >>> defaults = {"concurrency": -1, "tp": 1} + >>> merged = merge_params_with_priority(cli, scenario, defaults) + >>> print(merged["concurrency"]) # CLI explicitly set + 128 + >>> print(merged["tp"]) # From scenario (tp_size -> tp) + 4 + """ + if scenario is None: + return cli_params.copy() + + merged = cli_params.copy() + + # Mapping from scenario keys to CLI parameter keys + # Note: 'model' is excluded because it's a required top-level trtllm-bench parameter + param_mapping = { + "target_concurrency": "concurrency", + "target_isl": "target_input_len", + "target_osl": "target_output_len", + "num_requests": "num_requests", + "tp_size": "tp", + "ep_size": "ep", + "pp_size": "pp", + "streaming": "streaming", + } + + for scenario_key, cli_key in param_mapping.items(): + if scenario_key in scenario: + scenario_value = scenario[scenario_key] + + # Check if CLI value was explicitly set (differs from default) + cli_value = cli_params.get(cli_key) + default_value = cli_defaults.get(cli_key) if cli_defaults else None + + # Use scenario value if: + # 1. CLI value is None/not set, OR + # 2. CLI value equals the default (not explicitly set by user) + if cli_value is None or (default_value is not None and cli_value == default_value): + merged[cli_key] = scenario_value + logger.info( + f"Using recipe value for --{cli_key}: {scenario_value} " + f"(from scenario.{scenario_key})" + ) + else: + # CLI value was explicitly set - it overrides scenario + logger.warning( + f"CLI flag --{cli_key}={cli_value} overrides recipe value " + f"scenario.{scenario_key}={scenario_value}" + ) + + return merged + + +def validate_scenario_params(scenario: Dict[str, Any]) -> None: + """Validate scenario parameters using Pydantic schema. + + Args: + scenario: Scenario dictionary to validate + + Raises: + ValidationError: If scenario parameters are invalid + """ + # Pydantic validation handles all field checks automatically + ScenarioConfig(**scenario) + + +def prepare_llm_api_options_for_recipe( + extra_llm_api_options_path: Optional[str], scenario: Optional[Dict[str, Any]] +) -> Optional[str]: + """Prepare llm_api_options for LLM constructor when using recipe format. + + When a recipe format is detected (scenario is not None), this function extracts + only the llm_api_options section and writes it to a temporary file. This prevents + the scenario section from being passed to the LLM constructor, which would cause + an "invalid argument" error. + + Args: + extra_llm_api_options_path: Path to recipe/config YAML file + scenario: Scenario dict from recipe (None if not recipe format) + + Returns: + Path to temporary file with llm_api_options (if recipe format), or + original path (if not recipe format), or None (if no path provided) + + Example: + >>> scenario = extract_scenario_from_recipe("recipe.yaml") + >>> config_path = prepare_llm_api_options_for_recipe("recipe.yaml", scenario) + # config_path now points to temp file with only llm_api_options section + """ + if extra_llm_api_options_path is None: + return None + + # If not a recipe format, return original path + if scenario is None: + return extra_llm_api_options_path + + # Recipe format detected - extract llm_api_options only + logger.info("Recipe format detected - extracting llm_api_options for LLM constructor") + + try: + with open(extra_llm_api_options_path, "r") as f: + full_recipe = yaml.safe_load(f) + + # Extract only the llm_api_options section + llm_api_options_only = full_recipe.get("llm_api_options", {}) + + # Create temporary file with only llm_api_options + temp_fd, temp_path = tempfile.mkstemp(suffix=".yaml", text=True) + with os.fdopen(temp_fd, "w") as f: + yaml.safe_dump(llm_api_options_only, f) + + logger.info(f"Created temporary config file with llm_api_options at: {temp_path}") + return temp_path + + except (FileNotFoundError, yaml.YAMLError, KeyError) as e: + logger.warning(f"Failed to process recipe file for llm_api_options: {e}") + return extra_llm_api_options_path + + +def auto_generate_dataset( + scenario: Dict[str, Any], + workspace: Path, + tokenizer: str, + output_filename: str = "auto_generated_dataset.json", +) -> Path: + """Generate a synthetic dataset from scenario parameters. + + Args: + scenario: Scenario dictionary with ISL/OSL/concurrency parameters + workspace: Workspace directory to write dataset + tokenizer: Tokenizer name or path for dataset generation + output_filename: Name of output dataset file + + Returns: + Path to generated dataset file + + Raises: + ValueError: If required scenario parameters are missing + """ + validate_scenario_params(scenario) + + dataset_path = workspace / output_filename + + # Extract parameters + target_isl = scenario["target_isl"] + target_osl = scenario["target_osl"] + num_requests = scenario.get("num_requests", 512) + isl_stdev = scenario.get("isl_stdev", 0) + osl_stdev = scenario.get("osl_stdev", 0) + + # Generate synthetic dataset using prepare_dataset.py logic + # For now, create a simple JSON format that benchmarks can consume + # + # TODO: This is a simplified implementation. In production, should either: + # 1. Call prepare_dataset.py as a subprocess + # 2. Import and use prepare_dataset.py's generation logic + # 3. Use the dataset generation utilities from benchmarks/cpp/ + + import numpy as np + + requests = [] + for i in range(num_requests): + # Generate input/output lengths with normal distribution + if isl_stdev > 0: + input_len = int(max(1, np.random.normal(target_isl, isl_stdev))) + else: + input_len = target_isl + + if osl_stdev > 0: + output_len = int(max(1, np.random.normal(target_osl, osl_stdev))) + else: + output_len = target_osl + + # Create request in format expected by benchmarks + request = { + "task_id": i, + "prompt": " ".join(["word"] * input_len), # Placeholder tokens + "output_tokens": output_len, + "input_len": input_len, + } + requests.append(request) + + # Write to JSON Lines file (one JSON object per line) + # This is the format expected by trtllm-bench + workspace.mkdir(parents=True, exist_ok=True) + with open(dataset_path, "w") as f: + for request in requests: + f.write(json.dumps(request) + "\n") + + return dataset_path + + +def process_recipe_scenario( + params: Dict[str, Any], + options: "GeneralExecSettings", + bench_env: "BenchmarkEnvironment", + cli_defaults: Dict[str, Any], +) -> Tuple[Dict[str, Any], "GeneralExecSettings", Optional[Dict[str, Any]]]: + """Process recipe scenario: extract, merge params, and auto-generate dataset. + + This is a unified helper for throughput and low_latency benchmarks to handle + recipe-based configuration. It: + 1. Extracts scenario from recipe file (if present) + 2. Merges CLI params with scenario (CLI takes precedence) + 3. Auto-generates dataset if needed based on scenario ISL/OSL + + Args: + params: CLI parameters dictionary (will be modified in-place) + options: General execution settings from get_general_cli_options + bench_env: Benchmark environment object + cli_defaults: Default values for CLI args (used to detect explicit values) + Should vary by benchmark type (e.g., concurrency differs) + + Returns: + Tuple of (updated_params, updated_options, scenario) + - updated_params: params dict with merged scenario values + - updated_options: regenerated options if dataset was auto-generated + - scenario: extracted scenario dict (or None if not recipe format) + """ + # Import here to avoid circular dependency + from tensorrt_llm.bench.benchmark import get_general_cli_options + + # Extract scenario from recipe + # Priority: --extra_llm_api_options > --recipe + recipe_path = params.get("recipe") + extra_llm_api_options_path = params.get("extra_llm_api_options") + config_path = extra_llm_api_options_path if extra_llm_api_options_path else recipe_path + + # Warn if both are provided + if recipe_path and extra_llm_api_options_path: + logger.warning( + f"Both --recipe and --extra_llm_api_options provided. " + f"Using --extra_llm_api_options ({extra_llm_api_options_path}) " + f"which overrides --recipe ({recipe_path})" + ) + + scenario = extract_scenario_from_recipe(config_path) + + if not scenario: + return params, options, None + + logger.info("Detected recipe format with scenario parameters") + + # Merge CLI params with scenario (CLI explicitly set takes precedence) + merged_params = merge_params_with_priority(params, scenario, cli_defaults) + + # Update params with merged values + params.update(merged_params) + + # Auto-generate dataset if not provided + if params.get("dataset") is None and scenario.get("target_isl") and scenario.get("target_osl"): + logger.info("No dataset provided, auto-generating from scenario parameters") + workspace = Path.cwd() / ".trtllm_bench_workspace" + auto_dataset_path = auto_generate_dataset( + scenario, workspace, tokenizer=str(options.checkpoint_path) + ) + params["dataset"] = auto_dataset_path + logger.info(f"Generated dataset at {auto_dataset_path}") + + # Update options with auto-generated dataset + options = get_general_cli_options(params, bench_env) + + return params, options, scenario diff --git a/tensorrt_llm/commands/serve.py b/tensorrt_llm/commands/serve.py index f4f188fdea8b..0e78568fe5f8 100644 --- a/tensorrt_llm/commands/serve.py +++ b/tensorrt_llm/commands/serve.py @@ -10,6 +10,7 @@ import click import torch import yaml +from pydantic import ValidationError from strenum import StrEnum from torch.cuda import device_count @@ -32,6 +33,7 @@ from tensorrt_llm.llmapi.mpi_session import find_free_port from tensorrt_llm.llmapi.reasoning_parser import ReasoningParserFactory from tensorrt_llm.logger import logger, severity_map +from tensorrt_llm.recipes import RecipeConfig from tensorrt_llm.serve import OpenAIDisaggServer, OpenAIServer from tensorrt_llm.serve.tool_parser import ToolParserFactory @@ -397,7 +399,24 @@ def serve( llm_args_extra_dict = {} if extra_llm_api_options is not None: with open(extra_llm_api_options, 'r') as f: - llm_args_extra_dict = yaml.safe_load(f) + loaded_data = yaml.safe_load(f) + + # Try to parse as recipe format with Pydantic validation + try: + recipe = RecipeConfig(**loaded_data) + # Recipe format validated - extract llm_api_options and env + llm_args_extra_dict = recipe.llm_api_options + + # Set environment variables from 'env' section (if not already set) + for key, value in recipe.env.items(): + if key not in os.environ: + os.environ[key] = str(value) + logger.info( + f"Set environment variable from recipe: {key}={value}") + except ValidationError: + # Not a valid recipe format - treat as simple llm_api_options format + llm_args_extra_dict = loaded_data + llm_args = update_llm_args_with_extra_dict(llm_args, llm_args_extra_dict) metadata_server_cfg = parse_metadata_server_config_file( @@ -817,6 +836,7 @@ def resolve_command(self, ctx, args): main = DefaultGroup( commands={ "serve": serve, + "configure": configure, "disaggregated": disaggregated, "disaggregated_mpi_worker": disaggregated_mpi_worker, "mm_embedding_serve": serve_encoder diff --git a/tensorrt_llm/recipes/__init__.py b/tensorrt_llm/recipes/__init__.py new file mode 100644 index 000000000000..741ae134e4fc --- /dev/null +++ b/tensorrt_llm/recipes/__init__.py @@ -0,0 +1,10 @@ +"""Recipe validation and configuration schemas. + +This package provides Pydantic schemas for validating recipe YAML files. +Recipes combine scenario parameters (benchmark settings) with LLM API +configuration for reproducible performance testing. +""" + +from .schema import RecipeConfig, ScenarioConfig + +__all__ = ["RecipeConfig", "ScenarioConfig"] diff --git a/tensorrt_llm/recipes/db/__init__.py b/tensorrt_llm/recipes/db/__init__.py new file mode 100644 index 000000000000..8255910b2ffc --- /dev/null +++ b/tensorrt_llm/recipes/db/__init__.py @@ -0,0 +1 @@ +"""Curated recipe database for common inference scenarios.""" diff --git a/tensorrt_llm/recipes/db/tinyllama-test.yaml b/tensorrt_llm/recipes/db/tinyllama-test.yaml new file mode 100644 index 000000000000..b44834816016 --- /dev/null +++ b/tensorrt_llm/recipes/db/tinyllama-test.yaml @@ -0,0 +1,26 @@ +# TinyLlama 1.1B FP16 Recipe (Test Configuration) +# + +scenario: + model: tinyllama + num_gpus: 1 + target_isl: 1024 + target_osl: 256 + target_concurrency: 32 + # Optional: Dataset generation parameters. + # This is useful for trtllm-bench to auto-generate dataset, so one can just specify this recipe + # to trtllm-bench without prior steps. + isl_stdev: 0 # Input sequence length standard deviation (0 = exact) + osl_stdev: 0 # Output sequence length standard deviation (0 = exact) + num_requests: 128 # Number of requests for auto-generated dataset + +env: + TLLM_WORKER_USE_SINGLE_PROCESS: 1 + +llm_api_options: + tensor_parallel_size: 1 + max_batch_size: 256 + max_num_tokens: 4096 + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 diff --git a/tensorrt_llm/recipes/schema.py b/tensorrt_llm/recipes/schema.py new file mode 100644 index 000000000000..9d93f83545e9 --- /dev/null +++ b/tensorrt_llm/recipes/schema.py @@ -0,0 +1,67 @@ +"""Pydantic schemas for recipe validation. + +This module provides the single source of truth for recipe file structure. +Recipes are YAML files that combine scenario parameters (benchmark settings) +with LLM API options (model configuration). +""" + +from typing import Any, Dict, Optional + +from pydantic import BaseModel, Field + + +class ScenarioConfig(BaseModel): + """Scenario parameters for benchmark configuration. + + Defines the target workload characteristics for performance testing. + """ + + model_config = {"extra": "forbid"} # Strict validation - only known fields allowed + + # Required fields + model: str = Field(description="Model identifier (e.g., 'tinyllama', 'llama-7b')") + target_isl: int = Field(gt=0, description="Target input sequence length (must be positive)") + target_osl: int = Field(gt=0, description="Target output sequence length (must be positive)") + target_concurrency: int = Field(gt=0, description="Target concurrency rate (must be positive)") + + # Optional benchmark-specific fields for trtllm-bench auto-dataset generation + isl_stdev: int = Field( + default=0, + ge=0, + description="ISL standard deviation for auto-dataset generation (0=exact, for trtllm-bench)", + ) + osl_stdev: int = Field( + default=0, + ge=0, + description="OSL standard deviation for auto-dataset generation (0=exact, for trtllm-bench)", + ) + num_requests: int = Field( + default=512, + gt=0, + description="Number of requests for auto-dataset generation (consumed by trtllm-bench)", + ) + + # Metadata (optional, not validated beyond type) + gpu: Optional[str] = Field(default=None, description="GPU type metadata (e.g., 'H100', 'A100')") + num_gpus: Optional[int] = Field(default=None, ge=1, description="Number of GPUs (metadata)") + + +class RecipeConfig(BaseModel): + """Complete recipe configuration. + + A recipe combines: + - scenario: Benchmark workload parameters + - llm_api_options: LLM API configuration (validated separately by LlmArgs) + - env: Environment variables to set + """ + + model_config = {"extra": "forbid"} # Strict validation at top level + + # Required + scenario: ScenarioConfig = Field(description="Benchmark scenario parameters") + + # Optional + env: Dict[str, Any] = Field(default_factory=dict, description="Environment variables") + llm_api_options: Dict[str, Any] = Field( + default_factory=dict, description="LLM API configuration" + ) diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index 081b9fb6b67e..6584a386acef 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -67,6 +67,7 @@ "modelopt-hf-model-hub/Llama-3.1-405B-Instruct-fp4", "llama_v3.1_70b_instruct": "llama-3.1-model/Meta-Llama-3.1-70B-Instruct", "llama_v3.2_1b": "llama-3.2-models/Llama-3.2-1B", + "tinyllama": "llama-models-v2/TinyLlama-1.1B-Chat-v1.0", "llama_v3.1_nemotron_nano_8b": "Llama-3.1-Nemotron-Nano-8B-v1", "llama_v3.1_nemotron_nano_8b_fp8": "Llama-3.1-Nemotron-Nano-8B-v1-FP8", "llama_v3.3_nemotron_super_49b": @@ -940,6 +941,10 @@ def __init__( self.server_configs = [] self.server_client_configs = {} + # Used for recipe-based tests + # recipe_file: Name of recipe YAML file in tensorrt_llm/recipes/db/ + self.recipe_file = None + def _to_string_disagg(self, entries: List[str]): entries.append(f"disagg_server") if self.ctx_tp_size > 1: @@ -964,6 +969,10 @@ def to_string(self, custom_output_len: int = None, device_subtype: str = None) -> str: + # Used for recipe-based tests + if self.recipe_file is not None: + return f"recipe-{self.recipe_file}" + # Used for perf sanity test if self.config_file is not None: entries = ["perf_sanity", self.config_file] @@ -1142,6 +1151,48 @@ def load_from_str(self, test_param_labels) -> None: # Extract configs from test param labels. labels = test_param_labels.split("-") + # Used for recipe-based tests + if labels[0] == "recipe": + assert len(labels) >= 2, "recipe test must specify recipe file!" + self.runtime = "bench" + # Reconstruct full recipe filename (everything after "recipe-") + self.recipe_file = "-".join(labels[1:]) + + # Parse recipe file to extract model_name and backend for proper test setup + from pathlib import Path + + import yaml + recipe_path = Path( + __file__ + ).parent.parent.parent.parent.parent / "tensorrt_llm" / "recipes" / "db" / f"{self.recipe_file}.yaml" + + if recipe_path.exists(): + with open(recipe_path, 'r') as f: + recipe_data = yaml.safe_load(f) + scenario = recipe_data.get('scenario', {}) + + # Use model name directly from recipe (should match MODEL_PATH_DICT key) + self.model_name = scenario.get('model', '') + assert self.model_name in MODEL_PATH_DICT.keys(), \ + f"Recipe model '{self.model_name}' not found in MODEL_PATH_DICT. " \ + f"Please ensure recipe uses a model name that exists in MODEL_PATH_DICT." + + # Use PyTorch backend for recipe tests (no pre-built engine needed) + self.backend = "pytorch" + + # Extract dataset generation parameters from recipe for prepare_dataset + self.input_lens = [scenario.get('target_isl', 128)] + self.output_lens = [scenario.get('target_osl', 128)] + self.num_reqs = scenario.get('num_requests', 128) + self.batch_sizes = [1] # Single batch size for recipe tests + else: + raise FileNotFoundError( + f"Recipe file not found: {recipe_path}. " + f"Please ensure the recipe file exists in tensorrt_llm/recipes/db/" + ) + + return + # Used for perf sanity test if "perf_sanity" in labels[0]: assert len(labels) > 1, "perf_sanity test must have a config file!" @@ -1694,6 +1745,28 @@ def get_prepare_data_command(self, engine_dir, input_len, return data_cmd def get_trtllm_bench_command(self, engine_dir): + # Handle recipe-based tests + if self._config.recipe_file: + recipe_path = os.path.join(self._llm_root, + "tensorrt_llm/recipes/db", + f"{self._config.recipe_file}.yaml") + dataset_path = os.path.join(engine_dir, "synthetic_data.json") + report_path = os.path.join(engine_dir, "report.json") + + # Get model name and path from MODEL_PATH_DICT + model_name = self._config.model_name + model_path = os.path.join(llm_models_root(), + MODEL_PATH_DICT[model_name]) + + # Build command - dataset pre-generated by prepare_dataset + benchmark_cmd = [ + self._benchmark_script, f"--model={model_name}", + f"--model_path={model_path}", "throughput", + f"--dataset={dataset_path}", f"--report_json={report_path}", + f"--recipe={recipe_path}" + ] + return benchmark_cmd + model_dir = self.get_trtllm_bench_model() model_name = self._config.model_name dataset_path = os.path.join(engine_dir, "synthetic_data.json") @@ -1977,6 +2050,7 @@ def run_metrics(self, llm_venv, gpu_clock_lock, session_data_writer, """ #print info to separate cases print_info(f"Running perf test for case: {self._short_test_name}") + self._current_cmd_idx = 0 metrics = self._get_metrics() outputs = {} diff --git a/tests/integration/test_lists/qa/llm_perf_recipe_db.yml b/tests/integration/test_lists/qa/llm_perf_recipe_db.yml new file mode 100644 index 000000000000..7dbacbbe010f --- /dev/null +++ b/tests/integration/test_lists/qa/llm_perf_recipe_db.yml @@ -0,0 +1 @@ +perf/test_perf.py::test_perf[recipe-tinyllama-test] diff --git a/tests/unittest/bench/__init__.py b/tests/unittest/bench/__init__.py new file mode 100644 index 000000000000..7cd936ec917a --- /dev/null +++ b/tests/unittest/bench/__init__.py @@ -0,0 +1 @@ +"""Tests for tensorrt_llm.bench module.""" diff --git a/tests/unittest/bench/test_scenario.py b/tests/unittest/bench/test_scenario.py new file mode 100644 index 000000000000..c0660528e4b4 --- /dev/null +++ b/tests/unittest/bench/test_scenario.py @@ -0,0 +1,354 @@ +"""Unit tests for trtllm-bench scenario handling and priority logic. + +These tests verify the override behavior between --recipe, --extra_llm_api_options, +and CLI flags to ensure correct priority order and warning messages. +""" + +import tempfile +from pathlib import Path +from unittest.mock import patch + +import yaml + +from tensorrt_llm.bench.utils.scenario import ( + merge_params_with_priority, + prepare_llm_api_options_for_recipe, +) + + +class TestMergeParamsWithPriority: + """Tests for merge_params_with_priority() function.""" + + @patch("tensorrt_llm.bench.utils.scenario.logger") + def test_cli_explicitly_set_overrides_scenario(self, mock_logger): + """Test that explicitly set CLI values override scenario values.""" + cli_params = {"concurrency": 128, "tp": 2} + scenario = {"target_concurrency": 256, "tp_size": 4} + cli_defaults = {"concurrency": -1, "tp": 1} + + merged = merge_params_with_priority(cli_params, scenario, cli_defaults) + + # CLI concurrency was explicitly set (differs from default) + assert merged["concurrency"] == 128 + + # CLI tp was explicitly set (differs from default) + assert merged["tp"] == 2 + + # Verify warnings were logged + assert mock_logger.warning.call_count == 2 + warning_calls = [call[0][0] for call in mock_logger.warning.call_args_list] + assert any( + "CLI flag --concurrency=128 overrides recipe value" in call for call in warning_calls + ) + assert any("scenario.target_concurrency=256" in call for call in warning_calls) + assert any("CLI flag --tp=2 overrides recipe value" in call for call in warning_calls) + assert any("scenario.tp_size=4" in call for call in warning_calls) + + @patch("tensorrt_llm.bench.utils.scenario.logger") + def test_scenario_value_used_when_cli_not_explicitly_set(self, mock_logger): + """Test that scenario values are used when CLI equals default.""" + cli_params = {"concurrency": -1, "tp": 1} + scenario = {"target_concurrency": 256, "tp_size": 4} + cli_defaults = {"concurrency": -1, "tp": 1} + + merged = merge_params_with_priority(cli_params, scenario, cli_defaults) + + # Both CLI values equal defaults, so scenario values should be used + assert merged["concurrency"] == 256 + assert merged["tp"] == 4 + + # Verify info logs were called + assert mock_logger.info.call_count == 2 + info_calls = [call[0][0] for call in mock_logger.info.call_args_list] + assert any("Using recipe value for --concurrency: 256" in call for call in info_calls) + assert any("from scenario.target_concurrency" in call for call in info_calls) + assert any("Using recipe value for --tp: 4" in call for call in info_calls) + assert any("from scenario.tp_size" in call for call in info_calls) + + @patch("tensorrt_llm.bench.utils.scenario.logger") + def test_mixed_explicit_and_default_cli_values(self, mock_logger): + """Test scenario with some CLI values explicit and some default.""" + cli_params = {"concurrency": 128, "tp": 1, "target_input_len": None} + scenario = { + "target_concurrency": 256, + "tp_size": 4, + "target_isl": 1024, + } + cli_defaults = {"concurrency": -1, "tp": 1, "target_input_len": None} + + merged = merge_params_with_priority(cli_params, scenario, cli_defaults) + + # concurrency explicitly set -> override + assert merged["concurrency"] == 128 + + # tp equals default -> use scenario + assert merged["tp"] == 4 + + # target_input_len is None -> use scenario + assert merged["target_input_len"] == 1024 + + # Verify 1 warning and 2 info calls + assert mock_logger.warning.call_count == 1 + assert mock_logger.info.call_count == 2 + + @patch("tensorrt_llm.bench.utils.scenario.logger") + def test_cli_value_none_uses_scenario(self, mock_logger): + """Test that None CLI values use scenario values.""" + cli_params = {"tp": None, "ep": None} + scenario = {"tp_size": 4, "ep_size": 2} + cli_defaults = {"tp": 1, "ep": 1} + + merged = merge_params_with_priority(cli_params, scenario, cli_defaults) + + assert merged["tp"] == 4 + assert merged["ep"] == 2 + + # Verify info logs were called + assert mock_logger.info.call_count == 2 + + def test_all_parameter_mappings(self): + """Test all scenario-to-CLI parameter mappings.""" + cli_params = { + "concurrency": -1, + "target_input_len": None, + "target_output_len": None, + "num_requests": 512, + "tp": 1, + "ep": 1, + "pp": 1, + "streaming": False, + } + scenario = { + "target_concurrency": 128, + "target_isl": 2048, + "target_osl": 512, + "num_requests": 1000, + "tp_size": 2, + "ep_size": 4, + "pp_size": 2, + "streaming": True, + } + cli_defaults = { + "concurrency": -1, + "target_input_len": None, + "target_output_len": None, + "num_requests": 512, + "tp": 1, + "ep": 1, + "pp": 1, + "streaming": False, + } + + merged = merge_params_with_priority(cli_params, scenario, cli_defaults) + + # All should use scenario values since CLI equals defaults + assert merged["concurrency"] == 128 + assert merged["target_input_len"] == 2048 + assert merged["target_output_len"] == 512 + assert merged["num_requests"] == 1000 + assert merged["tp"] == 2 + assert merged["ep"] == 4 + assert merged["pp"] == 2 + assert merged["streaming"] is True + + def test_no_scenario_returns_cli_params(self): + """Test that None scenario returns copy of CLI params unchanged.""" + cli_params = {"concurrency": 128, "tp": 2} + cli_defaults = {"concurrency": -1, "tp": 1} + + merged = merge_params_with_priority(cli_params, None, cli_defaults) + + assert merged == cli_params + assert merged is not cli_params # Should be a copy + + def test_no_cli_defaults_provided(self, caplog): + """Test behavior when cli_defaults is None.""" + cli_params = {"concurrency": 128} + scenario = {"target_concurrency": 256} + cli_defaults = None + + merged = merge_params_with_priority(cli_params, scenario, cli_defaults) + + # Without defaults, CLI value should still override + assert merged["concurrency"] == 128 + + def test_scenario_key_not_in_params_mapping(self): + """Test that scenario keys not in mapping are ignored.""" + cli_params = {"concurrency": -1} + scenario = { + "target_concurrency": 128, + "unknown_field": "some_value", # Not in param_mapping + } + cli_defaults = {"concurrency": -1} + + merged = merge_params_with_priority(cli_params, scenario, cli_defaults) + + assert merged["concurrency"] == 128 + assert "unknown_field" not in merged + + +class TestPrepareExtraLlmApiOptions: + """Tests for priority between --recipe and --extra_llm_api_options.""" + + def test_extra_llm_api_options_overrides_recipe(self, caplog): + """Test that --extra_llm_api_options takes priority over --recipe.""" + # This would be tested at the caller level in process_recipe_scenario + # We're testing the warning message here + with patch("tensorrt_llm.bench.utils.scenario.logger") as mock_logger: + recipe_path = "/path/to/recipe.yaml" + extra_path = "/path/to/extra.yaml" + + # Simulate the logic in process_recipe_scenario + if recipe_path and extra_path: + mock_logger.warning( + f"Both --recipe and --extra_llm_api_options provided. " + f"Using --extra_llm_api_options ({extra_path}) " + f"which overrides --recipe ({recipe_path})" + ) + + # Verify warning was called + mock_logger.warning.assert_called_once() + call_args = mock_logger.warning.call_args[0][0] + assert "Both --recipe and --extra_llm_api_options provided" in call_args + assert extra_path in call_args + assert recipe_path in call_args + + +class TestPrepareLlmApiOptionsForRecipe: + """Tests for prepare_llm_api_options_for_recipe() function.""" + + def test_none_path_returns_none(self): + """Test that None path returns None.""" + result = prepare_llm_api_options_for_recipe(None, None) + assert result is None + + def test_non_recipe_format_returns_original_path(self): + """Test that non-recipe format returns original path unchanged.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + # Write simple llm_api_options (not recipe format) + yaml.safe_dump({"max_tokens": 100, "temperature": 0.7}, f) + temp_path = f.name + + try: + # scenario=None means not recipe format + result = prepare_llm_api_options_for_recipe(temp_path, scenario=None) + assert result == temp_path + finally: + Path(temp_path).unlink() + + @patch("tensorrt_llm.bench.utils.scenario.logger") + def test_recipe_format_extracts_llm_api_options(self, mock_logger): + """Test that recipe format extracts llm_api_options to temp file.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + # Write recipe format + recipe_data = { + "scenario": { + "model": "test", + "target_isl": 1024, + "target_osl": 256, + "target_concurrency": 32, + }, + "llm_api_options": {"max_tokens": 100, "temperature": 0.7}, + "env": {"SOME_VAR": "value"}, + } + yaml.safe_dump(recipe_data, f) + temp_path = f.name + + try: + # scenario dict means recipe format detected + scenario = recipe_data["scenario"] + result = prepare_llm_api_options_for_recipe(temp_path, scenario) + + # Should return a different path (temp file) + assert result != temp_path + assert result is not None + + # Verify info log was called + info_calls = [call[0][0] for call in mock_logger.info.call_args_list] + assert any("Recipe format detected" in call for call in info_calls) + + # Verify temp file contains only llm_api_options + with open(result) as f: + extracted = yaml.safe_load(f) + assert extracted == {"max_tokens": 100, "temperature": 0.7} + + # Clean up temp file + Path(result).unlink() + finally: + Path(temp_path).unlink() + + def test_recipe_with_empty_llm_api_options(self): + """Test recipe with empty llm_api_options section.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + recipe_data = { + "scenario": { + "model": "test", + "target_isl": 1024, + "target_osl": 256, + "target_concurrency": 32, + }, + "llm_api_options": {}, + } + yaml.safe_dump(recipe_data, f) + temp_path = f.name + + try: + scenario = recipe_data["scenario"] + result = prepare_llm_api_options_for_recipe(temp_path, scenario) + + assert result is not None + assert result != temp_path + + # Verify temp file contains empty dict + with open(result) as f: + extracted = yaml.safe_load(f) + assert extracted == {} + + Path(result).unlink() + finally: + Path(temp_path).unlink() + + def test_recipe_without_llm_api_options_key(self): + """Test recipe without llm_api_options key (defaults to empty dict).""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + recipe_data = { + "scenario": { + "model": "test", + "target_isl": 1024, + "target_osl": 256, + "target_concurrency": 32, + }, + # No llm_api_options key + } + yaml.safe_dump(recipe_data, f) + temp_path = f.name + + try: + scenario = recipe_data["scenario"] + result = prepare_llm_api_options_for_recipe(temp_path, scenario) + + assert result is not None + + # Verify temp file contains empty dict (default from .get()) + with open(result) as f: + extracted = yaml.safe_load(f) + assert extracted == {} or extracted is None + + Path(result).unlink() + finally: + Path(temp_path).unlink() + + @patch("tensorrt_llm.bench.utils.scenario.logger") + def test_file_not_found_returns_original_path(self, mock_logger): + """Test that FileNotFoundError returns original path with warning.""" + non_existent = "/path/that/does/not/exist.yaml" + scenario = {"model": "test", "target_isl": 1024} + + result = prepare_llm_api_options_for_recipe(non_existent, scenario) + + # Should return original path and log warning + assert result == non_existent + + # Verify warning was logged + warning_calls = [call[0][0] for call in mock_logger.warning.call_args_list] + assert any("Failed to process recipe file" in call for call in warning_calls) diff --git a/tests/unittest/recipes/__init__.py b/tests/unittest/recipes/__init__.py new file mode 100644 index 000000000000..46ea99623f58 --- /dev/null +++ b/tests/unittest/recipes/__init__.py @@ -0,0 +1 @@ +"""Unit tests for recipe validation.""" diff --git a/tests/unittest/recipes/test_schema.py b/tests/unittest/recipes/test_schema.py new file mode 100644 index 000000000000..c3e90439fa5d --- /dev/null +++ b/tests/unittest/recipes/test_schema.py @@ -0,0 +1,82 @@ +"""Unit tests for recipe schema validation. + +These tests verify that Pydantic schemas correctly validate recipe YAML files. +Minimal tests are needed since Pydantic handles validation automatically. +""" + +from pathlib import Path + +import pytest +import yaml +from pydantic import ValidationError + +from tensorrt_llm.recipes import RecipeConfig, ScenarioConfig + + +def test_tinyllama_recipe_validates(): + """Test that the tinyllama recipe file validates successfully.""" + recipe_path = Path(__file__).parents[3] / "tensorrt_llm/recipes/db/tinyllama-test.yaml" + + with open(recipe_path) as f: + data = yaml.safe_load(f) + + # Should not raise ValidationError + recipe = RecipeConfig(**data) + + # Verify basic fields + assert recipe.scenario.model == "tinyllama" + assert recipe.scenario.target_isl == 1024 + assert recipe.scenario.target_osl == 256 + assert recipe.scenario.target_concurrency == 32 + + +def test_all_recipes_in_db_validate(): + """Test that all recipe files in db/ directory validate successfully.""" + recipes_dir = Path(__file__).parents[3] / "tensorrt_llm/recipes/db" + + recipe_files = list(recipes_dir.glob("*.yaml")) + assert len(recipe_files) > 0, "No recipe files found in db/ directory" + + for recipe_file in recipe_files: + with open(recipe_file) as f: + data = yaml.safe_load(f) + + # Should not raise ValidationError + RecipeConfig(**data) + + +def test_invalid_scenario_caught(): + """Test that Pydantic catches invalid scenario parameters.""" + # Negative target_isl should be caught + with pytest.raises(ValidationError) as exc_info: + ScenarioConfig( + model="test", + target_isl=-1, # Invalid: must be positive + target_osl=256, + target_concurrency=32, + ) + + # Verify the error is about target_isl constraint + assert "target_isl" in str(exc_info.value) + + +def test_missing_required_fields(): + """Test that missing required fields are caught.""" + with pytest.raises(ValidationError) as exc_info: + ScenarioConfig( + model="test", + target_isl=1024, + # Missing target_osl and target_concurrency + ) + + error_str = str(exc_info.value) + assert "target_osl" in error_str or "target_concurrency" in error_str + + +def test_optional_fields_have_defaults(): + """Test that optional fields have correct default values.""" + scenario = ScenarioConfig(model="test", target_isl=1024, target_osl=256, target_concurrency=32) + + assert scenario.isl_stdev == 0 + assert scenario.osl_stdev == 0 + assert scenario.num_requests == 512