From b5bf18b27fd3967d4c4867f33f4af09a2a68b971 Mon Sep 17 00:00:00 2001 From: Eran Geva <19514940+MrGeva@users.noreply.github.com> Date: Sun, 28 Sep 2025 07:32:44 -0700 Subject: [PATCH 1/2] Add AD backend support for performance testing - Added AD backend support in test_perf.py - Added test to l0_perf.yml with proper configuration - Updated performance thresholds and regression handling - Added KV cache metrics and threshold support - Simplified regex handling for KV cache - Removed old AD perf test and trtllm_bench_backend_comparison - Fixed import coding style and various path/param issues Signed-off-by: Eran Geva <19514940+MrGeva@users.noreply.github.com> --- ..._Compute_Implementation_in_TensorRT-LLM.md | 2 +- jenkins/L0_Test.groovy | 16 +- .../auto_deploy/transform/library/kvcache.py | 7 + .../defs/perf/base_perf_pytorch.csv | 4 + .../defs/perf/sanity_perf_check.py | 108 ++- tests/integration/defs/perf/test_perf.py | 55 +- .../integration/test_lists/test-db/l0_a30.yml | 2 +- .../test_lists/test-db/l0_b200.yml | 2 +- .../test_lists/test-db/l0_h100.yml | 1 - .../test_lists/test-db/l0_perf.yml | 14 + .../unit/singlegpu/test_ad_trtllm_bench.py | 795 +----------------- 11 files changed, 211 insertions(+), 795 deletions(-) diff --git a/docs/source/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.md b/docs/source/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.md index 94070f280ffc..869a6d519684 100644 --- a/docs/source/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.md +++ b/docs/source/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.md @@ -166,7 +166,7 @@ prototype_controller = NativeGenerationController(sampling_params={ llm = ScaffoldingLlm( prototype_controller, - {NativeGenerationController.WorkerTag.GENERATION: llm_worker}, + {NativeGenerationController.WorkerTag.GENERATION: proposer_worker}, ) results = llm.generate(prompts) ``` diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index c474076f8947..c78686635da5 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1767,11 +1767,17 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO basePerfFilename = stageName.contains("PyTorch") ? "base_perf_pytorch.csv" : "base_perf.csv" basePerfPath = "${llmSrc}/tests/integration/defs/perf/${basePerfFilename}" stage("Check perf result") { - sh """ - python3 ${llmSrc}/tests/integration/defs/perf/sanity_perf_check.py \ - ${stageName}/perf_script_test_results.csv \ - ${basePerfPath} - """ + def perfCheckResult = sh( + script: """ + python3 ${llmSrc}/tests/integration/defs/perf/sanity_perf_check.py \ + ${stageName}/perf_script_test_results.csv \ + ${basePerfPath} + """, + returnStatus: true + ) + if (perfCheckResult != 0) { + error "Performance regression detected and failing the build (exit code: ${perfCheckResult})" + } } stage("Create perf report") { sh """ diff --git a/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py b/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py index 94ce7427d995..65f5421416d7 100644 --- a/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py +++ b/tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py @@ -12,6 +12,7 @@ from ...models.factory import ModelFactory from ...shim.interface import CachedSequenceInterface from ...transformations._graph import add_graph_input +from ...utils.logger import ad_logger from ...utils.node_utils import get_all_input_output_nodes, is_op from ..interface import ( BaseTransform, @@ -305,6 +306,12 @@ def _get_mem_info_in_mb(): self._log_info(f"After all_gather - new_num_pages: {new_num_pages}") cm.resize_cache(new_num_pages) + # Log the final cache size for performance measurement, do not remove this log. + final_cache_size_bytes = cm.current_cache_size_bytes() + final_cache_size_gb = final_cache_size_bytes / (1024**3) # Convert to GiB + ad_logger.info( + f"Final KV cache size after resize: {final_cache_size_gb:.2f} GiB ({new_num_pages} pages)" + ) # Free memory torch.cuda.empty_cache() diff --git a/tests/integration/defs/perf/base_perf_pytorch.csv b/tests/integration/defs/perf/base_perf_pytorch.csv index 8785f7587fc6..541128f802f2 100644 --- a/tests/integration/defs/perf/base_perf_pytorch.csv +++ b/tests/integration/defs/perf/base_perf_pytorch.csv @@ -2,3 +2,7 @@ network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,p "llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406 "llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618 "llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002 +"llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.3,50,INFERENCE_TIME,214410.6447 +"llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_kv_cache_size[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_kv_cache_size[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.3,50,KV_CACHE_SIZE,68.84 +"llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.3,10,SEQ_THROUGHPUT,38.2071 +"llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.3,10,TOKEN_THROUGHPUT,4890.5035 diff --git a/tests/integration/defs/perf/sanity_perf_check.py b/tests/integration/defs/perf/sanity_perf_check.py index e00c34ad180c..eabeaa24f08c 100644 --- a/tests/integration/defs/perf/sanity_perf_check.py +++ b/tests/integration/defs/perf/sanity_perf_check.py @@ -49,6 +49,96 @@ def write_patch(self, old_lines: list[str], new_lines: list[str], f'b/tests/integration/defs/perf/{base_perf_filename}'): f.write(diff_line) + def _check_autodeploy_failures(self, full_diff: pd.DataFrame, + base_perf: pd.DataFrame, + current_perf: pd.DataFrame) -> bool: + """ + Check if any of the performance regressions are from autodeploy tests. + Only considers actual regressions (worse performance), not improvements. + Returns True if there are autodeploy regressions, False otherwise. + """ + # Create mappings for network_name, threshold, absolute_threshold, and metric_type + base_network_mapping = dict( + zip(base_perf['perf_case_name'], base_perf['network_name'])) + current_network_mapping = dict( + zip(current_perf['perf_case_name'], current_perf['network_name'])) + base_threshold_mapping = dict( + zip(base_perf['perf_case_name'], base_perf['threshold'])) + base_abs_threshold_mapping = dict( + zip(base_perf['perf_case_name'], base_perf['absolute_threshold'])) + base_metric_type_mapping = dict( + zip(base_perf['perf_case_name'], base_perf['metric_type'])) + + # Check each performance difference + for idx, row in full_diff.iterrows(): + # Look up network_name from either base or current (they should be the same) + network_name = base_network_mapping.get( + idx) or current_network_mapping.get(idx) + + # Only check autodeploy tests + if network_name and "_autodeploy" in str(network_name): + # Check if this is actually a regression (worse performance) + if hasattr(row, 'perf_metric_base') and hasattr( + row, 'perf_metric_target'): + base_value = row.perf_metric_base + target_value = row.perf_metric_target + threshold = base_threshold_mapping.get(idx, 0) + abs_threshold = base_abs_threshold_mapping.get(idx, 50) + metric_type = base_metric_type_mapping.get(idx, '') + + # Skip if we don't have the necessary data + if pd.isna(base_value) or pd.isna(target_value): + continue + + # Determine if this is a regression based on metric type and threshold sign + is_regression = self._is_performance_regression( + base_value, target_value, threshold, abs_threshold, + metric_type) + + if is_regression: + return True + + return False + + def _is_performance_regression(self, base_value: float, target_value: float, + threshold: float, abs_threshold: float, + metric_type: str) -> bool: + """ + Determine if a performance change represents a regression (worse performance) + that exceeds the acceptable threshold. + + Args: + base_value: Baseline performance value + target_value: Current performance value + threshold: Performance threshold (sign indicates better direction) + abs_threshold: Absolute threshold for tolerance calculation + metric_type: Type of metric (for context) + + Returns: + True if target_value represents worse performance than base_value + AND the change exceeds the threshold + """ + import numpy as np + + # First check if the change exceeds the threshold (same logic as diff_tools.py) + # Use absolute value of threshold for relative tolerance calculation + rel_threshold = abs(threshold) + + # If values are within threshold tolerance, no significant change + if np.isclose(base_value, + target_value, + rtol=rel_threshold, + atol=abs(abs_threshold)): + return False + + # Now check if it's a regression (worse performance) in the expected direction + if threshold > 0: + # Positive threshold: lower is better - regression if target > base + return target_value > base_value + else: + # Negative threshold: higher is better - regression if target < base + return target_value < base_value + def __call__(self, *args, **kwargs): # Check if the base_perf_csv file exists if not self.base_perf_csv.exists(): @@ -71,9 +161,23 @@ def __call__(self, *args, **kwargs): print( "You can download the file and update base_perf.csv by `git apply `" ) - print("Sanity perf check failed, but it has been disabled") + + # Check if any of the failed tests are autodeploy tests + autodeploy_failures = self._check_autodeploy_failures( + full_diff, base_perf, current_perf) + + if autodeploy_failures: + print( + "Sanity perf check failed for autodeploy tests - failing the build" + ) + return 1 + else: + print( + "Sanity perf check failed, but it has been disabled for non-autodeploy tests" + ) return 0 if __name__ == '__main__': - SanityPerfCheck(sys.argv[1], sys.argv[2])() + exit_code = SanityPerfCheck(sys.argv[1], sys.argv[2])() + sys.exit(exit_code) diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py index b92689d0a669..12ba705b2647 100644 --- a/tests/integration/defs/perf/test_perf.py +++ b/tests/integration/defs/perf/test_perf.py @@ -408,6 +408,11 @@ def __init__( tp_size: int = 1, pp_size: int = 1, num_gpus: int = 1, + # _autodeploy backend specific parameters + ad_compile_backend: str = "torch-opt", + free_mem_ratio: float = 0.9, + extra_runtime: str = "trtllm", + skip_loading_weights: bool = False, ): # The model name. self.model_name = model_name @@ -461,6 +466,11 @@ def __init__( self.pp_size = pp_size # Number of GPUs. self.num_gpus = num_gpus + # _autodeploy backend specific parameters + self.ad_compile_backend = ad_compile_backend + self.free_mem_ratio = free_mem_ratio + self.extra_runtime = extra_runtime + self.skip_loading_weights = skip_loading_weights # Just build engines self.build_only = False @@ -507,6 +517,8 @@ def to_string(self, entries.append(f"bench") if self.backend == 'pytorch': entries.append(f"pytorch") + elif self.backend == '_autodeploy': + entries.append(f"_autodeploy") if self.streaming == "streaming": entries.append(f"streaming") elif self.runtime == "disagg_server": # trtllm-serve @@ -658,7 +670,8 @@ def load_from_str(self, test_param_labels) -> None: return self._load_from_str_disagg(labels) self.api = labels.pop(0) if labels[0] == "exe" else "" - self.backend = labels.pop(0) if labels[0] == "pytorch" else "" + self.backend = labels.pop(0) if labels[0] in ["pytorch", "_autodeploy" + ] else "" self.streaming = labels.pop(0) if labels[0] == "streaming" else "" self.static_batching = labels.pop( 0) if labels[0] == "static_batching" else "" @@ -1354,12 +1367,14 @@ def get_trtllm_bench_command(self, engine_dir): f"--report_json={report_path}", f"--kv_cache_free_gpu_mem_fraction={self._config.kv_cache_free_gpu_mem_fraction}", ] - if self._config.backend != "pytorch": + if self._config.backend == "pytorch": + benchmark_cmd += ["--backend=pytorch"] + elif self._config.backend == "_autodeploy": + benchmark_cmd += ["--backend=_autodeploy"] + else: benchmark_cmd += [ f"--backend=tensorrt", f"--engine_dir={engine_dir}" ] - else: - benchmark_cmd += ["--backend=pytorch"] if self._config.num_reqs > 0: benchmark_cmd += [f"--num_requests={self._config.num_reqs}"] if self._config.concurrency != -1: @@ -1385,6 +1400,28 @@ def get_trtllm_bench_command(self, engine_dir): with open(pytorch_config_path, 'w') as f: yaml.dump(config, f, default_flow_style=False) benchmark_cmd += [f"--extra_llm_api_options={pytorch_config_path}"] + elif self._config.backend == "_autodeploy": + import yaml + autodeploy_config_path = os.path.join(engine_dir, + "extra_llm_api_options.yaml") + if not os.path.exists(autodeploy_config_path): + os.makedirs(os.path.dirname(autodeploy_config_path), + exist_ok=True) + + # Create _autodeploy specific configuration + autodeploy_config = { + 'compile_backend': self._config.ad_compile_backend, + 'free_mem_ratio': self._config.free_mem_ratio, + 'runtime': self._config.extra_runtime, + 'skip_loading_weights': self._config.skip_loading_weights + } + + print_info(f"_autodeploy model config: {autodeploy_config}") + with open(autodeploy_config_path, 'w') as f: + yaml.dump(autodeploy_config, f, default_flow_style=False) + benchmark_cmd += [ + f"--extra_llm_api_options={autodeploy_config_path}" + ] return benchmark_cmd def get_gpt_manager_runtime_benchmark_command(self, engine_dir, bs, @@ -1499,8 +1536,8 @@ def get_commands(self): build_cmd = self.get_trtllm_build_command(engine_dir, checkpoint_dir) elif self._config.runtime == "bench": - if self._config.backend == "pytorch": - # Skip building process as it is pytorch backend") + if self._config.backend in ["pytorch", "_autodeploy"]: + # Skip building process as it is pytorch or _autodeploy backend") pass else: build_cmd = self.get_trtllm_bench_build_command(engine_dir) @@ -1561,7 +1598,7 @@ def get_perf_result(self, outputs: Dict[int, str]) -> float: # Make sure we have outputs. assert cmd_idx in outputs, f"Output log for command {cmd_idx} does not exist!" - # Use the regex to go through the log from the N-th command, where N = cmd_idx. + # Use all applicable regex patterns to go through the log from the N-th command, where N = cmd_idx. print_info( f"Searching for metric {metric_name} from output log of command {cmd_idx} ..." ) @@ -1769,9 +1806,9 @@ def _get_metrics(self) -> List[PerfTestMetric]: # Build command is the first command. cmd_idx = 0 if self._config.runtime != "bench" else 1 if self._config.runtime == "bench": - if self._config.backend == "pytorch": + if self._config.backend in ["pytorch", "_autodeploy"]: print_info( - f"Skip building process for {self._config.model_name} as it is pytorch backend" + f"Skip building process for {self._config.model_name} as it is {self._config.backend} backend" ) builder_metrics = [] else: diff --git a/tests/integration/test_lists/test-db/l0_a30.yml b/tests/integration/test_lists/test-db/l0_a30.yml index 64cbb936b629..1ee25d2496e6 100644 --- a/tests/integration/test_lists/test-db/l0_a30.yml +++ b/tests/integration/test_lists/test-db/l0_a30.yml @@ -19,7 +19,7 @@ l0_a30: - unittest/_torch/modeling -k "modeling_qwen" - unittest/_torch/modeling -k "modeling_qwen_moe" - unittest/_torch/modeling -k "modeling_out_of_tree" - - unittest/_torch/auto_deploy/unit/singlegpu -k "not test_trtllm_bench_backend_comparison" + - unittest/_torch/auto_deploy/unit/singlegpu - unittest/_torch/sampler/test_beam_search.py - test_e2e.py::test_openai_completions_with_logit_bias[torch_sampler] - test_e2e.py::test_openai_chat_with_logit_bias[torch_sampler] diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index cd34a690e819..b435b61e4239 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -73,7 +73,7 @@ l0_b200: - unittest/_torch/modeling -k "modeling_llama" - unittest/_torch/modeling -k "modeling_mixtral" - unittest/_torch/modeling -k "modeling_gpt_oss" - - unittest/_torch/auto_deploy/unit/singlegpu -k "not test_trtllm_bench_backend_comparison" + - unittest/_torch/auto_deploy/unit/singlegpu - condition: ranges: system_gpu_count: diff --git a/tests/integration/test_lists/test-db/l0_h100.yml b/tests/integration/test_lists/test-db/l0_h100.yml index 00333950a6f7..586161fa15b9 100644 --- a/tests/integration/test_lists/test-db/l0_h100.yml +++ b/tests/integration/test_lists/test-db/l0_h100.yml @@ -31,7 +31,6 @@ l0_h100: - unittest/_torch/modeling -k "modeling_nemotron" - unittest/_torch/modeling -k "modeling_gemma3" - unittest/_torch/modeling -k "modeling_gpt_oss" - - unittest/_torch/auto_deploy/unit/singlegpu/test_ad_trtllm_bench.py::test_trtllm_bench_backend_comparison - unittest/disaggregated/test_disagg_utils.py - unittest/disaggregated/test_router.py - unittest/disaggregated/test_remoteDictionary.py diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml index 21f5a11dfb07..b9b4b42db669 100644 --- a/tests/integration/test_lists/test-db/l0_perf.yml +++ b/tests/integration/test_lists/test-db/l0_perf.yml @@ -29,3 +29,17 @@ l0_perf: backend: pytorch tests: - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192] + - condition: + ranges: + system_gpu_count: + gte: 1 + lte: 1 + wildcards: + gpu: + - '*h100*' + linux_distribution_name: ubuntu* + terms: + stage: pre_merge + backend: pytorch + tests: + - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-_autodeploy-float16-input_output_len:128,128-reqs:8192] diff --git a/tests/unittest/_torch/auto_deploy/unit/singlegpu/test_ad_trtllm_bench.py b/tests/unittest/_torch/auto_deploy/unit/singlegpu/test_ad_trtllm_bench.py index e3fdea5b4ec5..246c74023ac2 100644 --- a/tests/unittest/_torch/auto_deploy/unit/singlegpu/test_ad_trtllm_bench.py +++ b/tests/unittest/_torch/auto_deploy/unit/singlegpu/test_ad_trtllm_bench.py @@ -1,72 +1,14 @@ -import json -import os -import re import subprocess import tempfile from pathlib import Path -from statistics import mean, median -from typing import List, Tuple import pytest import yaml from _model_test_utils import _hf_model_dir_or_hub_id +from click.testing import CliRunner from utils.cpp_paths import llm_root # noqa: F401 -# Tolerance for additional memory reduction after fwd pass (in MB) -POST_FWD_FREE_MEM_LOWER_SLACK_MB = 2000 - - -def remove_outliers_iqr(values: List[float]) -> List[float]: - """ - Remove outliers using the IQR (Interquartile Range) method. - Values outside Q1 - 1.5*IQR and Q3 + 1.5*IQR are considered outliers. - - Args: - values: List of numerical values - - Returns: - List of values with outliers removed - """ - if len(values) < 4: # Need at least 4 values for meaningful IQR - return values - - sorted_values = sorted(values) - n = len(sorted_values) - q1 = sorted_values[n // 4] - q3 = sorted_values[3 * n // 4] - iqr = q3 - q1 - - lower_bound = q1 - 1.5 * iqr - upper_bound = q3 + 1.5 * iqr - - filtered_values = [v for v in values if lower_bound <= v <= upper_bound] - - # Ensure we keep at least half the original values - if len(filtered_values) < len(values) // 2: - removed_str = f"{len(values) - len(filtered_values)}/{len(values)}" - print(f"āš ļø IQR filtering would remove too many values ({removed_str}), keeping all") - return values - - removed_count = len(values) - len(filtered_values) - if removed_count > 0: - bounds_str = f"{lower_bound:.2f} - {upper_bound:.2f}" - print(f"šŸ“Š Removed {removed_count} outliers using IQR method (bounds: {bounds_str})") - - return filtered_values - - -def calculate_robust_stats(values: List[float]) -> Tuple[float, float, int]: - """ - Calculate robust statistics after removing outliers. - - Args: - values: List of performance values - - Returns: - Tuple of (mean, median, count_after_outlier_removal) - """ - filtered_values = remove_outliers_iqr(values) - return mean(filtered_values), median(filtered_values), len(filtered_values) +from tensorrt_llm.commands.bench import main def tiny_llama_details(): @@ -76,228 +18,22 @@ def tiny_llama_details(): return model_path_or_name, model_name, model_path -def parse_kv_cache_metrics(log_output: str, free_mem_ratio: float = 0.8): - """Parse KV cache metrics from the benchmark log output.""" - metrics = {} - - # Simple patterns based on actual log format - patterns = { - "current_cache_size": r"Current cache size \(MB\):\s*(\d+)", - "free_mem_pre_mb": r"Free memory before forward pass \(MB\):\s*(\d+)", - "free_mem_post_mb": r"Free memory after forward pass \(MB\):\s*(\d+)", - } - - # Extract metrics using simple regex patterns - for metric_name, pattern in patterns.items(): - match = re.search(pattern, log_output, re.IGNORECASE) - if match: - value = int(match.group(1)) - metrics[metric_name] = value - print(f" āœ… Found {metric_name}: {value}") - else: - print(f" āŒ Could not find {metric_name}") - - try: - metrics["current_cache_size"] = metrics["current_cache_size"] * 1024 * 1024 - except KeyError: - print(" āŒ Could not find current_cache_size") - - # Calculate new_cache_size using the same formula as in resize_kv_cache - # new_cache_size = free_mem_post * 1024 * 1024 * free_mem_ratio + current_cache_size - if "free_mem_post_mb" in metrics and "current_cache_size" in metrics: - metrics["new_cache_size"] = int( - metrics["free_mem_post_mb"] * 1024 * 1024 * free_mem_ratio - + metrics["current_cache_size"] - ) - print( - f" āœ… Calculated new_cache_size: {metrics['new_cache_size']} (using free_mem_ratio={free_mem_ratio})" - ) - else: - print(" āŒ Cannot calculate new_cache_size - missing required metrics") - - return metrics - - -def run_benchmark( - model_name: str, - model_path: str, - dataset_path: str, - temp_dir: str, - backend: str = "_autodeploy", - report_json_path: str = None, - max_batch_size: int = 32, - num_hidden_layers: int = 2, - free_mem_ratio: float = 0.1, -): - """Run benchmark and capture KV cache metrics from log output.""" - - # Read the test config to get free_mem_ratio - config_path = f"{temp_dir}/extra_llm_api_options.yaml" - - # Build the command to run the benchmark - cmd = ["python", "-m", "tensorrt_llm.commands.bench", "--model", model_name] - - # If the model exists locally, then using the local copy will make the test robust to CI network issues - if os.path.isdir(model_path): - cmd.extend(["--model_path", model_path]) - - cmd.extend( - [ - "throughput", - "--backend", - backend, - "--dataset", - str(dataset_path), - "--max_batch_size", - str(max_batch_size), - ] - ) - - # Add report_json argument if path is provided - if report_json_path: - cmd.extend(["--report_json", report_json_path]) - - if backend == "_autodeploy": - # Add extra_llm_api_options only for autodeploy backend - cmd.extend(["--extra_llm_api_options", config_path]) - - # Run benchmark as subprocess to capture ALL output - env = os.environ.copy() - if backend == "pytorch": - env["TLLM_OVERRIDE_LAYER_NUM"] = str(num_hidden_layers) - print(f"šŸ“‹ Using TLLM_OVERRIDE_LAYER_NUM from env: {env['TLLM_OVERRIDE_LAYER_NUM']}") - cmd.extend(["--kv_cache_free_gpu_mem_fraction", str(free_mem_ratio)]) - print(f"šŸš€ Running benchmark command ({backend} backend): {' '.join(cmd)}") - result = subprocess.run(cmd, capture_output=True, text=True, env=env, timeout=600) - - # Check if the command succeeded - assert result.returncode == 0, ( - f"Benchmark failed with return code {result.returncode}:\n" - f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" - ) - - # Combine stdout and stderr for parsing - full_log_output = f"{result.stdout}\n{result.stderr}" - - # Parse KV cache metrics from the combined log output (only for autodeploy backend) - kv_cache_metrics = {} - if backend == "_autodeploy": - kv_cache_metrics = parse_kv_cache_metrics(full_log_output, free_mem_ratio) - print("šŸ“Š KV Cache Metrics parsed from logs:") - if kv_cache_metrics: - for key, value in kv_cache_metrics.items(): - if "mb" in key.lower(): - print(f" {key}: {value}MB") - else: - print(f" {key}: {value} bytes") - else: - print(" āš ļø No KV cache metrics were parsed successfully") - else: - print(f"šŸ“Š KV Cache Metrics: Skipped for {backend} backend") - - # Return parsed JSON report with KV cache metrics if requested - if report_json_path and Path(report_json_path).exists(): - with open(report_json_path, "r") as f: - report_data = json.load(f) - - # Add KV cache metrics to the report (only for autodeploy backend) - if backend == "_autodeploy": - report_data["kv_cache_metrics"] = kv_cache_metrics - report_data["backend"] = backend - return report_data - return None - - -def compare_backends_performance( - autodeploy_tokens_per_sec: float, - pytorch_tokens_per_sec: float, - relative_tolerance: float = 0.20, - absolute_tolerance: float = 10.0, -): - """ - Compare performance between autodeploy and pytorch backends. - Fails if autodeploy is significantly worse than pytorch. - - Args: - autodeploy_tokens_per_sec: Performance of autodeploy backend - pytorch_tokens_per_sec: Performance of pytorch backend - relative_tolerance: Relative tolerance (20% by default for backend comparison) - absolute_tolerance: Absolute tolerance (10 tokens/sec by default) - """ - # Calculate performance difference - performance_diff = pytorch_tokens_per_sec - autodeploy_tokens_per_sec - relative_diff = performance_diff / pytorch_tokens_per_sec if pytorch_tokens_per_sec > 0 else 0 +def run_benchmark(model_name: str, dataset_path: str, extra_llm_api_options_path: str): + runner = CliRunner() - print("=== BACKEND PERFORMANCE COMPARISON ===") - print(f"PyTorch backend: {pytorch_tokens_per_sec:.2f} tokens/sec/user") - print(f"Autodeploy backend: {autodeploy_tokens_per_sec:.2f} tokens/sec/user") - print(f"Performance difference: {performance_diff:.2f} tokens/sec ({relative_diff:.2%})") - - # If autodeploy is better than or equal to pytorch, always pass - if autodeploy_tokens_per_sec >= pytorch_tokens_per_sec: - print("āœ… Autodeploy backend matches or exceeds PyTorch backend performance") - return - - # Autodeploy is slower - check if it's within acceptable tolerance - within_relative_tolerance = relative_diff <= relative_tolerance - within_absolute_tolerance = performance_diff <= absolute_tolerance - - if within_relative_tolerance or within_absolute_tolerance: - print("āœ… Autodeploy backend performance within acceptable tolerance") - print( - f" Tolerance: {relative_tolerance:.2%} relative OR {absolute_tolerance:.2f} tokens/sec absolute" - ) - else: - assert False, ( - f"Autodeploy backend significantly underperforms compared to PyTorch! " - f"Autodeploy: {autodeploy_tokens_per_sec:.2f} tokens/sec/user, " - f"PyTorch: {pytorch_tokens_per_sec:.2f} tokens/sec/user, " - f"Performance gap: {performance_diff:.2f} tokens/sec ({relative_diff:.2%}), " - f"Tolerance: {relative_tolerance:.2%} relative OR {absolute_tolerance:.2f} tokens/sec absolute" - ) - - -def assert_performance_within_tolerance( - actual_tokens_per_sec: float, - golden_tokens_per_sec: float, - relative_tolerance: float = 0.15, - absolute_tolerance: float = 10.0, -): - """ - Assert that actual performance is within tolerance of golden result. - Only fails if performance is WORSE than golden - improvements always pass. - - Args: - actual_tokens_per_sec: Measured performance metric - golden_tokens_per_sec: Expected performance metric - relative_tolerance: Relative tolerance (15% by default) - absolute_tolerance: Absolute tolerance (10 tokens/sec by default) - """ - # If actual performance is better than or equal to golden, always pass - if actual_tokens_per_sec >= golden_tokens_per_sec: - print( - f"āœ… Performance improvement detected:" - f" {actual_tokens_per_sec:.2f} >= {golden_tokens_per_sec:.2f} tokens/sec/user" - ) - return - - # Performance is worse than golden - check if it's within acceptable tolerance - performance_drop = golden_tokens_per_sec - actual_tokens_per_sec - relative_drop = ( - performance_drop / golden_tokens_per_sec if golden_tokens_per_sec > 0 else float("inf") - ) - - # Performance should be within relative tolerance OR absolute tolerance - within_relative_tolerance = relative_drop <= relative_tolerance - within_absolute_tolerance = performance_drop <= absolute_tolerance - - assert within_relative_tolerance or within_absolute_tolerance, ( - f"Performance regression detected! " - f"Actual: {actual_tokens_per_sec:.2f} tokens/sec/user, " - f"Golden: {golden_tokens_per_sec:.2f} tokens/sec/user, " - f"Performance drop: {performance_drop:.2f} tokens/sec ({relative_drop:.2%}), " - f"Tolerance: {relative_tolerance:.2%} relative OR {absolute_tolerance:.2f} tokens/sec absolute" - ) + args = [ + "--model", + model_name, + "throughput", + "--backend", + "_autodeploy", + "--dataset", + dataset_path, + "--extra_llm_api_options", + f"{extra_llm_api_options_path}", + ] + result = runner.invoke(main, args, catch_exceptions=False) + assert result.exit_code == 0 def prepare_dataset(root_dir: str, temp_dir: str, model_path_or_name: str): @@ -337,473 +73,12 @@ def prepare_dataset(root_dir: str, temp_dir: str, model_path_or_name: str): return dataset_path -def calculate_expected_kv_cache_metrics(free_mem_ratio: float): - """Calculate expected KV cache metrics based on actual GPU memory.""" - try: - import torch - - if torch.cuda.is_available(): - # Get total GPU memory in MB - _, total_mem_bytes = torch.cuda.mem_get_info(0) - total_mem_mb = total_mem_bytes // (1024 * 1024) - - # Estimate expected values based on model size - # For TinyLlama-1.1B, model should be 2.2GB - estimated_model_size_mb = 2200 # Conservative estimate - # TODO: https://github.com/NVIDIA/TensorRT-LLM/issues/6335 check why there is extra consumption - extra_consumption_mb = 2700 - expected_free_mem_range = ( - total_mem_mb - estimated_model_size_mb - extra_consumption_mb, - total_mem_mb - estimated_model_size_mb, - ) - - # Current cache size is typically small initially (16MB range) - expected_current_cache_size = 16777216 - - # Free memory values should be in reasonable range - expected_free_mem_pre_range = expected_free_mem_range - # Allow extra headroom after forward pass to account for fragmentation/transient buffers. - lower_slack_mb = POST_FWD_FREE_MEM_LOWER_SLACK_MB - expected_free_mem_post_range = ( - max(0, expected_free_mem_range[0] - lower_slack_mb), - expected_free_mem_range[1], - ) - - print("šŸ“Š GPU Memory Analysis:") - print(f" Total GPU memory: {total_mem_mb}MB") - print( - f" Expected free memory range: {expected_free_mem_range[0]}-{expected_free_mem_range[1]}MB" - ) - - return { - "total_mem_mb": total_mem_mb, - "expected_current_cache_size": expected_current_cache_size, - "expected_free_mem_pre_range": expected_free_mem_pre_range, - "expected_free_mem_post_range": expected_free_mem_post_range, - "free_mem_ratio": free_mem_ratio, - } - else: - return None - except ImportError: - return None - - -def validate_kv_cache_metrics_dynamic(kv_cache_metrics: dict, expected_metrics: dict): - """Validate KV cache metrics using dynamic expected values.""" - - # Validate current_cache_size (should be relatively stable) - current_cache_size = kv_cache_metrics.get("current_cache_size") - expected_cache_size = expected_metrics["expected_current_cache_size"] - if current_cache_size: - cache_diff = abs(current_cache_size - expected_cache_size) / expected_cache_size - assert cache_diff <= 0.5, ( # 50% tolerance for cache size - f"Current cache size outside expected range: {current_cache_size} vs expected ~{expected_cache_size}" - ) - print(f" āœ… current_cache_size: {current_cache_size} bytes (within range)") - - # Validate free memory values are in reasonable ranges - free_mem_pre = kv_cache_metrics.get("free_mem_pre_mb") - free_mem_post = kv_cache_metrics.get("free_mem_post_mb") - - if free_mem_pre: - pre_range = expected_metrics["expected_free_mem_pre_range"] - assert pre_range[0] <= free_mem_pre <= pre_range[1], ( - f"Free memory before forward pass outside expected range: " - f"{free_mem_pre}MB not in range {pre_range[0]}-{pre_range[1]}MB" - ) - print(f" āœ… free_mem_pre_mb: {free_mem_pre}MB (within range)") - - if free_mem_post: - post_range = expected_metrics["expected_free_mem_post_range"] - assert post_range[0] <= free_mem_post <= post_range[1], ( - f"Free memory after forward pass outside expected range: " - f"{free_mem_post}MB not in range {post_range[0]}-{post_range[1]}MB" - ) - print(f" āœ… free_mem_post_mb: {free_mem_post}MB (within range)") - - # Validate memory reduction (pre should be > post) - if free_mem_pre and free_mem_post: - memory_reduction = free_mem_pre - free_mem_post - assert memory_reduction > 0, ( - f"Expected memory reduction during forward pass, got {memory_reduction}MB" - ) - print(f" āœ… Memory reduction during forward pass: {memory_reduction}MB") - - # Validate calculated new_cache_size - new_cache_size = kv_cache_metrics.get("new_cache_size") - if new_cache_size and free_mem_post and current_cache_size: - expected_new_cache = int( - free_mem_post * 1024 * 1024 * expected_metrics["free_mem_ratio"] + current_cache_size - ) - cache_size_diff = abs(new_cache_size - expected_new_cache) / expected_new_cache - assert cache_size_diff <= 0.01, ( # 1% tolerance for calculated value - f"Calculated new_cache_size mismatch: {new_cache_size} vs expected {expected_new_cache}" - ) - print(f" āœ… new_cache_size: {new_cache_size} bytes (calculation correct)") - - -def extract_performance_metric(report_data, report_name="benchmark"): - """Extract performance metric from a benchmark report with validation.""" - assert report_data is not None, f"Failed to capture {report_name} report" - assert "performance" in report_data, f"Performance metrics not found in {report_name} report" - - tokens_per_sec = report_data["performance"].get("output_throughput_per_user_tok_s") - assert tokens_per_sec is not None, ( - f"output_throughput_per_user_tok_s not found in {report_name} performance metrics" - ) - - return tokens_per_sec - - -def validate_and_extract_kv_cache_metrics(report_data, free_mem_ratio, require_metrics=True): - """ - Validate and extract KV cache metrics from report. - - Args: - report_data: The benchmark report data - free_mem_ratio: Free memory ratio for calculating expected metrics - require_metrics: If True, fail when metrics are missing. If False, just warn. - - Returns: - Tuple of (kv_cache_metrics, expected_metrics) or (None, None) if validation fails - """ - required_metrics = [ - "current_cache_size", - "free_mem_pre_mb", - "free_mem_post_mb", - "new_cache_size", - ] - - # Extract KV cache metrics - kv_cache_metrics = report_data.get("kv_cache_metrics", {}) - - if not kv_cache_metrics: - message = ( - "KV cache metrics not found! " - "The autodeploy backend must log memory statistics for this test to pass. " - f"Expected metrics: {', '.join(required_metrics)}" - ) - if require_metrics: - assert False, f"REQUIRED {message}" - else: - print(f"ā„¹ļø {message}") - assert False, "KV cache metrics are missing" - - # Check for missing metrics - missing_metrics = [metric for metric in required_metrics if metric not in kv_cache_metrics] - - if missing_metrics: - message = ( - f"Missing required KV cache metrics: {missing_metrics}. " - f"Found metrics: {list(kv_cache_metrics.keys())}. " - f"All of {required_metrics} are required for the test to pass." - ) - if require_metrics: - assert False, message - else: - print(f"ā„¹ļø KV cache validation skipped - {message}") - assert False, "KV cache metrics are missing" - - # Calculate expected metrics - expected_metrics = calculate_expected_kv_cache_metrics(free_mem_ratio) - assert expected_metrics, "Could not determine expected metrics for this GPU" - - return kv_cache_metrics, expected_metrics - - -def print_kv_cache_metrics(kv_cache_metrics): - """Print KV cache metrics in a formatted way.""" - print("=== KV CACHE METRICS (DYNAMIC VALIDATION) ===") - for metric_name, actual_value in kv_cache_metrics.items(): - if "mb" in metric_name.lower(): - print(f"{metric_name}: {actual_value}MB") - else: - print(f"{metric_name}: {actual_value} bytes") - - -def run_multiple_benchmarks_with_outlier_removal( - model_name: str, - model_path: str, - dataset_path: str, - temp_dir: str, - backend: str, - report_json_path: str, - max_batch_size: int, - num_hidden_layers: int, - free_mem_ratio: float, - num_iterations: int = 10, -) -> dict: - """ - Run benchmark multiple times and return averaged results with outlier removal. - - Args: - All the same args as run_benchmark, plus: - num_iterations: Number of times to run the benchmark (default 10) - - Returns: - Dictionary containing averaged performance metrics and KV cache metrics - """ - print(f"=== RUNNING {backend.upper()} BACKEND {num_iterations} TIMES WITH OUTLIER REMOVAL ===") - - performance_values = [] - all_kv_metrics = [] - successful_runs = 0 - - for i in range(num_iterations): - try: - print(f"šŸ”„ Iteration {i + 1}/{num_iterations}") - report_data = run_benchmark( - model_name, - model_path, - dataset_path, - temp_dir, - backend, - report_json_path, - max_batch_size, - num_hidden_layers, - free_mem_ratio, - ) - - if report_data and "performance" in report_data: - tokens_per_sec = extract_performance_metric(report_data, f"{backend}_iter_{i + 1}") - performance_values.append(tokens_per_sec) - - # Store KV cache metrics for autodeploy backend - if backend == "_autodeploy" and "kv_cache_metrics" in report_data: - all_kv_metrics.append(report_data["kv_cache_metrics"]) - - successful_runs += 1 - print(f" āœ… Iteration {i + 1}: {tokens_per_sec:.2f} tokens/sec/user") - else: - print(f" āŒ Iteration {i + 1}: Failed to get valid report") - - except Exception as e: - print(f" āŒ Iteration {i + 1}: Exception occurred: {e}") - continue - - if successful_runs < 3: # Need at least 3 successful runs - raise RuntimeError( - f"Only {successful_runs} successful benchmark runs out of {num_iterations}" - ) - - print(f"\nšŸ“Š Performance Summary ({successful_runs} successful runs):") - print(f"Raw values: {[f'{v:.2f}' for v in performance_values]}") - - # Calculate robust statistics - avg_perf, median_perf, count_after_filtering = calculate_robust_stats(performance_values) - - print(f"Average (after outlier removal): {avg_perf:.2f} tokens/sec/user") - print(f"Median: {median_perf:.2f} tokens/sec/user") - print(f"Values used for average: {count_after_filtering}/{len(performance_values)}") - - # Create averaged report similar to single run - averaged_report = { - "performance": { - "output_throughput_per_user_tok_s": avg_perf, - "median_throughput_per_user_tok_s": median_perf, - "raw_values": performance_values, - "successful_runs": successful_runs, - "values_after_filtering": count_after_filtering, - }, - "backend": backend, - } - - # For autodeploy backend, average KV cache metrics if available - if backend == "_autodeploy" and all_kv_metrics: - averaged_kv_metrics = {} - - # Average each metric across all runs - metric_names = [ - "current_cache_size", - "free_mem_pre_mb", - "free_mem_post_mb", - "new_cache_size", - ] - for metric_name in metric_names: - values = [] - for kv_metrics in all_kv_metrics: - if metric_name in kv_metrics: - values.append(kv_metrics[metric_name]) - - if values: - avg_value, _, _ = calculate_robust_stats(values) - averaged_kv_metrics[metric_name] = int(avg_value) - print(f" Averaged {metric_name}: {averaged_kv_metrics[metric_name]}") - - averaged_report["kv_cache_metrics"] = averaged_kv_metrics - - return averaged_report - - -def trtllm_bench_unified_comparison( - llm_root, # noqa: F811 - comparison_mode="backend", - free_mem_ratio=0.1, - num_hidden_layers=2, - max_batch_size=32, # below this value the kv cache resizing is skipped - golden_tokens_per_sec=1400, - backend_relative_tolerance=0.23, - backend_absolute_tolerance=250.0, - golden_relative_tolerance=0.1, - golden_absolute_tolerance=5.0, - num_iterations=10, -): - """ - Unified test that compares autodeploy backend performance in two modes: - - "backend": compares against pytorch backend performance - - "golden": compares against predefined golden performance values - - Runs multiple iterations to calculate robust averages and remove outliers. - - Args: - llm_root: Root directory for LLM models (pytest fixture) - comparison_mode: Either "backend" or "golden" to determine comparison type - free_mem_ratio: Ratio of free memory to use for KV cache - num_hidden_layers: Number of hidden layers for the model - max_batch_size: Maximum batch size for benchmarking - golden_tokens_per_sec: Golden performance value in tokens/sec/user - backend_relative_tolerance: Relative tolerance for backend comparison - backend_absolute_tolerance: Absolute tolerance for backend comparison - golden_relative_tolerance: Relative tolerance for golden comparison - golden_absolute_tolerance: Absolute tolerance for golden comparison - num_iterations: Number of benchmark iterations to run (default 10) - """ - model_path_or_name, model_name, model_path = tiny_llama_details() - - with tempfile.TemporaryDirectory() as temp_dir: - with open(f"{temp_dir}/extra_llm_api_options.yaml", "w") as f: - yaml.dump( - { - "model_kwargs": {"num_hidden_layers": num_hidden_layers}, - "cuda_graph_batch_sizes": [1, 2, 4, 8, 16, 32], - "compile_backend": "torch-opt", - "free_mem_ratio": free_mem_ratio, - "runtime": "trtllm", - }, - f, - ) - - dataset_path = prepare_dataset(llm_root, temp_dir, model_path_or_name) - - # Always run autodeploy backend with multiple iterations - autodeploy_report_path = f"{temp_dir}/autodeploy_report.json" - autodeploy_report = run_multiple_benchmarks_with_outlier_removal( - model_name, - model_path, - dataset_path, - temp_dir, - "_autodeploy", - autodeploy_report_path, - max_batch_size, - num_hidden_layers, - free_mem_ratio, - num_iterations, - ) - - # Extract autodeploy performance metrics - autodeploy_tokens_per_sec = extract_performance_metric(autodeploy_report, "autodeploy") - - # Validate and extract KV cache metrics (now required for both modes after user's changes) - kv_cache_metrics, expected_metrics = validate_and_extract_kv_cache_metrics( - autodeploy_report, free_mem_ratio, require_metrics=True - ) - - if comparison_mode == "backend": - # Backend comparison mode: also run pytorch backend with multiple iterations - pytorch_report_path = f"{temp_dir}/pytorch_report.json" - pytorch_report = run_multiple_benchmarks_with_outlier_removal( - model_name, - model_path, - dataset_path, - temp_dir, - "pytorch", - pytorch_report_path, - max_batch_size, - num_hidden_layers, - free_mem_ratio, - num_iterations, - ) - - # Extract pytorch performance metrics - pytorch_tokens_per_sec = extract_performance_metric(pytorch_report, "pytorch") - - # Compare backend performance - compare_backends_performance( - autodeploy_tokens_per_sec, - pytorch_tokens_per_sec, - relative_tolerance=backend_relative_tolerance, - absolute_tolerance=backend_absolute_tolerance, - ) - - # Validate KV cache metrics - validate_kv_cache_metrics_dynamic(kv_cache_metrics, expected_metrics) - print("āœ… KV Cache Metrics validation passed") - - print("=== BACKEND COMPARISON TEST PASSED ===") - ad_runs = autodeploy_report["performance"]["successful_runs"] - pt_runs = pytorch_report["performance"]["successful_runs"] - print( - f"Autodeploy: {autodeploy_tokens_per_sec:.2f} tokens/sec/user (avg of {ad_runs} runs)" - ) - print(f"PyTorch: {pytorch_tokens_per_sec:.2f} tokens/sec/user (avg of {pt_runs} runs)") - - # Print additional statistics - if "raw_values" in autodeploy_report["performance"]: - ad_values = autodeploy_report["performance"]["raw_values"] - print(f"Autodeploy raw values: {[f'{v:.2f}' for v in ad_values]}") - if "raw_values" in pytorch_report["performance"]: - pt_values = pytorch_report["performance"]["raw_values"] - print(f"PyTorch raw values: {[f'{v:.2f}' for v in pt_values]}") - - elif comparison_mode == "golden": - # Golden comparison mode: compare against golden values - print("=== PERFORMANCE METRICS ===") - ad_runs = autodeploy_report["performance"]["successful_runs"] - print( - f"Measured performance: {autodeploy_tokens_per_sec:.2f} tokens/sec/user (avg of {ad_runs} runs)" - ) - print(f"Golden performance: {golden_tokens_per_sec:.2f} tokens/sec/user") - - # Print additional statistics - if "raw_values" in autodeploy_report["performance"]: - ad_values = autodeploy_report["performance"]["raw_values"] - print(f"Autodeploy raw values: {[f'{v:.2f}' for v in ad_values]}") - - # Print KV cache metrics - print_kv_cache_metrics(kv_cache_metrics) - - # Performance validation - assert_performance_within_tolerance( - autodeploy_tokens_per_sec, - golden_tokens_per_sec, - relative_tolerance=golden_relative_tolerance, - absolute_tolerance=golden_absolute_tolerance, - ) - - # KV cache metrics validation - print( - f"Validating {len(kv_cache_metrics)} KV cache metrics against GPU-specific ranges..." - ) - validate_kv_cache_metrics_dynamic(kv_cache_metrics, expected_metrics) - - print("=== ALL TESTS PASSED ===") - ad_runs = autodeploy_report["performance"]["successful_runs"] - perf_str = f"Performance: āœ… {autodeploy_tokens_per_sec:.2f} tokens/sec/user" - print(f"{perf_str} (avg of {ad_runs} runs) within bounds") - print("KV Cache Metrics: āœ… All metrics within GPU-specific expected ranges") - - else: - raise ValueError( - f"Invalid comparison_mode: {comparison_mode}. Must be 'backend' or 'golden'" - ) - - -@pytest.mark.skip(reason="https://nvbugs/5542907") @pytest.mark.parametrize("compile_backend", ["torch-compile", "torch-opt", "torch-cudagraph"]) def test_trtllm_bench(llm_root, compile_backend): # noqa: F811 model_path_or_name, model_name, model_path = tiny_llama_details() with tempfile.TemporaryDirectory() as temp_dir: - with open(f"{temp_dir}/extra_llm_api_options.yaml", "w") as f: + extra_llm_api_options_path = f"{temp_dir}/extra_llm_api_options.yaml" + with open(extra_llm_api_options_path, "w") as f: yaml.dump( { "model_kwargs": {"num_hidden_layers": 2}, @@ -815,34 +90,4 @@ def test_trtllm_bench(llm_root, compile_backend): # noqa: F811 ) dataset_path = prepare_dataset(llm_root, temp_dir, model_path_or_name) - run_benchmark(model_name, model_path, dataset_path, temp_dir) - - -@pytest.mark.no_xdist -@pytest.mark.skip(reason="https://nvbugs/5458798") -def test_trtllm_bench_backend_comparison(llm_root): # noqa: F811 - """Test that compares autodeploy backend performance against pytorch backend - with given relative and absolute thresholds. - - This test runs both backends 10 times each, removes outliers using IQR method, - and compares the averaged performance to reduce impact of intermittent failures - and performance variability. - - It also checks the memory footprint of the autodeploy backend by parsing the - log output from the resize_kv_cache function and extracting the following metrics: - current_cache_size - the cache size before resize - free_mem_pre_mb - the free memory before forward pass - free_mem_post_mb - the free memory after forward pass - new_cache_size - the cache size after resize - - The following checks are performed: - 1. free_mem_pre_fw_pass and free_mem_post_fw_pass are in: - [Total mem - expected_model_size - extra_consumption, Total mem - expected_model_size] - 2. memory_reduction = free_mem_pre_fw_pass - free_mem_post_fw_pass > 0 - 3. expected_new_cache = free_mem_post * free_mem_ratio + current_cache_size - cache_size_diff = abs(new_cache_size - expected_new_cache) / expected_new_cache - assert cache_size_diff <= 0.01 - - extra_consumption_mb = 2700 - this is unexplained memory consumption to be investigated. - """ - trtllm_bench_unified_comparison(llm_root, comparison_mode="backend") + run_benchmark(model_name, dataset_path, extra_llm_api_options_path) From 040fe29657d4acfbb2911768f5f97f843abc1cfe Mon Sep 17 00:00:00 2001 From: Eran Geva <19514940+MrGeva@users.noreply.github.com> Date: Sun, 28 Sep 2025 07:35:27 -0700 Subject: [PATCH 2/2] fixed wrong rebase Signed-off-by: Eran Geva <19514940+MrGeva@users.noreply.github.com> --- ...g13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.md b/docs/source/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.md index 869a6d519684..94070f280ffc 100644 --- a/docs/source/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.md +++ b/docs/source/blogs/tech_blog/blog13_Inference_Time_Compute_Implementation_in_TensorRT-LLM.md @@ -166,7 +166,7 @@ prototype_controller = NativeGenerationController(sampling_params={ llm = ScaffoldingLlm( prototype_controller, - {NativeGenerationController.WorkerTag.GENERATION: proposer_worker}, + {NativeGenerationController.WorkerTag.GENERATION: llm_worker}, ) results = llm.generate(prompts) ```