[#7288][feat] Added AutoDeploy backend support to test_perf.py

jenkins/L0_Test.groovy

@@ -1767,11 +1767,17 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
             basePerfFilename = stageName.contains("PyTorch") ? "base_perf_pytorch.csv" : "base_perf.csv"
             basePerfPath = "${llmSrc}/tests/integration/defs/perf/${basePerfFilename}"
             stage("Check perf result") {
-                sh """
-                    python3 ${llmSrc}/tests/integration/defs/perf/sanity_perf_check.py \
-                    ${stageName}/perf_script_test_results.csv \
-                    ${basePerfPath}
-                """
+                def perfCheckResult = sh(
+                    script: """
+                        python3 ${llmSrc}/tests/integration/defs/perf/sanity_perf_check.py \
+                        ${stageName}/perf_script_test_results.csv \
+                        ${basePerfPath}
+                    """,
+                    returnStatus: true
+                )
+                if (perfCheckResult != 0) {
+                    error "Performance regression detected and failing the build (exit code: ${perfCheckResult})"
+                }
             }
             stage("Create perf report") {
                 sh """

tensorrt_llm/_torch/auto_deploy/transform/library/kvcache.py

@@ -12,6 +12,7 @@
 from ...models.factory import ModelFactory
 from ...shim.interface import CachedSequenceInterface
 from ...transformations._graph import add_graph_input
+from ...utils.logger import ad_logger
 from ...utils.node_utils import get_all_input_output_nodes, is_op
 from ..interface import (
     BaseTransform,
@@ -305,6 +306,12 @@ def _get_mem_info_in_mb():
         self._log_info(f"After all_gather - new_num_pages: {new_num_pages}")
 
         cm.resize_cache(new_num_pages)
+        # Log the final cache size for performance measurement, do not remove this log.
+        final_cache_size_bytes = cm.current_cache_size_bytes()
+        final_cache_size_gb = final_cache_size_bytes / (1024**3)  # Convert to GiB
+        ad_logger.info(
+            f"Final KV cache size after resize: {final_cache_size_gb:.2f} GiB ({new_num_pages} pages)"
+        )
 
         # Free memory
         torch.cuda.empty_cache()

tests/integration/defs/perf/base_perf_pytorch.csv

@@ -2,3 +2,7 @@ network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,p
 "llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406
 "llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618
 "llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002
+"llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.3,50,INFERENCE_TIME,214410.6447
+"llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_kv_cache_size[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_kv_cache_size[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.3,50,KV_CACHE_SIZE,68.84
+"llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.3,10,SEQ_THROUGHPUT,38.2071
+"llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-_autodeploy-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.3,10,TOKEN_THROUGHPUT,4890.5035

tests/integration/defs/perf/sanity_perf_check.py

@@ -49,6 +49,96 @@ def write_patch(self, old_lines: list[str], new_lines: list[str],
                     f'b/tests/integration/defs/perf/{base_perf_filename}'):
                 f.write(diff_line)
 
+    def _check_autodeploy_failures(self, full_diff: pd.DataFrame,
+                                   base_perf: pd.DataFrame,
+                                   current_perf: pd.DataFrame) -> bool:
+        """
+        Check if any of the performance regressions are from autodeploy tests.
+        Only considers actual regressions (worse performance), not improvements.
+        Returns True if there are autodeploy regressions, False otherwise.
+        """
+        # Create mappings for network_name, threshold, absolute_threshold, and metric_type
+        base_network_mapping = dict(
+            zip(base_perf['perf_case_name'], base_perf['network_name']))
+        current_network_mapping = dict(
+            zip(current_perf['perf_case_name'], current_perf['network_name']))
+        base_threshold_mapping = dict(
+            zip(base_perf['perf_case_name'], base_perf['threshold']))
+        base_abs_threshold_mapping = dict(
+            zip(base_perf['perf_case_name'], base_perf['absolute_threshold']))
+        base_metric_type_mapping = dict(
+            zip(base_perf['perf_case_name'], base_perf['metric_type']))
+
+        # Check each performance difference
+        for idx, row in full_diff.iterrows():
+            # Look up network_name from either base or current (they should be the same)
+            network_name = base_network_mapping.get(
+                idx) or current_network_mapping.get(idx)
+
+            # Only check autodeploy tests
+            if network_name and "_autodeploy" in str(network_name):
+                # Check if this is actually a regression (worse performance)
+                if hasattr(row, 'perf_metric_base') and hasattr(
+                        row, 'perf_metric_target'):
+                    base_value = row.perf_metric_base
+                    target_value = row.perf_metric_target
+                    threshold = base_threshold_mapping.get(idx, 0)
+                    abs_threshold = base_abs_threshold_mapping.get(idx, 50)
+                    metric_type = base_metric_type_mapping.get(idx, '')
+
+                    # Skip if we don't have the necessary data
+                    if pd.isna(base_value) or pd.isna(target_value):
+                        continue
+
+                    # Determine if this is a regression based on metric type and threshold sign
+                    is_regression = self._is_performance_regression(
+                        base_value, target_value, threshold, abs_threshold,
+                        metric_type)
+
+                    if is_regression:
+                        return True
+
+        return False
+
+    def _is_performance_regression(self, base_value: float, target_value: float,
+                                   threshold: float, abs_threshold: float,
+                                   metric_type: str) -> bool:
+        """
+        Determine if a performance change represents a regression (worse performance)
+        that exceeds the acceptable threshold.
+
+        Args:
+            base_value: Baseline performance value
+            target_value: Current performance value
+            threshold: Performance threshold (sign indicates better direction)
+            abs_threshold: Absolute threshold for tolerance calculation
+            metric_type: Type of metric (for context)
+
+        Returns:
+            True if target_value represents worse performance than base_value
+            AND the change exceeds the threshold
+        """
+        import numpy as np
+
+        # First check if the change exceeds the threshold (same logic as diff_tools.py)
+        # Use absolute value of threshold for relative tolerance calculation
+        rel_threshold = abs(threshold)
+
+        # If values are within threshold tolerance, no significant change
+        if np.isclose(base_value,
+                      target_value,
+                      rtol=rel_threshold,
+                      atol=abs(abs_threshold)):
+            return False
+
+        # Now check if it's a regression (worse performance) in the expected direction
+        if threshold > 0:
+            # Positive threshold: lower is better - regression if target > base
+            return target_value > base_value
+        else:
+            # Negative threshold: higher is better - regression if target < base
+            return target_value < base_value
+
     def __call__(self, *args, **kwargs):
         # Check if the base_perf_csv file exists
         if not self.base_perf_csv.exists():
@@ -71,9 +161,23 @@ def __call__(self, *args, **kwargs):
             print(
                 "You can download the file and update base_perf.csv by `git apply <patch file>`"
             )
-            print("Sanity perf check failed, but it has been disabled")
+
+            # Check if any of the failed tests are autodeploy tests
+            autodeploy_failures = self._check_autodeploy_failures(
+                full_diff, base_perf, current_perf)
+
+            if autodeploy_failures:
+                print(
+                    "Sanity perf check failed for autodeploy tests - failing the build"
+                )
+                return 1
+            else:
+                print(
+                    "Sanity perf check failed, but it has been disabled for non-autodeploy tests"
+                )
         return 0
 
 
 if __name__ == '__main__':
-    SanityPerfCheck(sys.argv[1], sys.argv[2])()
+    exit_code = SanityPerfCheck(sys.argv[1], sys.argv[2])()
+    sys.exit(exit_code)

tests/integration/defs/perf/test_perf.py

@@ -408,6 +408,11 @@ def __init__(
         tp_size: int = 1,
         pp_size: int = 1,
         num_gpus: int = 1,
+        # _autodeploy backend specific parameters
+        ad_compile_backend: str = "torch-opt",
+        free_mem_ratio: float = 0.9,
+        extra_runtime: str = "trtllm",
+        skip_loading_weights: bool = False,
     ):
         # The model name.
         self.model_name = model_name
@@ -461,6 +466,11 @@ def __init__(
         self.pp_size = pp_size
         # Number of GPUs.
         self.num_gpus = num_gpus
+        # _autodeploy backend specific parameters
+        self.ad_compile_backend = ad_compile_backend
+        self.free_mem_ratio = free_mem_ratio
+        self.extra_runtime = extra_runtime
+        self.skip_loading_weights = skip_loading_weights
         # Just build engines
         self.build_only = False
 
@@ -507,6 +517,8 @@ def to_string(self,
             entries.append(f"bench")
             if self.backend == 'pytorch':
                 entries.append(f"pytorch")
+            elif self.backend == '_autodeploy':
+                entries.append(f"_autodeploy")
             if self.streaming == "streaming":
                 entries.append(f"streaming")
         elif self.runtime == "disagg_server":  # trtllm-serve
@@ -658,7 +670,8 @@ def load_from_str(self, test_param_labels) -> None:
             return self._load_from_str_disagg(labels)
 
         self.api = labels.pop(0) if labels[0] == "exe" else ""
-        self.backend = labels.pop(0) if labels[0] == "pytorch" else ""
+        self.backend = labels.pop(0) if labels[0] in ["pytorch", "_autodeploy"
+                                                      ] else ""
         self.streaming = labels.pop(0) if labels[0] == "streaming" else ""
         self.static_batching = labels.pop(
             0) if labels[0] == "static_batching" else ""
@@ -1354,12 +1367,14 @@ def get_trtllm_bench_command(self, engine_dir):
             f"--report_json={report_path}",
             f"--kv_cache_free_gpu_mem_fraction={self._config.kv_cache_free_gpu_mem_fraction}",
         ]
-        if self._config.backend != "pytorch":
+        if self._config.backend == "pytorch":
+            benchmark_cmd += ["--backend=pytorch"]
+        elif self._config.backend == "_autodeploy":
+            benchmark_cmd += ["--backend=_autodeploy"]
+        else:
             benchmark_cmd += [
                 f"--backend=tensorrt", f"--engine_dir={engine_dir}"
             ]
-        else:
-            benchmark_cmd += ["--backend=pytorch"]
         if self._config.num_reqs > 0:
             benchmark_cmd += [f"--num_requests={self._config.num_reqs}"]
         if self._config.concurrency != -1:
@@ -1385,6 +1400,28 @@ def get_trtllm_bench_command(self, engine_dir):
             with open(pytorch_config_path, 'w') as f:
                 yaml.dump(config, f, default_flow_style=False)
             benchmark_cmd += [f"--extra_llm_api_options={pytorch_config_path}"]
+        elif self._config.backend == "_autodeploy":
+            import yaml
+            autodeploy_config_path = os.path.join(engine_dir,
+                                                  "extra_llm_api_options.yaml")
+            if not os.path.exists(autodeploy_config_path):
+                os.makedirs(os.path.dirname(autodeploy_config_path),
+                            exist_ok=True)
+
+            # Create _autodeploy specific configuration
+            autodeploy_config = {
+                'compile_backend': self._config.ad_compile_backend,
+                'free_mem_ratio': self._config.free_mem_ratio,
+                'runtime': self._config.extra_runtime,
+                'skip_loading_weights': self._config.skip_loading_weights
+            }
+
+            print_info(f"_autodeploy model config: {autodeploy_config}")
+            with open(autodeploy_config_path, 'w') as f:
+                yaml.dump(autodeploy_config, f, default_flow_style=False)
+            benchmark_cmd += [
+                f"--extra_llm_api_options={autodeploy_config_path}"
+            ]
         return benchmark_cmd
 
     def get_gpt_manager_runtime_benchmark_command(self, engine_dir, bs,
@@ -1499,8 +1536,8 @@ def get_commands(self):
             build_cmd = self.get_trtllm_build_command(engine_dir,
                                                       checkpoint_dir)
         elif self._config.runtime == "bench":
-            if self._config.backend == "pytorch":
-                # Skip building process as it is pytorch backend")
+            if self._config.backend in ["pytorch", "_autodeploy"]:
+                # Skip building process as it is pytorch or _autodeploy backend")
                 pass
             else:
                 build_cmd = self.get_trtllm_bench_build_command(engine_dir)
@@ -1561,7 +1598,7 @@ def get_perf_result(self, outputs: Dict[int, str]) -> float:
         # Make sure we have outputs.
         assert cmd_idx in outputs, f"Output log for command {cmd_idx} does not exist!"
 
-        # Use the regex to go through the log from the N-th command, where N = cmd_idx.
+        # Use all applicable regex patterns to go through the log from the N-th command, where N = cmd_idx.
         print_info(
             f"Searching for metric {metric_name} from output log of command {cmd_idx} ..."
         )
@@ -1769,9 +1806,9 @@ def _get_metrics(self) -> List[PerfTestMetric]:
         # Build command is the first command.
         cmd_idx = 0 if self._config.runtime != "bench" else 1
         if self._config.runtime == "bench":
-            if self._config.backend == "pytorch":
+            if self._config.backend in ["pytorch", "_autodeploy"]:
                 print_info(
-                    f"Skip building process for {self._config.model_name} as it is pytorch backend"
+                    f"Skip building process for {self._config.model_name} as it is {self._config.backend} backend"
                 )
                 builder_metrics = []
             else:

tests/integration/test_lists/test-db/l0_a30.yml

@@ -19,7 +19,7 @@ l0_a30:
   - unittest/_torch/modeling -k "modeling_qwen"
   - unittest/_torch/modeling -k "modeling_qwen_moe"
   - unittest/_torch/modeling -k "modeling_out_of_tree"
-  - unittest/_torch/auto_deploy/unit/singlegpu -k "not test_trtllm_bench_backend_comparison"
+  - unittest/_torch/auto_deploy/unit/singlegpu
   - unittest/_torch/sampler/test_beam_search.py
   - test_e2e.py::test_openai_completions_with_logit_bias[torch_sampler]
   - test_e2e.py::test_openai_chat_with_logit_bias[torch_sampler]

tests/integration/test_lists/test-db/l0_b200.yml

@@ -73,7 +73,7 @@ l0_b200:
   - unittest/_torch/modeling -k "modeling_llama"
   - unittest/_torch/modeling -k "modeling_mixtral"
   - unittest/_torch/modeling -k "modeling_gpt_oss"
-  - unittest/_torch/auto_deploy/unit/singlegpu -k "not test_trtllm_bench_backend_comparison"
+  - unittest/_torch/auto_deploy/unit/singlegpu
 - condition:
     ranges:
       system_gpu_count:

tests/integration/test_lists/test-db/l0_h100.yml

@@ -31,7 +31,6 @@ l0_h100:
   - unittest/_torch/modeling -k "modeling_nemotron"
   - unittest/_torch/modeling -k "modeling_gemma3"
   - unittest/_torch/modeling -k "modeling_gpt_oss"
-  - unittest/_torch/auto_deploy/unit/singlegpu/test_ad_trtllm_bench.py::test_trtllm_bench_backend_comparison
   - unittest/disaggregated/test_disagg_utils.py
   - unittest/disaggregated/test_router.py
   - unittest/disaggregated/test_remoteDictionary.py

tests/integration/test_lists/test-db/l0_perf.yml

@@ -29,3 +29,17 @@ l0_perf:
         backend: pytorch
     tests:
       - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192]
+  - condition:
+      ranges:
+        system_gpu_count:
+          gte: 1
+          lte: 1
+      wildcards:
+        gpu:
+          - '*h100*'
+        linux_distribution_name: ubuntu*
+      terms:
+        stage: pre_merge
+        backend: pytorch
+    tests:
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-_autodeploy-float16-input_output_len:128,128-reqs:8192]