From 8cc62cdcb4f4591ecc5863e81d78687c1607b964 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Mon, 14 Apr 2025 11:34:07 +0300
Subject: [PATCH 01/32] wip: Change perf regression

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/test_lists/test-db/l0_perf.yml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 731b4371ea0b..2366b705c3ec 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -7,16 +7,10 @@ l0_perf:
           lte: 1
       wildcards:
         gpu:
-          - '*a100*'
           - '*h100*'
         linux_distribution_name: ubuntu*
       terms:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[bert_base-cpp-plugin-float16-bs:32-input_len:32]
-      - perf/test_perf.py::test_perf[bert_base-cpp-ootb-float16-bs:32-input_len:32]
-      - perf/test_perf.py::test_perf[roberta_base-cpp-plugin-float16-bs:32-input_len:128+512]
-      - perf/test_perf.py::test_perf[gpt_350m-cppmanager-plugin_ifb-float16-bs:32-input_output_len:60,20]
-      - perf/test_perf.py::test_perf[gpt_350m-cppmanager-plugin_ifb-float16-gwp:0.0-bs:32-input_output_len:60,20]
-      - perf/test_perf.py::test_perf[gpt_350m-cppmanager-static_batching-plugin_ifb-float16-bs:32-input_output_len:60,20]
+      - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-maxbs:64-input_output_len:128,128+512,32]

From e413e67dde261770b46eb09f8cb3a8413ddc2e28 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Mon, 14 Apr 2025 13:31:00 +0300
Subject: [PATCH 02/32] Small fix

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 jenkins/L0_Test.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 5311418bde81..bc751bb2593f 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -1572,7 +1572,7 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null)
         "H100_PCIe-TensorRT-[Post-Merge]-2": ["h100-cr", "l0_h100", 2, 2],
         "B200_PCIe-Triton-Python-[Post-Merge]-1": ["b100-ts2", "l0_b200", 1, 1],
         "DGX_H100-4_GPUs-TensorRT-[Post-Merge]-1": ["dgx-h100-x4", "l0_dgx_h100", 1, 1, 4],
-        "A100_80GB_PCIE-TensorRT-Perf-1": ["a100-80gb-pcie", "l0_perf", 1, 1],
+        // "A100_80GB_PCIE-TensorRT-Perf-1": ["a100-80gb-pcie", "l0_perf", 1, 1],
         "H100_PCIe-TensorRT-Perf-1": ["h100-cr", "l0_perf", 1, 1],
         "DGX_H200-8_GPUs-PyTorch-[Post-Merge]-1": ["dgx-h200-x8", "l0_dgx_h200", 1, 1, 8],
         "DGX_H200-4_GPUs-PyTorch-[Post-Merge]-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 2, 4],

From ae6c3590b70683a95a14511e79ad5d5bba417a52 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Tue, 15 Apr 2025 11:11:47 +0300
Subject: [PATCH 03/32] Change to bench script

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/test_lists/test-db/l0_perf.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 2366b705c3ec..525e73cd9868 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,4 +13,4 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3_8b_instruct-cppmanager-exe-plugin_ifb-float16-maxbs:64-input_output_len:128,128+512,32]
+      - perf/test_perf.py::test_perf[llama_v3_8b_instruct-bench-float16-maxbs:64-input_output_len:128,128+512,32]

From 9826db47e4f2268b77e51fb2ea4d69bacac9d3f0 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Tue, 15 Apr 2025 14:04:39 +0300
Subject: [PATCH 04/32] Change model name

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/test_lists/test-db/l0_perf.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 525e73cd9868..45f77f9c58e3 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,4 +13,4 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3_8b_instruct-bench-float16-maxbs:64-input_output_len:128,128+512,32]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-maxbs:64-input_output_len:128,128+512,32]

From a534f03453b7a09105133ebd88db7985f3fbb4f2 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Wed, 16 Apr 2025 11:03:27 +0300
Subject: [PATCH 05/32] Print some debug logs

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/test_perf.py | 4 +++-
 tests/integration/defs/perf/utils.py     | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py
index c155c13c7efb..928eac52b702 100644
--- a/tests/integration/defs/perf/test_perf.py
+++ b/tests/integration/defs/perf/test_perf.py
@@ -1430,7 +1430,9 @@ def add_myelin_time_pass_to(input_env):
         old_llm_venv = add_myelin_time_pass_to(llm_venv._new_env)
         if self._config.runtime == 'bench':
             #prepare dataset first for trtllm-bench
-            print_info(f"Running command for generating dataset")
+            print_info(
+                f"Running command for generating dataset {self._current_cmd_idx}"
+            )
             outputs = self.run_ex("prepare_dataset",
                                   llm_venv,
                                   gpu_clock_lock,
diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py
index 129731947b2a..020df182ab39 100644
--- a/tests/integration/defs/perf/utils.py
+++ b/tests/integration/defs/perf/utils.py
@@ -463,6 +463,7 @@ def run_ex(self,
 
         # Only save perf result if the result is valid.
         if self._result_state == "valid":
+            print_info(f"Done with command {cmd_idx} - {full_test_name}")
             # Parse the perf result from the test outputs.
             if self._config.runtime == 'bench' and cmd_idx == 0:
                 print_info(

From 156abe15db72c3bb03e6c308eb9eaffa12f6400b Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Thu, 17 Apr 2025 12:27:46 +0300
Subject: [PATCH 06/32] Change test to fix run

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/test_perf.py         | 4 +---
 tests/integration/defs/perf/utils.py             | 1 -
 tests/integration/test_lists/test-db/l0_perf.yml | 2 +-
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/integration/defs/perf/test_perf.py b/tests/integration/defs/perf/test_perf.py
index 928eac52b702..c155c13c7efb 100644
--- a/tests/integration/defs/perf/test_perf.py
+++ b/tests/integration/defs/perf/test_perf.py
@@ -1430,9 +1430,7 @@ def add_myelin_time_pass_to(input_env):
         old_llm_venv = add_myelin_time_pass_to(llm_venv._new_env)
         if self._config.runtime == 'bench':
             #prepare dataset first for trtllm-bench
-            print_info(
-                f"Running command for generating dataset {self._current_cmd_idx}"
-            )
+            print_info(f"Running command for generating dataset")
             outputs = self.run_ex("prepare_dataset",
                                   llm_venv,
                                   gpu_clock_lock,
diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py
index 020df182ab39..129731947b2a 100644
--- a/tests/integration/defs/perf/utils.py
+++ b/tests/integration/defs/perf/utils.py
@@ -463,7 +463,6 @@ def run_ex(self,
 
         # Only save perf result if the result is valid.
         if self._result_state == "valid":
-            print_info(f"Done with command {cmd_idx} - {full_test_name}")
             # Parse the perf result from the test outputs.
             if self._config.runtime == 'bench' and cmd_idx == 0:
                 print_info(
diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 45f77f9c58e3..af09c0c422c6 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,4 +13,4 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-maxbs:64-input_output_len:128,128+512,32]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128+512]

From 42b7bae7cf21066485fd15df57a487f6a4a4cc24 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Thu, 17 Apr 2025 16:40:32 +0300
Subject: [PATCH 07/32] Another fix

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/test_lists/test-db/l0_perf.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index af09c0c422c6..9a3f78cda9c9 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,4 +13,4 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128+512]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128]

From 69e1d17f834cd183e79a416ac13ab86c6263e04f Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Mon, 21 Apr 2025 15:15:04 +0300
Subject: [PATCH 08/32] Update test and fix output

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/utils.py             | 1 +
 tests/integration/test_lists/test-db/l0_perf.yml | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py
index 129731947b2a..6a6f42b257e9 100644
--- a/tests/integration/defs/perf/utils.py
+++ b/tests/integration/defs/perf/utils.py
@@ -468,6 +468,7 @@ def run_ex(self,
                 print_info(
                     f"skip writing perf result when calling generating dataset in trtllm-bench"
                 )
+                outputs.pop(cmd_idx)
             else:
                 self._perf_result = self.get_perf_result(outputs)
 
diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 9a3f78cda9c9..2b6e76a6193a 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,4 +13,4 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:4096]

From 7140e9ba3dc6c257984873fcd13b8d7e5aca3793 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Tue, 22 Apr 2025 10:06:41 +0300
Subject: [PATCH 09/32] Change test

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/test_lists/test-db/l0_perf.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 2b6e76a6193a..25f3fe2a8f83 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,4 +13,4 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:4096]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:512,128-reqs:8192]

From f1d2a7cd66a1fff5ca8691a5f614fb538f574981 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Wed, 23 Apr 2025 12:58:19 +0300
Subject: [PATCH 10/32] Change tests

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/test_lists/test-db/l0_perf.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 25f3fe2a8f83..82cef6b8e6b6 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -14,3 +14,4 @@ l0_perf:
         backend: tensorrt
     tests:
       - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:512,128-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:512,128-reqs:4096]

From 94f9172e3de1a53e48e0871ef2b117808ac5d462 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Sun, 27 Apr 2025 16:45:39 +0300
Subject: [PATCH 11/32] Update sanity perf check to use pandas

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/diff_tools.py     |  87 +++++++++++
 .../defs/perf/sanity_perf_check.py            | 137 +++++-------------
 2 files changed, 124 insertions(+), 100 deletions(-)
 create mode 100644 tests/integration/defs/perf/diff_tools.py

diff --git a/tests/integration/defs/perf/diff_tools.py b/tests/integration/defs/perf/diff_tools.py
new file mode 100644
index 000000000000..4e36a02fe8c1
--- /dev/null
+++ b/tests/integration/defs/perf/diff_tools.py
@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+from io import StringIO
+
+import numpy as np
+import pandas as pd
+
+PERF_CASE_NAME = 'perf_case_name'
+PERF_METRIC = 'perf_metric'
+THRESHOLD = 'threshold'
+ABSOLUTE_THRESHOLD = 'absolute_threshold'
+METRIC_TYPE = 'metric_type'
+IGNORED_METRICS = {'BUILD_TIME'}
+
+
+def load_file(csv_file: str) -> pd.DataFrame:
+    return pd.read_csv(csv_file)
+
+
+def clean_missing(
+    base: pd.DataFrame, target: pd.DataFrame
+) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series]:
+    missing_from_target = base.index.difference(target.index)
+    missing_from_base = target.index.difference(base.index)
+
+    cleaned_base = base.drop(missing_from_target).sort_index()
+    cleaned_target = target.drop(missing_from_base).sort_index()
+    return cleaned_base, cleaned_target, base.loc[
+        missing_from_target], target.loc[missing_from_base]
+
+
+def get_diff_exceeding_threshold(
+        base: pd.DataFrame,
+        target: pd.DataFrame) -> tuple[np.array, pd.DataFrame]:
+    diff_exceeding_threshold = ~np.isclose(base[PERF_METRIC],
+                                           target[PERF_METRIC],
+                                           rtol=base[THRESHOLD],
+                                           atol=base[ABSOLUTE_THRESHOLD])
+    diff_exceeding_threshold = np.array([
+        diff and base[METRIC_TYPE][i] not in IGNORED_METRICS
+        for i, diff in enumerate(diff_exceeding_threshold)
+    ])
+    diff_mask = np.tile(diff_exceeding_threshold[:, None],
+                        (1, target.shape[-1]))
+    return diff_exceeding_threshold, target.where(diff_mask, base)
+
+
+def get_full_diff(base: pd.DataFrame, target: pd.DataFrame,
+                  missing_from_base: pd.Series, missing_from_target: pd.Series,
+                  diff_over_threshold: np.array) -> pd.DataFrame:
+    PERF_METRIC_BASE = f'{PERF_METRIC}_base'
+    PERF_METRIC_TARGET = f'{PERF_METRIC}_target'
+    thershold_diff = pd.merge(base,
+                              target,
+                              on=PERF_CASE_NAME,
+                              suffixes=['_base',
+                                        '_target'])[diff_over_threshold][[
+                                            PERF_METRIC_BASE, PERF_METRIC_TARGET
+                                        ]]
+    missing_from_base = missing_from_base.rename(
+        columns={PERF_METRIC: PERF_METRIC_TARGET})[[PERF_METRIC_TARGET]]
+    missing_from_target = missing_from_target.rename(
+        columns={PERF_METRIC: PERF_METRIC_BASE})[[PERF_METRIC_BASE]]
+    return pd.concat([thershold_diff, missing_from_base, missing_from_target])
+
+
+def get_diff(base: pd.DataFrame,
+             target: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]:
+    relevant_columns = base.columns
+    base = base.set_index(PERF_CASE_NAME)
+    target = target.set_index(PERF_CASE_NAME)
+    cleaned_base, cleaned_target, missing_from_target, missing_from_base = clean_missing(
+        base, target)
+    diff_over_threshold, new_df = get_diff_exceeding_threshold(
+        cleaned_base, cleaned_target)
+    full_diff = get_full_diff(cleaned_base, cleaned_target, missing_from_base,
+                              missing_from_target, diff_over_threshold)
+    return full_diff, pd.concat([new_df, missing_from_base
+                                 ]).reset_index()[relevant_columns]
+
+
+def get_csv_lines(df: pd.DataFrame) -> list[str]:
+    string_buffer = StringIO()
+    df.to_csv(string_buffer, index=False, lineterminator='\r\n'
+              )  # We want to be compatible with csv module default settings
+    string_buffer.seek(0)
+    return string_buffer.readlines()
diff --git a/tests/integration/defs/perf/sanity_perf_check.py b/tests/integration/defs/perf/sanity_perf_check.py
index f4e3829595cb..0ed96e4d86b8 100644
--- a/tests/integration/defs/perf/sanity_perf_check.py
+++ b/tests/integration/defs/perf/sanity_perf_check.py
@@ -12,121 +12,58 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import csv
-import os
+import difflib
 import sys
+from pathlib import Path
 
-# This is to prevent csv field size limit error
-maxInt = sys.maxsize
-while True:
-    try:
-        csv.field_size_limit(maxInt)
-        break
-    except OverflowError:
-        maxInt = int(maxInt / 10)
+import pandas as pd
+from diff_tools import get_csv_lines, get_diff, load_file
 
 
 class SanityPerfCheck():
 
-    # This is to prevent redundant messages and long logs.
-    USEFUL_METRICS = [
-        "original_test_name", "perf_case_name", "metric_type", "perf_metric",
-        "command", "sm_clk", "mem_clk", "start_timestamp", "end_timestamp",
-        "state", "threshold", "absolute_threshold"
-    ]
-
     def __init__(self, target_perf_csv, base_perf_csv=None, threshold=0.1):
-        self.target_perf_csv = target_perf_csv
-        self.base_perf_csv = base_perf_csv
+        self.target_perf_csv = Path(target_perf_csv)
+        self.base_perf_csv = Path(base_perf_csv)
         self.threshold = threshold
 
-    def _parse_result(self, csv_path):
-        result = {}
-        with open(csv_path) as csv_file:
-            parsed_csv_file = csv.DictReader(csv_file)
-            for row in parsed_csv_file:
-                if row['metric_type'] not in result:
-                    result[row['metric_type']] = {}
-                result[row['metric_type']][row['perf_case_name']] = float(
-                    row['perf_metric'])
-        return result
-
-    def _dump_csv_row(self, csv_path, metric_type, test_name):
-        with open(csv_path) as csv_file:
-            parsed_csv_file = csv.DictReader(csv_file)
-            for row in parsed_csv_file:
-                if row['metric_type'] == metric_type and row[
-                        'perf_case_name'] == test_name:
-                    print('=' * 40)
-                    print('Please fill below content into the base_perf.csv.')
-                    cleaned_row = []
-                    for k in self.USEFUL_METRICS:
-                        v = row[k]
-                        # Need to truncate the commands
-                        if k == "command":
-                            options = v.split(" ")
-                            cleaned_options = []
-                            for option in options:
-                                # Truncate workspace dir
-                                if "build.py" in option or "SessionBenchmark.cpp" in option:
-                                    cleaned_options.append("/".join(
-                                        option.split("/")[-5:]))
-                                # Remove engine_dir as it is not useful
-                                elif "--engine_dir=" not in option and "--output_dir=" not in option:
-                                    cleaned_options.append(option)
-                            cleaned_row.append(" ".join(cleaned_options))
-                        else:
-                            cleaned_row.append(v)
-
-                    print(",".join(['\"' + row + '\"' for row in cleaned_row]))
-                    print('=' * 40)
-                    break
+    def report_diff(self, full_diff: pd.DataFrame) -> None:
+        print("=" * 40)
+        for diff in full_diff.itertuples():
+            print(
+                f"perf_case_name: {diff.Index}, base->target: {diff.perf_metric_base}->{diff.perf_metric_target}"
+            )
+        print("=" * 40)
+
+    def write_patch(self, old_lines: list[str], new_lines: list[str],
+                    output_path: str) -> None:
+        with open(output_path, 'w') as f:
+            for diff_line in difflib.unified_diff(
+                    old_lines, new_lines,
+                    'a/tests/integration/defs/perf/base_perf.csv',
+                    'b/tests/integration/defs/perf/base_perf.csv'):
+                f.write(diff_line)
 
     def __call__(self, *args, **kwargs):
         # Check if the base_perf_csv file exists
-        if not os.path.exists(self.base_perf_csv):
+        if not self.base_perf_csv.exists():
             print(f"base_perf.csv doesn't exist, skip check the perf result.")
             return 0
 
-        base_result = self._parse_result(self.base_perf_csv)
-        target_result = self._parse_result(self.target_perf_csv)
-
-        success = True
-
-        for _, metric_type in enumerate(target_result):
-            # Engine build time is very CPU specific, so skip the check
-            if metric_type != "BUILD_TIME":
-                for _, test_name in enumerate(target_result[metric_type]):
-                    if metric_type not in base_result or test_name not in base_result[
-                            metric_type]:
-
-                        self._dump_csv_row(self.target_perf_csv, metric_type,
-                                           test_name),
-                        print(
-                            f"{metric_type} {test_name} doesn't exist in the base_perf.csv, please add it and rerun the pipeline."
-                        )
-                        success = False
-                    else:
-                        base_perf = base_result[metric_type][test_name]
-                        target_perf = target_result[metric_type][test_name]
-
-                        if target_perf > base_perf * (1 + self.threshold):
-                            # the mr perf is worse than baseline, there's perf regression.
-                            print(
-                                f"Perf Regression found on {metric_type} {test_name} where the current perf is {target_perf} while the baseline is {base_perf}."
-                            )
-                            success = False
-                        elif target_perf < base_perf * (1 - self.threshold):
-                            # the MR perf is better than baseline, please update the base_perf.csv
-                            self._dump_csv_row(self.target_perf_csv,
-                                               metric_type, test_name),
-                            print(
-                                f"Please update {metric_type} {test_name} into base_perf.csv and commit again. The outdated perf baseline is {base_perf} and the new perf baseline is {target_perf}"
-                            )
-                            success = False
-
-        if not success:
-            # We have temporarily disabled post perf sanity tests
+        base_perf = load_file(self.base_perf_csv.as_posix())
+        current_perf = load_file(self.target_perf_csv.as_posix())
+
+        full_diff, new_base = get_diff(base_perf, current_perf)
+        if not full_diff.empty:
+            self.report_diff(full_diff)
+            output_patch = self.target_perf_csv.with_name(
+                'perf_patch.patch').as_posix()
+            self.write_patch(get_csv_lines(base_perf), get_csv_lines(new_base),
+                             output_patch)
+            print(f"patch_file was written to {output_patch}")
+            print(
+                "You can download the file and update base_perf.csv by `git apply <patch file>`"
+            )
             print("Sanity perf check failed, but it has been disabled")
         return 0
 

From 820e023770f69155bbbde3d5df4fa5dfc98097bc Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Mon, 28 Apr 2025 11:21:36 +0300
Subject: [PATCH 12/32] Change tests

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/test_lists/test-db/l0_perf.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 82cef6b8e6b6..db36ab4001be 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -14,4 +14,4 @@ l0_perf:
         backend: tensorrt
     tests:
       - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:512,128-reqs:8192]
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:512,128-reqs:4096]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:512,128-reqs:8192]

From db5a21f9b88e22a069cf28d84c91760dff60bcbf Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Mon, 28 Apr 2025 14:28:23 +0300
Subject: [PATCH 13/32] Update README.md

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/README.md         | 55 ++++++++++++++++---
 .../test_lists/test-db/l0_perf.yml            |  4 +-
 2 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/tests/integration/defs/perf/README.md b/tests/integration/defs/perf/README.md
index 891aa1c0834b..1fb9570097b8 100644
--- a/tests/integration/defs/perf/README.md
+++ b/tests/integration/defs/perf/README.md
@@ -1,11 +1,48 @@
-Sanity Perf Check Introduction
+# Sanity Perf Check Introduction
 
-# Background
-The sanity perf check mechanism is the way of perf regression detection for L0 testing. We create the base_perf.csv which consists of the several models' perf baseline and use the sanity_perf_check.py to detect the perf regression.
-# Usage
-There're four typical scenarios for sanity perf check feature.
+## Background
+"Sanity perf check" is a mechanism to detect performance regressions in the L0 pipeline.
+The tests defined in `l0_perf.yaml` are the ones that are required to pass for every PR before merge.
 
-1. The newly added MR doesn't impact the models' perf, the perf check will pass w/o exception.
-2. The newly added MR introduces the new model into perf model list. The sanity check will trigger the exception and the author of this MR needs to add the perf into base_perf.csv.
-3. The newly added MR improves the existed models' perf and the MR author need to refresh the base_perf.csv data w/ new baseline.
-4. The newly added MR introduces the perf regression and the MR author needs to fix the issue and rerun the pipeline.
+### `base_perf.csv`
+The baseline for performance benchmarking is defined at `base_perf.csv` - this file contains the metrics that we verify regression on between CI runs.
+
+This file contains records in the following format:
+```
+perf_case_name metric_type	perf_metric	threshold absolute_threshold
+```
+
+To allow for some machine dependent variance in performance benchmarking we also define a `threshold` and an `absolute_threshold`. This ensures we do not fail on results that reside within legitimate variance thresholds.
+
+`threshold` is relative.
+
+## CI
+As part of our CI, the `test_perf.py` collects performance metrics for configurations defined in `l0_perf.yaml`. This step outputs a `perf_script_test_results.csv` containing the metrics collected for all configurations.
+
+After this step completes, the CI will run `sanity_perf_check.py`. This script will make sure that all differences in metrics from the run on this branch is within a designated threshold of the baseline (`base_perf.csv`).
+
+There're 4 possible results for this:
+1. The current HEAD impact on the performance for our setups is within accepted threshold - the perf check will **pass** w/o exception.
+2. The current HEAD introduces a new setup/metric in `l0_perf.yaml` or removes some of them. This will result in new metrics collected by `test_perf.py` which will **fail** `sanity_perf_check.py`. This requires an update for `base_perf.csv`.
+3. The current HEAD improves performance for at least one metric by more than the accepted threshold, which will **fail** `sanity_perf_check.py`. This requires an update for `base_perf.csv`
+4. The current HEAD introduces a regression to one of the metrics that is over the accepted threshold, which will **fail** `sanity_perf_check.py`. This will require to fix the current branch and rerun the pipeline.
+
+### Updating `base_perf.csv`
+If a CI run fails `sanity_perf_check.py`, it will upload a patch file as an artifact. This file can be applied to current branch using `git apply <patch_file>`.
+
+This patch will only update the metrics that had a difference which was over the accepted threshold. The patch will also remove/add metrics according to the removed or added tests.
+
+## Running locally
+Given a `target_perf_csv_path` you can compare it to another perf csv file.
+You can run it:
+```
+sanity_perf_check.py <target_perf_csv_path> <base_perf_csv_path>
+```
+** In the CI, `<base_perf_csv_path>` is the `base_perf.csv` file path mentioned above.
+
+Running this print the diffs between both performance results. It presents only:
+1. Metrics that have a diff bigger than the accepted threshold.
+2. Metrics missing in `base_perf_csv`.
+3. Metrics missing in `target_perf_csv`.
+
+If any diffs were found it will also generate a patch file to change `base_perf_csv`  with the new metrics, it will be written to the same directory as <target_perf_csv_path> resides in.
diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index db36ab4001be..9cb5f41919f2 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,5 +13,5 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:512,128-reqs:8192]
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:512,128-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:512,128-quant:fp8-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:512,128-quant:fp8-reqs:8192]

From 01275e998e055d3c372817d96df50f9295c13cd3 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Mon, 28 Apr 2025 16:22:58 +0300
Subject: [PATCH 14/32] Add tool for generating performance diff report

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/README.md         |  10 ++
 .../perf/create_perf_comparison_report.py     | 114 ++++++++++++++++++
 2 files changed, 124 insertions(+)
 create mode 100644 tests/integration/defs/perf/create_perf_comparison_report.py

diff --git a/tests/integration/defs/perf/README.md b/tests/integration/defs/perf/README.md
index 1fb9570097b8..3ee029a8c60d 100644
--- a/tests/integration/defs/perf/README.md
+++ b/tests/integration/defs/perf/README.md
@@ -46,3 +46,13 @@ Running this print the diffs between both performance results. It presents only:
 3. Metrics missing in `target_perf_csv`.
 
 If any diffs were found it will also generate a patch file to change `base_perf_csv`  with the new metrics, it will be written to the same directory as <target_perf_csv_path> resides in.
+
+
+## Generating diff report
+To view the difference between 2 reports, it is possible to generate a pdf report containing Bar graphs comparing the base and target value per-metric. Each metric will contain 2 bars per-each configuration (1 for the base and 1 for the target).
+To generate this report
+```
+python tests/integration/defs/perf/create_perf_comparison_report.py --base_csv_path=<path_to_base_csv> --target_csv_path=<path_to_target_csv> --output_path=<output_path_for_report>
+```
+
+This will create a pdf file at <output_path_for_report>.
diff --git a/tests/integration/defs/perf/create_perf_comparison_report.py b/tests/integration/defs/perf/create_perf_comparison_report.py
new file mode 100644
index 000000000000..c47b388f61fa
--- /dev/null
+++ b/tests/integration/defs/perf/create_perf_comparison_report.py
@@ -0,0 +1,114 @@
+import argparse
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import pandas as pd
+from matplotlib.backends.backend_pdf import PdfPages
+
+
+def shorten_names(merged: pd.DataFrame) -> tuple[dict[str, str], pd.DataFrame]:
+
+    def clean_name(name: str) -> str:
+        try:
+            machine = name.split('-')[0]
+            test = name.split('[')[1][:-1]
+            return f"{machine}-{test}"
+        except:
+            return name
+
+    name_mapping = {
+        k: f'configuration_{i+1}'
+        for i, k in enumerate(
+            set(clean_name(config) for config in merged['model_name']))
+    }
+    merged['model_name'] = merged['model_name'].apply(
+        lambda name: name_mapping[clean_name(name)])
+    return merged, name_mapping
+
+
+def write_name_mapping(name_mapping: dict[str, str], pdf: PdfPages) -> None:
+    fig, ax = plt.subplots(figsize=(max(len(n)
+                                        for n in name_mapping.keys()) * 0.3,
+                                    len(name_mapping) *
+                                    0.4))  # height depends on number of entries
+    ax.axis('off')
+
+    table_data = [["Original Name", "Short Name"]]
+    for original, short in name_mapping.items():
+        table_data.append([original, short])
+
+    plt.title("Long name to short name mapping")
+    table = ax.table(cellText=table_data, cellLoc='left', loc='center')
+    table.auto_set_font_size(False)
+    table.set_fontsize(10)
+    table.scale(1, 1.5)
+    pdf.savefig(fig)
+    plt.close(fig)
+
+
+def generate_plots(output_path: Path, name_mapping: dict[str, str],
+                   merged: pd.DataFrame) -> None:
+    metric_types = merged['metric_type'].unique()
+    with PdfPages(output_path.as_posix()) as pdf:
+        write_name_mapping(name_mapping, pdf)
+        for metric in metric_types:
+            metric_data = merged[merged['metric_type'] == metric]
+
+            plot_data = pd.DataFrame({
+                'model_name':
+                metric_data['model_name'],
+                'base':
+                metric_data['metric_value_base'],
+                'target':
+                metric_data['metric_value_target']
+            })
+
+            plot_data = plot_data.set_index('model_name')
+            plot_data.plot(kind='bar', figsize=(10, 6))
+
+            plt.title(f"Comparison for {metric}")
+            plt.ylabel("Metric Value")
+            plt.xlabel("Model Name")
+            plt.tight_layout()
+            pdf.savefig()
+            plt.close()
+
+
+def generate_perf_compare_report(base_perf: str, target_perf: str,
+                                 output_path: str):
+    base_df = pd.read_csv(base_perf)
+    target_df = pd.read_csv(target_perf)
+
+    merged = pd.merge(base_df,
+                      target_df,
+                      on=["model_name", "metric_type"],
+                      suffixes=('_base', '_target'),
+                      how='outer')
+    merged, name_mapping = shorten_names(merged)
+
+    generate_plots(Path(output_path), name_mapping, merged)
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(
+        description="Compare two CSVs and generate a PDF report.")
+    parser.add_argument("--base_csv_path",
+                        type=str,
+                        help="Path to the base CSV file")
+    parser.add_argument("--target_csv_path",
+                        type=str,
+                        help="Path to the target CSV file")
+    parser.add_argument("--output_path",
+                        type=str,
+                        help="Output path for report (pdf file)")
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_arguments()
+    generate_perf_compare_report(args.base_csv_path, args.target_csv_path,
+                                 args.output_path)
+
+
+if __name__ == '__main__':
+    main()

From 8994b373765765438916e1bd20e70aed143ab0c4 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Mon, 28 Apr 2025 17:15:04 +0300
Subject: [PATCH 15/32] Reduce output to stdout

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/utils.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py
index 6a6f42b257e9..f2af25680ba9 100644
--- a/tests/integration/defs/perf/utils.py
+++ b/tests/integration/defs/perf/utils.py
@@ -409,6 +409,8 @@ def run_ex(self,
         self._gpu_clock_lock = gpu_clock_lock
         tmpDir = temp_wd(self.get_working_dir())
 
+        is_prepare_dataset_cmd = self._config.runtime == 'bench' and cmd_idx == 0
+
         # Start the timer.
         self._start_timestamp = datetime.utcnow()
         try:
@@ -422,16 +424,17 @@ def run_ex(self,
                                 buf), self._gpu_clock_lock, tmpDir:
                             output = commands.run_cmd(cmd_idx, venv)
                             # Print the output log to buf.
-                            print(collect_and_clean_myelin_time(output))
+                            if not is_prepare_dataset_cmd:
+                                print(collect_and_clean_myelin_time(output))
                     else:
                         with contextlib.redirect_stdout(buf), tmpDir:
                             output = commands.run_cmd(cmd_idx, venv)
                             # Print the output log to buf.
-                            print(collect_and_clean_myelin_time(output))
+                            if not is_prepare_dataset_cmd:
+                                print(collect_and_clean_myelin_time(output))
 
                     # Print the output log to stdout and cache it.
-                    # skip the output log for prepare dataset command
-                    if 'prepare_dataset' not in commands.get_cmd_str(cmd_idx):
+                    if not is_prepare_dataset_cmd:
                         print(buf.getvalue())
                     outputs[cmd_idx] = buf.getvalue()
             else:
@@ -464,7 +467,7 @@ def run_ex(self,
         # Only save perf result if the result is valid.
         if self._result_state == "valid":
             # Parse the perf result from the test outputs.
-            if self._config.runtime == 'bench' and cmd_idx == 0:
+            if is_prepare_dataset_cmd:
                 print_info(
                     f"skip writing perf result when calling generating dataset in trtllm-bench"
                 )

From 4715cedd2e91bb2814e7de84b4d145d5ae5d802d Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Tue, 29 Apr 2025 17:01:36 +0300
Subject: [PATCH 16/32] Small change

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/integration/defs/perf/README.md b/tests/integration/defs/perf/README.md
index 3ee029a8c60d..df149500afac 100644
--- a/tests/integration/defs/perf/README.md
+++ b/tests/integration/defs/perf/README.md
@@ -2,7 +2,7 @@
 
 ## Background
 "Sanity perf check" is a mechanism to detect performance regressions in the L0 pipeline.
-The tests defined in `l0_perf.yaml` are the ones that are required to pass for every PR before merge.
+The tests defined in `l0_perf.yml` are the ones that are required to pass for every PR before merge.
 
 ### `base_perf.csv`
 The baseline for performance benchmarking is defined at `base_perf.csv` - this file contains the metrics that we verify regression on between CI runs.
@@ -17,13 +17,13 @@ To allow for some machine dependent variance in performance benchmarking we also
 `threshold` is relative.
 
 ## CI
-As part of our CI, the `test_perf.py` collects performance metrics for configurations defined in `l0_perf.yaml`. This step outputs a `perf_script_test_results.csv` containing the metrics collected for all configurations.
+As part of our CI, the `test_perf.py` collects performance metrics for configurations defined in `l0_perf.yml`. This step outputs a `perf_script_test_results.csv` containing the metrics collected for all configurations.
 
 After this step completes, the CI will run `sanity_perf_check.py`. This script will make sure that all differences in metrics from the run on this branch is within a designated threshold of the baseline (`base_perf.csv`).
 
 There're 4 possible results for this:
 1. The current HEAD impact on the performance for our setups is within accepted threshold - the perf check will **pass** w/o exception.
-2. The current HEAD introduces a new setup/metric in `l0_perf.yaml` or removes some of them. This will result in new metrics collected by `test_perf.py` which will **fail** `sanity_perf_check.py`. This requires an update for `base_perf.csv`.
+2. The current HEAD introduces a new setup/metric in `l0_perf.yml` or removes some of them. This will result in new metrics collected by `test_perf.py` which will **fail** `sanity_perf_check.py`. This requires an update for `base_perf.csv`.
 3. The current HEAD improves performance for at least one metric by more than the accepted threshold, which will **fail** `sanity_perf_check.py`. This requires an update for `base_perf.csv`
 4. The current HEAD introduces a regression to one of the metrics that is over the accepted threshold, which will **fail** `sanity_perf_check.py`. This will require to fix the current branch and rerun the pipeline.
 

From 37d1f2d274b819b0bb369bd88c4826fb3e0bfe32 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Tue, 29 Apr 2025 17:27:49 +0300
Subject: [PATCH 17/32] Change models

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/test_lists/test-db/l0_perf.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 9cb5f41919f2..118da3d12313 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,5 +13,5 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:512,128-quant:fp8-reqs:8192]
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:512,128-quant:fp8-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:512,128-quant:fp8-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-pytorch-float16-input_output_len:512,128-quant:fp8-reqs:8192]

From 8af03833deaa8bd60021d08a25e0192f20cc62c1 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Wed, 30 Apr 2025 00:04:06 +0300
Subject: [PATCH 18/32] Change tests

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/test_lists/test-db/l0_perf.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 118da3d12313..c31995250e34 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,5 +13,5 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:512,128-quant:fp8-reqs:8192]
-      - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-pytorch-float16-input_output_len:512,128-quant:fp8-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:128,128-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-pytorch-float16-input_output_len:128,128-reqs:8192]

From b42952b10b6f469e66a8c734f24e28a0c5343ddb Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Wed, 30 Apr 2025 09:41:34 +0300
Subject: [PATCH 19/32] Change tests

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/test_lists/test-db/l0_perf.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index c31995250e34..15cd07262dc7 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,5 +13,5 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-float16-input_output_len:128,128-reqs:8192]
-      - perf/test_perf.py::test_perf[llama_v3.1_8b-bench-pytorch-float16-input_output_len:128,128-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192]

From 259da5428b868a84c024ec16d4bd857ecde72f84 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Wed, 30 Apr 2025 16:13:30 +0300
Subject: [PATCH 20/32] Update script and tests

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/README.md         |  10 +-
 .../perf/create_perf_comparison_report.py     | 150 +++++++++++-------
 .../test_lists/test-db/l0_perf.yml            |   4 +-
 3 files changed, 104 insertions(+), 60 deletions(-)

diff --git a/tests/integration/defs/perf/README.md b/tests/integration/defs/perf/README.md
index df149500afac..caa318ab19f3 100644
--- a/tests/integration/defs/perf/README.md
+++ b/tests/integration/defs/perf/README.md
@@ -49,10 +49,14 @@ If any diffs were found it will also generate a patch file to change `base_perf_
 
 
 ## Generating diff report
-To view the difference between 2 reports, it is possible to generate a pdf report containing Bar graphs comparing the base and target value per-metric. Each metric will contain 2 bars per-each configuration (1 for the base and 1 for the target).
-To generate this report
+To view the difference between performance reports, it is possible to generate a pdf report containing Bar graphs comparing the perf metric value per-metric.
+Each metric will contain comparison bars per configuration.
+
+For example: If we run the script with 3 files and test 2 configurations per metric, we will have 2 groups of 3 bars - A group per-configuration, each group containing the 3 performance metrics reported in the 3 files.
+
+To generate this report:
 ```
-python tests/integration/defs/perf/create_perf_comparison_report.py --base_csv_path=<path_to_base_csv> --target_csv_path=<path_to_target_csv> --output_path=<output_path_for_report>
+python tests/integration/defs/perf/create_perf_comparison_report.py --output_path=<output_path_for_report> --files <csv file paths separated by spaces>
 ```
 
 This will create a pdf file at <output_path_for_report>.
diff --git a/tests/integration/defs/perf/create_perf_comparison_report.py b/tests/integration/defs/perf/create_perf_comparison_report.py
index c47b388f61fa..6e49417f954b 100644
--- a/tests/integration/defs/perf/create_perf_comparison_report.py
+++ b/tests/integration/defs/perf/create_perf_comparison_report.py
@@ -2,31 +2,28 @@
 from pathlib import Path
 
 import matplotlib.pyplot as plt
+import numpy as np
 import pandas as pd
 from matplotlib.backends.backend_pdf import PdfPages
+from matplotlib.figure import Figure
 
+TEST_NAME = 'test_name'
+METRIC_VALUE = 'perf_metric'
+METRIC_TYPE = 'metric_type'
+MEAN_COL = 'mean'
 
-def shorten_names(merged: pd.DataFrame) -> tuple[dict[str, str], pd.DataFrame]:
-
-    def clean_name(name: str) -> str:
-        try:
-            machine = name.split('-')[0]
-            test = name.split('[')[1][:-1]
-            return f"{machine}-{test}"
-        except:
-            return name
 
+def shorten_names(merged: pd.DataFrame) -> tuple[dict[str, str], pd.DataFrame]:
     name_mapping = {
         k: f'configuration_{i+1}'
-        for i, k in enumerate(
-            set(clean_name(config) for config in merged['model_name']))
+        for i, k in enumerate(set(config for config in merged[TEST_NAME]))
     }
-    merged['model_name'] = merged['model_name'].apply(
-        lambda name: name_mapping[clean_name(name)])
+    merged[TEST_NAME] = merged[TEST_NAME].apply(lambda name: name_mapping[name])
     return merged, name_mapping
 
 
-def write_name_mapping(name_mapping: dict[str, str], pdf: PdfPages) -> None:
+def write_name_mapping_table(name_mapping: dict[str, str],
+                             pdf: PdfPages) -> None:
     fig, ax = plt.subplots(figsize=(max(len(n)
                                         for n in name_mapping.keys()) * 0.3,
                                     len(name_mapping) *
@@ -46,58 +43,102 @@ def write_name_mapping(name_mapping: dict[str, str], pdf: PdfPages) -> None:
     plt.close(fig)
 
 
+def plot_metric(merged: pd.DataFrame, metric: str,
+                suffixes: set[str]) -> Figure:
+    metric_data = merged[merged[METRIC_TYPE] == metric]
+    relevant_metrics = {
+        MEAN_COL: metric_data[MEAN_COL]
+    } | {
+        suffix: metric_data[f"{METRIC_VALUE}_{suffix}"]
+        for suffix in suffixes
+    }
+
+    # Prepare the data: extract only the needed columns
+    plot_data = pd.DataFrame({
+        TEST_NAME: metric_data[TEST_NAME],
+    } | relevant_metrics)
+
+    plot_data = plot_data.set_index(TEST_NAME)
+
+    x = np.arange(len(plot_data))
+    width = 0.8 / len(relevant_metrics.keys())
+
+    fig, ax = plt.subplots(figsize=(10, 6))
+
+    for i, suffix in enumerate(relevant_metrics.keys()):
+        values = plot_data[suffix]
+        bar_positions = x + i * width
+        bars = ax.bar(bar_positions, values, width, label=suffix)
+
+        for bar in bars:
+            height = bar.get_height()
+            ax.text(bar.get_x() + bar.get_width() / 2,
+                    height + 0.01,
+                    f'{height:.2f}',
+                    ha='center',
+                    va='bottom',
+                    fontsize=8)
+
+    ax.set_title(f"Comparison for {metric}")
+    ax.set_ylabel("Metric Value")
+    ax.set_xlabel("Model Name")
+    ax.set_xticks(x + width * (len(suffixes) - 1) / 2)
+    ax.set_xticklabels(plot_data.index, rotation=45, ha='right')
+    ax.legend(title='Suffix')
+
+    return fig
+
+
 def generate_plots(output_path: Path, name_mapping: dict[str, str],
-                   merged: pd.DataFrame) -> None:
-    metric_types = merged['metric_type'].unique()
+                   merged: pd.DataFrame, suffixes: set[str]) -> None:
+    metric_types = merged[METRIC_TYPE].unique()
     with PdfPages(output_path.as_posix()) as pdf:
-        write_name_mapping(name_mapping, pdf)
+        write_name_mapping_table(name_mapping, pdf)
         for metric in metric_types:
-            metric_data = merged[merged['metric_type'] == metric]
-
-            plot_data = pd.DataFrame({
-                'model_name':
-                metric_data['model_name'],
-                'base':
-                metric_data['metric_value_base'],
-                'target':
-                metric_data['metric_value_target']
-            })
-
-            plot_data = plot_data.set_index('model_name')
-            plot_data.plot(kind='bar', figsize=(10, 6))
-
-            plt.title(f"Comparison for {metric}")
-            plt.ylabel("Metric Value")
-            plt.xlabel("Model Name")
+            fig = plot_metric(merged, metric, suffixes)
             plt.tight_layout()
-            pdf.savefig()
-            plt.close()
+            pdf.savefig(fig)
+            plt.close(fig)
+
 
+def parse_perf_data(
+        perf_files: list[str]) -> tuple[dict[str, str], pd.DataFrame, set[str]]:
+    perfs = {
+        Path(file_path).name: pd.read_csv(file_path)
+        for file_path in perf_files
+    }
 
-def generate_perf_compare_report(base_perf: str, target_perf: str,
-                                 output_path: str):
-    base_df = pd.read_csv(base_perf)
-    target_df = pd.read_csv(target_perf)
+    merged = pd.DataFrame(columns=[TEST_NAME, METRIC_TYPE])
+    suffixes: set[str] = set()
+    for file_path, df in perfs.items():
+        df = df.rename(
+            columns={
+                column: f'{column}_{file_path}'
+                for column in df.columns
+                if column not in (TEST_NAME, METRIC_TYPE)
+            })
+        merged = merged.merge(df, on=[TEST_NAME, METRIC_TYPE], how='outer')
+        suffixes.add(file_path)
 
-    merged = pd.merge(base_df,
-                      target_df,
-                      on=["model_name", "metric_type"],
-                      suffixes=('_base', '_target'),
-                      how='outer')
+    merged[MEAN_COL] = merged[[
+        f'{METRIC_VALUE}_{suffix}' for suffix in suffixes
+    ]].mean(axis=1)
     merged, name_mapping = shorten_names(merged)
 
-    generate_plots(Path(output_path), name_mapping, merged)
+    return name_mapping, merged, suffixes
+
+
+def generate_perf_compare_report(perf_files: list[str], output_path: str):
+    name_mapping, merged, suffixes = parse_perf_data(perf_files)
+    generate_plots(Path(output_path), name_mapping, merged, suffixes)
 
 
 def parse_arguments():
     parser = argparse.ArgumentParser(
-        description="Compare two CSVs and generate a PDF report.")
-    parser.add_argument("--base_csv_path",
-                        type=str,
-                        help="Path to the base CSV file")
-    parser.add_argument("--target_csv_path",
-                        type=str,
-                        help="Path to the target CSV file")
+        description="Create a report comparing multiple performance csvs")
+    parser.add_argument('--files',
+                        nargs='*',
+                        help="A list of csv files to compare")
     parser.add_argument("--output_path",
                         type=str,
                         help="Output path for report (pdf file)")
@@ -106,8 +147,7 @@ def parse_arguments():
 
 def main() -> None:
     args = parse_arguments()
-    generate_perf_compare_report(args.base_csv_path, args.target_csv_path,
-                                 args.output_path)
+    generate_perf_compare_report(args.files, args.output_path)
 
 
 if __name__ == '__main__':
diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 15cd07262dc7..79480c062e65 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,5 +13,5 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:8192]
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-quant:fp8-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-quant:fp8-reqs:8192]

From 9de7e979ac9a5cd90d7f3192e15f9c23266f8033 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Wed, 30 Apr 2025 16:15:28 +0300
Subject: [PATCH 21/32] Add base-perf

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/base_perf.csv        |  8 ++++++++
 tests/integration/defs/perf/diff_tools.py        | 11 +++++------
 tests/integration/test_lists/test-db/l0_perf.yml |  4 ++--
 3 files changed, 15 insertions(+), 8 deletions(-)
 create mode 100644 tests/integration/defs/perf/base_perf.csv

diff --git a/tests/integration/defs/perf/base_perf.csv b/tests/integration/defs/perf/base_perf.csv
new file mode 100644
index 000000000000..26c057fe55ec
--- /dev/null
+++ b/tests/integration/defs/perf/base_perf.csv
@@ -0,0 +1,8 @@
+,network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,perf_metric
+0,"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,30,BUILD_TIME,143.5976
+1,"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,106778.60992
+2,"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,76.72174
+3,"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,9820.38162
+4,"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406
+5,"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618
+6,"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002
diff --git a/tests/integration/defs/perf/diff_tools.py b/tests/integration/defs/perf/diff_tools.py
index 4e36a02fe8c1..bbe0582c871c 100644
--- a/tests/integration/defs/perf/diff_tools.py
+++ b/tests/integration/defs/perf/diff_tools.py
@@ -17,7 +17,7 @@ def load_file(csv_file: str) -> pd.DataFrame:
     return pd.read_csv(csv_file)
 
 
-def clean_missing(
+def get_intersecting_metrics(
     base: pd.DataFrame, target: pd.DataFrame
 ) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series]:
     missing_from_target = base.index.difference(target.index)
@@ -34,8 +34,8 @@ def get_diff_exceeding_threshold(
         target: pd.DataFrame) -> tuple[np.array, pd.DataFrame]:
     diff_exceeding_threshold = ~np.isclose(base[PERF_METRIC],
                                            target[PERF_METRIC],
-                                           rtol=base[THRESHOLD],
-                                           atol=base[ABSOLUTE_THRESHOLD])
+                                           rtol=abs(base[THRESHOLD]),
+                                           atol=abs(base[ABSOLUTE_THRESHOLD]))
     diff_exceeding_threshold = np.array([
         diff and base[METRIC_TYPE][i] not in IGNORED_METRICS
         for i, diff in enumerate(diff_exceeding_threshold)
@@ -69,7 +69,7 @@ def get_diff(base: pd.DataFrame,
     relevant_columns = base.columns
     base = base.set_index(PERF_CASE_NAME)
     target = target.set_index(PERF_CASE_NAME)
-    cleaned_base, cleaned_target, missing_from_target, missing_from_base = clean_missing(
+    cleaned_base, cleaned_target, missing_from_target, missing_from_base = get_intersecting_metrics(
         base, target)
     diff_over_threshold, new_df = get_diff_exceeding_threshold(
         cleaned_base, cleaned_target)
@@ -81,7 +81,6 @@ def get_diff(base: pd.DataFrame,
 
 def get_csv_lines(df: pd.DataFrame) -> list[str]:
     string_buffer = StringIO()
-    df.to_csv(string_buffer, index=False, lineterminator='\r\n'
-              )  # We want to be compatible with csv module default settings
+    df.to_csv(string_buffer, index=False)
     string_buffer.seek(0)
     return string_buffer.readlines()
diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 79480c062e65..15cd07262dc7 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -13,5 +13,5 @@ l0_perf:
         stage: pre_merge
         backend: tensorrt
     tests:
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-quant:fp8-reqs:8192]
-      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-quant:fp8-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:8192]
+      - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192]

From 77c75f4355e2f40701703b687d78e2b5f615b525 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Sun, 4 May 2025 13:53:31 +0300
Subject: [PATCH 22/32] Change base_perf

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/base_perf.csv | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/integration/defs/perf/base_perf.csv b/tests/integration/defs/perf/base_perf.csv
index 26c057fe55ec..cf29103ce9f2 100644
--- a/tests/integration/defs/perf/base_perf.csv
+++ b/tests/integration/defs/perf/base_perf.csv
@@ -1,8 +1,8 @@
-,network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,perf_metric
-0,"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,30,BUILD_TIME,143.5976
-1,"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,106778.60992
-2,"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,76.72174
-3,"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,9820.38162
-4,"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406
-5,"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618
-6,"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002
+network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,perf_metric
+"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,30,BUILD_TIME,143.5976
+"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,106778.60992
+"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,76.72174
+"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,9820.38162
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002

From 2d20d09d91be56a4b0b27119aae90c7ba986909b Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Sun, 4 May 2025 17:32:59 +0300
Subject: [PATCH 23/32] Some output changes

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 jenkins/L0_Test.groovy               | 8 ++++++++
 tests/integration/defs/perf/utils.py | 2 --
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index bc751bb2593f..005695f8989f 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -1237,6 +1237,14 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
                     ${llmSrc}/tests/integration/defs/perf/base_perf.csv
                 """
             }
+            stage("Create perf report") {
+                sh """
+                    python3 ${llmSrc}/tests/integration/defs/perf/create_perf_comparison_report.py \
+                    --output-path ${stageName}/report.pdf \
+                    --files ${stageName}/perf_script_test_results.csv \
+                    ${llmSrc}/tests/integration/defs/perf/base_perf.csv
+                """
+            }
         }
     }
 }
diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py
index f2af25680ba9..f1325669b8b4 100644
--- a/tests/integration/defs/perf/utils.py
+++ b/tests/integration/defs/perf/utils.py
@@ -536,8 +536,6 @@ def _write_result(self, full_test_name: str,
             "original_test_name":
             original_test_name
             if original_test_name is not None else full_test_name,
-            "raw_result":
-            raw_result,
             "perf_metric":
             self._perf_result,
             "total_time__sec":

From 4b59d171d5a813f188deed39e0bc8790a0114c44 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Mon, 5 May 2025 09:42:26 +0300
Subject: [PATCH 24/32] Remove raw result

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/utils.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py
index f1325669b8b4..a4e66f5bde3f 100644
--- a/tests/integration/defs/perf/utils.py
+++ b/tests/integration/defs/perf/utils.py
@@ -536,6 +536,8 @@ def _write_result(self, full_test_name: str,
             "original_test_name":
             original_test_name
             if original_test_name is not None else full_test_name,
+            "raw_result":
+            raw_result,
             "perf_metric":
             self._perf_result,
             "total_time__sec":
@@ -564,8 +566,7 @@ def _write_result(self, full_test_name: str,
         if "csv" in session_data_writer._output_formats:
             csv_name = "perf_script_test_results.csv"
             cvs_result_dict = {**test_description_dict, **test_result_dict}
-            cvs_result_dict["raw_result"] = cvs_result_dict[
-                "raw_result"].replace("\n", "\\n")
+            cvs_result_dict.pop("raw_result")
             write_csv(output_dir,
                       csv_name, [cvs_result_dict],
                       list(cvs_result_dict.keys()),

From 7e36121bdcf929a4e82821387bb83f86add206ae Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Mon, 5 May 2025 14:34:13 +0300
Subject: [PATCH 25/32] Small fix

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 jenkins/L0_Test.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 005695f8989f..17ff25f1f391 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -1240,7 +1240,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
             stage("Create perf report") {
                 sh """
                     python3 ${llmSrc}/tests/integration/defs/perf/create_perf_comparison_report.py \
-                    --output-path ${stageName}/report.pdf \
+                    --output_path ${stageName}/report.pdf \
                     --files ${stageName}/perf_script_test_results.csv \
                     ${llmSrc}/tests/integration/defs/perf/base_perf.csv
                 """

From a9a7609ad416c1d218a99788f3361533900f200f Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Mon, 5 May 2025 17:29:39 +0300
Subject: [PATCH 26/32] Remove raw result from csv and yaml

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py
index a4e66f5bde3f..029cc09a40bd 100644
--- a/tests/integration/defs/perf/utils.py
+++ b/tests/integration/defs/perf/utils.py
@@ -536,8 +536,6 @@ def _write_result(self, full_test_name: str,
             "original_test_name":
             original_test_name
             if original_test_name is not None else full_test_name,
-            "raw_result":
-            raw_result,
             "perf_metric":
             self._perf_result,
             "total_time__sec":
@@ -566,7 +564,9 @@ def _write_result(self, full_test_name: str,
         if "csv" in session_data_writer._output_formats:
             csv_name = "perf_script_test_results.csv"
             cvs_result_dict = {**test_description_dict, **test_result_dict}
-            cvs_result_dict.pop("raw_result")
+            if "raw_result" in cvs_result_dict:
+                cvs_result_dict["raw_result"] = cvs_result_dict[
+                    "raw_result"].replace("\n", "\\n")
             write_csv(output_dir,
                       csv_name, [cvs_result_dict],
                       list(cvs_result_dict.keys()),

From ae105e71e77a375b9c051cde43b21c85443f8a4a Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Thu, 8 May 2025 16:00:06 +0300
Subject: [PATCH 27/32] Split tests to 2 stages

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 jenkins/L0_Test.groovy                           |  1 +
 tests/integration/test_lists/test-db/l0_perf.yml | 14 ++++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 17ff25f1f391..523b21331a71 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -1582,6 +1582,7 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null)
         "DGX_H100-4_GPUs-TensorRT-[Post-Merge]-1": ["dgx-h100-x4", "l0_dgx_h100", 1, 1, 4],
         // "A100_80GB_PCIE-TensorRT-Perf-1": ["a100-80gb-pcie", "l0_perf", 1, 1],
         "H100_PCIe-TensorRT-Perf-1": ["h100-cr", "l0_perf", 1, 1],
+        "H100_PCIe-PyTorch-Perf-1": ["h100-cr", "l0_perf", 1, 1],
         "DGX_H200-8_GPUs-PyTorch-[Post-Merge]-1": ["dgx-h200-x8", "l0_dgx_h200", 1, 1, 8],
         "DGX_H200-4_GPUs-PyTorch-[Post-Merge]-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 2, 4],
         "DGX_H200-4_GPUs-PyTorch-[Post-Merge]-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 2, 4],
diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml
index 15cd07262dc7..21f5a11dfb07 100644
--- a/tests/integration/test_lists/test-db/l0_perf.yml
+++ b/tests/integration/test_lists/test-db/l0_perf.yml
@@ -14,4 +14,18 @@ l0_perf:
         backend: tensorrt
     tests:
       - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:8192]
+
+  - condition:
+      ranges:
+        system_gpu_count:
+          gte: 1
+          lte: 1
+      wildcards:
+        gpu:
+          - '*h100*'
+        linux_distribution_name: ubuntu*
+      terms:
+        stage: pre_merge
+        backend: pytorch
+    tests:
       - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192]

From dd6eec26bff0e6660801f73806b544a6af7c8cf6 Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Thu, 8 May 2025 18:35:28 +0300
Subject: [PATCH 28/32] Change perf_case_name in base_perf

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/base_perf.csv | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/integration/defs/perf/base_perf.csv b/tests/integration/defs/perf/base_perf.csv
index cf29103ce9f2..2b507989faab 100644
--- a/tests/integration/defs/perf/base_perf.csv
+++ b/tests/integration/defs/perf/base_perf.csv
@@ -3,6 +3,6 @@ network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,p
 "llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,106778.60992
 "llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,76.72174
 "llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,9820.38162
-"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406
-"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618
-"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002

From bdef182588279d9d057bab669c53f7507899346b Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Sun, 11 May 2025 10:27:17 +0300
Subject: [PATCH 29/32] Update perf files based on backend

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 jenkins/L0_Test.groovy                            |  6 ++++--
 tests/integration/defs/perf/base_perf.csv         |  3 ---
 tests/integration/defs/perf/base_perf_pytorch.csv |  4 ++++
 tests/integration/defs/perf/sanity_perf_check.py  | 12 +++++++-----
 tests/integration/defs/perf/utils.py              |  3 ++-
 5 files changed, 17 insertions(+), 11 deletions(-)
 create mode 100644 tests/integration/defs/perf/base_perf_pytorch.csv

diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy
index 523b21331a71..27046d09829c 100644
--- a/jenkins/L0_Test.groovy
+++ b/jenkins/L0_Test.groovy
@@ -1230,11 +1230,13 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         }
 
         if (perfMode) {
+            basePerfFilename = stageName.contains("PyTorch") ? "base_perf_pytorch.csv" : "base_perf.csv"
+            basePerfPath = "${llmSrc}/tests/integration/defs/perf/${basePerfFilename}"
             stage("Check perf result") {
                 sh """
                     python3 ${llmSrc}/tests/integration/defs/perf/sanity_perf_check.py \
                     ${stageName}/perf_script_test_results.csv \
-                    ${llmSrc}/tests/integration/defs/perf/base_perf.csv
+                    ${basePerfPath}
                 """
             }
             stage("Create perf report") {
@@ -1242,7 +1244,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
                     python3 ${llmSrc}/tests/integration/defs/perf/create_perf_comparison_report.py \
                     --output_path ${stageName}/report.pdf \
                     --files ${stageName}/perf_script_test_results.csv \
-                    ${llmSrc}/tests/integration/defs/perf/base_perf.csv
+                    ${basePerfPath}
                 """
             }
         }
diff --git a/tests/integration/defs/perf/base_perf.csv b/tests/integration/defs/perf/base_perf.csv
index 2b507989faab..3d997d72eed4 100644
--- a/tests/integration/defs/perf/base_perf.csv
+++ b/tests/integration/defs/perf/base_perf.csv
@@ -3,6 +3,3 @@ network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,p
 "llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,106778.60992
 "llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,76.72174
 "llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,9820.38162
-"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406
-"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618
-"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002
diff --git a/tests/integration/defs/perf/base_perf_pytorch.csv b/tests/integration/defs/perf/base_perf_pytorch.csv
new file mode 100644
index 000000000000..bb2206bd3cf5
--- /dev/null
+++ b/tests/integration/defs/perf/base_perf_pytorch.csv
@@ -0,0 +1,4 @@
+network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,perf_metric
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002
diff --git a/tests/integration/defs/perf/sanity_perf_check.py b/tests/integration/defs/perf/sanity_perf_check.py
index 0ed96e4d86b8..620afeb76f6a 100644
--- a/tests/integration/defs/perf/sanity_perf_check.py
+++ b/tests/integration/defs/perf/sanity_perf_check.py
@@ -36,18 +36,20 @@ def report_diff(self, full_diff: pd.DataFrame) -> None:
         print("=" * 40)
 
     def write_patch(self, old_lines: list[str], new_lines: list[str],
-                    output_path: str) -> None:
+                    output_path: str, base_perf_filename: str) -> None:
         with open(output_path, 'w') as f:
             for diff_line in difflib.unified_diff(
                     old_lines, new_lines,
-                    'a/tests/integration/defs/perf/base_perf.csv',
-                    'b/tests/integration/defs/perf/base_perf.csv'):
+                    f'a/tests/integration/defs/perf/{base_perf_filename}',
+                    f'b/tests/integration/defs/perf/{base_perf_filename}'):
                 f.write(diff_line)
 
     def __call__(self, *args, **kwargs):
         # Check if the base_perf_csv file exists
         if not self.base_perf_csv.exists():
-            print(f"base_perf.csv doesn't exist, skip check the perf result.")
+            print(
+                f"{self.base_perf_csv.name} doesn't exist, skip check the perf result."
+            )
             return 0
 
         base_perf = load_file(self.base_perf_csv.as_posix())
@@ -59,7 +61,7 @@ def __call__(self, *args, **kwargs):
             output_patch = self.target_perf_csv.with_name(
                 'perf_patch.patch').as_posix()
             self.write_patch(get_csv_lines(base_perf), get_csv_lines(new_base),
-                             output_patch)
+                             output_patch, self.base_perf_csv.name)
             print(f"patch_file was written to {output_patch}")
             print(
                 "You can download the file and update base_perf.csv by `git apply <patch file>`"
diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py
index 029cc09a40bd..a0f5da2144f7 100644
--- a/tests/integration/defs/perf/utils.py
+++ b/tests/integration/defs/perf/utils.py
@@ -409,7 +409,8 @@ def run_ex(self,
         self._gpu_clock_lock = gpu_clock_lock
         tmpDir = temp_wd(self.get_working_dir())
 
-        is_prepare_dataset_cmd = self._config.runtime == 'bench' and cmd_idx == 0
+        is_prepare_dataset_cmd = 'prepare_dataset' in commands.get_cmd_str(
+            cmd_idx)
 
         # Start the timer.
         self._start_timestamp = datetime.utcnow()

From cfdb293f6442dc5e6848271b4f3dee341679974d Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Sun, 11 May 2025 17:07:32 +0300
Subject: [PATCH 30/32] Revert print removal

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py
index a0f5da2144f7..9ca08f4d029d 100644
--- a/tests/integration/defs/perf/utils.py
+++ b/tests/integration/defs/perf/utils.py
@@ -425,14 +425,14 @@ def run_ex(self,
                                 buf), self._gpu_clock_lock, tmpDir:
                             output = commands.run_cmd(cmd_idx, venv)
                             # Print the output log to buf.
-                            if not is_prepare_dataset_cmd:
-                                print(collect_and_clean_myelin_time(output))
+                            # if not is_prepare_dataset_cmd:
+                            print(collect_and_clean_myelin_time(output))
                     else:
                         with contextlib.redirect_stdout(buf), tmpDir:
                             output = commands.run_cmd(cmd_idx, venv)
                             # Print the output log to buf.
-                            if not is_prepare_dataset_cmd:
-                                print(collect_and_clean_myelin_time(output))
+                            # if not is_prepare_dataset_cmd:
+                            print(collect_and_clean_myelin_time(output))
 
                     # Print the output log to stdout and cache it.
                     if not is_prepare_dataset_cmd:

From c6da68a02e65fa91226fc343aa14b44a768b324e Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Tue, 27 May 2025 14:53:58 +0300
Subject: [PATCH 31/32] CR and fix test

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/base_perf.csv            |  8 ++++----
 tests/integration/defs/perf/base_perf_pytorch.csv    |  6 +++---
 .../defs/perf/create_perf_comparison_report.py       |  5 +++--
 tests/integration/defs/perf/diff_tools.py            | 12 ++++++------
 tests/integration/defs/perf/sanity_perf_check.py     | 11 ++++++++---
 5 files changed, 24 insertions(+), 18 deletions(-)

diff --git a/tests/integration/defs/perf/base_perf.csv b/tests/integration/defs/perf/base_perf.csv
index 3d997d72eed4..b4f374c1cc79 100644
--- a/tests/integration/defs/perf/base_perf.csv
+++ b/tests/integration/defs/perf/base_perf.csv
@@ -1,5 +1,5 @@
 network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,perf_metric
-"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,30,BUILD_TIME,143.5976
-"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,106778.60992
-"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,76.72174
-"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,9820.38162
+"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf-1/perf/test_perf.py::test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,30,BUILD_TIME,143.5976
+"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf-1/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,106778.60992
+"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf-1/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,76.72174
+"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf-1/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,9820.38162
diff --git a/tests/integration/defs/perf/base_perf_pytorch.csv b/tests/integration/defs/perf/base_perf_pytorch.csv
index bb2206bd3cf5..8785f7587fc6 100644
--- a/tests/integration/defs/perf/base_perf_pytorch.csv
+++ b/tests/integration/defs/perf/base_perf_pytorch.csv
@@ -1,4 +1,4 @@
 network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,perf_metric
-"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406
-"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618
-"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618
+"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002
diff --git a/tests/integration/defs/perf/create_perf_comparison_report.py b/tests/integration/defs/perf/create_perf_comparison_report.py
index 6e49417f954b..d36ec0a23dca 100644
--- a/tests/integration/defs/perf/create_perf_comparison_report.py
+++ b/tests/integration/defs/perf/create_perf_comparison_report.py
@@ -128,12 +128,13 @@ def parse_perf_data(
     return name_mapping, merged, suffixes
 
 
-def generate_perf_compare_report(perf_files: list[str], output_path: str):
+def generate_perf_compare_report(perf_files: list[str],
+                                 output_path: str) -> None:
     name_mapping, merged, suffixes = parse_perf_data(perf_files)
     generate_plots(Path(output_path), name_mapping, merged, suffixes)
 
 
-def parse_arguments():
+def parse_arguments() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         description="Create a report comparing multiple performance csvs")
     parser.add_argument('--files',
diff --git a/tests/integration/defs/perf/diff_tools.py b/tests/integration/defs/perf/diff_tools.py
index bbe0582c871c..ce072fdcfa82 100644
--- a/tests/integration/defs/perf/diff_tools.py
+++ b/tests/integration/defs/perf/diff_tools.py
@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from io import StringIO
 
 import numpy as np
@@ -53,10 +51,12 @@ def get_full_diff(base: pd.DataFrame, target: pd.DataFrame,
     thershold_diff = pd.merge(base,
                               target,
                               on=PERF_CASE_NAME,
-                              suffixes=['_base',
-                                        '_target'])[diff_over_threshold][[
-                                            PERF_METRIC_BASE, PERF_METRIC_TARGET
-                                        ]]
+                              how='outer',
+                              suffixes=['_base', '_target'])
+    if not thershold_diff.empty:
+        thershold_diff = thershold_diff[diff_over_threshold][[
+            PERF_METRIC_BASE, PERF_METRIC_TARGET
+        ]]
     missing_from_base = missing_from_base.rename(
         columns={PERF_METRIC: PERF_METRIC_TARGET})[[PERF_METRIC_TARGET]]
     missing_from_target = missing_from_target.rename(
diff --git a/tests/integration/defs/perf/sanity_perf_check.py b/tests/integration/defs/perf/sanity_perf_check.py
index 620afeb76f6a..e00c34ad180c 100644
--- a/tests/integration/defs/perf/sanity_perf_check.py
+++ b/tests/integration/defs/perf/sanity_perf_check.py
@@ -30,9 +30,14 @@ def __init__(self, target_perf_csv, base_perf_csv=None, threshold=0.1):
     def report_diff(self, full_diff: pd.DataFrame) -> None:
         print("=" * 40)
         for diff in full_diff.itertuples():
-            print(
-                f"perf_case_name: {diff.Index}, base->target: {diff.perf_metric_base}->{diff.perf_metric_target}"
-            )
+            if pd.isna(diff.perf_metric_base):
+                print(f"perf_case_name: {diff.Index} is missing from base")
+            elif pd.isna(diff.perf_metric_target):
+                print(f"perf_case_name: {diff.Index} is missing from target")
+            else:
+                print(
+                    f"perf_case_name: {diff.Index}, base->target: {diff.perf_metric_base}->{diff.perf_metric_target}"
+                )
         print("=" * 40)
 
     def write_patch(self, old_lines: list[str], new_lines: list[str],

From 8ea826edad75424eaec0fa689df18009be674cad Mon Sep 17 00:00:00 2001
From: Amir Klein <203507526+amirkl94@users.noreply.github.com>
Date: Thu, 29 May 2025 10:55:08 +0300
Subject: [PATCH 32/32] Add requirements for report generation script

Signed-off-by: Amir Klein <203507526+amirkl94@users.noreply.github.com>
---
 tests/integration/defs/perf/README.md        | 6 +++++-
 tests/integration/defs/perf/requirements.txt | 3 +++
 2 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 tests/integration/defs/perf/requirements.txt

diff --git a/tests/integration/defs/perf/README.md b/tests/integration/defs/perf/README.md
index caa318ab19f3..569063f22ae0 100644
--- a/tests/integration/defs/perf/README.md
+++ b/tests/integration/defs/perf/README.md
@@ -34,7 +34,11 @@ This patch will only update the metrics that had a difference which was over the
 
 ## Running locally
 Given a `target_perf_csv_path` you can compare it to another perf csv file.
-You can run it:
+First make sure you install the dependencies:
+```
+pip install -r tests/integration/defs/perf/requirements.txt
+```
+Then, you can run it with:
 ```
 sanity_perf_check.py <target_perf_csv_path> <base_perf_csv_path>
 ```
diff --git a/tests/integration/defs/perf/requirements.txt b/tests/integration/defs/perf/requirements.txt
new file mode 100644
index 000000000000..4d7329f9d923
--- /dev/null
+++ b/tests/integration/defs/perf/requirements.txt
@@ -0,0 +1,3 @@
+pandas
+numpy<2
+matplotlib