diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 5311418bde81..27046d09829c 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1230,11 +1230,21 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } if (perfMode) { + basePerfFilename = stageName.contains("PyTorch") ? "base_perf_pytorch.csv" : "base_perf.csv" + basePerfPath = "${llmSrc}/tests/integration/defs/perf/${basePerfFilename}" stage("Check perf result") { sh """ python3 ${llmSrc}/tests/integration/defs/perf/sanity_perf_check.py \ ${stageName}/perf_script_test_results.csv \ - ${llmSrc}/tests/integration/defs/perf/base_perf.csv + ${basePerfPath} + """ + } + stage("Create perf report") { + sh """ + python3 ${llmSrc}/tests/integration/defs/perf/create_perf_comparison_report.py \ + --output_path ${stageName}/report.pdf \ + --files ${stageName}/perf_script_test_results.csv \ + ${basePerfPath} """ } } @@ -1572,8 +1582,9 @@ def launchTestJobs(pipeline, testFilter, dockerNode=null) "H100_PCIe-TensorRT-[Post-Merge]-2": ["h100-cr", "l0_h100", 2, 2], "B200_PCIe-Triton-Python-[Post-Merge]-1": ["b100-ts2", "l0_b200", 1, 1], "DGX_H100-4_GPUs-TensorRT-[Post-Merge]-1": ["dgx-h100-x4", "l0_dgx_h100", 1, 1, 4], - "A100_80GB_PCIE-TensorRT-Perf-1": ["a100-80gb-pcie", "l0_perf", 1, 1], + // "A100_80GB_PCIE-TensorRT-Perf-1": ["a100-80gb-pcie", "l0_perf", 1, 1], "H100_PCIe-TensorRT-Perf-1": ["h100-cr", "l0_perf", 1, 1], + "H100_PCIe-PyTorch-Perf-1": ["h100-cr", "l0_perf", 1, 1], "DGX_H200-8_GPUs-PyTorch-[Post-Merge]-1": ["dgx-h200-x8", "l0_dgx_h200", 1, 1, 8], "DGX_H200-4_GPUs-PyTorch-[Post-Merge]-1": ["dgx-h200-x4", "l0_dgx_h200", 1, 2, 4], "DGX_H200-4_GPUs-PyTorch-[Post-Merge]-2": ["dgx-h200-x4", "l0_dgx_h200", 2, 2, 4], diff --git a/tests/integration/defs/perf/README.md b/tests/integration/defs/perf/README.md index 891aa1c0834b..569063f22ae0 100644 --- a/tests/integration/defs/perf/README.md +++ b/tests/integration/defs/perf/README.md @@ -1,11 +1,66 @@ -Sanity Perf Check Introduction +# Sanity Perf Check Introduction -# Background -The sanity perf check mechanism is the way of perf regression detection for L0 testing. We create the base_perf.csv which consists of the several models' perf baseline and use the sanity_perf_check.py to detect the perf regression. -# Usage -There're four typical scenarios for sanity perf check feature. +## Background +"Sanity perf check" is a mechanism to detect performance regressions in the L0 pipeline. +The tests defined in `l0_perf.yml` are the ones that are required to pass for every PR before merge. -1. The newly added MR doesn't impact the models' perf, the perf check will pass w/o exception. -2. The newly added MR introduces the new model into perf model list. The sanity check will trigger the exception and the author of this MR needs to add the perf into base_perf.csv. -3. The newly added MR improves the existed models' perf and the MR author need to refresh the base_perf.csv data w/ new baseline. -4. The newly added MR introduces the perf regression and the MR author needs to fix the issue and rerun the pipeline. +### `base_perf.csv` +The baseline for performance benchmarking is defined at `base_perf.csv` - this file contains the metrics that we verify regression on between CI runs. + +This file contains records in the following format: +``` +perf_case_name metric_type perf_metric threshold absolute_threshold +``` + +To allow for some machine dependent variance in performance benchmarking we also define a `threshold` and an `absolute_threshold`. This ensures we do not fail on results that reside within legitimate variance thresholds. + +`threshold` is relative. + +## CI +As part of our CI, the `test_perf.py` collects performance metrics for configurations defined in `l0_perf.yml`. This step outputs a `perf_script_test_results.csv` containing the metrics collected for all configurations. + +After this step completes, the CI will run `sanity_perf_check.py`. This script will make sure that all differences in metrics from the run on this branch is within a designated threshold of the baseline (`base_perf.csv`). + +There're 4 possible results for this: +1. The current HEAD impact on the performance for our setups is within accepted threshold - the perf check will **pass** w/o exception. +2. The current HEAD introduces a new setup/metric in `l0_perf.yml` or removes some of them. This will result in new metrics collected by `test_perf.py` which will **fail** `sanity_perf_check.py`. This requires an update for `base_perf.csv`. +3. The current HEAD improves performance for at least one metric by more than the accepted threshold, which will **fail** `sanity_perf_check.py`. This requires an update for `base_perf.csv` +4. The current HEAD introduces a regression to one of the metrics that is over the accepted threshold, which will **fail** `sanity_perf_check.py`. This will require to fix the current branch and rerun the pipeline. + +### Updating `base_perf.csv` +If a CI run fails `sanity_perf_check.py`, it will upload a patch file as an artifact. This file can be applied to current branch using `git apply `. + +This patch will only update the metrics that had a difference which was over the accepted threshold. The patch will also remove/add metrics according to the removed or added tests. + +## Running locally +Given a `target_perf_csv_path` you can compare it to another perf csv file. +First make sure you install the dependencies: +``` +pip install -r tests/integration/defs/perf/requirements.txt +``` +Then, you can run it with: +``` +sanity_perf_check.py +``` +** In the CI, `` is the `base_perf.csv` file path mentioned above. + +Running this print the diffs between both performance results. It presents only: +1. Metrics that have a diff bigger than the accepted threshold. +2. Metrics missing in `base_perf_csv`. +3. Metrics missing in `target_perf_csv`. + +If any diffs were found it will also generate a patch file to change `base_perf_csv` with the new metrics, it will be written to the same directory as resides in. + + +## Generating diff report +To view the difference between performance reports, it is possible to generate a pdf report containing Bar graphs comparing the perf metric value per-metric. +Each metric will contain comparison bars per configuration. + +For example: If we run the script with 3 files and test 2 configurations per metric, we will have 2 groups of 3 bars - A group per-configuration, each group containing the 3 performance metrics reported in the 3 files. + +To generate this report: +``` +python tests/integration/defs/perf/create_perf_comparison_report.py --output_path= --files +``` + +This will create a pdf file at . diff --git a/tests/integration/defs/perf/base_perf.csv b/tests/integration/defs/perf/base_perf.csv new file mode 100644 index 000000000000..b4f374c1cc79 --- /dev/null +++ b/tests/integration/defs/perf/base_perf.csv @@ -0,0 +1,5 @@ +network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,perf_metric +"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf-1/perf/test_perf.py::test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_build_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,30,BUILD_TIME,143.5976 +"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf-1/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,106778.60992 +"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf-1/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,76.72174 +"llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-TensorRT-Perf-1/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,9820.38162 diff --git a/tests/integration/defs/perf/base_perf_pytorch.csv b/tests/integration/defs/perf/base_perf_pytorch.csv new file mode 100644 index 000000000000..8785f7587fc6 --- /dev/null +++ b/tests/integration/defs/perf/base_perf_pytorch.csv @@ -0,0 +1,4 @@ +network_name,perf_case_name,test_name,threshold,absolute_threshold,metric_type,perf_metric +"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_inference_time[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",0.1,50,INFERENCE_TIME,99133.65406 +"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_seq_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,SEQ_THROUGHPUT,82.63618 +"llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192","H100_PCIe-PyTorch-Perf-1/perf/test_perf.py::test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]","test_perf_metric_token_throughput[llama_v3.1_8b_instruct-bench-pytorch-float16-maxbs:512-maxnt:2048-input_output_len:128,128-reqs:8192]",-0.1,10,TOKEN_THROUGHPUT,10577.431520000002 diff --git a/tests/integration/defs/perf/create_perf_comparison_report.py b/tests/integration/defs/perf/create_perf_comparison_report.py new file mode 100644 index 000000000000..d36ec0a23dca --- /dev/null +++ b/tests/integration/defs/perf/create_perf_comparison_report.py @@ -0,0 +1,155 @@ +import argparse +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from matplotlib.backends.backend_pdf import PdfPages +from matplotlib.figure import Figure + +TEST_NAME = 'test_name' +METRIC_VALUE = 'perf_metric' +METRIC_TYPE = 'metric_type' +MEAN_COL = 'mean' + + +def shorten_names(merged: pd.DataFrame) -> tuple[dict[str, str], pd.DataFrame]: + name_mapping = { + k: f'configuration_{i+1}' + for i, k in enumerate(set(config for config in merged[TEST_NAME])) + } + merged[TEST_NAME] = merged[TEST_NAME].apply(lambda name: name_mapping[name]) + return merged, name_mapping + + +def write_name_mapping_table(name_mapping: dict[str, str], + pdf: PdfPages) -> None: + fig, ax = plt.subplots(figsize=(max(len(n) + for n in name_mapping.keys()) * 0.3, + len(name_mapping) * + 0.4)) # height depends on number of entries + ax.axis('off') + + table_data = [["Original Name", "Short Name"]] + for original, short in name_mapping.items(): + table_data.append([original, short]) + + plt.title("Long name to short name mapping") + table = ax.table(cellText=table_data, cellLoc='left', loc='center') + table.auto_set_font_size(False) + table.set_fontsize(10) + table.scale(1, 1.5) + pdf.savefig(fig) + plt.close(fig) + + +def plot_metric(merged: pd.DataFrame, metric: str, + suffixes: set[str]) -> Figure: + metric_data = merged[merged[METRIC_TYPE] == metric] + relevant_metrics = { + MEAN_COL: metric_data[MEAN_COL] + } | { + suffix: metric_data[f"{METRIC_VALUE}_{suffix}"] + for suffix in suffixes + } + + # Prepare the data: extract only the needed columns + plot_data = pd.DataFrame({ + TEST_NAME: metric_data[TEST_NAME], + } | relevant_metrics) + + plot_data = plot_data.set_index(TEST_NAME) + + x = np.arange(len(plot_data)) + width = 0.8 / len(relevant_metrics.keys()) + + fig, ax = plt.subplots(figsize=(10, 6)) + + for i, suffix in enumerate(relevant_metrics.keys()): + values = plot_data[suffix] + bar_positions = x + i * width + bars = ax.bar(bar_positions, values, width, label=suffix) + + for bar in bars: + height = bar.get_height() + ax.text(bar.get_x() + bar.get_width() / 2, + height + 0.01, + f'{height:.2f}', + ha='center', + va='bottom', + fontsize=8) + + ax.set_title(f"Comparison for {metric}") + ax.set_ylabel("Metric Value") + ax.set_xlabel("Model Name") + ax.set_xticks(x + width * (len(suffixes) - 1) / 2) + ax.set_xticklabels(plot_data.index, rotation=45, ha='right') + ax.legend(title='Suffix') + + return fig + + +def generate_plots(output_path: Path, name_mapping: dict[str, str], + merged: pd.DataFrame, suffixes: set[str]) -> None: + metric_types = merged[METRIC_TYPE].unique() + with PdfPages(output_path.as_posix()) as pdf: + write_name_mapping_table(name_mapping, pdf) + for metric in metric_types: + fig = plot_metric(merged, metric, suffixes) + plt.tight_layout() + pdf.savefig(fig) + plt.close(fig) + + +def parse_perf_data( + perf_files: list[str]) -> tuple[dict[str, str], pd.DataFrame, set[str]]: + perfs = { + Path(file_path).name: pd.read_csv(file_path) + for file_path in perf_files + } + + merged = pd.DataFrame(columns=[TEST_NAME, METRIC_TYPE]) + suffixes: set[str] = set() + for file_path, df in perfs.items(): + df = df.rename( + columns={ + column: f'{column}_{file_path}' + for column in df.columns + if column not in (TEST_NAME, METRIC_TYPE) + }) + merged = merged.merge(df, on=[TEST_NAME, METRIC_TYPE], how='outer') + suffixes.add(file_path) + + merged[MEAN_COL] = merged[[ + f'{METRIC_VALUE}_{suffix}' for suffix in suffixes + ]].mean(axis=1) + merged, name_mapping = shorten_names(merged) + + return name_mapping, merged, suffixes + + +def generate_perf_compare_report(perf_files: list[str], + output_path: str) -> None: + name_mapping, merged, suffixes = parse_perf_data(perf_files) + generate_plots(Path(output_path), name_mapping, merged, suffixes) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Create a report comparing multiple performance csvs") + parser.add_argument('--files', + nargs='*', + help="A list of csv files to compare") + parser.add_argument("--output_path", + type=str, + help="Output path for report (pdf file)") + return parser.parse_args() + + +def main() -> None: + args = parse_arguments() + generate_perf_compare_report(args.files, args.output_path) + + +if __name__ == '__main__': + main() diff --git a/tests/integration/defs/perf/diff_tools.py b/tests/integration/defs/perf/diff_tools.py new file mode 100644 index 000000000000..ce072fdcfa82 --- /dev/null +++ b/tests/integration/defs/perf/diff_tools.py @@ -0,0 +1,86 @@ +from io import StringIO + +import numpy as np +import pandas as pd + +PERF_CASE_NAME = 'perf_case_name' +PERF_METRIC = 'perf_metric' +THRESHOLD = 'threshold' +ABSOLUTE_THRESHOLD = 'absolute_threshold' +METRIC_TYPE = 'metric_type' +IGNORED_METRICS = {'BUILD_TIME'} + + +def load_file(csv_file: str) -> pd.DataFrame: + return pd.read_csv(csv_file) + + +def get_intersecting_metrics( + base: pd.DataFrame, target: pd.DataFrame +) -> tuple[pd.DataFrame, pd.DataFrame, pd.Series, pd.Series]: + missing_from_target = base.index.difference(target.index) + missing_from_base = target.index.difference(base.index) + + cleaned_base = base.drop(missing_from_target).sort_index() + cleaned_target = target.drop(missing_from_base).sort_index() + return cleaned_base, cleaned_target, base.loc[ + missing_from_target], target.loc[missing_from_base] + + +def get_diff_exceeding_threshold( + base: pd.DataFrame, + target: pd.DataFrame) -> tuple[np.array, pd.DataFrame]: + diff_exceeding_threshold = ~np.isclose(base[PERF_METRIC], + target[PERF_METRIC], + rtol=abs(base[THRESHOLD]), + atol=abs(base[ABSOLUTE_THRESHOLD])) + diff_exceeding_threshold = np.array([ + diff and base[METRIC_TYPE][i] not in IGNORED_METRICS + for i, diff in enumerate(diff_exceeding_threshold) + ]) + diff_mask = np.tile(diff_exceeding_threshold[:, None], + (1, target.shape[-1])) + return diff_exceeding_threshold, target.where(diff_mask, base) + + +def get_full_diff(base: pd.DataFrame, target: pd.DataFrame, + missing_from_base: pd.Series, missing_from_target: pd.Series, + diff_over_threshold: np.array) -> pd.DataFrame: + PERF_METRIC_BASE = f'{PERF_METRIC}_base' + PERF_METRIC_TARGET = f'{PERF_METRIC}_target' + thershold_diff = pd.merge(base, + target, + on=PERF_CASE_NAME, + how='outer', + suffixes=['_base', '_target']) + if not thershold_diff.empty: + thershold_diff = thershold_diff[diff_over_threshold][[ + PERF_METRIC_BASE, PERF_METRIC_TARGET + ]] + missing_from_base = missing_from_base.rename( + columns={PERF_METRIC: PERF_METRIC_TARGET})[[PERF_METRIC_TARGET]] + missing_from_target = missing_from_target.rename( + columns={PERF_METRIC: PERF_METRIC_BASE})[[PERF_METRIC_BASE]] + return pd.concat([thershold_diff, missing_from_base, missing_from_target]) + + +def get_diff(base: pd.DataFrame, + target: pd.DataFrame) -> tuple[pd.DataFrame, pd.DataFrame]: + relevant_columns = base.columns + base = base.set_index(PERF_CASE_NAME) + target = target.set_index(PERF_CASE_NAME) + cleaned_base, cleaned_target, missing_from_target, missing_from_base = get_intersecting_metrics( + base, target) + diff_over_threshold, new_df = get_diff_exceeding_threshold( + cleaned_base, cleaned_target) + full_diff = get_full_diff(cleaned_base, cleaned_target, missing_from_base, + missing_from_target, diff_over_threshold) + return full_diff, pd.concat([new_df, missing_from_base + ]).reset_index()[relevant_columns] + + +def get_csv_lines(df: pd.DataFrame) -> list[str]: + string_buffer = StringIO() + df.to_csv(string_buffer, index=False) + string_buffer.seek(0) + return string_buffer.readlines() diff --git a/tests/integration/defs/perf/requirements.txt b/tests/integration/defs/perf/requirements.txt new file mode 100644 index 000000000000..4d7329f9d923 --- /dev/null +++ b/tests/integration/defs/perf/requirements.txt @@ -0,0 +1,3 @@ +pandas +numpy<2 +matplotlib diff --git a/tests/integration/defs/perf/sanity_perf_check.py b/tests/integration/defs/perf/sanity_perf_check.py index f4e3829595cb..e00c34ad180c 100644 --- a/tests/integration/defs/perf/sanity_perf_check.py +++ b/tests/integration/defs/perf/sanity_perf_check.py @@ -12,121 +12,65 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import csv -import os +import difflib import sys +from pathlib import Path -# This is to prevent csv field size limit error -maxInt = sys.maxsize -while True: - try: - csv.field_size_limit(maxInt) - break - except OverflowError: - maxInt = int(maxInt / 10) +import pandas as pd +from diff_tools import get_csv_lines, get_diff, load_file class SanityPerfCheck(): - # This is to prevent redundant messages and long logs. - USEFUL_METRICS = [ - "original_test_name", "perf_case_name", "metric_type", "perf_metric", - "command", "sm_clk", "mem_clk", "start_timestamp", "end_timestamp", - "state", "threshold", "absolute_threshold" - ] - def __init__(self, target_perf_csv, base_perf_csv=None, threshold=0.1): - self.target_perf_csv = target_perf_csv - self.base_perf_csv = base_perf_csv + self.target_perf_csv = Path(target_perf_csv) + self.base_perf_csv = Path(base_perf_csv) self.threshold = threshold - def _parse_result(self, csv_path): - result = {} - with open(csv_path) as csv_file: - parsed_csv_file = csv.DictReader(csv_file) - for row in parsed_csv_file: - if row['metric_type'] not in result: - result[row['metric_type']] = {} - result[row['metric_type']][row['perf_case_name']] = float( - row['perf_metric']) - return result - - def _dump_csv_row(self, csv_path, metric_type, test_name): - with open(csv_path) as csv_file: - parsed_csv_file = csv.DictReader(csv_file) - for row in parsed_csv_file: - if row['metric_type'] == metric_type and row[ - 'perf_case_name'] == test_name: - print('=' * 40) - print('Please fill below content into the base_perf.csv.') - cleaned_row = [] - for k in self.USEFUL_METRICS: - v = row[k] - # Need to truncate the commands - if k == "command": - options = v.split(" ") - cleaned_options = [] - for option in options: - # Truncate workspace dir - if "build.py" in option or "SessionBenchmark.cpp" in option: - cleaned_options.append("/".join( - option.split("/")[-5:])) - # Remove engine_dir as it is not useful - elif "--engine_dir=" not in option and "--output_dir=" not in option: - cleaned_options.append(option) - cleaned_row.append(" ".join(cleaned_options)) - else: - cleaned_row.append(v) - - print(",".join(['\"' + row + '\"' for row in cleaned_row])) - print('=' * 40) - break + def report_diff(self, full_diff: pd.DataFrame) -> None: + print("=" * 40) + for diff in full_diff.itertuples(): + if pd.isna(diff.perf_metric_base): + print(f"perf_case_name: {diff.Index} is missing from base") + elif pd.isna(diff.perf_metric_target): + print(f"perf_case_name: {diff.Index} is missing from target") + else: + print( + f"perf_case_name: {diff.Index}, base->target: {diff.perf_metric_base}->{diff.perf_metric_target}" + ) + print("=" * 40) + + def write_patch(self, old_lines: list[str], new_lines: list[str], + output_path: str, base_perf_filename: str) -> None: + with open(output_path, 'w') as f: + for diff_line in difflib.unified_diff( + old_lines, new_lines, + f'a/tests/integration/defs/perf/{base_perf_filename}', + f'b/tests/integration/defs/perf/{base_perf_filename}'): + f.write(diff_line) def __call__(self, *args, **kwargs): # Check if the base_perf_csv file exists - if not os.path.exists(self.base_perf_csv): - print(f"base_perf.csv doesn't exist, skip check the perf result.") + if not self.base_perf_csv.exists(): + print( + f"{self.base_perf_csv.name} doesn't exist, skip check the perf result." + ) return 0 - base_result = self._parse_result(self.base_perf_csv) - target_result = self._parse_result(self.target_perf_csv) - - success = True - - for _, metric_type in enumerate(target_result): - # Engine build time is very CPU specific, so skip the check - if metric_type != "BUILD_TIME": - for _, test_name in enumerate(target_result[metric_type]): - if metric_type not in base_result or test_name not in base_result[ - metric_type]: - - self._dump_csv_row(self.target_perf_csv, metric_type, - test_name), - print( - f"{metric_type} {test_name} doesn't exist in the base_perf.csv, please add it and rerun the pipeline." - ) - success = False - else: - base_perf = base_result[metric_type][test_name] - target_perf = target_result[metric_type][test_name] - - if target_perf > base_perf * (1 + self.threshold): - # the mr perf is worse than baseline, there's perf regression. - print( - f"Perf Regression found on {metric_type} {test_name} where the current perf is {target_perf} while the baseline is {base_perf}." - ) - success = False - elif target_perf < base_perf * (1 - self.threshold): - # the MR perf is better than baseline, please update the base_perf.csv - self._dump_csv_row(self.target_perf_csv, - metric_type, test_name), - print( - f"Please update {metric_type} {test_name} into base_perf.csv and commit again. The outdated perf baseline is {base_perf} and the new perf baseline is {target_perf}" - ) - success = False - - if not success: - # We have temporarily disabled post perf sanity tests + base_perf = load_file(self.base_perf_csv.as_posix()) + current_perf = load_file(self.target_perf_csv.as_posix()) + + full_diff, new_base = get_diff(base_perf, current_perf) + if not full_diff.empty: + self.report_diff(full_diff) + output_patch = self.target_perf_csv.with_name( + 'perf_patch.patch').as_posix() + self.write_patch(get_csv_lines(base_perf), get_csv_lines(new_base), + output_patch, self.base_perf_csv.name) + print(f"patch_file was written to {output_patch}") + print( + "You can download the file and update base_perf.csv by `git apply `" + ) print("Sanity perf check failed, but it has been disabled") return 0 diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py index 129731947b2a..9ca08f4d029d 100644 --- a/tests/integration/defs/perf/utils.py +++ b/tests/integration/defs/perf/utils.py @@ -409,6 +409,9 @@ def run_ex(self, self._gpu_clock_lock = gpu_clock_lock tmpDir = temp_wd(self.get_working_dir()) + is_prepare_dataset_cmd = 'prepare_dataset' in commands.get_cmd_str( + cmd_idx) + # Start the timer. self._start_timestamp = datetime.utcnow() try: @@ -422,16 +425,17 @@ def run_ex(self, buf), self._gpu_clock_lock, tmpDir: output = commands.run_cmd(cmd_idx, venv) # Print the output log to buf. + # if not is_prepare_dataset_cmd: print(collect_and_clean_myelin_time(output)) else: with contextlib.redirect_stdout(buf), tmpDir: output = commands.run_cmd(cmd_idx, venv) # Print the output log to buf. + # if not is_prepare_dataset_cmd: print(collect_and_clean_myelin_time(output)) # Print the output log to stdout and cache it. - # skip the output log for prepare dataset command - if 'prepare_dataset' not in commands.get_cmd_str(cmd_idx): + if not is_prepare_dataset_cmd: print(buf.getvalue()) outputs[cmd_idx] = buf.getvalue() else: @@ -464,10 +468,11 @@ def run_ex(self, # Only save perf result if the result is valid. if self._result_state == "valid": # Parse the perf result from the test outputs. - if self._config.runtime == 'bench' and cmd_idx == 0: + if is_prepare_dataset_cmd: print_info( f"skip writing perf result when calling generating dataset in trtllm-bench" ) + outputs.pop(cmd_idx) else: self._perf_result = self.get_perf_result(outputs) @@ -532,8 +537,6 @@ def _write_result(self, full_test_name: str, "original_test_name": original_test_name if original_test_name is not None else full_test_name, - "raw_result": - raw_result, "perf_metric": self._perf_result, "total_time__sec": @@ -562,8 +565,9 @@ def _write_result(self, full_test_name: str, if "csv" in session_data_writer._output_formats: csv_name = "perf_script_test_results.csv" cvs_result_dict = {**test_description_dict, **test_result_dict} - cvs_result_dict["raw_result"] = cvs_result_dict[ - "raw_result"].replace("\n", "\\n") + if "raw_result" in cvs_result_dict: + cvs_result_dict["raw_result"] = cvs_result_dict[ + "raw_result"].replace("\n", "\\n") write_csv(output_dir, csv_name, [cvs_result_dict], list(cvs_result_dict.keys()), diff --git a/tests/integration/test_lists/test-db/l0_perf.yml b/tests/integration/test_lists/test-db/l0_perf.yml index 731b4371ea0b..21f5a11dfb07 100644 --- a/tests/integration/test_lists/test-db/l0_perf.yml +++ b/tests/integration/test_lists/test-db/l0_perf.yml @@ -7,16 +7,25 @@ l0_perf: lte: 1 wildcards: gpu: - - '*a100*' - '*h100*' linux_distribution_name: ubuntu* terms: stage: pre_merge backend: tensorrt tests: - - perf/test_perf.py::test_perf[bert_base-cpp-plugin-float16-bs:32-input_len:32] - - perf/test_perf.py::test_perf[bert_base-cpp-ootb-float16-bs:32-input_len:32] - - perf/test_perf.py::test_perf[roberta_base-cpp-plugin-float16-bs:32-input_len:128+512] - - perf/test_perf.py::test_perf[gpt_350m-cppmanager-plugin_ifb-float16-bs:32-input_output_len:60,20] - - perf/test_perf.py::test_perf[gpt_350m-cppmanager-plugin_ifb-float16-gwp:0.0-bs:32-input_output_len:60,20] - - perf/test_perf.py::test_perf[gpt_350m-cppmanager-static_batching-plugin_ifb-float16-bs:32-input_output_len:60,20] + - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-float16-input_output_len:128,128-reqs:8192] + + - condition: + ranges: + system_gpu_count: + gte: 1 + lte: 1 + wildcards: + gpu: + - '*h100*' + linux_distribution_name: ubuntu* + terms: + stage: pre_merge + backend: pytorch + tests: + - perf/test_perf.py::test_perf[llama_v3.1_8b_instruct-bench-pytorch-float16-input_output_len:128,128-reqs:8192]